Pyspark- DataframesTransformations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "02eb6a99",
   "metadata": {},
   "source": [
    "#### This notebook is about the Transformation to be done on DFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fe76488c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkConf, SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c1e3bac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "employees = [\n",
    "    (1, \"Scott\", \"Tiger\", 1000.0, \n",
    "      \"united states\", \"+1 123 456 7890\", \"123 45 6789\"\n",
    "    ),\n",
    "     (2, \"Henry\", \"Ford\", 1250.0, \n",
    "      \"India\", \"+91 234 567 8901\", \"456 78 9123\"\n",
    "     ),\n",
    "     (3, \"Nick\", \"Junior\", 750.0, \n",
    "      \"united KINGDOM\", \"+44 111 111 1111\", \"222 33 4444\"\n",
    "     ),\n",
    "     (4, \"Bill\", \"Gomes\", 1500.0, \n",
    "      \"AUSTRALIA\", \"+61 987 654 3210\", \"789 12 6118\"\n",
    "     )\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b0b61326",
   "metadata": {},
   "outputs": [],
   "source": [
    "productPath = \"/home/solverbot/spark-warehouse/retail_db/products/part-00000\"\n",
    "orderitemPath = \"/home/solverbot/spark-warehouse/retail_db/order_items/part-00000\"\n",
    "ordersPath = \"/home/solverbot/spark-warehouse/retail_db/orders/part-00000.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "baa1b48d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/11/22 04:41:09 WARN Utils: Your hostname, codeStation resolves to a loopback address: 127.0.1.1; using 172.17.0.1 instead (on interface docker0)\n",
      "22/11/22 04:41:09 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/11/22 04:41:11 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "#What is the difference between Session and Context?\n",
    "#SC is part of the Spark session that is established above\n",
    "spark = SparkSession.builder.appName('DF Tranformations').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f609fb46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://172.17.0.1:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>DF Tranformations</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local[*] appName=DF Tranformations>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc = spark.sparkContext\n",
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4fb19e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "employeesDF = spark. \\\n",
    "    createDataFrame(employees,\n",
    "                    schema=\"\"\"employee_id INT, first_name STRING, \n",
    "                    last_name STRING, salary FLOAT, nationality STRING,\n",
    "                    phone_number STRING, ssn STRING\"\"\"\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e5e70e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "be6049b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:==============>                                            (1 + 3) / 4]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+-----+\n",
      "|upper_nationality|count|\n",
      "+-----------------+-----+\n",
      "|    UNITED STATES|    1|\n",
      "|            INDIA|    1|\n",
      "|   UNITED KINGDOM|    1|\n",
      "|        AUSTRALIA|    1|\n",
      "+-----------------+-----+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "employeesDF. \\\n",
    "    groupBy(upper(col(\"nationality\")).alias('upper_nationality')). \\\n",
    "    count(). \\\n",
    "    show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "92622c04",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "orderItemDF = spark.read.csv(\"/home/solverbot/spark-warehouse/retail_db/order_items/\",inferSchema=True).toDF(\"order_item_id\",\"order_item_order_id\",\"product_id\", \"qty\",\"product_cost\",\"order_subtotal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25050fca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "orderDF = spark.read.csv(\"/home/solverbot/spark-warehouse/retail_db/orders\",inferSchema=True) \\\n",
    "            .toDF(\"order_id\",\"order_date\",\"order_customer_id\",\"order_status\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "885564ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+---------------+\n",
      "|order_id|         order_date|order_customer_id|   order_status|\n",
      "+--------+-------------------+-----------------+---------------+\n",
      "|       1|2013-07-25 00:00:00|            11599|         CLOSED|\n",
      "|       2|2013-07-25 00:00:00|              256|PENDING_PAYMENT|\n",
      "+--------+-------------------+-----------------+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8f3db55d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|order_item_id|order_item_order_id|product_id|qty|product_cost|order_subtotal|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|            1|                  1|       957|  1|      299.98|        299.98|\n",
      "|            2|                  2|      1073|  1|      199.99|        199.99|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderItemDF.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c4d25568",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- order_id: integer (nullable = true)\n",
      " |-- order_date: timestamp (nullable = true)\n",
      " |-- order_customer_id: integer (nullable = true)\n",
      " |-- order_status: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Note, even the numbers are read as strings when the DF is created\n",
    "orderDF.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f3b5bd1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- order_item_id: integer (nullable = true)\n",
      " |-- order_item_order_id: integer (nullable = true)\n",
      " |-- product_id: integer (nullable = true)\n",
      " |-- qty: integer (nullable = true)\n",
      " |-- product_cost: double (nullable = true)\n",
      " |-- order_subtotal: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderItemDF.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7d5f5005",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method filter in module pyspark.sql.dataframe:\n",
      "\n",
      "filter(condition: 'ColumnOrName') -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Filters rows using the given condition.\n",
      "    \n",
      "    :func:`where` is an alias for :func:`filter`.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    condition : :class:`Column` or str\n",
      "        a :class:`Column` of :class:`types.BooleanType`\n",
      "        or a string of SQL expression.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df.filter(df.age > 3).collect()\n",
      "    [Row(age=5, name='Bob')]\n",
      "    >>> df.where(df.age == 2).collect()\n",
      "    [Row(age=2, name='Alice')]\n",
      "    \n",
      "    >>> df.filter(\"age > 3\").collect()\n",
      "    [Row(age=5, name='Bob')]\n",
      "    >>> df.where(\"age = 2\").collect()\n",
      "    [Row(age=2, name='Alice')]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b53a348a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+---------------+\n",
      "|order_id|         order_date|order_customer_id|   order_status|\n",
      "+--------+-------------------+-----------------+---------------+\n",
      "|       1|2013-07-25 00:00:00|            11599|         CLOSED|\n",
      "|       2|2013-07-25 00:00:00|              256|PENDING_PAYMENT|\n",
      "+--------+-------------------+-----------------+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "filterOrder = orderDF.filter(orderDF.order_status != \"COMPLETE\")\n",
    "filterOrder.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "e2b3e616",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+------------+\n",
      "|order_id|         order_date|order_customer_id|order_status|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "|       1|2013-07-25 00:00:00|            11599|      CLOSED|\n",
      "|       3|2013-07-25 00:00:00|            12111|    COMPLETE|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.where('order_status IN (\"COMPLETE\",\"CLOSED\")').show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "c9e479e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|(order_status = COMPLETE)|\n",
      "+-------------------------+\n",
      "|                    false|\n",
      "|                    false|\n",
      "+-------------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.selectExpr('order_status =\"COMPLETE\"').alias('filtered').show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "0e0229d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on Column in module pyspark.sql.column object:\n",
      "\n",
      "class Column(builtins.object)\n",
      " |  Column(jc: py4j.java_gateway.JavaObject) -> None\n",
      " |  \n",
      " |  A column in a DataFrame.\n",
      " |  \n",
      " |  :class:`Column` instances can be created by::\n",
      " |  \n",
      " |      # 1. Select a column out of a DataFrame\n",
      " |  \n",
      " |      df.colName\n",
      " |      df[\"colName\"]\n",
      " |  \n",
      " |      # 2. Create from an expression\n",
      " |      df.colName + 1\n",
      " |      1 / df.colName\n",
      " |  \n",
      " |  .. versionadded:: 1.3.0\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __add__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __and__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __bool__ = __nonzero__(self) -> None\n",
      " |  \n",
      " |  __contains__(self, item: Any) -> None\n",
      " |      # container operators\n",
      " |  \n",
      " |  __div__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __eq__(self, other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary function\n",
      " |  \n",
      " |  __ge__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __getattr__(self, item: Any) -> 'Column'\n",
      " |  \n",
      " |  __getitem__(self, k: Any) -> 'Column'\n",
      " |  \n",
      " |  __gt__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __init__(self, jc: py4j.java_gateway.JavaObject) -> None\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  __invert__ = _(self: 'Column') -> 'Column'\n",
      " |  \n",
      " |  __iter__(self) -> None\n",
      " |  \n",
      " |  __le__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __lt__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __mod__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __mul__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __ne__(self, other: Any) -> 'Column'\n",
      " |      binary function\n",
      " |  \n",
      " |  __neg__ = _(self: 'Column') -> 'Column'\n",
      " |  \n",
      " |  __nonzero__(self) -> None\n",
      " |  \n",
      " |  __or__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __pow__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary function\n",
      " |  \n",
      " |  __radd__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __rand__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __rdiv__ = _(self: 'Column', other: Union[ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __repr__(self) -> str\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __rmod__ = _(self: 'Column', other: Union[ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __rmul__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __ror__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __rpow__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary function\n",
      " |  \n",
      " |  __rsub__ = _(self: 'Column', other: Union[ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __rtruediv__ = _(self: 'Column', other: Union[ForwardRef('LiteralType'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __sub__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  __truediv__ = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      binary operator\n",
      " |  \n",
      " |  alias(self, *alias: str, **kwargs: Any) -> 'Column'\n",
      " |      Returns this column aliased with a new name or names (in the case of expressions that\n",
      " |      return more than one column, such as explode).\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      alias : str\n",
      " |          desired column names (collects all positional arguments passed)\n",
      " |      \n",
      " |      Other Parameters\n",
      " |      ----------------\n",
      " |      metadata: dict\n",
      " |          a dict of information to be stored in ``metadata`` attribute of the\n",
      " |          corresponding :class:`StructField <pyspark.sql.types.StructField>` (optional, keyword\n",
      " |          only argument)\n",
      " |      \n",
      " |          .. versionchanged:: 2.2.0\n",
      " |             Added optional ``metadata`` argument.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.select(df.age.alias(\"age2\")).collect()\n",
      " |      [Row(age2=2), Row(age2=5)]\n",
      " |      >>> df.select(df.age.alias(\"age3\", metadata={'max': 99})).schema['age3'].metadata['max']\n",
      " |      99\n",
      " |  \n",
      " |  asc = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on ascending order of the column.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.asc()).collect()\n",
      " |      [Row(name='Alice'), Row(name='Tom')]\n",
      " |  \n",
      " |  asc_nulls_first = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on ascending order of the column, and null values\n",
      " |      return before non-null values.\n",
      " |      \n",
      " |      .. versionadded:: 2.4.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()\n",
      " |      [Row(name=None), Row(name='Alice'), Row(name='Tom')]\n",
      " |  \n",
      " |  asc_nulls_last = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on ascending order of the column, and null values\n",
      " |      appear after non-null values.\n",
      " |      \n",
      " |      .. versionadded:: 2.4.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()\n",
      " |      [Row(name='Alice'), Row(name='Tom'), Row(name=None)]\n",
      " |  \n",
      " |  astype = cast(self, dataType)\n",
      " |      :func:`astype` is an alias for :func:`cast`.\n",
      " |      \n",
      " |      .. versionadded:: 1.4\n",
      " |  \n",
      " |  between(self, lowerBound: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DateTimeLiteral'), ForwardRef('DecimalLiteral')], upperBound: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DateTimeLiteral'), ForwardRef('DecimalLiteral')]) -> 'Column'\n",
      " |      True if the current column is between the lower bound and upper bound, inclusive.\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.select(df.name, df.age.between(2, 4)).show()\n",
      " |      +-----+---------------------------+\n",
      " |      | name|((age >= 2) AND (age <= 4))|\n",
      " |      +-----+---------------------------+\n",
      " |      |Alice|                       true|\n",
      " |      |  Bob|                      false|\n",
      " |      +-----+---------------------------+\n",
      " |  \n",
      " |  bitwiseAND = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      Compute bitwise AND of this expression with another expression.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other\n",
      " |          a value or :class:`Column` to calculate bitwise and(&) with\n",
      " |          this :class:`Column`.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(a=170, b=75)])\n",
      " |      >>> df.select(df.a.bitwiseAND(df.b)).collect()\n",
      " |      [Row((a & b)=10)]\n",
      " |  \n",
      " |  bitwiseOR = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      Compute bitwise OR of this expression with another expression.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other\n",
      " |          a value or :class:`Column` to calculate bitwise or(|) with\n",
      " |          this :class:`Column`.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(a=170, b=75)])\n",
      " |      >>> df.select(df.a.bitwiseOR(df.b)).collect()\n",
      " |      [Row((a | b)=235)]\n",
      " |  \n",
      " |  bitwiseXOR = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      Compute bitwise XOR of this expression with another expression.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other\n",
      " |          a value or :class:`Column` to calculate bitwise xor(^) with\n",
      " |          this :class:`Column`.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(a=170, b=75)])\n",
      " |      >>> df.select(df.a.bitwiseXOR(df.b)).collect()\n",
      " |      [Row((a ^ b)=225)]\n",
      " |  \n",
      " |  cast(self, dataType: Union[pyspark.sql.types.DataType, str]) -> 'Column'\n",
      " |      Casts the column into type ``dataType``.\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.select(df.age.cast(\"string\").alias('ages')).collect()\n",
      " |      [Row(ages='2'), Row(ages='5')]\n",
      " |      >>> df.select(df.age.cast(StringType()).alias('ages')).collect()\n",
      " |      [Row(ages='2'), Row(ages='5')]\n",
      " |  \n",
      " |  contains = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      Contains the other element. Returns a boolean :class:`Column` based on a string match.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other\n",
      " |          string in line. A value as a literal or a :class:`Column`.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.contains('o')).collect()\n",
      " |      [Row(age=5, name='Bob')]\n",
      " |  \n",
      " |  desc = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on the descending order of the column.\n",
      " |      \n",
      " |      .. versionadded:: 2.4.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.desc()).collect()\n",
      " |      [Row(name='Tom'), Row(name='Alice')]\n",
      " |  \n",
      " |  desc_nulls_first = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on the descending order of the column, and null values\n",
      " |      appear before non-null values.\n",
      " |      \n",
      " |      .. versionadded:: 2.4.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()\n",
      " |      [Row(name=None), Row(name='Tom'), Row(name='Alice')]\n",
      " |  \n",
      " |  desc_nulls_last = _(self: 'Column') -> 'Column'\n",
      " |      Returns a sort expression based on the descending order of the column, and null values\n",
      " |      appear after non-null values.\n",
      " |      \n",
      " |      .. versionadded:: 2.4.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], [\"name\", \"height\"])\n",
      " |      >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()\n",
      " |      [Row(name='Tom'), Row(name='Alice'), Row(name=None)]\n",
      " |  \n",
      " |  dropFields(self, *fieldNames: str) -> 'Column'\n",
      " |      An expression that drops fields in :class:`StructType` by name.\n",
      " |      This is a no-op if schema doesn't contain field name(s).\n",
      " |      \n",
      " |      .. versionadded:: 3.1.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> from pyspark.sql.functions import col, lit\n",
      " |      >>> df = spark.createDataFrame([\n",
      " |      ...     Row(a=Row(b=1, c=2, d=3, e=Row(f=4, g=5, h=6)))])\n",
      " |      >>> df.withColumn('a', df['a'].dropFields('b')).show()\n",
      " |      +-----------------+\n",
      " |      |                a|\n",
      " |      +-----------------+\n",
      " |      |{2, 3, {4, 5, 6}}|\n",
      " |      +-----------------+\n",
      " |      \n",
      " |      >>> df.withColumn('a', df['a'].dropFields('b', 'c')).show()\n",
      " |      +--------------+\n",
      " |      |             a|\n",
      " |      +--------------+\n",
      " |      |{3, {4, 5, 6}}|\n",
      " |      +--------------+\n",
      " |      \n",
      " |      This method supports dropping multiple nested fields directly e.g.\n",
      " |      \n",
      " |      >>> df.withColumn(\"a\", col(\"a\").dropFields(\"e.g\", \"e.h\")).show()\n",
      " |      +--------------+\n",
      " |      |             a|\n",
      " |      +--------------+\n",
      " |      |{1, 2, 3, {4}}|\n",
      " |      +--------------+\n",
      " |      \n",
      " |      However, if you are going to add/replace multiple nested fields,\n",
      " |      it is preferred to extract out the nested struct before\n",
      " |      adding/replacing multiple fields e.g.\n",
      " |      \n",
      " |      >>> df.select(col(\"a\").withField(\n",
      " |      ...     \"e\", col(\"a.e\").dropFields(\"g\", \"h\")).alias(\"a\")\n",
      " |      ... ).show()\n",
      " |      +--------------+\n",
      " |      |             a|\n",
      " |      +--------------+\n",
      " |      |{1, 2, 3, {4}}|\n",
      " |      +--------------+\n",
      " |  \n",
      " |  endswith = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      String ends with. Returns a boolean :class:`Column` based on a string match.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other : :class:`Column` or str\n",
      " |          string at end of line (do not use a regex `$`)\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.endswith('ice')).collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |      >>> df.filter(df.name.endswith('ice$')).collect()\n",
      " |      []\n",
      " |  \n",
      " |  eqNullSafe = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      Equality test that is safe for null values.\n",
      " |      \n",
      " |      .. versionadded:: 2.3.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other\n",
      " |          a value or :class:`Column`\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df1 = spark.createDataFrame([\n",
      " |      ...     Row(id=1, value='foo'),\n",
      " |      ...     Row(id=2, value=None)\n",
      " |      ... ])\n",
      " |      >>> df1.select(\n",
      " |      ...     df1['value'] == 'foo',\n",
      " |      ...     df1['value'].eqNullSafe('foo'),\n",
      " |      ...     df1['value'].eqNullSafe(None)\n",
      " |      ... ).show()\n",
      " |      +-------------+---------------+----------------+\n",
      " |      |(value = foo)|(value <=> foo)|(value <=> NULL)|\n",
      " |      +-------------+---------------+----------------+\n",
      " |      |         true|           true|           false|\n",
      " |      |         null|          false|            true|\n",
      " |      +-------------+---------------+----------------+\n",
      " |      >>> df2 = spark.createDataFrame([\n",
      " |      ...     Row(value = 'bar'),\n",
      " |      ...     Row(value = None)\n",
      " |      ... ])\n",
      " |      >>> df1.join(df2, df1[\"value\"] == df2[\"value\"]).count()\n",
      " |      0\n",
      " |      >>> df1.join(df2, df1[\"value\"].eqNullSafe(df2[\"value\"])).count()\n",
      " |      1\n",
      " |      >>> df2 = spark.createDataFrame([\n",
      " |      ...     Row(id=1, value=float('NaN')),\n",
      " |      ...     Row(id=2, value=42.0),\n",
      " |      ...     Row(id=3, value=None)\n",
      " |      ... ])\n",
      " |      >>> df2.select(\n",
      " |      ...     df2['value'].eqNullSafe(None),\n",
      " |      ...     df2['value'].eqNullSafe(float('NaN')),\n",
      " |      ...     df2['value'].eqNullSafe(42.0)\n",
      " |      ... ).show()\n",
      " |      +----------------+---------------+----------------+\n",
      " |      |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|\n",
      " |      +----------------+---------------+----------------+\n",
      " |      |           false|           true|           false|\n",
      " |      |           false|          false|            true|\n",
      " |      |            true|          false|           false|\n",
      " |      +----------------+---------------+----------------+\n",
      " |      \n",
      " |      Notes\n",
      " |      -----\n",
      " |      Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the\n",
      " |      `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_\n",
      " |      for details.\n",
      " |  \n",
      " |  getField(self, name: Any) -> 'Column'\n",
      " |      An expression that gets a field by name in a :class:`StructType`.\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(r=Row(a=1, b=\"b\"))])\n",
      " |      >>> df.select(df.r.getField(\"b\")).show()\n",
      " |      +---+\n",
      " |      |r.b|\n",
      " |      +---+\n",
      " |      |  b|\n",
      " |      +---+\n",
      " |      >>> df.select(df.r.a).show()\n",
      " |      +---+\n",
      " |      |r.a|\n",
      " |      +---+\n",
      " |      |  1|\n",
      " |      +---+\n",
      " |  \n",
      " |  getItem(self, key: Any) -> 'Column'\n",
      " |      An expression that gets an item at position ``ordinal`` out of a list,\n",
      " |      or gets an item by key out of a dict.\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df = spark.createDataFrame([([1, 2], {\"key\": \"value\"})], [\"l\", \"d\"])\n",
      " |      >>> df.select(df.l.getItem(0), df.d.getItem(\"key\")).show()\n",
      " |      +----+------+\n",
      " |      |l[0]|d[key]|\n",
      " |      +----+------+\n",
      " |      |   1| value|\n",
      " |      +----+------+\n",
      " |  \n",
      " |  ilike = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      SQL ILIKE expression (case insensitive LIKE). Returns a boolean :class:`Column`\n",
      " |      based on a case insensitive match.\n",
      " |      \n",
      " |      .. versionadded:: 3.3.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other : str\n",
      " |          a SQL LIKE pattern\n",
      " |      \n",
      " |      See Also\n",
      " |      --------\n",
      " |      pyspark.sql.Column.rlike\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.ilike('%Ice')).collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |  \n",
      " |  isNotNull = _(self: 'Column') -> 'Column'\n",
      " |      True if the current expression is NOT null.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])\n",
      " |      >>> df.filter(df.height.isNotNull()).collect()\n",
      " |      [Row(name='Tom', height=80)]\n",
      " |  \n",
      " |  isNull = _(self: 'Column') -> 'Column'\n",
      " |      True if the current expression is null.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])\n",
      " |      >>> df.filter(df.height.isNull()).collect()\n",
      " |      [Row(name='Alice', height=None)]\n",
      " |  \n",
      " |  isin(self, *cols: Any) -> 'Column'\n",
      " |      A boolean expression that is evaluated to true if the value of this\n",
      " |      expression is contained by the evaluated values of the arguments.\n",
      " |      \n",
      " |      .. versionadded:: 1.5.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df[df.name.isin(\"Bob\", \"Mike\")].collect()\n",
      " |      [Row(age=5, name='Bob')]\n",
      " |      >>> df[df.age.isin([1, 2, 3])].collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |  \n",
      " |  like = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other : str\n",
      " |          a SQL LIKE pattern\n",
      " |      \n",
      " |      See Also\n",
      " |      --------\n",
      " |      pyspark.sql.Column.rlike\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.like('Al%')).collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |  \n",
      " |  name = alias(self, *alias, **kwargs)\n",
      " |      :func:`name` is an alias for :func:`alias`.\n",
      " |      \n",
      " |      .. versionadded:: 2.0\n",
      " |  \n",
      " |  otherwise(self, value: Any) -> 'Column'\n",
      " |      Evaluates a list of conditions and returns one of multiple possible result expressions.\n",
      " |      If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.\n",
      " |      \n",
      " |      .. versionadded:: 1.4.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      value\n",
      " |          a literal value, or a :class:`Column` expression.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import functions as F\n",
      " |      >>> df.select(df.name, F.when(df.age > 3, 1).otherwise(0)).show()\n",
      " |      +-----+-------------------------------------+\n",
      " |      | name|CASE WHEN (age > 3) THEN 1 ELSE 0 END|\n",
      " |      +-----+-------------------------------------+\n",
      " |      |Alice|                                    0|\n",
      " |      |  Bob|                                    1|\n",
      " |      +-----+-------------------------------------+\n",
      " |      \n",
      " |      See Also\n",
      " |      --------\n",
      " |      pyspark.sql.functions.when\n",
      " |  \n",
      " |  over(self, window: 'WindowSpec') -> 'Column'\n",
      " |      Define a windowing column.\n",
      " |      \n",
      " |      .. versionadded:: 1.4.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      window : :class:`WindowSpec`\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      :class:`Column`\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Window\n",
      " |      >>> window = Window.partitionBy(\"name\").orderBy(\"age\")                 .rowsBetween(Window.unboundedPreceding, Window.currentRow)\n",
      " |      >>> from pyspark.sql.functions import rank, min\n",
      " |      >>> from pyspark.sql.functions import desc\n",
      " |      >>> df.withColumn(\"rank\", rank().over(window))                 .withColumn(\"min\", min('age').over(window)).sort(desc(\"age\")).show()\n",
      " |      +---+-----+----+---+\n",
      " |      |age| name|rank|min|\n",
      " |      +---+-----+----+---+\n",
      " |      |  5|  Bob|   1|  5|\n",
      " |      |  2|Alice|   1|  2|\n",
      " |      +---+-----+----+---+\n",
      " |  \n",
      " |  rlike = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex\n",
      " |      match.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other : str\n",
      " |          an extended regex expression\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.rlike('ice$')).collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |  \n",
      " |  startswith = _(self: 'Column', other: Union[ForwardRef('Column'), ForwardRef('LiteralType'), ForwardRef('DecimalLiteral'), ForwardRef('DateTimeLiteral')]) -> 'Column'\n",
      " |      String starts with. Returns a boolean :class:`Column` based on a string match.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      other : :class:`Column` or str\n",
      " |          string at start of line (do not use a regex `^`)\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.filter(df.name.startswith('Al')).collect()\n",
      " |      [Row(age=2, name='Alice')]\n",
      " |      >>> df.filter(df.name.startswith('^Al')).collect()\n",
      " |      []\n",
      " |  \n",
      " |  substr(self, startPos: Union[int, ForwardRef('Column')], length: Union[int, ForwardRef('Column')]) -> 'Column'\n",
      " |      Return a :class:`Column` which is a substring of the column.\n",
      " |      \n",
      " |      .. versionadded:: 1.3.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      startPos : :class:`Column` or int\n",
      " |          start position\n",
      " |      length : :class:`Column` or int\n",
      " |          length of the substring\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> df.select(df.name.substr(1, 3).alias(\"col\")).collect()\n",
      " |      [Row(col='Ali'), Row(col='Bob')]\n",
      " |  \n",
      " |  when(self, condition: 'Column', value: Any) -> 'Column'\n",
      " |      Evaluates a list of conditions and returns one of multiple possible result expressions.\n",
      " |      If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.\n",
      " |      \n",
      " |      .. versionadded:: 1.4.0\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      condition : :class:`Column`\n",
      " |          a boolean :class:`Column` expression.\n",
      " |      value\n",
      " |          a literal value, or a :class:`Column` expression.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import functions as F\n",
      " |      >>> df.select(df.name, F.when(df.age > 4, 1).when(df.age < 3, -1).otherwise(0)).show()\n",
      " |      +-----+------------------------------------------------------------+\n",
      " |      | name|CASE WHEN (age > 4) THEN 1 WHEN (age < 3) THEN -1 ELSE 0 END|\n",
      " |      +-----+------------------------------------------------------------+\n",
      " |      |Alice|                                                          -1|\n",
      " |      |  Bob|                                                           1|\n",
      " |      +-----+------------------------------------------------------------+\n",
      " |      \n",
      " |      See Also\n",
      " |      --------\n",
      " |      pyspark.sql.functions.when\n",
      " |  \n",
      " |  withField(self, fieldName: str, col: 'Column') -> 'Column'\n",
      " |      An expression that adds/replaces a field in :class:`StructType` by name.\n",
      " |      \n",
      " |      .. versionadded:: 3.1.0\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> from pyspark.sql import Row\n",
      " |      >>> from pyspark.sql.functions import lit\n",
      " |      >>> df = spark.createDataFrame([Row(a=Row(b=1, c=2))])\n",
      " |      >>> df.withColumn('a', df['a'].withField('b', lit(3))).select('a.b').show()\n",
      " |      +---+\n",
      " |      |  b|\n",
      " |      +---+\n",
      " |      |  3|\n",
      " |      +---+\n",
      " |      >>> df.withColumn('a', df['a'].withField('d', lit(4))).select('a.d').show()\n",
      " |      +---+\n",
      " |      |  d|\n",
      " |      +---+\n",
      " |      |  4|\n",
      " |      +---+\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes defined here:\n",
      " |  \n",
      " |  __hash__ = None\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.order_status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e4c929b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+------------+\n",
      "|order_id|         order_date|order_customer_id|order_status|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "|       1|2013-07-25 00:00:00|            11599|      CLOSED|\n",
      "|       3|2013-07-25 00:00:00|            12111|    COMPLETE|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.where((orderDF.order_status == 'COMPLETE').__or__(orderDF.order_status=='CLOSED')).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "442c29d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+------------+\n",
      "|order_id|         order_date|order_customer_id|order_status|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "|   20919|2013-12-01 00:00:00|              383|    COMPLETE|\n",
      "|   20922|2013-12-01 00:00:00|             9720|    COMPLETE|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#There is dateformat which needs to be understood\n",
    "orderDF.where((orderDF.order_status == 'COMPLETE').__and__(orderDF.order_date.startswith('2013-12'))).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3e3a653a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+------------+\n",
      "|order_id|         order_date|order_customer_id|order_status|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "|   25882|2014-01-01 00:00:00|             4598|    COMPLETE|\n",
      "|   25888|2014-01-01 00:00:00|             6735|    COMPLETE|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.where((orderDF.order_status=='COMPLETE').__and__(orderDF.order_date.like(\"2014-01%\"))).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "aabf184f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|order_item_id|order_item_order_id|product_id|qty|product_cost|order_subtotal|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|            3|                  2|       502|  5|       250.0|          50.0|\n",
      "|            5|                  4|       897|  2|       49.98|         24.99|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderItemDF.where(orderItemDF.order_subtotal != orderItemDF.qty * orderItemDF.product_cost).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c90b3f85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|order_item_id|order_item_order_id|product_id|qty|product_cost|order_subtotal|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "|            3|                  2|       502|  5|       250.0|          50.0|\n",
      "|            5|                  4|       897|  2|       49.98|         24.99|\n",
      "+-------------+-------------------+----------+---+------------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderItemDF.filter(orderItemDF.order_subtotal != orderItemDF.qty * orderItemDF.product_cost).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b6e7d4b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------------------------------------------------------+\n",
      "|CASE WHEN (order_status IN (COMPLETE, CLOSED)) THEN COMPLETED ELSE PENDING END|\n",
      "+------------------------------------------------------------------------------+\n",
      "|                                                                     COMPLETED|\n",
      "|                                                                       PENDING|\n",
      "+------------------------------------------------------------------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.selectExpr(\"CASE WHEN order_status IN ('COMPLETE','CLOSED') THEN 'COMPLETED' ELSE 'PENDING' END\").show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "17250338",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method join in module pyspark.sql.dataframe:\n",
      "\n",
      "join(other: 'DataFrame', on: Union[str, List[str], pyspark.sql.column.Column, List[pyspark.sql.column.Column], NoneType] = None, how: Optional[str] = None) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Joins with another :class:`DataFrame`, using the given join expression.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    other : :class:`DataFrame`\n",
      "        Right side of the join\n",
      "    on : str, list or :class:`Column`, optional\n",
      "        a string for the join column name, a list of column names,\n",
      "        a join expression (Column), or a list of Columns.\n",
      "        If `on` is a string or a list of strings indicating the name of the join column(s),\n",
      "        the column(s) must exist on both sides, and this performs an equi-join.\n",
      "    how : str, optional\n",
      "        default ``inner``. Must be one of: ``inner``, ``cross``, ``outer``,\n",
      "        ``full``, ``fullouter``, ``full_outer``, ``left``, ``leftouter``, ``left_outer``,\n",
      "        ``right``, ``rightouter``, ``right_outer``, ``semi``, ``leftsemi``, ``left_semi``,\n",
      "        ``anti``, ``leftanti`` and ``left_anti``.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    The following performs a full outer join between ``df1`` and ``df2``.\n",
      "    \n",
      "    >>> from pyspark.sql.functions import desc\n",
      "    >>> df.join(df2, df.name == df2.name, 'outer').select(df.name, df2.height)                 .sort(desc(\"name\")).collect()\n",
      "    [Row(name='Bob', height=85), Row(name='Alice', height=None), Row(name=None, height=80)]\n",
      "    \n",
      "    >>> df.join(df2, 'name', 'outer').select('name', 'height').sort(desc(\"name\")).collect()\n",
      "    [Row(name='Tom', height=80), Row(name='Bob', height=85), Row(name='Alice', height=None)]\n",
      "    \n",
      "    >>> cond = [df.name == df3.name, df.age == df3.age]\n",
      "    >>> df.join(df3, cond, 'outer').select(df.name, df3.age).collect()\n",
      "    [Row(name='Alice', age=2), Row(name='Bob', age=5)]\n",
      "    \n",
      "    >>> df.join(df2, 'name').select(df.name, df2.height).collect()\n",
      "    [Row(name='Bob', height=85)]\n",
      "    \n",
      "    >>> df.join(df4, ['name', 'age']).select(df.name, df.age).collect()\n",
      "    [Row(name='Bob', age=5)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.join)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e2f5186e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "orderJoined = orderDF.join(orderItemDF, orderDF.order_id == orderItemDF.order_item_id, 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "47044ac0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Row(order_id=1, order_date=datetime.datetime(2013, 7, 25, 0, 0), order_customer_id=11599, order_status='CLOSED', order_item_id=1, order_item_order_id=1, product_id=957, qty=1, product_cost=299.98, order_subtotal=299.98),\n",
       " Row(order_id=1, order_date=datetime.datetime(2013, 7, 25, 0, 0), order_customer_id=11599, order_status='CLOSED', order_item_id=1, order_item_order_id=1, product_id=957, qty=1, product_cost=299.98, order_subtotal=299.98)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orderJoined.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e436f108",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method groupBy in module pyspark.sql.dataframe:\n",
      "\n",
      "groupBy(*cols: 'ColumnOrName') -> 'GroupedData' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Groups the :class:`DataFrame` using the specified columns,\n",
      "    so we can run aggregation on them. See :class:`GroupedData`\n",
      "    for all the available aggregate functions.\n",
      "    \n",
      "    :func:`groupby` is an alias for :func:`groupBy`.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    cols : list, str or :class:`Column`\n",
      "        columns to group by.\n",
      "        Each element should be a column name (string) or an expression (:class:`Column`).\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df.groupBy().avg().collect()\n",
      "    [Row(avg(age)=3.5)]\n",
      "    >>> sorted(df.groupBy('name').agg({'age': 'mean'}).collect())\n",
      "    [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]\n",
      "    >>> sorted(df.groupBy(df.name).avg().collect())\n",
      "    [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]\n",
      "    >>> sorted(df.groupBy(['name', df.age]).count().collect())\n",
      "    [Row(name='Alice', age=2, count=1), Row(name='Bob', age=5, count=1)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.groupBy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f71a2ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "orderJoinedGroup= orderJoined.groupBy('order_id').agg(sum(col(\"order_subtotal\")).alias('order_total'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e037f206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----------+\n",
      "|order_id|order_total|\n",
      "+--------+-----------+\n",
      "|     148|      99.99|\n",
      "|     463|     399.98|\n",
      "+--------+-----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderJoinedGroup.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2a72bda1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method sort in module pyspark.sql.dataframe:\n",
      "\n",
      "sort(*cols: Union[str, pyspark.sql.column.Column, List[Union[str, pyspark.sql.column.Column]]], **kwargs: Any) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Returns a new :class:`DataFrame` sorted by the specified column(s).\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    cols : str, list, or :class:`Column`, optional\n",
      "         list of :class:`Column` or column names to sort by.\n",
      "    \n",
      "    Other Parameters\n",
      "    ----------------\n",
      "    ascending : bool or list, optional\n",
      "        boolean or list of boolean (default ``True``).\n",
      "        Sort ascending vs. descending. Specify list for multiple sort orders.\n",
      "        If a list is specified, length of the list must equal length of the `cols`.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df.sort(df.age.desc()).collect()\n",
      "    [Row(age=5, name='Bob'), Row(age=2, name='Alice')]\n",
      "    >>> df.sort(\"age\", ascending=False).collect()\n",
      "    [Row(age=5, name='Bob'), Row(age=2, name='Alice')]\n",
      "    >>> df.orderBy(df.age.desc()).collect()\n",
      "    [Row(age=5, name='Bob'), Row(age=2, name='Alice')]\n",
      "    >>> from pyspark.sql.functions import *\n",
      "    >>> df.sort(asc(\"age\")).collect()\n",
      "    [Row(age=2, name='Alice'), Row(age=5, name='Bob')]\n",
      "    >>> df.orderBy(desc(\"age\"), \"name\").collect()\n",
      "    [Row(age=5, name='Bob'), Row(age=2, name='Alice')]\n",
      "    >>> df.orderBy([\"age\", \"name\"], ascending=[0, 1]).collect()\n",
      "    [Row(age=5, name='Bob'), Row(age=2, name='Alice')]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderJoinedGroup.sort)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "65a46de7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 179:============================>                            (1 + 1) / 2]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----------+\n",
      "|order_id|order_total|\n",
      "+--------+-----------+\n",
      "|       1|     599.96|\n",
      "|       9|     599.96|\n",
      "|   18498|     399.98|\n",
      "|   26623|     399.98|\n",
      "|   26708|     399.98|\n",
      "|   27760|     399.98|\n",
      "|   38723|     399.98|\n",
      "|   30361|     399.98|\n",
      "|    1645|     399.98|\n",
      "|   30970|     399.98|\n",
      "|    3749|     399.98|\n",
      "|   29719|     399.98|\n",
      "|    6336|     399.98|\n",
      "|   10206|     399.98|\n",
      "|   30654|     399.98|\n",
      "|   13840|     399.98|\n",
      "|     463|     399.98|\n",
      "|   31912|     399.98|\n",
      "|    1829|     399.98|\n",
      "|   32396|     399.98|\n",
      "+--------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "orderJoinedGroup.sort(col(\"order_total\").desc()).dropna().show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "408821a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method select in module pyspark.sql.dataframe:\n",
      "\n",
      "select(*cols: 'ColumnOrName') -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Projects a set of expressions and returns a new :class:`DataFrame`.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    cols : str, :class:`Column`, or list\n",
      "        column names (string) or expressions (:class:`Column`).\n",
      "        If one of the column names is '*', that column is expanded to include all columns\n",
      "        in the current :class:`DataFrame`.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df.select('*').collect()\n",
      "    [Row(age=2, name='Alice'), Row(age=5, name='Bob')]\n",
      "    >>> df.select('name', 'age').collect()\n",
      "    [Row(name='Alice', age=2), Row(name='Bob', age=5)]\n",
      "    >>> df.select(df.name, (df.age + 10).alias('age')).collect()\n",
      "    [Row(name='Alice', age=12), Row(name='Bob', age=15)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.select)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "092a3b60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+---------------+\n",
      "|order_id|   order_status|\n",
      "+--------+---------------+\n",
      "|       1|         CLOSED|\n",
      "|       2|PENDING_PAYMENT|\n",
      "+--------+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.select(col('order_id'),col('order_status')).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "2650e7dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+---------------+\n",
      "|order_id|   order_status|\n",
      "+--------+---------------+\n",
      "|       1|         CLOSED|\n",
      "|       2|PENDING_PAYMENT|\n",
      "+--------+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.select('order_id','order_status').show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "fac0b678",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+\n",
      "|order_id|         order_date|\n",
      "+--------+-------------------+\n",
      "|       1|2013-07-25 00:00:00|\n",
      "|       2|2013-07-25 00:00:00|\n",
      "+--------+-------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.select(orderDF.order_id, orderDF.order_date).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b99b7dc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orderDF.rdd.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "91c4877f",
   "metadata": {},
   "outputs": [],
   "source": [
    "orderDF.createTempView(\"sqlView\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b726084e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|   order_status|\n",
      "+---------------+\n",
      "|         CLOSED|\n",
      "|PENDING_PAYMENT|\n",
      "+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#spark sql can work on only views, which are equivalent to the views \n",
    "spark.sql(\"SELECT order_status FROM sqlView\").show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1b3341ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+-----+\n",
      "|   order_status|count|\n",
      "+---------------+-----+\n",
      "|         CLOSED| 7558|\n",
      "|PENDING_PAYMENT|15033|\n",
      "|       COMPLETE|22903|\n",
      "|           null|68893|\n",
      "|        ON_HOLD| 3798|\n",
      "|     PROCESSING| 8276|\n",
      "| PAYMENT_REVIEW|  729|\n",
      "|        PENDING| 7610|\n",
      "|SUSPECTED_FRAUD| 1558|\n",
      "|       CANCELED| 1428|\n",
      "+---------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.cube(\"order_status\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "322b3304",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on module pyspark.sql.functions in pyspark.sql:\n",
      "\n",
      "NAME\n",
      "    pyspark.sql.functions - A collections of builtin functions\n",
      "\n",
      "FUNCTIONS\n",
      "    abs(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the absolute value.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    acos(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes inverse cosine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            inverse cosine of `col`, as if computed by `java.lang.Math.acos()`\n",
      "    \n",
      "    acosh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes inverse hyperbolic cosine of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "    \n",
      "    add_months(start: 'ColumnOrName', months: Union[ForwardRef('ColumnOrName'), int]) -> pyspark.sql.column.Column\n",
      "        Returns the date that is `months` months after `start`\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08', 2)], ['dt', 'add'])\n",
      "        >>> df.select(add_months(df.dt, 1).alias('next_month')).collect()\n",
      "        [Row(next_month=datetime.date(2015, 5, 8))]\n",
      "        >>> df.select(add_months(df.dt, df.add.cast('integer')).alias('next_month')).collect()\n",
      "        [Row(next_month=datetime.date(2015, 6, 8))]\n",
      "    \n",
      "    aggregate(col: 'ColumnOrName', initialValue: 'ColumnOrName', merge: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column], finish: Optional[Callable[[pyspark.sql.column.Column], pyspark.sql.column.Column]] = None) -> pyspark.sql.column.Column\n",
      "        Applies a binary operator to an initial state and all elements in the array,\n",
      "        and reduces this to a single state. The final state is converted into the final result\n",
      "        by applying a finish function.\n",
      "        \n",
      "        Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "        :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "        Python ``UserDefinedFunctions`` are not supported\n",
      "        (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        initialValue : :class:`~pyspark.sql.Column` or str\n",
      "            initial value. Name of column or expression\n",
      "        merge : function\n",
      "            a binary function ``(acc: Column, x: Column) -> Column...`` returning expression\n",
      "            of the same type as ``zero``\n",
      "        finish : function\n",
      "            an optional unary function ``(x: Column) -> Column: ...``\n",
      "            used to convert accumulated value.\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], (\"id\", \"values\"))\n",
      "        >>> df.select(aggregate(\"values\", lit(0.0), lambda acc, x: acc + x).alias(\"sum\")).show()\n",
      "        +----+\n",
      "        | sum|\n",
      "        +----+\n",
      "        |42.0|\n",
      "        +----+\n",
      "        \n",
      "        >>> def merge(acc, x):\n",
      "        ...     count = acc.count + 1\n",
      "        ...     sum = acc.sum + x\n",
      "        ...     return struct(count.alias(\"count\"), sum.alias(\"sum\"))\n",
      "        >>> df.select(\n",
      "        ...     aggregate(\n",
      "        ...         \"values\",\n",
      "        ...         struct(lit(0).alias(\"count\"), lit(0.0).alias(\"sum\")),\n",
      "        ...         merge,\n",
      "        ...         lambda acc: acc.sum / acc.count,\n",
      "        ...     ).alias(\"mean\")\n",
      "        ... ).show()\n",
      "        +----+\n",
      "        |mean|\n",
      "        +----+\n",
      "        | 8.4|\n",
      "        +----+\n",
      "    \n",
      "    approxCountDistinct(col: 'ColumnOrName', rsd: Optional[float] = None) -> pyspark.sql.column.Column\n",
      "        .. deprecated:: 2.1.0\n",
      "            Use :func:`approx_count_distinct` instead.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    approx_count_distinct(col: 'ColumnOrName', rsd: Optional[float] = None) -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns a new :class:`~pyspark.sql.Column` for approximate distinct count\n",
      "        of column `col`.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "        rsd : float, optional\n",
      "            maximum relative standard deviation allowed (default = 0.05).\n",
      "            For rsd < 0.01, it is more efficient to use :func:`count_distinct`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.agg(approx_count_distinct(df.age).alias('distinct_ages')).collect()\n",
      "        [Row(distinct_ages=2)]\n",
      "    \n",
      "    array(*cols: Union[ForwardRef('ColumnOrName'), List[ForwardRef('ColumnOrName_')], Tuple[ForwardRef('ColumnOrName_'), ...]]) -> pyspark.sql.column.Column\n",
      "        Creates a new array column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            column names or :class:`~pyspark.sql.Column`\\s that have\n",
      "            the same data type.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(array('age', 'age').alias(\"arr\")).collect()\n",
      "        [Row(arr=[2, 2]), Row(arr=[5, 5])]\n",
      "        >>> df.select(array([df.age, df.age]).alias(\"arr\")).collect()\n",
      "        [Row(arr=[2, 2]), Row(arr=[5, 5])]\n",
      "    \n",
      "    array_contains(col: 'ColumnOrName', value: Any) -> pyspark.sql.column.Column\n",
      "        Collection function: returns null if the array is null, true if the array contains the\n",
      "        given value, and false otherwise.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        value :\n",
      "            value or column to check for in array\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([\"a\", \"b\", \"c\"],), ([],)], ['data'])\n",
      "        >>> df.select(array_contains(df.data, \"a\")).collect()\n",
      "        [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]\n",
      "        >>> df.select(array_contains(df.data, lit(\"a\"))).collect()\n",
      "        [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]\n",
      "    \n",
      "    array_distinct(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: removes duplicate values from the array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data'])\n",
      "        >>> df.select(array_distinct(df.data)).collect()\n",
      "        [Row(array_distinct(data)=[1, 2, 3]), Row(array_distinct(data)=[4, 5])]\n",
      "    \n",
      "    array_except(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns an array of the elements in col1 but not in col2,\n",
      "        without duplicates.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        col2 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> df = spark.createDataFrame([Row(c1=[\"b\", \"a\", \"c\"], c2=[\"c\", \"d\", \"a\", \"f\"])])\n",
      "        >>> df.select(array_except(df.c1, df.c2)).collect()\n",
      "        [Row(array_except(c1, c2)=['b'])]\n",
      "    \n",
      "    array_intersect(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns an array of the elements in the intersection of col1 and col2,\n",
      "        without duplicates.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        col2 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> df = spark.createDataFrame([Row(c1=[\"b\", \"a\", \"c\"], c2=[\"c\", \"d\", \"a\", \"f\"])])\n",
      "        >>> df.select(array_intersect(df.c1, df.c2)).collect()\n",
      "        [Row(array_intersect(c1, c2)=['a', 'c'])]\n",
      "    \n",
      "    array_join(col: 'ColumnOrName', delimiter: str, null_replacement: Optional[str] = None) -> pyspark.sql.column.Column\n",
      "        Concatenates the elements of `column` using the `delimiter`. Null values are replaced with\n",
      "        `null_replacement` if set, otherwise they are ignored.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([\"a\", \"b\", \"c\"],), ([\"a\", None],)], ['data'])\n",
      "        >>> df.select(array_join(df.data, \",\").alias(\"joined\")).collect()\n",
      "        [Row(joined='a,b,c'), Row(joined='a')]\n",
      "        >>> df.select(array_join(df.data, \",\", \"NULL\").alias(\"joined\")).collect()\n",
      "        [Row(joined='a,b,c'), Row(joined='a,NULL')]\n",
      "    \n",
      "    array_max(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns the maximum value of the array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])\n",
      "        >>> df.select(array_max(df.data).alias('max')).collect()\n",
      "        [Row(max=3), Row(max=10)]\n",
      "    \n",
      "    array_min(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns the minimum value of the array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])\n",
      "        >>> df.select(array_min(df.data).alias('min')).collect()\n",
      "        [Row(min=1), Row(min=-1)]\n",
      "    \n",
      "    array_position(col: 'ColumnOrName', value: Any) -> pyspark.sql.column.Column\n",
      "        Collection function: Locates the position of the first occurrence of the given value\n",
      "        in the given array. Returns null if either of the arguments are null.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The position is not zero based, but 1 based index. Returns 0 if the given\n",
      "        value could not be found in the array.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([\"c\", \"b\", \"a\"],), ([],)], ['data'])\n",
      "        >>> df.select(array_position(df.data, \"a\")).collect()\n",
      "        [Row(array_position(data, a)=3), Row(array_position(data, a)=0)]\n",
      "    \n",
      "    array_remove(col: 'ColumnOrName', element: Any) -> pyspark.sql.column.Column\n",
      "        Collection function: Remove all elements that equal to element from the given array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        element :\n",
      "            element to be removed from the array\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([],)], ['data'])\n",
      "        >>> df.select(array_remove(df.data, 1)).collect()\n",
      "        [Row(array_remove(data, 1)=[2, 3]), Row(array_remove(data, 1)=[])]\n",
      "    \n",
      "    array_repeat(col: 'ColumnOrName', count: Union[ForwardRef('ColumnOrName'), int]) -> pyspark.sql.column.Column\n",
      "        Collection function: creates an array containing a column repeated count times.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            column name or column that contains the element to be repeated\n",
      "        count : :class:`~pyspark.sql.Column` or str or int\n",
      "            column name, column, or int containing the number of times to repeat the first argument\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('ab',)], ['data'])\n",
      "        >>> df.select(array_repeat(df.data, 3).alias('r')).collect()\n",
      "        [Row(r=['ab', 'ab', 'ab'])]\n",
      "    \n",
      "    array_sort(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: sorts the input array in ascending order. The elements of the input array\n",
      "        must be orderable. Null elements will be placed at the end of the returned array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])\n",
      "        >>> df.select(array_sort(df.data).alias('r')).collect()\n",
      "        [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])]\n",
      "    \n",
      "    array_union(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns an array of the elements in the union of col1 and col2,\n",
      "        without duplicates.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        col2 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> df = spark.createDataFrame([Row(c1=[\"b\", \"a\", \"c\"], c2=[\"c\", \"d\", \"a\", \"f\"])])\n",
      "        >>> df.select(array_union(df.c1, df.c2)).collect()\n",
      "        [Row(array_union(c1, c2)=['b', 'a', 'c', 'd', 'f'])]\n",
      "    \n",
      "    arrays_overlap(a1: 'ColumnOrName', a2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns true if the arrays contain any common non-null element; if not,\n",
      "        returns null if both the arrays are non-empty and any of them contains a null element; returns\n",
      "        false otherwise.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([\"a\", \"b\"], [\"b\", \"c\"]), ([\"a\"], [\"b\", \"c\"])], ['x', 'y'])\n",
      "        >>> df.select(arrays_overlap(df.x, df.y).alias(\"overlap\")).collect()\n",
      "        [Row(overlap=True), Row(overlap=False)]\n",
      "    \n",
      "    arrays_zip(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Returns a merged array of structs in which the N-th struct contains all\n",
      "        N-th values of input arrays.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            columns of arrays to be merged.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import arrays_zip\n",
      "        >>> df = spark.createDataFrame([(([1, 2, 3], [2, 3, 4]))], ['vals1', 'vals2'])\n",
      "        >>> df.select(arrays_zip(df.vals1, df.vals2).alias('zipped')).collect()\n",
      "        [Row(zipped=[Row(vals1=1, vals2=2), Row(vals1=2, vals2=3), Row(vals1=3, vals2=4)])]\n",
      "    \n",
      "    asc(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the ascending order of the given column name.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    asc_nulls_first(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the ascending order of the given\n",
      "        column name, and null values return before non-null values.\n",
      "        \n",
      "        .. versionadded:: 2.4\n",
      "    \n",
      "    asc_nulls_last(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the ascending order of the given\n",
      "        column name, and null values appear after non-null values.\n",
      "        \n",
      "        .. versionadded:: 2.4\n",
      "    \n",
      "    ascii(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the numeric value of the first character of the string column.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    asin(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes inverse sine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "        \n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            inverse sine of `col`, as if computed by `java.lang.Math.asin()`\n",
      "    \n",
      "    asinh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes inverse hyperbolic sine of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "    \n",
      "    assert_true(col: 'ColumnOrName', errMsg: Union[pyspark.sql.column.Column, str, NoneType] = None) -> pyspark.sql.column.Column\n",
      "        Returns null if the input column is true; throws an exception with the provided error message\n",
      "        otherwise.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            column name or column that represents the input column to test\n",
      "        errMsg : :class:`~pyspark.sql.Column` or str\n",
      "            A Python string literal or column containing the error message\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])\n",
      "        >>> df.select(assert_true(df.a < df.b).alias('r')).collect()\n",
      "        [Row(r=None)]\n",
      "        >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])\n",
      "        >>> df.select(assert_true(df.a < df.b, df.a).alias('r')).collect()\n",
      "        [Row(r=None)]\n",
      "        >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])\n",
      "        >>> df.select(assert_true(df.a < df.b, 'error').alias('r')).collect()\n",
      "        [Row(r=None)]\n",
      "    \n",
      "    atan(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Compute inverse tangent of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            inverse tangent of `col`, as if computed by `java.lang.Math.atan()`\n",
      "    \n",
      "    atan2(col1: Union[ForwardRef('ColumnOrName'), float], col2: Union[ForwardRef('ColumnOrName'), float]) -> pyspark.sql.column.Column\n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : str, :class:`~pyspark.sql.Column` or float\n",
      "            coordinate on y-axis\n",
      "        col2 : str, :class:`~pyspark.sql.Column` or float\n",
      "            coordinate on x-axis\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            the `theta` component of the point\n",
      "            (`r`, `theta`)\n",
      "            in polar coordinates that corresponds to the point\n",
      "            (`x`, `y`) in Cartesian coordinates,\n",
      "            as if computed by `java.lang.Math.atan2()`\n",
      "    \n",
      "    atanh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes inverse hyperbolic tangent of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "    \n",
      "    avg(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the average of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    base64(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the BASE64 encoding of a binary column and returns it as a string column.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    bin(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the string representation of the binary value of the given column.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(bin(df.age).alias('c')).collect()\n",
      "        [Row(c='10'), Row(c='101')]\n",
      "    \n",
      "    bit_length(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the bit length for the specified string column.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            Source column or strings\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            Bit length of the col\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import bit_length\n",
      "        >>> spark.createDataFrame([('cat',), ( '🐈',)], ['cat']) \\\n",
      "        ...      .select(bit_length('cat')).collect()\n",
      "            [Row(bit_length(cat)=24), Row(bit_length(cat)=32)]\n",
      "    \n",
      "    bitwiseNOT(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes bitwise not.\n",
      "        \n",
      "        .. deprecated:: 3.2.0\n",
      "            Use :func:`bitwise_not` instead.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    bitwise_not(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes bitwise not.\n",
      "        \n",
      "        .. versionadded:: 3.2\n",
      "    \n",
      "    broadcast(df: pyspark.sql.dataframe.DataFrame) -> pyspark.sql.dataframe.DataFrame\n",
      "        Marks a DataFrame as small enough for use in broadcast joins.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    bround(col: 'ColumnOrName', scale: int = 0) -> pyspark.sql.column.Column\n",
      "        Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0\n",
      "        or at integral part when `scale` < 0.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([(2.5,)], ['a']).select(bround('a', 0).alias('r')).collect()\n",
      "        [Row(r=2.0)]\n",
      "    \n",
      "    bucket(numBuckets: Union[pyspark.sql.column.Column, int], col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Partition transform function: A transform for any type that partitions\n",
      "        by a hash of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.writeTo(\"catalog.db.table\").partitionedBy(  # doctest: +SKIP\n",
      "        ...     bucket(42, \"ts\")\n",
      "        ... ).createOrReplace()\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This function can be used only in combination with\n",
      "        :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`\n",
      "        method of the `DataFrameWriterV2`.\n",
      "    \n",
      "    cbrt(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the cube-root of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    ceil(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the ceiling of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    coalesce(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the first column that is not null.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> cDf = spark.createDataFrame([(None, None), (1, None), (None, 2)], (\"a\", \"b\"))\n",
      "        >>> cDf.show()\n",
      "        +----+----+\n",
      "        |   a|   b|\n",
      "        +----+----+\n",
      "        |null|null|\n",
      "        |   1|null|\n",
      "        |null|   2|\n",
      "        +----+----+\n",
      "        \n",
      "        >>> cDf.select(coalesce(cDf[\"a\"], cDf[\"b\"])).show()\n",
      "        +--------------+\n",
      "        |coalesce(a, b)|\n",
      "        +--------------+\n",
      "        |          null|\n",
      "        |             1|\n",
      "        |             2|\n",
      "        +--------------+\n",
      "        \n",
      "        >>> cDf.select('*', coalesce(cDf[\"a\"], lit(0.0))).show()\n",
      "        +----+----+----------------+\n",
      "        |   a|   b|coalesce(a, 0.0)|\n",
      "        +----+----+----------------+\n",
      "        |null|null|             0.0|\n",
      "        |   1|null|             1.0|\n",
      "        |null|   2|             0.0|\n",
      "        +----+----+----------------+\n",
      "    \n",
      "    col(col: str) -> pyspark.sql.column.Column\n",
      "        Returns a :class:`~pyspark.sql.Column` based on the given column name.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> col('x')\n",
      "        Column<'x'>\n",
      "        >>> column('x')\n",
      "        Column<'x'>\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    collect_list(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns a list of objects with duplicates.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic because the order of collected results depends\n",
      "        on the order of the rows which may be non-deterministic after a shuffle.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))\n",
      "        >>> df2.agg(collect_list('age')).collect()\n",
      "        [Row(collect_list(age)=[2, 5, 5])]\n",
      "    \n",
      "    collect_set(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns a set of objects with duplicate elements eliminated.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic because the order of collected results depends\n",
      "        on the order of the rows which may be non-deterministic after a shuffle.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))\n",
      "        >>> df2.agg(array_sort(collect_set('age')).alias('c')).collect()\n",
      "        [Row(c=[2, 5])]\n",
      "    \n",
      "    column = col(col: str) -> pyspark.sql.column.Column\n",
      "        Returns a :class:`~pyspark.sql.Column` based on the given column name.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> col('x')\n",
      "        Column<'x'>\n",
      "        >>> column('x')\n",
      "        Column<'x'>\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    concat(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Concatenates multiple input columns together into a single column.\n",
      "        The function works with strings, binary and compatible array columns.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])\n",
      "        >>> df.select(concat(df.s, df.d).alias('s')).collect()\n",
      "        [Row(s='abcd123')]\n",
      "        \n",
      "        >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c'])\n",
      "        >>> df.select(concat(df.a, df.b, df.c).alias(\"arr\")).collect()\n",
      "        [Row(arr=[1, 2, 3, 4, 5]), Row(arr=None)]\n",
      "    \n",
      "    concat_ws(sep: str, *cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Concatenates multiple input string columns together into a single string column,\n",
      "        using the given separator.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])\n",
      "        >>> df.select(concat_ws('-', df.s, df.d).alias('s')).collect()\n",
      "        [Row(s='abcd-123')]\n",
      "    \n",
      "    conv(col: 'ColumnOrName', fromBase: int, toBase: int) -> pyspark.sql.column.Column\n",
      "        Convert a number in a string column from one base to another.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(\"010101\",)], ['n'])\n",
      "        >>> df.select(conv(df.n, 2, 16).alias('hex')).collect()\n",
      "        [Row(hex='15')]\n",
      "    \n",
      "    corr(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for\n",
      "        ``col1`` and ``col2``.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> a = range(20)\n",
      "        >>> b = [2 * x for x in range(20)]\n",
      "        >>> df = spark.createDataFrame(zip(a, b), [\"a\", \"b\"])\n",
      "        >>> df.agg(corr(\"a\", \"b\").alias('c')).collect()\n",
      "        [Row(c=1.0)]\n",
      "    \n",
      "    cos(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes cosine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            cosine of the angle, as if computed by `java.lang.Math.cos()`.\n",
      "    \n",
      "    cosh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes hyperbolic cosine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            hyperbolic angle\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()`\n",
      "    \n",
      "    cot(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes cotangent of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            Angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            Cotangent of the angle.\n",
      "    \n",
      "    count(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the number of items in a group.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    countDistinct(col: 'ColumnOrName', *cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``.\n",
      "        \n",
      "        An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct`\n",
      "        directly.\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "    \n",
      "    count_distinct(col: 'ColumnOrName', *cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new :class:`Column` for distinct count of ``col`` or ``cols``.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.agg(count_distinct(df.age, df.name).alias('c')).collect()\n",
      "        [Row(c=2)]\n",
      "        \n",
      "        >>> df.agg(count_distinct(\"age\", \"name\").alias('c')).collect()\n",
      "        [Row(c=2)]\n",
      "    \n",
      "    covar_pop(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and\n",
      "        ``col2``.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> a = [1] * 10\n",
      "        >>> b = [1] * 10\n",
      "        >>> df = spark.createDataFrame(zip(a, b), [\"a\", \"b\"])\n",
      "        >>> df.agg(covar_pop(\"a\", \"b\").alias('c')).collect()\n",
      "        [Row(c=0.0)]\n",
      "    \n",
      "    covar_samp(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and\n",
      "        ``col2``.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> a = [1] * 10\n",
      "        >>> b = [1] * 10\n",
      "        >>> df = spark.createDataFrame(zip(a, b), [\"a\", \"b\"])\n",
      "        >>> df.agg(covar_samp(\"a\", \"b\").alias('c')).collect()\n",
      "        [Row(c=0.0)]\n",
      "    \n",
      "    crc32(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the cyclic redundancy check value  (CRC32) of a binary column and\n",
      "        returns the value as a bigint.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC',)], ['a']).select(crc32('a').alias('crc32')).collect()\n",
      "        [Row(crc32=2743272264)]\n",
      "    \n",
      "    create_map(*cols: Union[ForwardRef('ColumnOrName'), List[ForwardRef('ColumnOrName_')], Tuple[ForwardRef('ColumnOrName_'), ...]]) -> pyspark.sql.column.Column\n",
      "        Creates a new map column.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            column names or :class:`~pyspark.sql.Column`\\s that are\n",
      "            grouped as key-value pairs, e.g. (key1, value1, key2, value2, ...).\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(create_map('name', 'age').alias(\"map\")).collect()\n",
      "        [Row(map={'Alice': 2}), Row(map={'Bob': 5})]\n",
      "        >>> df.select(create_map([df.name, df.age]).alias(\"map\")).collect()\n",
      "        [Row(map={'Alice': 2}), Row(map={'Bob': 5})]\n",
      "    \n",
      "    csc(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes cosecant of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            Angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            Cosecant of the angle.\n",
      "    \n",
      "    cume_dist() -> pyspark.sql.column.Column\n",
      "        Window function: returns the cumulative distribution of values within a window partition,\n",
      "        i.e. the fraction of rows that are below the current row.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    current_date() -> pyspark.sql.column.Column\n",
      "        Returns the current date at the start of query evaluation as a :class:`DateType` column.\n",
      "        All calls of current_date within the same query return the same value.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    current_timestamp() -> pyspark.sql.column.Column\n",
      "        Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`\n",
      "        column. All calls of current_timestamp within the same query return the same value.\n",
      "    \n",
      "    date_add(start: 'ColumnOrName', days: Union[ForwardRef('ColumnOrName'), int]) -> pyspark.sql.column.Column\n",
      "        Returns the date that is `days` days after `start`\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08', 2,)], ['dt', 'add'])\n",
      "        >>> df.select(date_add(df.dt, 1).alias('next_date')).collect()\n",
      "        [Row(next_date=datetime.date(2015, 4, 9))]\n",
      "        >>> df.select(date_add(df.dt, df.add.cast('integer')).alias('next_date')).collect()\n",
      "        [Row(next_date=datetime.date(2015, 4, 10))]\n",
      "    \n",
      "    date_format(date: 'ColumnOrName', format: str) -> pyspark.sql.column.Column\n",
      "        Converts a date/timestamp/string to a value of string in the format specified by the date\n",
      "        format given by the second argument.\n",
      "        \n",
      "        A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All\n",
      "        pattern letters of `datetime pattern`_. can be used.\n",
      "        \n",
      "        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        Whenever possible, use specialized functions like `year`.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(date_format('dt', 'MM/dd/yyy').alias('date')).collect()\n",
      "        [Row(date='04/08/2015')]\n",
      "    \n",
      "    date_sub(start: 'ColumnOrName', days: Union[ForwardRef('ColumnOrName'), int]) -> pyspark.sql.column.Column\n",
      "        Returns the date that is `days` days before `start`\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08', 2,)], ['dt', 'sub'])\n",
      "        >>> df.select(date_sub(df.dt, 1).alias('prev_date')).collect()\n",
      "        [Row(prev_date=datetime.date(2015, 4, 7))]\n",
      "        >>> df.select(date_sub(df.dt, df.sub.cast('integer')).alias('prev_date')).collect()\n",
      "        [Row(prev_date=datetime.date(2015, 4, 6))]\n",
      "    \n",
      "    date_trunc(format: str, timestamp: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns timestamp truncated to the unit specified by the format.\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        format : str\n",
      "            'year', 'yyyy', 'yy' to truncate by year,\n",
      "            'month', 'mon', 'mm' to truncate by month,\n",
      "            'day', 'dd' to truncate by day,\n",
      "            Other options are:\n",
      "            'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'\n",
      "        timestamp : :class:`~pyspark.sql.Column` or str\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 05:02:11',)], ['t'])\n",
      "        >>> df.select(date_trunc('year', df.t).alias('year')).collect()\n",
      "        [Row(year=datetime.datetime(1997, 1, 1, 0, 0))]\n",
      "        >>> df.select(date_trunc('mon', df.t).alias('month')).collect()\n",
      "        [Row(month=datetime.datetime(1997, 2, 1, 0, 0))]\n",
      "    \n",
      "    datediff(end: 'ColumnOrName', start: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the number of days from `start` to `end`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2'])\n",
      "        >>> df.select(datediff(df.d2, df.d1).alias('diff')).collect()\n",
      "        [Row(diff=32)]\n",
      "    \n",
      "    dayofmonth(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the day of the month of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(dayofmonth('dt').alias('day')).collect()\n",
      "        [Row(day=8)]\n",
      "    \n",
      "    dayofweek(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the day of the week of a given date as integer.\n",
      "        Ranges from 1 for a Sunday through to 7 for a Saturday\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(dayofweek('dt').alias('day')).collect()\n",
      "        [Row(day=4)]\n",
      "    \n",
      "    dayofyear(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the day of the year of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(dayofyear('dt').alias('day')).collect()\n",
      "        [Row(day=98)]\n",
      "    \n",
      "    days(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Partition transform function: A transform for timestamps and dates\n",
      "        to partition data into days.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.writeTo(\"catalog.db.table\").partitionedBy(  # doctest: +SKIP\n",
      "        ...     days(\"ts\")\n",
      "        ... ).createOrReplace()\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This function can be used only in combination with\n",
      "        :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`\n",
      "        method of the `DataFrameWriterV2`.\n",
      "    \n",
      "    decode(col: 'ColumnOrName', charset: str) -> pyspark.sql.column.Column\n",
      "        Computes the first argument into a string from a binary using the provided character set\n",
      "        (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    degrees(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Converts an angle measured in radians to an approximately equivalent angle\n",
      "        measured in degrees.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            angle in degrees, as if computed by `java.lang.Math.toDegrees()`\n",
      "    \n",
      "    dense_rank() -> pyspark.sql.column.Column\n",
      "        Window function: returns the rank of rows within a window partition, without any gaps.\n",
      "        \n",
      "        The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking\n",
      "        sequence when there are ties. That is, if you were ranking a competition using dense_rank\n",
      "        and had three people tie for second place, you would say that all three were in second\n",
      "        place and that the next person came in third. Rank would give me sequential numbers, making\n",
      "        the person that came in third place (after the ties) would register as coming in fifth.\n",
      "        \n",
      "        This is equivalent to the DENSE_RANK function in SQL.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    desc(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the descending order of the given column name.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    desc_nulls_first(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the descending order of the given\n",
      "        column name, and null values appear before non-null values.\n",
      "        \n",
      "        .. versionadded:: 2.4\n",
      "    \n",
      "    desc_nulls_last(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a sort expression based on the descending order of the given\n",
      "        column name, and null values appear after non-null values.\n",
      "        \n",
      "        .. versionadded:: 2.4\n",
      "    \n",
      "    element_at(col: 'ColumnOrName', extraction: Any) -> pyspark.sql.column.Column\n",
      "        Collection function: Returns element of array at given index in extraction if col is array.\n",
      "        Returns value for the given key in extraction if col is map.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing array or map\n",
      "        extraction :\n",
      "            index to check for in array or key to check for in map\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The position is not zero based, but 1 based index.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([\"a\", \"b\", \"c\"],)], ['data'])\n",
      "        >>> df.select(element_at(df.data, 1)).collect()\n",
      "        [Row(element_at(data, 1)='a')]\n",
      "        \n",
      "        >>> df = spark.createDataFrame([({\"a\": 1.0, \"b\": 2.0},)], ['data'])\n",
      "        >>> df.select(element_at(df.data, lit(\"a\"))).collect()\n",
      "        [Row(element_at(data, a)=1.0)]\n",
      "    \n",
      "    encode(col: 'ColumnOrName', charset: str) -> pyspark.sql.column.Column\n",
      "        Computes the first argument into a binary from a string using the provided character set\n",
      "        (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    exists(col: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Returns whether a predicate holds for one or more elements in the array.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            ``(x: Column) -> Column: ...``  returning the Boolean expression.\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        :return: a :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],(\"key\", \"values\"))\n",
      "        >>> df.select(exists(\"values\", lambda x: x < 0).alias(\"any_negative\")).show()\n",
      "        +------------+\n",
      "        |any_negative|\n",
      "        +------------+\n",
      "        |       false|\n",
      "        |        true|\n",
      "        +------------+\n",
      "    \n",
      "    exp(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the exponential of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    explode(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new row for each element in the given array or map.\n",
      "        Uses the default column name `col` for elements in the array and\n",
      "        `key` and `value` for elements in the map unless specified otherwise.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={\"a\": \"b\"})])\n",
      "        >>> eDF.select(explode(eDF.intlist).alias(\"anInt\")).collect()\n",
      "        [Row(anInt=1), Row(anInt=2), Row(anInt=3)]\n",
      "        \n",
      "        >>> eDF.select(explode(eDF.mapfield).alias(\"key\", \"value\")).show()\n",
      "        +---+-----+\n",
      "        |key|value|\n",
      "        +---+-----+\n",
      "        |  a|    b|\n",
      "        +---+-----+\n",
      "    \n",
      "    explode_outer(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new row for each element in the given array or map.\n",
      "        Unlike explode, if the array/map is null or empty then null is produced.\n",
      "        Uses the default column name `col` for elements in the array and\n",
      "        `key` and `value` for elements in the map unless specified otherwise.\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame(\n",
      "        ...     [(1, [\"foo\", \"bar\"], {\"x\": 1.0}), (2, [], {}), (3, None, None)],\n",
      "        ...     (\"id\", \"an_array\", \"a_map\")\n",
      "        ... )\n",
      "        >>> df.select(\"id\", \"an_array\", explode_outer(\"a_map\")).show()\n",
      "        +---+----------+----+-----+\n",
      "        | id|  an_array| key|value|\n",
      "        +---+----------+----+-----+\n",
      "        |  1|[foo, bar]|   x|  1.0|\n",
      "        |  2|        []|null| null|\n",
      "        |  3|      null|null| null|\n",
      "        +---+----------+----+-----+\n",
      "        \n",
      "        >>> df.select(\"id\", \"a_map\", explode_outer(\"an_array\")).show()\n",
      "        +---+----------+----+\n",
      "        | id|     a_map| col|\n",
      "        +---+----------+----+\n",
      "        |  1|{x -> 1.0}| foo|\n",
      "        |  1|{x -> 1.0}| bar|\n",
      "        |  2|        {}|null|\n",
      "        |  3|      null|null|\n",
      "        +---+----------+----+\n",
      "    \n",
      "    expm1(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the exponential of the given value minus one.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    expr(str: str) -> pyspark.sql.column.Column\n",
      "        Parses the expression string into the column that it represents\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(expr(\"length(name)\")).collect()\n",
      "        [Row(length(name)=5), Row(length(name)=3)]\n",
      "    \n",
      "    factorial(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the factorial of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(5,)], ['n'])\n",
      "        >>> df.select(factorial(df.n).alias('f')).collect()\n",
      "        [Row(f=120)]\n",
      "    \n",
      "    filter(col: 'ColumnOrName', f: Union[Callable[[pyspark.sql.column.Column], pyspark.sql.column.Column], Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]]) -> pyspark.sql.column.Column\n",
      "        Returns an array of elements for which a predicate holds in a given array.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            A function that returns the Boolean expression.\n",
      "            Can take one of the following forms:\n",
      "        \n",
      "            - Unary ``(x: Column) -> Column: ...``\n",
      "            - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is\n",
      "                a 0-based index of the element.\n",
      "        \n",
      "            and can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame(\n",
      "        ...     [(1, [\"2018-09-20\",  \"2019-02-03\", \"2019-07-01\", \"2020-06-01\"])],\n",
      "        ...     (\"key\", \"values\")\n",
      "        ... )\n",
      "        >>> def after_second_quarter(x):\n",
      "        ...     return month(to_date(x)) > 6\n",
      "        >>> df.select(\n",
      "        ...     filter(\"values\", after_second_quarter).alias(\"after_second_quarter\")\n",
      "        ... ).show(truncate=False)\n",
      "        +------------------------+\n",
      "        |after_second_quarter    |\n",
      "        +------------------------+\n",
      "        |[2018-09-20, 2019-07-01]|\n",
      "        +------------------------+\n",
      "    \n",
      "    first(col: 'ColumnOrName', ignorenulls: bool = False) -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the first value in a group.\n",
      "        \n",
      "        The function by default returns the first values it sees. It will return the first non-null\n",
      "        value it sees when ignoreNulls is set to true. If all values are null, then null is returned.\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic because its results depends on the order of the\n",
      "        rows which may be non-deterministic after a shuffle.\n",
      "    \n",
      "    flatten(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: creates a single array from an array of arrays.\n",
      "        If a structure of nested arrays is deeper than two levels,\n",
      "        only one level of nesting is removed.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],), ([None, [4, 5]],)], ['data'])\n",
      "        >>> df.select(flatten(df.data).alias('r')).collect()\n",
      "        [Row(r=[1, 2, 3, 4, 5, 6]), Row(r=None)]\n",
      "    \n",
      "    floor(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the floor of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    forall(col: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Returns whether a predicate holds for every element in the array.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            ``(x: Column) -> Column: ...``  returning the Boolean expression.\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame(\n",
      "        ...     [(1, [\"bar\"]), (2, [\"foo\", \"bar\"]), (3, [\"foobar\", \"foo\"])],\n",
      "        ...     (\"key\", \"values\")\n",
      "        ... )\n",
      "        >>> df.select(forall(\"values\", lambda x: x.rlike(\"foo\")).alias(\"all_foo\")).show()\n",
      "        +-------+\n",
      "        |all_foo|\n",
      "        +-------+\n",
      "        |  false|\n",
      "        |  false|\n",
      "        |   true|\n",
      "        +-------+\n",
      "    \n",
      "    format_number(col: 'ColumnOrName', d: int) -> pyspark.sql.column.Column\n",
      "        Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places\n",
      "        with HALF_EVEN round mode, and returns the result as a string.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            the column name of the numeric value to be formatted\n",
      "        d : int\n",
      "            the N decimal places\n",
      "        \n",
      "        >>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect()\n",
      "        [Row(v='5.0000')]\n",
      "    \n",
      "    format_string(format: str, *cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Formats the arguments in printf-style and returns the result as a string column.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        format : str\n",
      "            string that can contain embedded format tags and used as result column's value\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            column names or :class:`~pyspark.sql.Column`\\s to be used in formatting\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(5, \"hello\")], ['a', 'b'])\n",
      "        >>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()\n",
      "        [Row(v='5 hello')]\n",
      "    \n",
      "    from_csv(col: 'ColumnOrName', schema: Union[pyspark.sql.types.StructType, pyspark.sql.column.Column, str], options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Parses a column containing a CSV string to a row with the specified schema.\n",
      "        Returns `null`, in the case of an unparseable string.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            a column or column name in CSV format\n",
      "        schema :class:`~pyspark.sql.Column` or str\n",
      "            a column, or Python string literal with schema in DDL format, to use when parsing the CSV column.\n",
      "        options : dict, optional\n",
      "            options to control parsing. accepts the same options as the CSV datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-csv.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> data = [(\"1,2,3\",)]\n",
      "        >>> df = spark.createDataFrame(data, (\"value\",))\n",
      "        >>> df.select(from_csv(df.value, \"a INT, b INT, c INT\").alias(\"csv\")).collect()\n",
      "        [Row(csv=Row(a=1, b=2, c=3))]\n",
      "        >>> value = data[0][0]\n",
      "        >>> df.select(from_csv(df.value, schema_of_csv(value)).alias(\"csv\")).collect()\n",
      "        [Row(csv=Row(_c0=1, _c1=2, _c2=3))]\n",
      "        >>> data = [(\"   abc\",)]\n",
      "        >>> df = spark.createDataFrame(data, (\"value\",))\n",
      "        >>> options = {'ignoreLeadingWhiteSpace': True}\n",
      "        >>> df.select(from_csv(df.value, \"s string\", options).alias(\"csv\")).collect()\n",
      "        [Row(csv=Row(s='abc'))]\n",
      "    \n",
      "    from_json(col: 'ColumnOrName', schema: Union[pyspark.sql.types.ArrayType, pyspark.sql.types.StructType, pyspark.sql.column.Column, str], options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType`\n",
      "        as keys type, :class:`StructType` or :class:`ArrayType` with\n",
      "        the specified schema. Returns `null`, in the case of an unparseable string.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            a column or column name in JSON format\n",
      "        schema : :class:`DataType` or str\n",
      "            a StructType, ArrayType of StructType or Python string literal with a DDL-formatted string\n",
      "            to use when parsing the json column\n",
      "        options : dict, optional\n",
      "            options to control parsing. accepts the same options as the json datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-json.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.types import *\n",
      "        >>> data = [(1, '''{\"a\": 1}''')]\n",
      "        >>> schema = StructType([StructField(\"a\", IntegerType())])\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(from_json(df.value, schema).alias(\"json\")).collect()\n",
      "        [Row(json=Row(a=1))]\n",
      "        >>> df.select(from_json(df.value, \"a INT\").alias(\"json\")).collect()\n",
      "        [Row(json=Row(a=1))]\n",
      "        >>> df.select(from_json(df.value, \"MAP<STRING,INT>\").alias(\"json\")).collect()\n",
      "        [Row(json={'a': 1})]\n",
      "        >>> data = [(1, '''[{\"a\": 1}]''')]\n",
      "        >>> schema = ArrayType(StructType([StructField(\"a\", IntegerType())]))\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(from_json(df.value, schema).alias(\"json\")).collect()\n",
      "        [Row(json=[Row(a=1)])]\n",
      "        >>> schema = schema_of_json(lit('''{\"a\": 0}'''))\n",
      "        >>> df.select(from_json(df.value, schema).alias(\"json\")).collect()\n",
      "        [Row(json=Row(a=None))]\n",
      "        >>> data = [(1, '''[1, 2, 3]''')]\n",
      "        >>> schema = ArrayType(IntegerType())\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(from_json(df.value, schema).alias(\"json\")).collect()\n",
      "        [Row(json=[1, 2, 3])]\n",
      "    \n",
      "    from_unixtime(timestamp: 'ColumnOrName', format: str = 'yyyy-MM-dd HH:mm:ss') -> pyspark.sql.column.Column\n",
      "        Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string\n",
      "        representing the timestamp of that moment in the current system time zone in the given\n",
      "        format.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.conf.set(\"spark.sql.session.timeZone\", \"America/Los_Angeles\")\n",
      "        >>> time_df = spark.createDataFrame([(1428476400,)], ['unix_time'])\n",
      "        >>> time_df.select(from_unixtime('unix_time').alias('ts')).collect()\n",
      "        [Row(ts='2015-04-08 00:00:00')]\n",
      "        >>> spark.conf.unset(\"spark.sql.session.timeZone\")\n",
      "    \n",
      "    from_utc_timestamp(timestamp: 'ColumnOrName', tz: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function\n",
      "        takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and\n",
      "        renders that timestamp as a timestamp in the given time zone.\n",
      "        \n",
      "        However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not\n",
      "        timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to\n",
      "        the given timezone.\n",
      "        \n",
      "        This function may return confusing result if the input is a string with timezone, e.g.\n",
      "        '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp\n",
      "        according to the timezone in the string, and finally display the result by converting the\n",
      "        timestamp to string according to the session local timezone.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        timestamp : :class:`~pyspark.sql.Column` or str\n",
      "            the column that contains timestamps\n",
      "        tz : :class:`~pyspark.sql.Column` or str\n",
      "            A string detailing the time zone ID that the input should be adjusted to. It should\n",
      "            be in the format of either region-based zone IDs or zone offsets. Region IDs must\n",
      "            have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in\n",
      "            the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are\n",
      "            supported as aliases of '+00:00'. Other short names are not recommended to use\n",
      "            because they can be ambiguous.\n",
      "        \n",
      "            .. versionchanged:: 2.4\n",
      "               `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])\n",
      "        >>> df.select(from_utc_timestamp(df.ts, \"PST\").alias('local_time')).collect()\n",
      "        [Row(local_time=datetime.datetime(1997, 2, 28, 2, 30))]\n",
      "        >>> df.select(from_utc_timestamp(df.ts, df.tz).alias('local_time')).collect()\n",
      "        [Row(local_time=datetime.datetime(1997, 2, 28, 19, 30))]\n",
      "    \n",
      "    get_json_object(col: 'ColumnOrName', path: str) -> pyspark.sql.column.Column\n",
      "        Extracts json object from a json string based on json path specified, and returns json string\n",
      "        of the extracted json object. It will return null if the input json string is invalid.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            string column in json format\n",
      "        path : str\n",
      "            path to the json object to extract\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> data = [(\"1\", '''{\"f1\": \"value1\", \"f2\": \"value2\"}'''), (\"2\", '''{\"f1\": \"value12\"}''')]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"jstring\"))\n",
      "        >>> df.select(df.key, get_json_object(df.jstring, '$.f1').alias(\"c0\"), \\\n",
      "        ...                   get_json_object(df.jstring, '$.f2').alias(\"c1\") ).collect()\n",
      "        [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]\n",
      "    \n",
      "    greatest(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the greatest value of the list of column names, skipping null values.\n",
      "        This function takes at least 2 parameters. It will return null iff all parameters are null.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])\n",
      "        >>> df.select(greatest(df.a, df.b, df.c).alias(\"greatest\")).collect()\n",
      "        [Row(greatest=4)]\n",
      "    \n",
      "    grouping(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated\n",
      "        or not, returns 1 for aggregated or 0 for not aggregated in the result set.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.cube(\"name\").agg(grouping(\"name\"), sum(\"age\")).orderBy(\"name\").show()\n",
      "        +-----+--------------+--------+\n",
      "        | name|grouping(name)|sum(age)|\n",
      "        +-----+--------------+--------+\n",
      "        | null|             1|       7|\n",
      "        |Alice|             0|       2|\n",
      "        |  Bob|             0|       5|\n",
      "        +-----+--------------+--------+\n",
      "    \n",
      "    grouping_id(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the level of grouping, equals to\n",
      "        \n",
      "           (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The list of columns should match with grouping columns exactly, or empty (means all\n",
      "        the grouping columns).\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.cube(\"name\").agg(grouping_id(), sum(\"age\")).orderBy(\"name\").show()\n",
      "        +-----+-------------+--------+\n",
      "        | name|grouping_id()|sum(age)|\n",
      "        +-----+-------------+--------+\n",
      "        | null|            1|       7|\n",
      "        |Alice|            0|       2|\n",
      "        |  Bob|            0|       5|\n",
      "        +-----+-------------+--------+\n",
      "    \n",
      "    hash(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the hash code of given columns, and returns the result as an int column.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC',)], ['a']).select(hash('a').alias('hash')).collect()\n",
      "        [Row(hash=-757602832)]\n",
      "    \n",
      "    hex(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`,\n",
      "        :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or\n",
      "        :class:`pyspark.sql.types.LongType`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC', 3)], ['a', 'b']).select(hex('a'), hex('b')).collect()\n",
      "        [Row(hex(a)='414243', hex(b)='3')]\n",
      "    \n",
      "    hour(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the hours of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> import datetime\n",
      "        >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])\n",
      "        >>> df.select(hour('ts').alias('hour')).collect()\n",
      "        [Row(hour=13)]\n",
      "    \n",
      "    hours(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Partition transform function: A transform for timestamps\n",
      "        to partition data into hours.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.writeTo(\"catalog.db.table\").partitionedBy(   # doctest: +SKIP\n",
      "        ...     hours(\"ts\")\n",
      "        ... ).createOrReplace()\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This function can be used only in combination with\n",
      "        :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`\n",
      "        method of the `DataFrameWriterV2`.\n",
      "    \n",
      "    hypot(col1: Union[ForwardRef('ColumnOrName'), float], col2: Union[ForwardRef('ColumnOrName'), float]) -> pyspark.sql.column.Column\n",
      "        Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    initcap(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Translate the first letter of each word to upper case in the sentence.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ab cd',)], ['a']).select(initcap(\"a\").alias('v')).collect()\n",
      "        [Row(v='Ab Cd')]\n",
      "    \n",
      "    input_file_name() -> pyspark.sql.column.Column\n",
      "        Creates a string column for the file name of the current Spark task.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    instr(str: 'ColumnOrName', substr: str) -> pyspark.sql.column.Column\n",
      "        Locate the position of the first occurrence of substr column in the given string.\n",
      "        Returns null if either of the arguments are null.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The position is not zero based, but 1 based index. Returns 0 if substr\n",
      "        could not be found in str.\n",
      "        \n",
      "        >>> df = spark.createDataFrame([('abcd',)], ['s',])\n",
      "        >>> df.select(instr(df.s, 'b').alias('s')).collect()\n",
      "        [Row(s=2)]\n",
      "    \n",
      "    isnan(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        An expression that returns true iff the column is NaN.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], (\"a\", \"b\"))\n",
      "        >>> df.select(isnan(\"a\").alias(\"r1\"), isnan(df.a).alias(\"r2\")).collect()\n",
      "        [Row(r1=False, r2=False), Row(r1=True, r2=True)]\n",
      "    \n",
      "    isnull(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        An expression that returns true iff the column is null.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, None), (None, 2)], (\"a\", \"b\"))\n",
      "        >>> df.select(isnull(\"a\").alias(\"r1\"), isnull(df.a).alias(\"r2\")).collect()\n",
      "        [Row(r1=False, r2=False), Row(r1=True, r2=True)]\n",
      "    \n",
      "    json_tuple(col: 'ColumnOrName', *fields: str) -> pyspark.sql.column.Column\n",
      "        Creates a new row for a json column according to the given field names.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            string column in json format\n",
      "        fields : str\n",
      "            fields to extract\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> data = [(\"1\", '''{\"f1\": \"value1\", \"f2\": \"value2\"}'''), (\"2\", '''{\"f1\": \"value12\"}''')]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"jstring\"))\n",
      "        >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect()\n",
      "        [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]\n",
      "    \n",
      "    kurtosis(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the kurtosis of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    lag(col: 'ColumnOrName', offset: int = 1, default: Optional[Any] = None) -> pyspark.sql.column.Column\n",
      "        Window function: returns the value that is `offset` rows before the current row, and\n",
      "        `default` if there is less than `offset` rows before the current row. For example,\n",
      "        an `offset` of one will return the previous row at any given point in the window partition.\n",
      "        \n",
      "        This is equivalent to the LAG function in SQL.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        offset : int, optional\n",
      "            number of row to extend\n",
      "        default : optional\n",
      "            default value\n",
      "    \n",
      "    last(col: 'ColumnOrName', ignorenulls: bool = False) -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the last value in a group.\n",
      "        \n",
      "        The function by default returns the last values it sees. It will return the last non-null\n",
      "        value it sees when ignoreNulls is set to true. If all values are null, then null is returned.\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic because its results depends on the order of the\n",
      "        rows which may be non-deterministic after a shuffle.\n",
      "    \n",
      "    last_day(date: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the last day of the month which the given date belongs to.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-10',)], ['d'])\n",
      "        >>> df.select(last_day(df.d).alias('date')).collect()\n",
      "        [Row(date=datetime.date(1997, 2, 28))]\n",
      "    \n",
      "    lead(col: 'ColumnOrName', offset: int = 1, default: Optional[Any] = None) -> pyspark.sql.column.Column\n",
      "        Window function: returns the value that is `offset` rows after the current row, and\n",
      "        `default` if there is less than `offset` rows after the current row. For example,\n",
      "        an `offset` of one will return the next row at any given point in the window partition.\n",
      "        \n",
      "        This is equivalent to the LEAD function in SQL.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        offset : int, optional\n",
      "            number of row to extend\n",
      "        default : optional\n",
      "            default value\n",
      "    \n",
      "    least(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the least value of the list of column names, skipping null values.\n",
      "        This function takes at least 2 parameters. It will return null iff all parameters are null.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            column names or columns to be compared\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])\n",
      "        >>> df.select(least(df.a, df.b, df.c).alias(\"least\")).collect()\n",
      "        [Row(least=1)]\n",
      "    \n",
      "    length(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the character length of string data or number of bytes of binary data.\n",
      "        The length of character data includes the trailing spaces. The length of binary data\n",
      "        includes binary zeros.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC ',)], ['a']).select(length('a').alias('length')).collect()\n",
      "        [Row(length=4)]\n",
      "    \n",
      "    levenshtein(left: 'ColumnOrName', right: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the Levenshtein distance of the two given strings.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df0 = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r'])\n",
      "        >>> df0.select(levenshtein('l', 'r').alias('d')).collect()\n",
      "        [Row(d=3)]\n",
      "    \n",
      "    lit(col: Any) -> pyspark.sql.column.Column\n",
      "        Creates a :class:`~pyspark.sql.Column` of literal value.\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(lit(5).alias('height')).withColumn('spark_user', lit(True)).take(1)\n",
      "        [Row(height=5, spark_user=True)]\n",
      "    \n",
      "    locate(substr: str, str: 'ColumnOrName', pos: int = 1) -> pyspark.sql.column.Column\n",
      "        Locate the position of the first occurrence of substr in a string column, after position pos.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        substr : str\n",
      "            a string\n",
      "        str : :class:`~pyspark.sql.Column` or str\n",
      "            a Column of :class:`pyspark.sql.types.StringType`\n",
      "        pos : int, optional\n",
      "            start position (zero based)\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The position is not zero based, but 1 based index. Returns 0 if substr\n",
      "        could not be found in str.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd',)], ['s',])\n",
      "        >>> df.select(locate('b', df.s, 1).alias('s')).collect()\n",
      "        [Row(s=2)]\n",
      "    \n",
      "    log(arg1: Union[ForwardRef('ColumnOrName'), float], arg2: Optional[ForwardRef('ColumnOrName')] = None) -> pyspark.sql.column.Column\n",
      "        Returns the first argument-based logarithm of the second argument.\n",
      "        \n",
      "        If there is only one argument, then this takes the natural logarithm of the argument.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(log(10.0, df.age).alias('ten')).rdd.map(lambda l: str(l.ten)[:7]).collect()\n",
      "        ['0.30102', '0.69897']\n",
      "        \n",
      "        >>> df.select(log(df.age).alias('e')).rdd.map(lambda l: str(l.e)[:7]).collect()\n",
      "        ['0.69314', '1.60943']\n",
      "    \n",
      "    log10(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the logarithm of the given value in Base 10.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    log1p(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the natural logarithm of the given value plus one.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    log2(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the base-2 logarithm of the argument.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([(4,)], ['a']).select(log2('a').alias('log2')).collect()\n",
      "        [Row(log2=2.0)]\n",
      "    \n",
      "    lower(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Converts a string expression to lower case.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    lpad(col: 'ColumnOrName', len: int, pad: str) -> pyspark.sql.column.Column\n",
      "        Left-pad the string column to width `len` with `pad`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd',)], ['s',])\n",
      "        >>> df.select(lpad(df.s, 6, '#').alias('s')).collect()\n",
      "        [Row(s='##abcd')]\n",
      "    \n",
      "    ltrim(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Trim the spaces from left end for the specified string value.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    make_date(year: 'ColumnOrName', month: 'ColumnOrName', day: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a column with a date built from the year, month and day columns.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        year : :class:`~pyspark.sql.Column` or str\n",
      "            The year to build the date\n",
      "        month : :class:`~pyspark.sql.Column` or str\n",
      "            The month to build the date\n",
      "        day : :class:`~pyspark.sql.Column` or str\n",
      "            The day to build the date\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(2020, 6, 26)], ['Y', 'M', 'D'])\n",
      "        >>> df.select(make_date(df.Y, df.M, df.D).alias(\"datefield\")).collect()\n",
      "        [Row(datefield=datetime.date(2020, 6, 26))]\n",
      "    \n",
      "    map_concat(*cols: Union[ForwardRef('ColumnOrName'), List[ForwardRef('ColumnOrName_')], Tuple[ForwardRef('ColumnOrName_'), ...]]) -> pyspark.sql.column.Column\n",
      "        Returns the union of all the given maps.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : :class:`~pyspark.sql.Column` or str\n",
      "            column names or :class:`~pyspark.sql.Column`\\s\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import map_concat\n",
      "        >>> df = spark.sql(\"SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2\")\n",
      "        >>> df.select(map_concat(\"map1\", \"map2\").alias(\"map3\")).show(truncate=False)\n",
      "        +------------------------+\n",
      "        |map3                    |\n",
      "        +------------------------+\n",
      "        |{1 -> a, 2 -> b, 3 -> c}|\n",
      "        +------------------------+\n",
      "    \n",
      "    map_entries(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Returns an unordered array of all entries in the given map.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import map_entries\n",
      "        >>> df = spark.sql(\"SELECT map(1, 'a', 2, 'b') as data\")\n",
      "        >>> df.select(map_entries(\"data\").alias(\"entries\")).show()\n",
      "        +----------------+\n",
      "        |         entries|\n",
      "        +----------------+\n",
      "        |[{1, a}, {2, b}]|\n",
      "        +----------------+\n",
      "    \n",
      "    map_filter(col: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Returns a map whose key-value pairs satisfy a predicate.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            a binary function ``(k: Column, v: Column) -> Column...``\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, {\"foo\": 42.0, \"bar\": 1.0, \"baz\": 32.0})], (\"id\", \"data\"))\n",
      "        >>> df.select(map_filter(\n",
      "        ...     \"data\", lambda _, v: v > 30.0).alias(\"data_filtered\")\n",
      "        ... ).show(truncate=False)\n",
      "        +--------------------------+\n",
      "        |data_filtered             |\n",
      "        +--------------------------+\n",
      "        |{baz -> 32.0, foo -> 42.0}|\n",
      "        +--------------------------+\n",
      "    \n",
      "    map_from_arrays(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Creates a new map from two arrays.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing a set of keys. All elements should not be null\n",
      "        col2 : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing a set of values\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([2, 5], ['a', 'b'])], ['k', 'v'])\n",
      "        >>> df.select(map_from_arrays(df.k, df.v).alias(\"map\")).show()\n",
      "        +----------------+\n",
      "        |             map|\n",
      "        +----------------+\n",
      "        |{2 -> a, 5 -> b}|\n",
      "        +----------------+\n",
      "    \n",
      "    map_from_entries(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Returns a map created from the given array of entries.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import map_from_entries\n",
      "        >>> df = spark.sql(\"SELECT array(struct(1, 'a'), struct(2, 'b')) as data\")\n",
      "        >>> df.select(map_from_entries(\"data\").alias(\"map\")).show()\n",
      "        +----------------+\n",
      "        |             map|\n",
      "        +----------------+\n",
      "        |{1 -> a, 2 -> b}|\n",
      "        +----------------+\n",
      "    \n",
      "    map_keys(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Returns an unordered array containing the keys of the map.\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import map_keys\n",
      "        >>> df = spark.sql(\"SELECT map(1, 'a', 2, 'b') as data\")\n",
      "        >>> df.select(map_keys(\"data\").alias(\"keys\")).show()\n",
      "        +------+\n",
      "        |  keys|\n",
      "        +------+\n",
      "        |[1, 2]|\n",
      "        +------+\n",
      "    \n",
      "    map_values(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Returns an unordered array containing the values of the map.\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import map_values\n",
      "        >>> df = spark.sql(\"SELECT map(1, 'a', 2, 'b') as data\")\n",
      "        >>> df.select(map_values(\"data\").alias(\"values\")).show()\n",
      "        +------+\n",
      "        |values|\n",
      "        +------+\n",
      "        |[a, b]|\n",
      "        +------+\n",
      "    \n",
      "    map_zip_with(col1: 'ColumnOrName', col2: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Merge two given maps, key-wise into a single map using a function.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col1 : :class:`~pyspark.sql.Column` or str\n",
      "            name of the first column or expression\n",
      "        col2 : :class:`~pyspark.sql.Column` or str\n",
      "            name of the second column or expression\n",
      "        f : function\n",
      "            a ternary function ``(k: Column, v1: Column, v2: Column) -> Column...``\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([\n",
      "        ...     (1, {\"IT\": 24.0, \"SALES\": 12.00}, {\"IT\": 2.0, \"SALES\": 1.4})],\n",
      "        ...     (\"id\", \"base\", \"ratio\")\n",
      "        ... )\n",
      "        >>> df.select(map_zip_with(\n",
      "        ...     \"base\", \"ratio\", lambda k, v1, v2: round(v1 * v2, 2)).alias(\"updated_data\")\n",
      "        ... ).show(truncate=False)\n",
      "        +---------------------------+\n",
      "        |updated_data               |\n",
      "        +---------------------------+\n",
      "        |{SALES -> 16.8, IT -> 48.0}|\n",
      "        +---------------------------+\n",
      "    \n",
      "    max(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the maximum value of the expression in a group.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    max_by(col: 'ColumnOrName', ord: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the value associated with the maximum value of ord.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            target column that the value will be returned\n",
      "        ord : :class:`~pyspark.sql.Column` or str\n",
      "            column to be maximized\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            value associated with the maximum value of ord.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([\n",
      "        ...     (\"Java\", 2012, 20000), (\"dotNET\", 2012, 5000),\n",
      "        ...     (\"dotNET\", 2013, 48000), (\"Java\", 2013, 30000)],\n",
      "        ...     schema=(\"course\", \"year\", \"earnings\"))\n",
      "        >>> df.groupby(\"course\").agg(max_by(\"year\", \"earnings\")).show()\n",
      "        +------+----------------------+\n",
      "        |course|max_by(year, earnings)|\n",
      "        +------+----------------------+\n",
      "        |  Java|                  2013|\n",
      "        |dotNET|                  2013|\n",
      "        +------+----------------------+\n",
      "    \n",
      "    md5(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the MD5 digest and returns the value as a 32 character hex string.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC',)], ['a']).select(md5('a').alias('hash')).collect()\n",
      "        [Row(hash='902fbdd2b1df0c4f70b4a5d23525e932')]\n",
      "    \n",
      "    mean(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the average of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    min(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the minimum value of the expression in a group.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    min_by(col: 'ColumnOrName', ord: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the value associated with the minimum value of ord.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            target column that the value will be returned\n",
      "        ord : :class:`~pyspark.sql.Column` or str\n",
      "            column to be minimized\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            value associated with the minimum value of ord.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([\n",
      "        ...     (\"Java\", 2012, 20000), (\"dotNET\", 2012, 5000),\n",
      "        ...     (\"dotNET\", 2013, 48000), (\"Java\", 2013, 30000)],\n",
      "        ...     schema=(\"course\", \"year\", \"earnings\"))\n",
      "        >>> df.groupby(\"course\").agg(min_by(\"year\", \"earnings\")).show()\n",
      "        +------+----------------------+\n",
      "        |course|min_by(year, earnings)|\n",
      "        +------+----------------------+\n",
      "        |  Java|                  2012|\n",
      "        |dotNET|                  2012|\n",
      "        +------+----------------------+\n",
      "    \n",
      "    minute(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the minutes of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> import datetime\n",
      "        >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])\n",
      "        >>> df.select(minute('ts').alias('minute')).collect()\n",
      "        [Row(minute=8)]\n",
      "    \n",
      "    monotonically_increasing_id() -> pyspark.sql.column.Column\n",
      "        A column that generates monotonically increasing 64-bit integers.\n",
      "        \n",
      "        The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.\n",
      "        The current implementation puts the partition ID in the upper 31 bits, and the record number\n",
      "        within each partition in the lower 33 bits. The assumption is that the data frame has\n",
      "        less than 1 billion partitions, and each partition has less than 8 billion records.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic because its result depends on partition IDs.\n",
      "        \n",
      "        As an example, consider a :class:`DataFrame` with two partitions, each with 3 records.\n",
      "        This expression would return the following IDs:\n",
      "        0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.\n",
      "        \n",
      "        >>> df0 = sc.parallelize(range(2), 2).mapPartitions(lambda x: [(1,), (2,), (3,)]).toDF(['col1'])\n",
      "        >>> df0.select(monotonically_increasing_id().alias('id')).collect()\n",
      "        [Row(id=0), Row(id=1), Row(id=2), Row(id=8589934592), Row(id=8589934593), Row(id=8589934594)]\n",
      "    \n",
      "    month(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the month of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(month('dt').alias('month')).collect()\n",
      "        [Row(month=4)]\n",
      "    \n",
      "    months(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Partition transform function: A transform for timestamps and dates\n",
      "        to partition data into months.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.writeTo(\"catalog.db.table\").partitionedBy(\n",
      "        ...     months(\"ts\")\n",
      "        ... ).createOrReplace()  # doctest: +SKIP\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This function can be used only in combination with\n",
      "        :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`\n",
      "        method of the `DataFrameWriterV2`.\n",
      "    \n",
      "    months_between(date1: 'ColumnOrName', date2: 'ColumnOrName', roundOff: bool = True) -> pyspark.sql.column.Column\n",
      "        Returns number of months between dates date1 and date2.\n",
      "        If date1 is later than date2, then the result is positive.\n",
      "        A whole number is returned if both inputs have the same day of month or both are the last day\n",
      "        of their respective months. Otherwise, the difference is calculated assuming 31 days per month.\n",
      "        The result is rounded off to 8 digits unless `roundOff` is set to `False`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2'])\n",
      "        >>> df.select(months_between(df.date1, df.date2).alias('months')).collect()\n",
      "        [Row(months=3.94959677)]\n",
      "        >>> df.select(months_between(df.date1, df.date2, False).alias('months')).collect()\n",
      "        [Row(months=3.9495967741935485)]\n",
      "    \n",
      "    nanvl(col1: 'ColumnOrName', col2: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns col1 if it is not NaN, or col2 if col1 is NaN.\n",
      "        \n",
      "        Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`).\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], (\"a\", \"b\"))\n",
      "        >>> df.select(nanvl(\"a\", \"b\").alias(\"r1\"), nanvl(df.a, df.b).alias(\"r2\")).collect()\n",
      "        [Row(r1=1.0, r2=1.0), Row(r1=2.0, r2=2.0)]\n",
      "    \n",
      "    next_day(date: 'ColumnOrName', dayOfWeek: str) -> pyspark.sql.column.Column\n",
      "        Returns the first date which is later than the value of the date column.\n",
      "        \n",
      "        Day of the week parameter is case insensitive, and accepts:\n",
      "            \"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\".\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-07-27',)], ['d'])\n",
      "        >>> df.select(next_day(df.d, 'Sun').alias('date')).collect()\n",
      "        [Row(date=datetime.date(2015, 8, 2))]\n",
      "    \n",
      "    nth_value(col: 'ColumnOrName', offset: int, ignoreNulls: Optional[bool] = False) -> pyspark.sql.column.Column\n",
      "        Window function: returns the value that is the `offset`\\th row of the window frame\n",
      "        (counting from 1), and `null` if the size of window frame is less than `offset` rows.\n",
      "        \n",
      "        It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to\n",
      "        true. If all values are null, then null is returned.\n",
      "        \n",
      "        This is equivalent to the nth_value function in SQL.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        offset : int, optional\n",
      "            number of row to use as the value\n",
      "        ignoreNulls : bool, optional\n",
      "            indicates the Nth value should skip null in the\n",
      "            determination of which row to use\n",
      "    \n",
      "    ntile(n: int) -> pyspark.sql.column.Column\n",
      "        Window function: returns the ntile group id (from 1 to `n` inclusive)\n",
      "        in an ordered window partition. For example, if `n` is 4, the first\n",
      "        quarter of the rows will get value 1, the second quarter will get 2,\n",
      "        the third quarter will get 3, and the last quarter will get 4.\n",
      "        \n",
      "        This is equivalent to the NTILE function in SQL.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        n : int\n",
      "            an integer\n",
      "    \n",
      "    octet_length(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the byte length for the specified string column.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            Source column or strings\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            Byte length of the col\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import octet_length\n",
      "        >>> spark.createDataFrame([('cat',), ( '🐈',)], ['cat']) \\\n",
      "        ...      .select(octet_length('cat')).collect()\n",
      "            [Row(octet_length(cat)=3), Row(octet_length(cat)=4)]\n",
      "    \n",
      "    overlay(src: 'ColumnOrName', replace: 'ColumnOrName', pos: Union[ForwardRef('ColumnOrName'), int], len: Union[ForwardRef('ColumnOrName'), int] = -1) -> pyspark.sql.column.Column\n",
      "        Overlay the specified portion of `src` with `replace`,\n",
      "        starting from byte position `pos` of `src` and proceeding for `len` bytes.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        src : :class:`~pyspark.sql.Column` or str\n",
      "            column name or column containing the string that will be replaced\n",
      "        replace : :class:`~pyspark.sql.Column` or str\n",
      "            column name or column containing the substitution string\n",
      "        pos : :class:`~pyspark.sql.Column` or str or int\n",
      "            column name, column, or int containing the starting position in src\n",
      "        len : :class:`~pyspark.sql.Column` or str or int\n",
      "            column name, column, or int containing the number of bytes to replace in src\n",
      "            string by 'replace' defaults to -1, which represents the length of the 'replace' string\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(\"SPARK_SQL\", \"CORE\")], (\"x\", \"y\"))\n",
      "        >>> df.select(overlay(\"x\", \"y\", 7).alias(\"overlayed\")).collect()\n",
      "        [Row(overlayed='SPARK_CORE')]\n",
      "        >>> df.select(overlay(\"x\", \"y\", 7, 0).alias(\"overlayed\")).collect()\n",
      "        [Row(overlayed='SPARK_CORESQL')]\n",
      "        >>> df.select(overlay(\"x\", \"y\", 7, 2).alias(\"overlayed\")).collect()\n",
      "        [Row(overlayed='SPARK_COREL')]\n",
      "    \n",
      "    percent_rank() -> pyspark.sql.column.Column\n",
      "        Window function: returns the relative rank (i.e. percentile) of rows within a window partition.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    percentile_approx(col: 'ColumnOrName', percentage: Union[pyspark.sql.column.Column, float, List[float], Tuple[float]], accuracy: Union[pyspark.sql.column.Column, float] = 10000) -> pyspark.sql.column.Column\n",
      "        Returns the approximate `percentile` of the numeric column `col` which is the smallest value\n",
      "        in the ordered `col` values (sorted from least to greatest) such that no more than `percentage`\n",
      "        of `col` values is less than the value or equal to that value.\n",
      "        The value of percentage must be between 0.0 and 1.0.\n",
      "        \n",
      "        The accuracy parameter (default: 10000)\n",
      "        is a positive numeric literal which controls approximation accuracy at the cost of memory.\n",
      "        Higher value of accuracy yields better accuracy, 1.0/accuracy is the relative error\n",
      "        of the approximation.\n",
      "        \n",
      "        When percentage is an array, each value of the percentage array must be between 0.0 and 1.0.\n",
      "        In this case, returns the approximate percentile array of column col\n",
      "        at the given percentage array.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> key = (col(\"id\") % 3).alias(\"key\")\n",
      "        >>> value = (randn(42) + key * 10).alias(\"value\")\n",
      "        >>> df = spark.range(0, 1000, 1, 1).select(key, value)\n",
      "        >>> df.select(\n",
      "        ...     percentile_approx(\"value\", [0.25, 0.5, 0.75], 1000000).alias(\"quantiles\")\n",
      "        ... ).printSchema()\n",
      "        root\n",
      "         |-- quantiles: array (nullable = true)\n",
      "         |    |-- element: double (containsNull = false)\n",
      "        \n",
      "        >>> df.groupBy(\"key\").agg(\n",
      "        ...     percentile_approx(\"value\", 0.5, lit(1000000)).alias(\"median\")\n",
      "        ... ).printSchema()\n",
      "        root\n",
      "         |-- key: long (nullable = true)\n",
      "         |-- median: double (nullable = true)\n",
      "    \n",
      "    posexplode(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new row for each element with position in the given array or map.\n",
      "        Uses the default column name `pos` for position, and `col` for elements in the\n",
      "        array and `key` and `value` for elements in the map unless specified otherwise.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={\"a\": \"b\"})])\n",
      "        >>> eDF.select(posexplode(eDF.intlist)).collect()\n",
      "        [Row(pos=0, col=1), Row(pos=1, col=2), Row(pos=2, col=3)]\n",
      "        \n",
      "        >>> eDF.select(posexplode(eDF.mapfield)).show()\n",
      "        +---+---+-----+\n",
      "        |pos|key|value|\n",
      "        +---+---+-----+\n",
      "        |  0|  a|    b|\n",
      "        +---+---+-----+\n",
      "    \n",
      "    posexplode_outer(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns a new row for each element with position in the given array or map.\n",
      "        Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced.\n",
      "        Uses the default column name `pos` for position, and `col` for elements in the\n",
      "        array and `key` and `value` for elements in the map unless specified otherwise.\n",
      "        \n",
      "        .. versionadded:: 2.3.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame(\n",
      "        ...     [(1, [\"foo\", \"bar\"], {\"x\": 1.0}), (2, [], {}), (3, None, None)],\n",
      "        ...     (\"id\", \"an_array\", \"a_map\")\n",
      "        ... )\n",
      "        >>> df.select(\"id\", \"an_array\", posexplode_outer(\"a_map\")).show()\n",
      "        +---+----------+----+----+-----+\n",
      "        | id|  an_array| pos| key|value|\n",
      "        +---+----------+----+----+-----+\n",
      "        |  1|[foo, bar]|   0|   x|  1.0|\n",
      "        |  2|        []|null|null| null|\n",
      "        |  3|      null|null|null| null|\n",
      "        +---+----------+----+----+-----+\n",
      "        >>> df.select(\"id\", \"a_map\", posexplode_outer(\"an_array\")).show()\n",
      "        +---+----------+----+----+\n",
      "        | id|     a_map| pos| col|\n",
      "        +---+----------+----+----+\n",
      "        |  1|{x -> 1.0}|   0| foo|\n",
      "        |  1|{x -> 1.0}|   1| bar|\n",
      "        |  2|        {}|null|null|\n",
      "        |  3|      null|null|null|\n",
      "        +---+----------+----+----+\n",
      "    \n",
      "    pow(col1: Union[ForwardRef('ColumnOrName'), float], col2: Union[ForwardRef('ColumnOrName'), float]) -> pyspark.sql.column.Column\n",
      "        Returns the value of the first argument raised to the power of the second argument.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    product(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the product of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : str, :class:`Column`\n",
      "            column containing values to be multiplied together\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.range(1, 10).toDF('x').withColumn('mod3', col('x') % 3)\n",
      "        >>> prods = df.groupBy('mod3').agg(product('x').alias('product'))\n",
      "        >>> prods.orderBy('mod3').show()\n",
      "        +----+-------+\n",
      "        |mod3|product|\n",
      "        +----+-------+\n",
      "        |   0|  162.0|\n",
      "        |   1|   28.0|\n",
      "        |   2|   80.0|\n",
      "        +----+-------+\n",
      "    \n",
      "    quarter(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the quarter of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(quarter('dt').alias('quarter')).collect()\n",
      "        [Row(quarter=2)]\n",
      "    \n",
      "    radians(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Converts an angle measured in degrees to an approximately equivalent angle\n",
      "        measured in radians.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            angle in degrees\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            angle in radians, as if computed by `java.lang.Math.toRadians()`\n",
      "    \n",
      "    raise_error(errMsg: Union[pyspark.sql.column.Column, str]) -> pyspark.sql.column.Column\n",
      "        Throws an exception with the provided error message.\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        errMsg : :class:`~pyspark.sql.Column` or str\n",
      "            A Python string literal or column containing the error message\n",
      "        \n",
      "        .. versionadded:: 3.1\n",
      "    \n",
      "    rand(seed: Optional[int] = None) -> pyspark.sql.column.Column\n",
      "        Generates a random column with independent and identically distributed (i.i.d.) samples\n",
      "        uniformly distributed in [0.0, 1.0).\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic in general case.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.withColumn('rand', rand(seed=42) * 3).collect()\n",
      "        [Row(age=2, name='Alice', rand=2.4052597283576684),\n",
      "         Row(age=5, name='Bob', rand=2.3913904055683974)]\n",
      "    \n",
      "    randn(seed: Optional[int] = None) -> pyspark.sql.column.Column\n",
      "        Generates a column with independent and identically distributed (i.i.d.) samples from\n",
      "        the standard normal distribution.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic in general case.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.withColumn('randn', randn(seed=42)).collect()\n",
      "        [Row(age=2, name='Alice', randn=1.1027054481455365),\n",
      "        Row(age=5, name='Bob', randn=0.7400395449950132)]\n",
      "    \n",
      "    rank() -> pyspark.sql.column.Column\n",
      "        Window function: returns the rank of rows within a window partition.\n",
      "        \n",
      "        The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking\n",
      "        sequence when there are ties. That is, if you were ranking a competition using dense_rank\n",
      "        and had three people tie for second place, you would say that all three were in second\n",
      "        place and that the next person came in third. Rank would give me sequential numbers, making\n",
      "        the person that came in third place (after the ties) would register as coming in fifth.\n",
      "        \n",
      "        This is equivalent to the RANK function in SQL.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    regexp_extract(str: 'ColumnOrName', pattern: str, idx: int) -> pyspark.sql.column.Column\n",
      "        Extract a specific group matched by a Java regex, from the specified string column.\n",
      "        If the regex did not match, or the specified group did not match, an empty string is returned.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('100-200',)], ['str'])\n",
      "        >>> df.select(regexp_extract('str', r'(\\d+)-(\\d+)', 1).alias('d')).collect()\n",
      "        [Row(d='100')]\n",
      "        >>> df = spark.createDataFrame([('foo',)], ['str'])\n",
      "        >>> df.select(regexp_extract('str', r'(\\d+)', 1).alias('d')).collect()\n",
      "        [Row(d='')]\n",
      "        >>> df = spark.createDataFrame([('aaaac',)], ['str'])\n",
      "        >>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()\n",
      "        [Row(d='')]\n",
      "    \n",
      "    regexp_replace(str: 'ColumnOrName', pattern: str, replacement: str) -> pyspark.sql.column.Column\n",
      "        Replace all substrings of the specified string value that match regexp with rep.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('100-200',)], ['str'])\n",
      "        >>> df.select(regexp_replace('str', r'(\\d+)', '--').alias('d')).collect()\n",
      "        [Row(d='-----')]\n",
      "    \n",
      "    repeat(col: 'ColumnOrName', n: int) -> pyspark.sql.column.Column\n",
      "        Repeats a string column n times, and returns it as a new string column.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('ab',)], ['s',])\n",
      "        >>> df.select(repeat(df.s, 3).alias('s')).collect()\n",
      "        [Row(s='ababab')]\n",
      "    \n",
      "    reverse(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns a reversed string or an array with reverse order of elements.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('Spark SQL',)], ['data'])\n",
      "        >>> df.select(reverse(df.data).alias('s')).collect()\n",
      "        [Row(s='LQS krapS')]\n",
      "        >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data'])\n",
      "        >>> df.select(reverse(df.data).alias('r')).collect()\n",
      "        [Row(r=[3, 1, 2]), Row(r=[1]), Row(r=[])]\n",
      "    \n",
      "    rint(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the double value that is closest in value to the argument and\n",
      "        is equal to a mathematical integer.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    round(col: 'ColumnOrName', scale: int = 0) -> pyspark.sql.column.Column\n",
      "        Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0\n",
      "        or at integral part when `scale` < 0.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([(2.5,)], ['a']).select(round('a', 0).alias('r')).collect()\n",
      "        [Row(r=3.0)]\n",
      "    \n",
      "    row_number() -> pyspark.sql.column.Column\n",
      "        Window function: returns a sequential number starting at 1 within a window partition.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    rpad(col: 'ColumnOrName', len: int, pad: str) -> pyspark.sql.column.Column\n",
      "        Right-pad the string column to width `len` with `pad`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd',)], ['s',])\n",
      "        >>> df.select(rpad(df.s, 6, '#').alias('s')).collect()\n",
      "        [Row(s='abcd##')]\n",
      "    \n",
      "    rtrim(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Trim the spaces from right end for the specified string value.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    schema_of_csv(csv: 'ColumnOrName', options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Parses a CSV string and infers its schema in DDL format.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        csv : :class:`~pyspark.sql.Column` or str\n",
      "            a CSV string or a foldable string column containing a CSV string.\n",
      "        options : dict, optional\n",
      "            options to control parsing. accepts the same options as the CSV datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-csv.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.range(1)\n",
      "        >>> df.select(schema_of_csv(lit('1|a'), {'sep':'|'}).alias(\"csv\")).collect()\n",
      "        [Row(csv='STRUCT<_c0: INT, _c1: STRING>')]\n",
      "        >>> df.select(schema_of_csv('1|a', {'sep':'|'}).alias(\"csv\")).collect()\n",
      "        [Row(csv='STRUCT<_c0: INT, _c1: STRING>')]\n",
      "    \n",
      "    schema_of_json(json: 'ColumnOrName', options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Parses a JSON string and infers its schema in DDL format.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        json : :class:`~pyspark.sql.Column` or str\n",
      "            a JSON string or a foldable string column containing a JSON string.\n",
      "        options : dict, optional\n",
      "            options to control parsing. accepts the same options as the JSON datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-json.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "            .. versionchanged:: 3.0\n",
      "               It accepts `options` parameter to control schema inferring.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.range(1)\n",
      "        >>> df.select(schema_of_json(lit('{\"a\": 0}')).alias(\"json\")).collect()\n",
      "        [Row(json='STRUCT<a: BIGINT>')]\n",
      "        >>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'})\n",
      "        >>> df.select(schema.alias(\"json\")).collect()\n",
      "        [Row(json='STRUCT<a: BIGINT>')]\n",
      "    \n",
      "    sec(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes secant of the input column.\n",
      "        \n",
      "        .. versionadded:: 3.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            Angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            Secant of the angle.\n",
      "    \n",
      "    second(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the seconds of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> import datetime\n",
      "        >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])\n",
      "        >>> df.select(second('ts').alias('second')).collect()\n",
      "        [Row(second=15)]\n",
      "    \n",
      "    sentences(string: 'ColumnOrName', language: Optional[ForwardRef('ColumnOrName')] = None, country: Optional[ForwardRef('ColumnOrName')] = None) -> pyspark.sql.column.Column\n",
      "        Splits a string into arrays of sentences, where each sentence is an array of words.\n",
      "        The 'language' and 'country' arguments are optional, and if omitted, the default locale is used.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        string : :class:`~pyspark.sql.Column` or str\n",
      "            a string to be split\n",
      "        language : :class:`~pyspark.sql.Column` or str, optional\n",
      "            a language of the locale\n",
      "        country : :class:`~pyspark.sql.Column` or str, optional\n",
      "            a country of the locale\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([[\"This is an example sentence.\"]], [\"string\"])\n",
      "        >>> df.select(sentences(df.string, lit(\"en\"), lit(\"US\"))).show(truncate=False)\n",
      "        +-----------------------------------+\n",
      "        |sentences(string, en, US)          |\n",
      "        +-----------------------------------+\n",
      "        |[[This, is, an, example, sentence]]|\n",
      "        +-----------------------------------+\n",
      "    \n",
      "    sequence(start: 'ColumnOrName', stop: 'ColumnOrName', step: Optional[ForwardRef('ColumnOrName')] = None) -> pyspark.sql.column.Column\n",
      "        Generate a sequence of integers from `start` to `stop`, incrementing by `step`.\n",
      "        If `step` is not set, incrementing by 1 if `start` is less than or equal to `stop`,\n",
      "        otherwise -1.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df1 = spark.createDataFrame([(-2, 2)], ('C1', 'C2'))\n",
      "        >>> df1.select(sequence('C1', 'C2').alias('r')).collect()\n",
      "        [Row(r=[-2, -1, 0, 1, 2])]\n",
      "        >>> df2 = spark.createDataFrame([(4, -4, -2)], ('C1', 'C2', 'C3'))\n",
      "        >>> df2.select(sequence('C1', 'C2', 'C3').alias('r')).collect()\n",
      "        [Row(r=[4, 2, 0, -2, -4])]\n",
      "    \n",
      "    session_window(timeColumn: 'ColumnOrName', gapDuration: Union[pyspark.sql.column.Column, str]) -> pyspark.sql.column.Column\n",
      "        Generates session window given a timestamp specifying column.\n",
      "        Session window is one of dynamic windows, which means the length of window is varying\n",
      "        according to the given inputs. The length of session window is defined as \"the timestamp\n",
      "        of latest input of the session + gap duration\", so when the new inputs are bound to the\n",
      "        current session window, the end time of session window can be expanded according to the new\n",
      "        inputs.\n",
      "        Windows can support microsecond precision. Windows in the order of months are not supported.\n",
      "        For a streaming query, you may use the function `current_timestamp` to generate windows on\n",
      "        processing time.\n",
      "        gapDuration is provided as strings, e.g. '1 second', '1 day 12 hours', '2 minutes'. Valid\n",
      "        interval strings are 'week', 'day', 'hour', 'minute', 'second', 'millisecond', 'microsecond'.\n",
      "        It could also be a Column which can be evaluated to gap duration dynamically based on the\n",
      "        input row.\n",
      "        The output column will be a struct called 'session_window' by default with the nested columns\n",
      "        'start' and 'end', where 'start' and 'end' will be of :class:`pyspark.sql.types.TimestampType`.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        timeColumn : :class:`~pyspark.sql.Column` or str\n",
      "            The column name or column to use as the timestamp for windowing by time.\n",
      "            The time column must be of TimestampType.\n",
      "        gapDuration : :class:`~pyspark.sql.Column` or str\n",
      "            A Python string literal or column specifying the timeout of the session. It could be\n",
      "            static value, e.g. `10 minutes`, `1 second`, or an expression/UDF that specifies gap\n",
      "            duration dynamically based on the input row.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(\"2016-03-11 09:00:07\", 1)]).toDF(\"date\", \"val\")\n",
      "        >>> w = df.groupBy(session_window(\"date\", \"5 seconds\")).agg(sum(\"val\").alias(\"sum\"))\n",
      "        >>> w.select(w.session_window.start.cast(\"string\").alias(\"start\"),\n",
      "        ...          w.session_window.end.cast(\"string\").alias(\"end\"), \"sum\").collect()\n",
      "        [Row(start='2016-03-11 09:00:07', end='2016-03-11 09:00:12', sum=1)]\n",
      "        >>> w = df.groupBy(session_window(\"date\", lit(\"5 seconds\"))).agg(sum(\"val\").alias(\"sum\"))\n",
      "        >>> w.select(w.session_window.start.cast(\"string\").alias(\"start\"),\n",
      "        ...          w.session_window.end.cast(\"string\").alias(\"end\"), \"sum\").collect()\n",
      "        [Row(start='2016-03-11 09:00:07', end='2016-03-11 09:00:12', sum=1)]\n",
      "    \n",
      "    sha1(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the hex string result of SHA-1.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC',)], ['a']).select(sha1('a').alias('hash')).collect()\n",
      "        [Row(hash='3c01bdbb26f358bab27f267924aa2c9a03fcfdb8')]\n",
      "    \n",
      "    sha2(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384,\n",
      "        and SHA-512). The numBits indicates the desired bit length of the result, which must have a\n",
      "        value of 224, 256, 384, 512, or 0 (which is equivalent to 256).\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> digests = df.select(sha2(df.name, 256).alias('s')).collect()\n",
      "        >>> digests[0]\n",
      "        Row(s='3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043')\n",
      "        >>> digests[1]\n",
      "        Row(s='cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961')\n",
      "    \n",
      "    shiftLeft(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        Shift the given value numBits left.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        .. deprecated:: 3.2.0\n",
      "            Use :func:`shiftleft` instead.\n",
      "    \n",
      "    shiftRight(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        (Signed) shift the given value numBits right.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        .. deprecated:: 3.2.0\n",
      "            Use :func:`shiftright` instead.\n",
      "    \n",
      "    shiftRightUnsigned(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        Unsigned shift the given value numBits right.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        .. deprecated:: 3.2.0\n",
      "            Use :func:`shiftrightunsigned` instead.\n",
      "    \n",
      "    shiftleft(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        Shift the given value numBits left.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([(21,)], ['a']).select(shiftleft('a', 1).alias('r')).collect()\n",
      "        [Row(r=42)]\n",
      "    \n",
      "    shiftright(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        (Signed) shift the given value numBits right.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([(42,)], ['a']).select(shiftright('a', 1).alias('r')).collect()\n",
      "        [Row(r=21)]\n",
      "    \n",
      "    shiftrightunsigned(col: 'ColumnOrName', numBits: int) -> pyspark.sql.column.Column\n",
      "        Unsigned shift the given value numBits right.\n",
      "        \n",
      "        .. versionadded:: 3.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(-42,)], ['a'])\n",
      "        >>> df.select(shiftrightunsigned('a', 1).alias('r')).collect()\n",
      "        [Row(r=9223372036854775787)]\n",
      "    \n",
      "    shuffle(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: Generates a random permutation of the given array.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The function is non-deterministic.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([1, 20, 3, 5],), ([1, 20, None, 3],)], ['data'])\n",
      "        >>> df.select(shuffle(df.data).alias('s')).collect()  # doctest: +SKIP\n",
      "        [Row(s=[3, 1, 5, 20]), Row(s=[20, None, 3, 1])]\n",
      "    \n",
      "    signum(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the signum of the given value.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    sin(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes sine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            sine of the angle, as if computed by `java.lang.Math.sin()`\n",
      "    \n",
      "    sinh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes hyperbolic sine of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            hyperbolic angle\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            hyperbolic sine of the given value,\n",
      "            as if computed by `java.lang.Math.sinh()`\n",
      "    \n",
      "    size(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Collection function: returns the length of the array or map stored in the column.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data'])\n",
      "        >>> df.select(size(df.data)).collect()\n",
      "        [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)]\n",
      "    \n",
      "    skewness(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the skewness of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    slice(x: 'ColumnOrName', start: Union[ForwardRef('ColumnOrName'), int], length: Union[ForwardRef('ColumnOrName'), int]) -> pyspark.sql.column.Column\n",
      "        Collection function: returns an array containing  all the elements in `x` from index `start`\n",
      "        (array indices start at 1, or from the end if `start` is negative) with the specified `length`.\n",
      "        \n",
      "        .. versionadded:: 2.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        x : :class:`~pyspark.sql.Column` or str\n",
      "            column name or column containing the array to be sliced\n",
      "        start : :class:`~pyspark.sql.Column` or str or int\n",
      "            column name, column, or int containing the starting index\n",
      "        length : :class:`~pyspark.sql.Column` or str or int\n",
      "            column name, column, or int containing the length of the slice\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])\n",
      "        >>> df.select(slice(df.x, 2, 2).alias(\"sliced\")).collect()\n",
      "        [Row(sliced=[2, 3]), Row(sliced=[5])]\n",
      "    \n",
      "    sort_array(col: 'ColumnOrName', asc: bool = True) -> pyspark.sql.column.Column\n",
      "        Collection function: sorts the input array in ascending or descending order according\n",
      "        to the natural ordering of the array elements. Null elements will be placed at the beginning\n",
      "        of the returned array in ascending order or at the end of the returned array in descending\n",
      "        order.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        asc : bool, optional\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])\n",
      "        >>> df.select(sort_array(df.data).alias('r')).collect()\n",
      "        [Row(r=[None, 1, 2, 3]), Row(r=[1]), Row(r=[])]\n",
      "        >>> df.select(sort_array(df.data, asc=False).alias('r')).collect()\n",
      "        [Row(r=[3, 2, 1, None]), Row(r=[1]), Row(r=[])]\n",
      "    \n",
      "    soundex(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Returns the SoundEx encoding for a string\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(\"Peters\",),(\"Uhrbach\",)], ['name'])\n",
      "        >>> df.select(soundex(df.name).alias(\"soundex\")).collect()\n",
      "        [Row(soundex='P362'), Row(soundex='U612')]\n",
      "    \n",
      "    spark_partition_id() -> pyspark.sql.column.Column\n",
      "        A column for partition ID.\n",
      "        \n",
      "        .. versionadded:: 1.6.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This is non deterministic because it depends on data partitioning and task scheduling.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.repartition(1).select(spark_partition_id().alias(\"pid\")).collect()\n",
      "        [Row(pid=0), Row(pid=0)]\n",
      "    \n",
      "    split(str: 'ColumnOrName', pattern: str, limit: int = -1) -> pyspark.sql.column.Column\n",
      "        Splits str around matches of the given pattern.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        str : :class:`~pyspark.sql.Column` or str\n",
      "            a string expression to split\n",
      "        pattern : str\n",
      "            a string representing a regular expression. The regex string should be\n",
      "            a Java regular expression.\n",
      "        limit : int, optional\n",
      "            an integer which controls the number of times `pattern` is applied.\n",
      "        \n",
      "            * ``limit > 0``: The resulting array's length will not be more than `limit`, and the\n",
      "                             resulting array's last entry will contain all input beyond the last\n",
      "                             matched pattern.\n",
      "            * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting\n",
      "                              array can be of any size.\n",
      "        \n",
      "            .. versionchanged:: 3.0\n",
      "               `split` now takes an optional `limit` field. If not provided, default limit value is -1.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',])\n",
      "        >>> df.select(split(df.s, '[ABC]', 2).alias('s')).collect()\n",
      "        [Row(s=['one', 'twoBthreeC'])]\n",
      "        >>> df.select(split(df.s, '[ABC]', -1).alias('s')).collect()\n",
      "        [Row(s=['one', 'two', 'three', ''])]\n",
      "    \n",
      "    sqrt(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes the square root of the specified float value.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    stddev(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: alias for stddev_samp.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    stddev_pop(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns population standard deviation of\n",
      "        the expression in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    stddev_samp(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the unbiased sample standard deviation of\n",
      "        the expression in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    struct(*cols: Union[ForwardRef('ColumnOrName'), List[ForwardRef('ColumnOrName_')], Tuple[ForwardRef('ColumnOrName_'), ...]]) -> pyspark.sql.column.Column\n",
      "        Creates a new struct column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : list, set, str or :class:`~pyspark.sql.Column`\n",
      "            column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(struct('age', 'name').alias(\"struct\")).collect()\n",
      "        [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]\n",
      "        >>> df.select(struct([df.age, df.name]).alias(\"struct\")).collect()\n",
      "        [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]\n",
      "    \n",
      "    substring(str: 'ColumnOrName', pos: int, len: int) -> pyspark.sql.column.Column\n",
      "        Substring starts at `pos` and is of length `len` when str is String type or\n",
      "        returns the slice of byte array that starts at `pos` in byte and is of length `len`\n",
      "        when str is Binary type.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The position is not zero based, but 1 based index.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('abcd',)], ['s',])\n",
      "        >>> df.select(substring(df.s, 1, 2).alias('s')).collect()\n",
      "        [Row(s='ab')]\n",
      "    \n",
      "    substring_index(str: 'ColumnOrName', delim: str, count: int) -> pyspark.sql.column.Column\n",
      "        Returns the substring from string str before count occurrences of the delimiter delim.\n",
      "        If count is positive, everything the left of the final delimiter (counting from left) is\n",
      "        returned. If count is negative, every to the right of the final delimiter (counting from the\n",
      "        right) is returned. substring_index performs a case-sensitive match when searching for delim.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('a.b.c.d',)], ['s'])\n",
      "        >>> df.select(substring_index(df.s, '.', 2).alias('s')).collect()\n",
      "        [Row(s='a.b')]\n",
      "        >>> df.select(substring_index(df.s, '.', -3).alias('s')).collect()\n",
      "        [Row(s='b.c.d')]\n",
      "    \n",
      "    sum(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the sum of all values in the expression.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    sumDistinct(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the sum of distinct values in the expression.\n",
      "        \n",
      "        .. deprecated:: 3.2.0\n",
      "            Use :func:`sum_distinct` instead.\n",
      "        \n",
      "        .. versionadded:: 1.3\n",
      "    \n",
      "    sum_distinct(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the sum of distinct values in the expression.\n",
      "        \n",
      "        .. versionadded:: 3.2\n",
      "    \n",
      "    tan(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes tangent of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            angle in radians\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            tangent of the given value, as if computed by `java.lang.Math.tan()`\n",
      "    \n",
      "    tanh(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Computes hyperbolic tangent of the input column.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            hyperbolic angle\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "            hyperbolic tangent of the given value\n",
      "            as if computed by `java.lang.Math.tanh()`\n",
      "    \n",
      "    timestamp_seconds(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.functions import timestamp_seconds\n",
      "        >>> spark.conf.set(\"spark.sql.session.timeZone\", \"America/Los_Angeles\")\n",
      "        >>> time_df = spark.createDataFrame([(1230219000,)], ['unix_time'])\n",
      "        >>> time_df.select(timestamp_seconds(time_df.unix_time).alias('ts')).show()\n",
      "        +-------------------+\n",
      "        |                 ts|\n",
      "        +-------------------+\n",
      "        |2008-12-25 07:30:00|\n",
      "        +-------------------+\n",
      "        >>> spark.conf.unset(\"spark.sql.session.timeZone\")\n",
      "    \n",
      "    toDegrees(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        .. deprecated:: 2.1.0\n",
      "            Use :func:`degrees` instead.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    toRadians(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        .. deprecated:: 2.1.0\n",
      "            Use :func:`radians` instead.\n",
      "        \n",
      "        .. versionadded:: 1.4\n",
      "    \n",
      "    to_csv(col: 'ColumnOrName', options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Converts a column containing a :class:`StructType` into a CSV string.\n",
      "        Throws an exception, in the case of an unsupported type.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing a struct.\n",
      "        options: dict, optional\n",
      "            options to control converting. accepts the same options as the CSV datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-csv.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> data = [(1, Row(age=2, name='Alice'))]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_csv(df.value).alias(\"csv\")).collect()\n",
      "        [Row(csv='2,Alice')]\n",
      "    \n",
      "    to_date(col: 'ColumnOrName', format: Optional[str] = None) -> pyspark.sql.column.Column\n",
      "        Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType`\n",
      "        using the optionally specified format. Specify formats according to `datetime pattern`_.\n",
      "        By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format\n",
      "        is omitted. Equivalent to ``col.cast(\"date\")``.\n",
      "        \n",
      "        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html\n",
      "        \n",
      "        .. versionadded:: 2.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])\n",
      "        >>> df.select(to_date(df.t).alias('date')).collect()\n",
      "        [Row(date=datetime.date(1997, 2, 28))]\n",
      "        \n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])\n",
      "        >>> df.select(to_date(df.t, 'yyyy-MM-dd HH:mm:ss').alias('date')).collect()\n",
      "        [Row(date=datetime.date(1997, 2, 28))]\n",
      "    \n",
      "    to_json(col: 'ColumnOrName', options: Optional[Dict[str, str]] = None) -> pyspark.sql.column.Column\n",
      "        Converts a column containing a :class:`StructType`, :class:`ArrayType` or a :class:`MapType`\n",
      "        into a JSON string. Throws an exception, in the case of an unsupported type.\n",
      "        \n",
      "        .. versionadded:: 2.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column containing a struct, an array or a map.\n",
      "        options : dict, optional\n",
      "            options to control converting. accepts the same options as the JSON datasource.\n",
      "            See `Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-json.html#data-source-option>`_\n",
      "            in the version you use.\n",
      "            Additionally the function supports the `pretty` option which enables\n",
      "            pretty JSON generation.\n",
      "        \n",
      "            .. # noqa\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql import Row\n",
      "        >>> from pyspark.sql.types import *\n",
      "        >>> data = [(1, Row(age=2, name='Alice'))]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_json(df.value).alias(\"json\")).collect()\n",
      "        [Row(json='{\"age\":2,\"name\":\"Alice\"}')]\n",
      "        >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_json(df.value).alias(\"json\")).collect()\n",
      "        [Row(json='[{\"age\":2,\"name\":\"Alice\"},{\"age\":3,\"name\":\"Bob\"}]')]\n",
      "        >>> data = [(1, {\"name\": \"Alice\"})]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_json(df.value).alias(\"json\")).collect()\n",
      "        [Row(json='{\"name\":\"Alice\"}')]\n",
      "        >>> data = [(1, [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}])]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_json(df.value).alias(\"json\")).collect()\n",
      "        [Row(json='[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]')]\n",
      "        >>> data = [(1, [\"Alice\", \"Bob\"])]\n",
      "        >>> df = spark.createDataFrame(data, (\"key\", \"value\"))\n",
      "        >>> df.select(to_json(df.value).alias(\"json\")).collect()\n",
      "        [Row(json='[\"Alice\",\"Bob\"]')]\n",
      "    \n",
      "    to_timestamp(col: 'ColumnOrName', format: Optional[str] = None) -> pyspark.sql.column.Column\n",
      "        Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.TimestampType`\n",
      "        using the optionally specified format. Specify formats according to `datetime pattern`_.\n",
      "        By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format\n",
      "        is omitted. Equivalent to ``col.cast(\"timestamp\")``.\n",
      "        \n",
      "        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html\n",
      "        \n",
      "        .. versionadded:: 2.2.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])\n",
      "        >>> df.select(to_timestamp(df.t).alias('dt')).collect()\n",
      "        [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]\n",
      "        \n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])\n",
      "        >>> df.select(to_timestamp(df.t, 'yyyy-MM-dd HH:mm:ss').alias('dt')).collect()\n",
      "        [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]\n",
      "    \n",
      "    to_utc_timestamp(timestamp: 'ColumnOrName', tz: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function\n",
      "        takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given\n",
      "        timezone, and renders that timestamp as a timestamp in UTC.\n",
      "        \n",
      "        However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not\n",
      "        timezone-agnostic. So in Spark this function just shift the timestamp value from the given\n",
      "        timezone to UTC timezone.\n",
      "        \n",
      "        This function may return confusing result if the input is a string with timezone, e.g.\n",
      "        '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp\n",
      "        according to the timezone in the string, and finally display the result by converting the\n",
      "        timestamp to string according to the session local timezone.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        timestamp : :class:`~pyspark.sql.Column` or str\n",
      "            the column that contains timestamps\n",
      "        tz : :class:`~pyspark.sql.Column` or str\n",
      "            A string detailing the time zone ID that the input should be adjusted to. It should\n",
      "            be in the format of either region-based zone IDs or zone offsets. Region IDs must\n",
      "            have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in\n",
      "            the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are\n",
      "            upported as aliases of '+00:00'. Other short names are not recommended to use\n",
      "            because they can be ambiguous.\n",
      "        \n",
      "            .. versionchanged:: 2.4.0\n",
      "               `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])\n",
      "        >>> df.select(to_utc_timestamp(df.ts, \"PST\").alias('utc_time')).collect()\n",
      "        [Row(utc_time=datetime.datetime(1997, 2, 28, 18, 30))]\n",
      "        >>> df.select(to_utc_timestamp(df.ts, df.tz).alias('utc_time')).collect()\n",
      "        [Row(utc_time=datetime.datetime(1997, 2, 28, 1, 30))]\n",
      "    \n",
      "    transform(col: 'ColumnOrName', f: Union[Callable[[pyspark.sql.column.Column], pyspark.sql.column.Column], Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]]) -> pyspark.sql.column.Column\n",
      "        Returns an array of elements after applying a transformation to each element in the input array.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            a function that is applied to each element of the input array.\n",
      "            Can take one of the following forms:\n",
      "        \n",
      "            - Unary ``(x: Column) -> Column: ...``\n",
      "            - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is\n",
      "                a 0-based index of the element.\n",
      "        \n",
      "            and can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], (\"key\", \"values\"))\n",
      "        >>> df.select(transform(\"values\", lambda x: x * 2).alias(\"doubled\")).show()\n",
      "        +------------+\n",
      "        |     doubled|\n",
      "        +------------+\n",
      "        |[2, 4, 6, 8]|\n",
      "        +------------+\n",
      "        \n",
      "        >>> def alternate(x, i):\n",
      "        ...     return when(i % 2 == 0, x).otherwise(-x)\n",
      "        >>> df.select(transform(\"values\", alternate).alias(\"alternated\")).show()\n",
      "        +--------------+\n",
      "        |    alternated|\n",
      "        +--------------+\n",
      "        |[1, -2, 3, -4]|\n",
      "        +--------------+\n",
      "    \n",
      "    transform_keys(col: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Applies a function to every key-value pair in a map and returns\n",
      "        a map with the results of those applications as the new keys for the pairs.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            a binary function ``(k: Column, v: Column) -> Column...``\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, {\"foo\": -2.0, \"bar\": 2.0})], (\"id\", \"data\"))\n",
      "        >>> df.select(transform_keys(\n",
      "        ...     \"data\", lambda k, _: upper(k)).alias(\"data_upper\")\n",
      "        ... ).show(truncate=False)\n",
      "        +-------------------------+\n",
      "        |data_upper               |\n",
      "        +-------------------------+\n",
      "        |{BAR -> 2.0, FOO -> -2.0}|\n",
      "        +-------------------------+\n",
      "    \n",
      "    transform_values(col: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Applies a function to every key-value pair in a map and returns\n",
      "        a map with the results of those applications as the new values for the pairs.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        col : :class:`~pyspark.sql.Column` or str\n",
      "            name of column or expression\n",
      "        f : function\n",
      "            a binary function ``(k: Column, v: Column) -> Column...``\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, {\"IT\": 10.0, \"SALES\": 2.0, \"OPS\": 24.0})], (\"id\", \"data\"))\n",
      "        >>> df.select(transform_values(\n",
      "        ...     \"data\", lambda k, v: when(k.isin(\"IT\", \"OPS\"), v + 10.0).otherwise(v)\n",
      "        ... ).alias(\"new_data\")).show(truncate=False)\n",
      "        +---------------------------------------+\n",
      "        |new_data                               |\n",
      "        +---------------------------------------+\n",
      "        |{OPS -> 34.0, IT -> 20.0, SALES -> 2.0}|\n",
      "        +---------------------------------------+\n",
      "    \n",
      "    translate(srcCol: 'ColumnOrName', matching: str, replace: str) -> pyspark.sql.column.Column\n",
      "        A function translate any character in the `srcCol` by a character in `matching`.\n",
      "        The characters in `replace` is corresponding to the characters in `matching`.\n",
      "        The translate will happen when any character in the string matching with the character\n",
      "        in the `matching`.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', \"rnlt\", \"123\") \\\n",
      "        ...     .alias('r')).collect()\n",
      "        [Row(r='1a2s3ae')]\n",
      "    \n",
      "    trim(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Trim the spaces from both ends for the specified string column.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    trunc(date: 'ColumnOrName', format: str) -> pyspark.sql.column.Column\n",
      "        Returns date truncated to the unit specified by the format.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        date : :class:`~pyspark.sql.Column` or str\n",
      "        format : str\n",
      "            'year', 'yyyy', 'yy' to truncate by year,\n",
      "            or 'month', 'mon', 'mm' to truncate by month\n",
      "            Other options are: 'week', 'quarter'\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('1997-02-28',)], ['d'])\n",
      "        >>> df.select(trunc(df.d, 'year').alias('year')).collect()\n",
      "        [Row(year=datetime.date(1997, 1, 1))]\n",
      "        >>> df.select(trunc(df.d, 'mon').alias('month')).collect()\n",
      "        [Row(month=datetime.date(1997, 2, 1))]\n",
      "    \n",
      "    udf(f: Union[Callable[..., Any], ForwardRef('DataTypeOrString'), NoneType] = None, returnType: 'DataTypeOrString' = StringType()) -> Union[ForwardRef('UserDefinedFunctionLike'), Callable[[Callable[..., Any]], ForwardRef('UserDefinedFunctionLike')]]\n",
      "        Creates a user defined function (UDF).\n",
      "        \n",
      "        .. versionadded:: 1.3.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        f : function\n",
      "            python function if used as a standalone function\n",
      "        returnType : :class:`pyspark.sql.types.DataType` or str\n",
      "            the return type of the user-defined function. The value can be either a\n",
      "            :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> from pyspark.sql.types import IntegerType\n",
      "        >>> slen = udf(lambda s: len(s), IntegerType())\n",
      "        >>> @udf\n",
      "        ... def to_upper(s):\n",
      "        ...     if s is not None:\n",
      "        ...         return s.upper()\n",
      "        ...\n",
      "        >>> @udf(returnType=IntegerType())\n",
      "        ... def add_one(x):\n",
      "        ...     if x is not None:\n",
      "        ...         return x + 1\n",
      "        ...\n",
      "        >>> df = spark.createDataFrame([(1, \"John Doe\", 21)], (\"id\", \"name\", \"age\"))\n",
      "        >>> df.select(slen(\"name\").alias(\"slen(name)\"), to_upper(\"name\"), add_one(\"age\")).show()\n",
      "        +----------+--------------+------------+\n",
      "        |slen(name)|to_upper(name)|add_one(age)|\n",
      "        +----------+--------------+------------+\n",
      "        |         8|      JOHN DOE|          22|\n",
      "        +----------+--------------+------------+\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        The user-defined functions are considered deterministic by default. Due to\n",
      "        optimization, duplicate invocations may be eliminated or the function may even be invoked\n",
      "        more times than it is present in the query. If your function is not deterministic, call\n",
      "        `asNondeterministic` on the user defined function. E.g.:\n",
      "        \n",
      "        >>> from pyspark.sql.types import IntegerType\n",
      "        >>> import random\n",
      "        >>> random_udf = udf(lambda: int(random.random() * 100), IntegerType()).asNondeterministic()\n",
      "        \n",
      "        The user-defined functions do not support conditional expressions or short circuiting\n",
      "        in boolean expressions and it ends up with being executed all internally. If the functions\n",
      "        can fail on special rows, the workaround is to incorporate the condition into the functions.\n",
      "        \n",
      "        The user-defined functions do not take keyword arguments on the calling side.\n",
      "    \n",
      "    unbase64(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Decodes a BASE64 encoded string column and returns it as a binary column.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    unhex(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Inverse of hex. Interprets each pair of characters as a hexadecimal number\n",
      "        and converts to the byte representation of number.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('414243',)], ['a']).select(unhex('a')).collect()\n",
      "        [Row(unhex(a)=bytearray(b'ABC'))]\n",
      "    \n",
      "    unix_timestamp(timestamp: Optional[ForwardRef('ColumnOrName')] = None, format: str = 'yyyy-MM-dd HH:mm:ss') -> pyspark.sql.column.Column\n",
      "        Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)\n",
      "        to Unix time stamp (in seconds), using the default timezone and the default\n",
      "        locale, return null if fail.\n",
      "        \n",
      "        if `timestamp` is None, then it returns current timestamp.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.conf.set(\"spark.sql.session.timeZone\", \"America/Los_Angeles\")\n",
      "        >>> time_df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> time_df.select(unix_timestamp('dt', 'yyyy-MM-dd').alias('unix_time')).collect()\n",
      "        [Row(unix_time=1428476400)]\n",
      "        >>> spark.conf.unset(\"spark.sql.session.timeZone\")\n",
      "    \n",
      "    upper(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Converts a string expression to upper case.\n",
      "        \n",
      "        .. versionadded:: 1.5\n",
      "    \n",
      "    var_pop(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the population variance of the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    var_samp(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: returns the unbiased sample variance of\n",
      "        the values in a group.\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    variance(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Aggregate function: alias for var_samp\n",
      "        \n",
      "        .. versionadded:: 1.6\n",
      "    \n",
      "    weekofyear(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the week number of a given date as integer.\n",
      "        A week is considered to start on a Monday and week 1 is the first week with more than 3 days,\n",
      "        as defined by ISO 8601\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(weekofyear(df.dt).alias('week')).collect()\n",
      "        [Row(week=15)]\n",
      "    \n",
      "    when(condition: pyspark.sql.column.Column, value: Any) -> pyspark.sql.column.Column\n",
      "        Evaluates a list of conditions and returns one of multiple possible result expressions.\n",
      "        If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched\n",
      "        conditions.\n",
      "        \n",
      "        .. versionadded:: 1.4.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        condition : :class:`~pyspark.sql.Column`\n",
      "            a boolean :class:`~pyspark.sql.Column` expression.\n",
      "        value :\n",
      "            a literal value, or a :class:`~pyspark.sql.Column` expression.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.select(when(df['age'] == 2, 3).otherwise(4).alias(\"age\")).collect()\n",
      "        [Row(age=3), Row(age=4)]\n",
      "        \n",
      "        >>> df.select(when(df.age == 2, df.age + 1).alias(\"age\")).collect()\n",
      "        [Row(age=3), Row(age=None)]\n",
      "    \n",
      "    window(timeColumn: 'ColumnOrName', windowDuration: str, slideDuration: Optional[str] = None, startTime: Optional[str] = None) -> pyspark.sql.column.Column\n",
      "        Bucketize rows into one or more time windows given a timestamp specifying column. Window\n",
      "        starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window\n",
      "        [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in\n",
      "        the order of months are not supported.\n",
      "        \n",
      "        The time column must be of :class:`pyspark.sql.types.TimestampType`.\n",
      "        \n",
      "        Durations are provided as strings, e.g. '1 second', '1 day 12 hours', '2 minutes'. Valid\n",
      "        interval strings are 'week', 'day', 'hour', 'minute', 'second', 'millisecond', 'microsecond'.\n",
      "        If the ``slideDuration`` is not provided, the windows will be tumbling windows.\n",
      "        \n",
      "        The startTime is the offset with respect to 1970-01-01 00:00:00 UTC with which to start\n",
      "        window intervals. For example, in order to have hourly tumbling windows that start 15 minutes\n",
      "        past the hour, e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`.\n",
      "        \n",
      "        The output column will be a struct called 'window' by default with the nested columns 'start'\n",
      "        and 'end', where 'start' and 'end' will be of :class:`pyspark.sql.types.TimestampType`.\n",
      "        \n",
      "        .. versionadded:: 2.0.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        timeColumn : :class:`~pyspark.sql.Column`\n",
      "            The column or the expression to use as the timestamp for windowing by time.\n",
      "            The time column must be of TimestampType.\n",
      "        windowDuration : str\n",
      "            A string specifying the width of the window, e.g. `10 minutes`,\n",
      "            `1 second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for\n",
      "            valid duration identifiers. Note that the duration is a fixed length of\n",
      "            time, and does not vary over time according to a calendar. For example,\n",
      "            `1 day` always means 86,400,000 milliseconds, not a calendar day.\n",
      "        slideDuration : str, optional\n",
      "            A new window will be generated every `slideDuration`. Must be less than\n",
      "            or equal to the `windowDuration`. Check\n",
      "            `org.apache.spark.unsafe.types.CalendarInterval` for valid duration\n",
      "            identifiers. This duration is likewise absolute, and does not vary\n",
      "            according to a calendar.\n",
      "        startTime : str, optional\n",
      "            The offset with respect to 1970-01-01 00:00:00 UTC with which to start\n",
      "            window intervals. For example, in order to have hourly tumbling windows that\n",
      "            start 15 minutes past the hour, e.g. 12:15-13:15, 13:15-14:15... provide\n",
      "            `startTime` as `15 minutes`.\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> import datetime\n",
      "        >>> df = spark.createDataFrame(\n",
      "        ...     [(datetime.datetime(2016, 3, 11, 9, 0, 7), 1)],\n",
      "        ... ).toDF(\"date\", \"val\")\n",
      "        >>> w = df.groupBy(window(\"date\", \"5 seconds\")).agg(sum(\"val\").alias(\"sum\"))\n",
      "        >>> w.select(w.window.start.cast(\"string\").alias(\"start\"),\n",
      "        ...          w.window.end.cast(\"string\").alias(\"end\"), \"sum\").collect()\n",
      "        [Row(start='2016-03-11 09:00:05', end='2016-03-11 09:00:10', sum=1)]\n",
      "    \n",
      "    xxhash64(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm,\n",
      "        and returns the result as a long column.\n",
      "        \n",
      "        .. versionadded:: 3.0.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> spark.createDataFrame([('ABC',)], ['a']).select(xxhash64('a').alias('hash')).collect()\n",
      "        [Row(hash=4105715581806190027)]\n",
      "    \n",
      "    year(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Extract the year of a given date as integer.\n",
      "        \n",
      "        .. versionadded:: 1.5.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "        >>> df.select(year('dt').alias('year')).collect()\n",
      "        [Row(year=2015)]\n",
      "    \n",
      "    years(col: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "        Partition transform function: A transform for timestamps and dates\n",
      "        to partition data into years.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df.writeTo(\"catalog.db.table\").partitionedBy(  # doctest: +SKIP\n",
      "        ...     years(\"ts\")\n",
      "        ... ).createOrReplace()\n",
      "        \n",
      "        Notes\n",
      "        -----\n",
      "        This function can be used only in combination with\n",
      "        :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`\n",
      "        method of the `DataFrameWriterV2`.\n",
      "    \n",
      "    zip_with(left: 'ColumnOrName', right: 'ColumnOrName', f: Callable[[pyspark.sql.column.Column, pyspark.sql.column.Column], pyspark.sql.column.Column]) -> pyspark.sql.column.Column\n",
      "        Merge two given arrays, element-wise, into a single array using a function.\n",
      "        If one array is shorter, nulls are appended at the end to match the length of the longer\n",
      "        array, before applying the function.\n",
      "        \n",
      "        .. versionadded:: 3.1.0\n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        left : :class:`~pyspark.sql.Column` or str\n",
      "            name of the first column or expression\n",
      "        right : :class:`~pyspark.sql.Column` or str\n",
      "            name of the second column or expression\n",
      "        f : function\n",
      "            a binary function ``(x1: Column, x2: Column) -> Column...``\n",
      "            Can use methods of :class:`~pyspark.sql.Column`, functions defined in\n",
      "            :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.\n",
      "            Python ``UserDefinedFunctions`` are not supported\n",
      "            (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        :class:`~pyspark.sql.Column`\n",
      "        \n",
      "        Examples\n",
      "        --------\n",
      "        >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], (\"id\", \"xs\", \"ys\"))\n",
      "        >>> df.select(zip_with(\"xs\", \"ys\", lambda x, y: x ** y).alias(\"powers\")).show(truncate=False)\n",
      "        +---------------------------+\n",
      "        |powers                     |\n",
      "        +---------------------------+\n",
      "        |[1.0, 9.0, 625.0, 262144.0]|\n",
      "        +---------------------------+\n",
      "        \n",
      "        >>> df = spark.createDataFrame([(1, [\"foo\", \"bar\"], [1, 2, 3])], (\"id\", \"xs\", \"ys\"))\n",
      "        >>> df.select(zip_with(\"xs\", \"ys\", lambda x, y: concat_ws(\"_\", x, y)).alias(\"xs_ys\")).show()\n",
      "        +-----------------+\n",
      "        |            xs_ys|\n",
      "        +-----------------+\n",
      "        |[foo_1, bar_2, 3]|\n",
      "        +-----------------+\n",
      "\n",
      "DATA\n",
      "    Any = typing.Any\n",
      "        Special type indicating an unconstrained type.\n",
      "        \n",
      "        - Any is compatible with every type.\n",
      "        - Any assumed to have all methods.\n",
      "        - All values assumed to be instances of Any.\n",
      "        \n",
      "        Note that all the above statements are true from the point of view of\n",
      "        static type checkers. At runtime, Any should not be used with instance\n",
      "        or class checks.\n",
      "    \n",
      "    Callable = typing.Callable\n",
      "        Callable type; Callable[[int], str] is a function of (int) -> str.\n",
      "        \n",
      "        The subscription syntax must always be used with exactly two\n",
      "        values: the argument list and the return type.  The argument list\n",
      "        must be a list of types or ellipsis; the return type must be a single type.\n",
      "        \n",
      "        There is no syntax to indicate optional or keyword arguments,\n",
      "        such function types are rarely used as callback types.\n",
      "    \n",
      "    Dict = typing.Dict\n",
      "        A generic version of dict.\n",
      "    \n",
      "    Iterable = typing.Iterable\n",
      "        A generic version of collections.abc.Iterable.\n",
      "    \n",
      "    List = typing.List\n",
      "        A generic version of list.\n",
      "    \n",
      "    Optional = typing.Optional\n",
      "        Optional type.\n",
      "        \n",
      "        Optional[X] is equivalent to Union[X, None].\n",
      "    \n",
      "    TYPE_CHECKING = False\n",
      "    Tuple = typing.Tuple\n",
      "        Tuple type; Tuple[X, Y] is the cross-product type of X and Y.\n",
      "        \n",
      "        Example: Tuple[T1, T2] is a tuple of two elements corresponding\n",
      "        to type variables T1 and T2.  Tuple[int, float, str] is a tuple\n",
      "        of an int, a float and a string.\n",
      "        \n",
      "        To specify a variable-length tuple of homogeneous type, use Tuple[T, ...].\n",
      "    \n",
      "    Union = typing.Union\n",
      "        Union type; Union[X, Y] means either X or Y.\n",
      "        \n",
      "        To define a union, use e.g. Union[int, str].  Details:\n",
      "        - The arguments must be types and there must be at least one.\n",
      "        - None as an argument is a special case and is replaced by\n",
      "          type(None).\n",
      "        - Unions of unions are flattened, e.g.::\n",
      "        \n",
      "            Union[Union[int, str], float] == Union[int, str, float]\n",
      "        \n",
      "        - Unions of a single argument vanish, e.g.::\n",
      "        \n",
      "            Union[int] == int  # The constructor actually returns int\n",
      "        \n",
      "        - Redundant arguments are skipped, e.g.::\n",
      "        \n",
      "            Union[int, str, int] == Union[int, str]\n",
      "        \n",
      "        - When comparing unions, the argument order is ignored, e.g.::\n",
      "        \n",
      "            Union[int, str] == Union[str, int]\n",
      "        \n",
      "        - You cannot subclass or instantiate a union.\n",
      "        - You can use Optional[X] as a shorthand for Union[X, None].\n",
      "    \n",
      "    ValuesView = typing.ValuesView\n",
      "        A generic version of collections.abc.ValuesView.\n",
      "\n",
      "FILE\n",
      "    /home/solverbot/.local/lib/python3.10/site-packages/pyspark/sql/functions.py\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(pyspark.sql.functions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b7df96ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function date_format in module pyspark.sql.functions:\n",
      "\n",
      "date_format(date: 'ColumnOrName', format: str) -> pyspark.sql.column.Column\n",
      "    Converts a date/timestamp/string to a value of string in the format specified by the date\n",
      "    format given by the second argument.\n",
      "    \n",
      "    A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All\n",
      "    pattern letters of `datetime pattern`_. can be used.\n",
      "    \n",
      "    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html\n",
      "    \n",
      "    .. versionadded:: 1.5.0\n",
      "    \n",
      "    Notes\n",
      "    -----\n",
      "    Whenever possible, use specialized functions like `year`.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])\n",
      "    >>> df.select(date_format('dt', 'MM/dd/yyy').alias('date')).collect()\n",
      "    [Row(date='04/08/2015')]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(date_format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3df52f91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+\n",
      "|order_month|\n",
      "+-----------+\n",
      "|     201307|\n",
      "|     201307|\n",
      "+-----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderDF.select(date_format('order_date','yyyyMM').alias(\"order_month\")).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5a3b4890",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method withColumn in module pyspark.sql.dataframe:\n",
      "\n",
      "withColumn(colName: str, col: pyspark.sql.column.Column) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Returns a new :class:`DataFrame` by adding a column or replacing the\n",
      "    existing column that has the same name.\n",
      "    \n",
      "    The column expression must be an expression over this :class:`DataFrame`; attempting to add\n",
      "    a column from some other :class:`DataFrame` will raise an error.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    colName : str\n",
      "        string, name of the new column.\n",
      "    col : :class:`Column`\n",
      "        a :class:`Column` expression for the new column.\n",
      "    \n",
      "    Notes\n",
      "    -----\n",
      "    This method introduces a projection internally. Therefore, calling it multiple\n",
      "    times, for instance, via loops in order to add multiple columns can generate big\n",
      "    plans which can cause performance issues and even `StackOverflowException`.\n",
      "    To avoid this, use :func:`select` with the multiple columns at once.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df.withColumn('age2', df.age + 2).collect()\n",
      "    [Row(age=2, name='Alice', age2=4), Row(age=5, name='Bob', age2=7)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.withColumn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "5a2f1650",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method alias in module pyspark.sql.dataframe:\n",
      "\n",
      "alias(alias: str) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Returns a new :class:`DataFrame` with an alias set.\n",
      "    \n",
      "    .. versionadded:: 1.3.0\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    alias : str\n",
      "        an alias name to be set for the :class:`DataFrame`.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> from pyspark.sql.functions import *\n",
      "    >>> df_as1 = df.alias(\"df_as1\")\n",
      "    >>> df_as2 = df.alias(\"df_as2\")\n",
      "    >>> joined_df = df_as1.join(df_as2, col(\"df_as1.name\") == col(\"df_as2.name\"), 'inner')\n",
      "    >>> joined_df.select(\"df_as1.name\", \"df_as2.name\", \"df_as2.age\")                 .sort(desc(\"df_as1.name\")).collect()\n",
      "    [Row(name='Bob', name='Bob', age=5), Row(name='Alice', name='Alice', age=2)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(orderDF.alias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "d7b6dd1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = orderDF.alias('o')\n",
    "oi = orderItemDF.alias('oi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "bbdaca46",
   "metadata": {},
   "outputs": [],
   "source": [
    "joinedOrders=o.where(\"order_status IN ('COMPLETE','CLOSED')\").join(oi,o.order_id == oi.order_item_order_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "88b90039",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+-------------+----------+\n",
      "|order_id|         order_date|order_customer_id|order_item_id|product_id|\n",
      "+--------+-------------------+-----------------+-------------+----------+\n",
      "|       1|2013-07-25 00:00:00|            11599|            1|       957|\n",
      "|       1|2013-07-25 00:00:00|            11599|            1|       957|\n",
      "+--------+-------------------+-----------------+-------------+----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "joinedOrders.select('order_id','order_date','order_customer_id','order_item_id','product_id').show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "eeeb9626",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 39:=============================>                            (1 + 1) / 2]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+------------+--------------+-------------------+\n",
      "|order_id|product_id|product_cost|order_subtotal|         order_date|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "|       3|      null|        null|          null|2013-07-25 00:00:00|\n",
      "|       6|      null|        null|          null|2013-07-25 00:00:00|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.where(\"order_status IN ('COMPLETE','CLOSED')\"). \\\n",
    "    join(oi,o.order_id == oi.order_item_order_id, 'left'). \\\n",
    "    where(orderItemDF.order_item_order_id.isNull()). \\\n",
    "    select('order_id','product_id','product_cost','order_subtotal','order_date').show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "38857f85",
   "metadata": {},
   "outputs": [],
   "source": [
    "leftJoin = o.where(\"order_status IN ('COMPLETE','CLOSED')\"). \\\n",
    "    join(oi,o.order_id == oi.order_item_order_id, 'left'). \\\n",
    "    where(orderItemDF.order_item_order_id.isNotNull()). \\\n",
    "    select('order_id','product_id','product_cost','order_subtotal','order_date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "4bf2141b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+------------+--------------+-------------------+\n",
      "|order_id|product_id|product_cost|order_subtotal|         order_date|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "|       3|      null|        null|          null|2013-07-25 00:00:00|\n",
      "|       3|      null|        null|          null|2013-07-25 00:00:00|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.where(\"order_status IN ('COMPLETE','CLOSED')\"). \\\n",
    "    join(oi,o.order_id == oi.order_item_order_id, 'left'). \\\n",
    "    where(orderItemDF.order_item_order_id.isNull()). \\\n",
    "    select('order_id','product_id','product_cost','order_subtotal','order_date'). \\\n",
    "    where('order_id == 3').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "c03babf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------+-----------------+------------+\n",
      "|order_id|         order_date|order_customer_id|order_status|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "|       6|2013-07-25 00:00:00|             7130|    COMPLETE|\n",
      "|       6|2013-07-25 00:00:00|             7130|    COMPLETE|\n",
      "+--------+-------------------+-----------------+------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.where('order_id = 6').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "8e9a64a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+------------+--------------+-------------------+\n",
      "|order_id|product_id|product_cost|order_subtotal|         order_date|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "|       6|      null|        null|          null|2013-07-25 00:00:00|\n",
      "|       6|      null|        null|          null|2013-07-25 00:00:00|\n",
      "+--------+----------+------------+--------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.where(\"order_status IN ('COMPLETE','CLOSED')\"). \\\n",
    "    join(oi,o.order_id == oi.order_item_order_id, 'left'). \\\n",
    "    where(orderItemDF.order_item_order_id.isNull()). \\\n",
    "    select('order_id','product_id','product_cost','order_subtotal','order_date'). \\\n",
    "    where('order_id == 6').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "2bc78411",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+------------+\n",
      "|product_id|product_revn|\n",
      "+----------+------------+\n",
      "|       897|      3473.6|\n",
      "|       804|      2658.7|\n",
      "+----------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 70:=============================>                            (1 + 1) / 2]\r",
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "leftJoin.groupBy('product_id').agg(round(sum('order_subtotal'),1).alias('product_revn')).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "8caa4769",
   "metadata": {},
   "outputs": [],
   "source": [
    "orderReven = leftJoin.groupBy('order_id','product_id').agg(round(sum('order_subtotal'),1).alias('product_revn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "19172131",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 112:============================>                            (1 + 1) / 2]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+------------+\n",
      "|order_id|product_id|product_revn|\n",
      "+--------+----------+------------+\n",
      "|     236|       365|        60.0|\n",
      "|     256|       365|       120.0|\n",
      "+--------+----------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "orderReven.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "46d1e084",
   "metadata": {},
   "outputs": [],
   "source": [
    "productRevn = leftJoin.groupBy('product_id','order_date').agg(round(sum('order_subtotal'),1).alias('product_revn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "75397fba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 94:=============================>                            (1 + 1) / 2]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------------------+------------+\n",
      "|product_id|         order_date|product_revn|\n",
      "+----------+-------------------+------------+\n",
      "|      1004|2014-03-04 00:00:00|     16399.2|\n",
      "|      1004|2013-12-06 00:00:00|     15999.2|\n",
      "|      1004|2014-04-08 00:00:00|     15999.2|\n",
      "|      1004|2013-11-03 00:00:00|     15599.2|\n",
      "|      1004|2013-11-30 00:00:00|     15599.2|\n",
      "|      1004|2013-08-17 00:00:00|     15199.2|\n",
      "|      1004|2014-01-30 00:00:00|     14799.3|\n",
      "|      1004|2014-07-20 00:00:00|     14799.3|\n",
      "|      1004|2013-12-11 00:00:00|     14799.3|\n",
      "|      1004|2013-09-05 00:00:00|     14399.3|\n",
      "|      1004|2014-05-12 00:00:00|     13999.3|\n",
      "|      1004|2013-12-22 00:00:00|     13999.3|\n",
      "|      1004|2014-02-06 00:00:00|     13999.3|\n",
      "|      1004|2013-11-07 00:00:00|     13599.3|\n",
      "|      1004|2014-04-02 00:00:00|     13599.3|\n",
      "|      1004|2014-06-19 00:00:00|     13599.3|\n",
      "|      1004|2014-01-11 00:00:00|     13599.3|\n",
      "|      1004|2013-10-06 00:00:00|     13599.3|\n",
      "|      1004|2014-05-16 00:00:00|     13599.3|\n",
      "|      1004|2014-04-03 00:00:00|     13199.3|\n",
      "+----------+-------------------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "productRevn.orderBy(col('product_revn').desc()).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "3a87d43a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------------------+------------+\n",
      "|product_id|         order_date|product_revn|\n",
      "+----------+-------------------+------------+\n",
      "|      1073|2014-07-24 00:00:00|      4199.8|\n",
      "|      1073|2014-07-23 00:00:00|      2599.9|\n",
      "|      1073|2014-07-22 00:00:00|      4399.8|\n",
      "|      1073|2014-07-21 00:00:00|      7799.6|\n",
      "|      1073|2014-07-20 00:00:00|      5799.7|\n",
      "|      1073|2014-07-19 00:00:00|      4399.8|\n",
      "|      1073|2014-07-18 00:00:00|      2999.8|\n",
      "|      1073|2014-07-17 00:00:00|      2799.9|\n",
      "|      1073|2014-07-16 00:00:00|      4399.8|\n",
      "|      1073|2014-07-15 00:00:00|      6799.7|\n",
      "|      1073|2014-07-14 00:00:00|      2999.8|\n",
      "|      1073|2014-07-13 00:00:00|      3399.8|\n",
      "|      1073|2014-07-12 00:00:00|      3599.8|\n",
      "|      1073|2014-07-11 00:00:00|      2399.9|\n",
      "|      1073|2014-07-10 00:00:00|      5599.7|\n",
      "|      1073|2014-07-09 00:00:00|      3799.8|\n",
      "|      1073|2014-07-08 00:00:00|      4799.8|\n",
      "|      1073|2014-07-07 00:00:00|      2799.9|\n",
      "|      1073|2014-07-06 00:00:00|      1000.0|\n",
      "|      1073|2014-07-05 00:00:00|      4399.8|\n",
      "+----------+-------------------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "productRevn.orderBy(col('product_id').desc(),col('order_date').desc()).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "7e3d35db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+------------+\n",
      "|order_id|product_id|product_revn|\n",
      "+--------+----------+------------+\n",
      "|   68703|       208|      2000.0|\n",
      "|   68724|       208|      2000.0|\n",
      "|   68736|       208|      2000.0|\n",
      "|   68778|       208|      2000.0|\n",
      "|   68806|       208|      2000.0|\n",
      "|   68821|       208|      2000.0|\n",
      "|   68837|       208|      2000.0|\n",
      "|   68848|       208|      2000.0|\n",
      "|   68858|       208|      2000.0|\n",
      "|   68859|       208|      2000.0|\n",
      "|   68883|       208|      2000.0|\n",
      "|    9084|      1004|      1599.9|\n",
      "|   11105|      1004|      1599.9|\n",
      "|   14539|      1004|      1599.9|\n",
      "|   30299|      1004|      1599.9|\n",
      "|   44891|      1004|      1599.9|\n",
      "|       5|       957|      1199.9|\n",
      "|     730|      1004|      1199.9|\n",
      "|    2760|      1004|      1199.9|\n",
      "|    3531|       957|      1199.9|\n",
      "+--------+----------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "orderReven.orderBy(col('product_revn').desc(),col('order_id')).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "77c8c7a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+----------+\n",
      "|   order_status|stat_count|\n",
      "+---------------+----------+\n",
      "|PENDING_PAYMENT|     15033|\n",
      "|       COMPLETE|     22903|\n",
      "+---------------+----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.groupBy('order_status').agg(count('order_status').alias('stat_count')).show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "c3f5749f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+----------+\n",
      "|   order_status|stat_count|\n",
      "+---------------+----------+\n",
      "|       COMPLETE|     22903|\n",
      "|PENDING_PAYMENT|     15033|\n",
      "|     PROCESSING|      8276|\n",
      "|        PENDING|      7610|\n",
      "|         CLOSED|      7558|\n",
      "|        ON_HOLD|      3798|\n",
      "|SUSPECTED_FRAUD|      1558|\n",
      "|       CANCELED|      1428|\n",
      "| PAYMENT_REVIEW|       729|\n",
      "+---------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "o.groupBy('order_status').agg(count('order_status').alias('stat_count')). \\\n",
    "    orderBy(col('stat_count').desc()).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "c970c11e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 130:>                                                        (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "productRevn.write.csv('product_revenue.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73cb783c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}