demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Polars fast UUID4 string generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import polars.selectors as cs\n",
    "import numpy as np\n",
    "import uuid\n",
    "\n",
    "import polars_uuid4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.20.7'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Make dataframe of with 10 million random numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>Random numbers</th><th>A string column</th></tr><tr><td>u32</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>9999995</td><td>0.342875</td><td>&quot;value&quot;</td></tr><tr><td>9999996</td><td>0.283626</td><td>&quot;value&quot;</td></tr><tr><td>9999997</td><td>0.91639</td><td>&quot;value&quot;</td></tr><tr><td>9999998</td><td>0.299616</td><td>&quot;value&quot;</td></tr><tr><td>9999999</td><td>0.460211</td><td>&quot;value&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────┬────────────────┬─────────────────┐\n",
       "│ index   ┆ Random numbers ┆ A string column │\n",
       "│ ---     ┆ ---            ┆ ---             │\n",
       "│ u32     ┆ f64            ┆ str             │\n",
       "╞═════════╪════════════════╪═════════════════╡\n",
       "│ 9999995 ┆ 0.342875       ┆ value           │\n",
       "│ 9999996 ┆ 0.283626       ┆ value           │\n",
       "│ 9999997 ┆ 0.91639        ┆ value           │\n",
       "│ 9999998 ┆ 0.299616       ┆ value           │\n",
       "│ 9999999 ┆ 0.460211       ┆ value           │\n",
       "└─────────┴────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.DataFrame({\n",
    "    'Random numbers': np.random.rand(10000000),\n",
    "    'A string column': \"value\",\n",
    "}).with_row_index()\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create 10 million UUID4s\n",
    " * with_uuid4() accepts a variable so you can set the name of the series, defaults to uuid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (10_000_000, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>Random numbers</th><th>A string column</th><th>uuid</th></tr><tr><td>u32</td><td>f64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>0</td><td>0.431903</td><td>&quot;value&quot;</td><td>&quot;{57cfa3fd-01a5…</td></tr><tr><td>1</td><td>0.198707</td><td>&quot;value&quot;</td><td>&quot;{3e418a42-db42…</td></tr><tr><td>2</td><td>0.626431</td><td>&quot;value&quot;</td><td>&quot;{1e16aeb2-0675…</td></tr><tr><td>3</td><td>0.790102</td><td>&quot;value&quot;</td><td>&quot;{e1129c0a-38e1…</td></tr><tr><td>4</td><td>0.907382</td><td>&quot;value&quot;</td><td>&quot;{8ad58341-ab23…</td></tr><tr><td>5</td><td>0.995303</td><td>&quot;value&quot;</td><td>&quot;{83ed9d53-30a5…</td></tr><tr><td>6</td><td>0.998931</td><td>&quot;value&quot;</td><td>&quot;{2ce35a0f-9981…</td></tr><tr><td>7</td><td>0.836289</td><td>&quot;value&quot;</td><td>&quot;{655d0891-0f1b…</td></tr><tr><td>8</td><td>0.872352</td><td>&quot;value&quot;</td><td>&quot;{77fec4e7-1a23…</td></tr><tr><td>9</td><td>0.529137</td><td>&quot;value&quot;</td><td>&quot;{912c7ff7-0a12…</td></tr><tr><td>10</td><td>0.322931</td><td>&quot;value&quot;</td><td>&quot;{d6402d0d-b5ab…</td></tr><tr><td>11</td><td>0.456256</td><td>&quot;value&quot;</td><td>&quot;{26c89cc9-d740…</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>9999988</td><td>0.006378</td><td>&quot;value&quot;</td><td>&quot;{ddc657e6-2fa7…</td></tr><tr><td>9999989</td><td>0.50514</td><td>&quot;value&quot;</td><td>&quot;{3a7f87a4-23de…</td></tr><tr><td>9999990</td><td>0.708277</td><td>&quot;value&quot;</td><td>&quot;{b51b0665-32a0…</td></tr><tr><td>9999991</td><td>0.743679</td><td>&quot;value&quot;</td><td>&quot;{5fe2070b-9d4c…</td></tr><tr><td>9999992</td><td>0.937289</td><td>&quot;value&quot;</td><td>&quot;{11b6f029-6d44…</td></tr><tr><td>9999993</td><td>0.763785</td><td>&quot;value&quot;</td><td>&quot;{44b87135-d0a7…</td></tr><tr><td>9999994</td><td>0.913705</td><td>&quot;value&quot;</td><td>&quot;{9127c91c-2a4f…</td></tr><tr><td>9999995</td><td>0.342875</td><td>&quot;value&quot;</td><td>&quot;{4dcc6d5e-97da…</td></tr><tr><td>9999996</td><td>0.283626</td><td>&quot;value&quot;</td><td>&quot;{3b34e5ff-1047…</td></tr><tr><td>9999997</td><td>0.91639</td><td>&quot;value&quot;</td><td>&quot;{d32b1a17-50ba…</td></tr><tr><td>9999998</td><td>0.299616</td><td>&quot;value&quot;</td><td>&quot;{71ad3545-fe92…</td></tr><tr><td>9999999</td><td>0.460211</td><td>&quot;value&quot;</td><td>&quot;{5ca39c0a-9993…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (10_000_000, 4)\n",
       "┌─────────┬────────────────┬─────────────────┬───────────────────────────────────┐\n",
       "│ index   ┆ Random numbers ┆ A string column ┆ uuid                              │\n",
       "│ ---     ┆ ---            ┆ ---             ┆ ---                               │\n",
       "│ u32     ┆ f64            ┆ str             ┆ str                               │\n",
       "╞═════════╪════════════════╪═════════════════╪═══════════════════════════════════╡\n",
       "│ 0       ┆ 0.431903       ┆ value           ┆ {57cfa3fd-01a5-43a1-a1f1-d8f13a7… │\n",
       "│ 1       ┆ 0.198707       ┆ value           ┆ {3e418a42-db42-49c2-bf30-706e64b… │\n",
       "│ 2       ┆ 0.626431       ┆ value           ┆ {1e16aeb2-0675-4d13-afc0-b2bb17c… │\n",
       "│ 3       ┆ 0.790102       ┆ value           ┆ {e1129c0a-38e1-40c5-844c-75caa60… │\n",
       "│ 4       ┆ 0.907382       ┆ value           ┆ {8ad58341-ab23-4c71-8c9b-fae1322… │\n",
       "│ …       ┆ …              ┆ …               ┆ …                                 │\n",
       "│ 9999995 ┆ 0.342875       ┆ value           ┆ {4dcc6d5e-97da-4359-b4be-2b33700… │\n",
       "│ 9999996 ┆ 0.283626       ┆ value           ┆ {3b34e5ff-1047-48de-9b6d-834fbb6… │\n",
       "│ 9999997 ┆ 0.91639        ┆ value           ┆ {d32b1a17-50ba-418e-97c9-a50c2db… │\n",
       "│ 9999998 ┆ 0.299616       ┆ value           ┆ {71ad3545-fe92-49a9-8d79-5bb256e… │\n",
       "│ 9999999 ┆ 0.460211       ┆ value           ┆ {5ca39c0a-9993-468a-9a56-096b290… │\n",
       "└─────────┴────────────────┴─────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.uuid.with_uuid4()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Works with a lazy frame too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>Random numbers</th><th>A string column</th><th>uuid</th></tr><tr><td>u32</td><td>f64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>9999995</td><td>0.185959</td><td>&quot;value&quot;</td><td>&quot;{c4baf1ce-98c5…</td></tr><tr><td>9999996</td><td>0.005801</td><td>&quot;value&quot;</td><td>&quot;{172ddf3c-ea9b…</td></tr><tr><td>9999997</td><td>0.606094</td><td>&quot;value&quot;</td><td>&quot;{3dc75c0d-19fd…</td></tr><tr><td>9999998</td><td>0.268984</td><td>&quot;value&quot;</td><td>&quot;{f9a4f709-a2e9…</td></tr><tr><td>9999999</td><td>0.22677</td><td>&quot;value&quot;</td><td>&quot;{75f6c83d-a693…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌─────────┬────────────────┬─────────────────┬───────────────────────────────────┐\n",
       "│ index   ┆ Random numbers ┆ A string column ┆ uuid                              │\n",
       "│ ---     ┆ ---            ┆ ---             ┆ ---                               │\n",
       "│ u32     ┆ f64            ┆ str             ┆ str                               │\n",
       "╞═════════╪════════════════╪═════════════════╪═══════════════════════════════════╡\n",
       "│ 9999995 ┆ 0.185959       ┆ value           ┆ {c4baf1ce-98c5-47ed-bf0f-ce0ca1f… │\n",
       "│ 9999996 ┆ 0.005801       ┆ value           ┆ {172ddf3c-ea9b-4ec9-bd5a-be58147… │\n",
       "│ 9999997 ┆ 0.606094       ┆ value           ┆ {3dc75c0d-19fd-4cf1-abe4-58d04f9… │\n",
       "│ 9999998 ┆ 0.268984       ┆ value           ┆ {f9a4f709-a2e9-41c5-8b12-e3aa723… │\n",
       "│ 9999999 ┆ 0.22677        ┆ value           ┆ {75f6c83d-a693-4336-8619-517a9c1… │\n",
       "└─────────┴────────────────┴─────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.LazyFrame({\n",
    "    'Random numbers': np.random.rand(10000000),\n",
    "    'A string column': \"value\",\n",
    "}).with_row_index().uuid.with_uuid4().collect()\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### My old way to generate a UUID4 for each row\n",
    "  * Gets job done.  Creates a UUID4 for each row.\n",
    "  * Uses python uuid module.\n",
    "  * Takes a long time (in the polars world).\n",
    "    * 20.7 s ± 91 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.4 s ± 160 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "uuids = [\"{\"+str(uuid.uuid4())+\"}\" for i in range(len(df))]\n",
    "uuid_series = pl.Series(name=\"python_UUID\", values=uuids)\n",
    "df.with_columns(\n",
    "    uuid_series\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Using pl_uuid to generate a UUID4 for each row \n",
    "  * Gets job done.  Creates a UUID4 for each row.\n",
    "  * Uses rust uuid crate.\n",
    "  * Much easier to understand/simpler code.\n",
    "  * ~ 40x faster than using python's uuid module to generate UUID4 when the last column in the df is already a string\n",
    "  * 512 ms ± 6.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512 ms ± 6.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "df.uuid.with_uuid4()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Not quite as fast if there isnt an existing string column in the dataframe\n",
    "  * 644 ms ± 6.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>Random numbers</th></tr><tr><td>u32</td><td>f64</td></tr></thead><tbody><tr><td>9999995</td><td>0.313362</td></tr><tr><td>9999996</td><td>0.679717</td></tr><tr><td>9999997</td><td>0.076164</td></tr><tr><td>9999998</td><td>0.853126</td></tr><tr><td>9999999</td><td>0.892428</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────────┬────────────────┐\n",
       "│ index   ┆ Random numbers │\n",
       "│ ---     ┆ ---            │\n",
       "│ u32     ┆ f64            │\n",
       "╞═════════╪════════════════╡\n",
       "│ 9999995 ┆ 0.313362       │\n",
       "│ 9999996 ┆ 0.679717       │\n",
       "│ 9999997 ┆ 0.076164       │\n",
       "│ 9999998 ┆ 0.853126       │\n",
       "│ 9999999 ┆ 0.892428       │\n",
       "└─────────┴────────────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.DataFrame({\n",
    "    'Random numbers': np.random.rand(10000000),\n",
    "}).with_row_index()\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "644 ms ± 6.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "df.uuid.with_uuid4()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}