raster_benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "import numpy as np\n",
    "from rasterio.enums import Resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Raster_benchmarking.generate_rasters import generate_nlm_rasters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Benchmarks- Run all cells, and just change the num_files_to_open variable to desired benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_files_to_open = 100     #Change to desired input number <--------------!!!!!!!!!!!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating Benchmark Data (seeded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_nlm_rasters('./Raster_benchmarking/data', layers=num_files_to_open)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running DGGS Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Raster_benchmarking.DGGS_funcs import h3index_raster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "discrete_dir = Path('Raster_benchmarking/data/discrete')\n",
    "continuous_dir = Path('Raster_benchmarking/data/continuous')\n",
    "output_dir = Path('Raster_benchmarking/data/dggs')\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "mode = lambda x: pd.Series.mode(x)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Continuous files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 141 ms\n",
      "Wall time: 4.32 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = Parallel(n_jobs=-1)(delayed(h3index_raster)(file, output_dir, stem='continuous', operation=mode) for file in list(sorted(continuous_dir.glob('*.asc')))[:num_files_to_open])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Discrete files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 125 ms\n",
      "Wall time: 4.84 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = Parallel(n_jobs=-1)(delayed(h3index_raster)(file, output_dir, stem='discrete', operation=mode) for file in list(sorted(discrete_dir.glob('*.asc')))[:num_files_to_open])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join and Classify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Raster_benchmarking.DGGS_funcs import classify, summing, final_plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_files = list(output_dir.glob('continuous*.parquet'))\n",
    "d_files = list(output_dir.glob('discrete*.parquet'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Continuous files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing files: 100%|██████████| 100/100 [00:01<00:00, 53.26it/s]\n",
      "Joining DataFrames: 100%|██████████| 99/99 [00:00<00:00, 2189.52it/s]\n"
     ]
    }
   ],
   "source": [
    "cont_df = classify(c_files, num_files_to_open, scale=100)\n",
    "cont_df = summing(cont_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Discrete files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing files: 100%|██████████| 100/100 [00:01<00:00, 50.75it/s]\n",
      "Joining DataFrames: 100%|██████████| 99/99 [00:00<00:00, 2106.20it/s]\n"
     ]
    }
   ],
   "source": [
    "disc_df = classify(d_files, num_files_to_open, scale=100)\n",
    "disc_df = summing(disc_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running Raster Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Raster_benchmarking.Raster_funcs import process_rasters, compute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('Raster_benchmarking/data/raster', exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking & joining files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Continuous files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 156 ms\n",
      "Wall time: 960 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "process_rasters('Raster_benchmarking/data/continuous', num_files_to_open , output_file=f'Raster_benchmarking/data/raster/continuous_{num_files_to_open}.tif', nodata=np.iinfo(np.uint8).max, dtype=np.uint8, resampling=Resampling.nearest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Discrete files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 78.1 ms\n",
      "Wall time: 333 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "process_rasters('Raster_benchmarking/data/discrete', num_files_to_open , output_file=f'Raster_benchmarking/data/raster/discrete_{num_files_to_open}.tif', nodata=np.iinfo(np.uint8).max, dtype=np.uint8, resampling=Resampling.nearest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Continuous files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 297 ms\n",
      "Wall time: 6.77 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "compute(Path(f'Raster_benchmarking/data/raster/continuous_{num_files_to_open}.tif'), band_limit=num_files_to_open)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Discrete files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 734 ms\n",
      "Wall time: 7.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "compute(Path(f'Raster_benchmarking/data/raster/discrete_{num_files_to_open}.tif'), band_limit=num_files_to_open)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dggs",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}