From 132b14bc0c9291d3b5f0513f05bcfb4bd1c8ee58 Mon Sep 17 00:00:00 2001
From: "Iker S. Requerey" <iker.sanchez@vizzuality.com>
Date: Thu, 28 Nov 2024 13:04:59 +0000
Subject: [PATCH] replace nan with null

---
 .../notebooks/04_prepare_tabular_data.ipynb   | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/data-processing/notebooks/04_prepare_tabular_data.ipynb b/data-processing/notebooks/04_prepare_tabular_data.ipynb
index 0559f32..ee07e96 100644
--- a/data-processing/notebooks/04_prepare_tabular_data.ipynb
+++ b/data-processing/notebooks/04_prepare_tabular_data.ipynb
@@ -80,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -117,7 +117,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -151,29 +151,29 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'Agricultural drought exposure': <datasets.datasets.Dataset object at 0x7f79df77a3f0>,\n",
-      " 'Agricultural drought hazard': <datasets.datasets.Dataset object at 0x7f79df77a420>,\n",
-      " 'Boundaries': <datasets.datasets.Dataset object at 0x7f79df77a480>,\n",
-      " 'Contextual layers': <datasets.datasets.Dataset object at 0x7f79df77a540>,\n",
-      " 'EO-based flood exposure': <datasets.datasets.Dataset object at 0x7f79df77a600>,\n",
-      " 'EO-based flood hazard': <datasets.datasets.Dataset object at 0x7f79df77a4b0>,\n",
-      " 'Hydrographic data': <datasets.datasets.Dataset object at 0x7f79df77a4e0>,\n",
-      " 'Hydrometeorological Data': <datasets.datasets.Dataset object at 0x7f79df77a690>,\n",
-      " 'In-situ Data': <datasets.datasets.Dataset object at 0x7f79df77a6c0>,\n",
-      " 'Meteorological drought exposure': <datasets.datasets.Dataset object at 0x7f79df77a6f0>,\n",
-      " 'Meteorological drought hazard': <datasets.datasets.Dataset object at 0x7f79df77a720>,\n",
-      " 'Model-based flood exposure': <datasets.datasets.Dataset object at 0x7f79df77a750>,\n",
-      " 'Model-based flood hazard': <datasets.datasets.Dataset object at 0x7f79df77a780>,\n",
-      " 'Populated infrastructures': <datasets.datasets.Dataset object at 0x7f79df77a7b0>,\n",
-      " 'Transportation Network Infrastructures': <datasets.datasets.Dataset object at 0x7f79df77a7e0>,\n",
-      " 'Water-related infrastructures': <datasets.datasets.Dataset object at 0x7f79df77a810>}\n"
+      "{'Agricultural drought exposure': <datasets.datasets.Dataset object at 0x7f3df02993d0>,\n",
+      " 'Agricultural drought hazard': <datasets.datasets.Dataset object at 0x7f3d16b4e630>,\n",
+      " 'Boundaries': <datasets.datasets.Dataset object at 0x7f3d4bf21850>,\n",
+      " 'Contextual layers': <datasets.datasets.Dataset object at 0x7f3d16bca630>,\n",
+      " 'EO-based flood exposure': <datasets.datasets.Dataset object at 0x7f3d16bca600>,\n",
+      " 'EO-based flood hazard': <datasets.datasets.Dataset object at 0x7f3d16bca660>,\n",
+      " 'Hydrographic data': <datasets.datasets.Dataset object at 0x7f3d16bca720>,\n",
+      " 'Hydrometeorological Data': <datasets.datasets.Dataset object at 0x7f3d16bca7e0>,\n",
+      " 'In-situ Data': <datasets.datasets.Dataset object at 0x7f3d16bca690>,\n",
+      " 'Meteorological drought exposure': <datasets.datasets.Dataset object at 0x7f3d16bca6c0>,\n",
+      " 'Meteorological drought hazard': <datasets.datasets.Dataset object at 0x7f3d16bca870>,\n",
+      " 'Model-based flood exposure': <datasets.datasets.Dataset object at 0x7f3d16bca8a0>,\n",
+      " 'Model-based flood hazard': <datasets.datasets.Dataset object at 0x7f3d16bca8d0>,\n",
+      " 'Populated infrastructures': <datasets.datasets.Dataset object at 0x7f3d16bca900>,\n",
+      " 'Transportation Network Infrastructures': <datasets.datasets.Dataset object at 0x7f3d16bca930>,\n",
+      " 'Water-related infrastructures': <datasets.datasets.Dataset object at 0x7f3d16bca960>}\n"
      ]
     }
    ],
@@ -191,20 +191,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'Administrative Boundaries - adm0': <datasets.datasets.Layer at 0x7f79df77a930>,\n",
-       " 'Administrative Boundaries - adm1': <datasets.datasets.Layer at 0x7f79df77a840>,\n",
-       " 'Administrative Boundaries - adm2': <datasets.datasets.Layer at 0x7f79df77b110>,\n",
-       " 'Administrative Boundaries - adm3': <datasets.datasets.Layer at 0x7f79df77b140>,\n",
-       " 'Hydrological Basins': <datasets.datasets.Layer at 0x7f79df77b1a0>}"
+       "{'Administrative Boundaries - adm0': <datasets.datasets.Layer at 0x7f3d16bc84d0>,\n",
+       " 'Administrative Boundaries - adm1': <datasets.datasets.Layer at 0x7f3d16bcbef0>,\n",
+       " 'Administrative Boundaries - adm2': <datasets.datasets.Layer at 0x7f3d16bcab10>,\n",
+       " 'Administrative Boundaries - adm3': <datasets.datasets.Layer at 0x7f3d16bcaa20>,\n",
+       " 'Hydrological Basins': <datasets.datasets.Layer at 0x7f3d16bcacf0>}"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -335,7 +335,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -352,7 +352,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -408,13 +408,14 @@
     "\n",
     "        for _index, row in df.iterrows():\n",
     "            code = str(row[code_column])\n",
+    "            y_values = convert_string_to_float_list(row[\"y_axis_values\"])\n",
     "\n",
     "            chart_data = {\n",
     "                \"id\": chart_id,\n",
     "                \"location_code\": str(row[code_column]),\n",
     "                \"year\": int(row[\"year\"]),\n",
     "                \"x_values\": literal_eval(row[\"x_axis_values\"]),\n",
-    "                \"y_values\": convert_string_to_float_list(row[\"y_axis_values\"]),\n",
+    "                \"y_values\": [None if math.isnan(value) else value for value in y_values],\n",
     "                \"createdAt\": datetime.now().isoformat(),\n",
     "                \"updatedAt\": datetime.now().isoformat(),\n",
     "                \"layer\": layer_id,\n",