g-research-lightgbm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d5146163",
   "metadata": {
    "papermill": {
     "duration": 0.025652,
     "end_time": "2022-01-26T12:03:08.234785",
     "exception": false,
     "start_time": "2022-01-26T12:03:08.209133",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Loading the data\n",
    "\n",
    "References: [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12a34c81",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:03:08.310665Z",
     "iopub.status.busy": "2022-01-26T12:03:08.303194Z",
     "iopub.status.idle": "2022-01-26T12:03:10.751905Z",
     "shell.execute_reply": "2022-01-26T12:03:10.752688Z",
     "shell.execute_reply.started": "2022-01-26T10:27:05.643366Z"
    },
    "papermill": {
     "duration": 2.484151,
     "end_time": "2022-01-26T12:03:10.753039",
     "exception": false,
     "start_time": "2022-01-26T12:03:08.268888",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import gc\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from numba import jit\n",
    "from lightgbm import LGBMRegressor\n",
    "\n",
    "import gresearch_crypto\n",
    "\n",
    "# from cuml import ForestInference\n",
    "\n",
    "\n",
    "TRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\n",
    "TRAIN_SUPPLEMENTAL_CSV = '../input/g-research-crypto-forecasting/supplemental_train.csv'\n",
    "ASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\n",
    "\n",
    "SEED = 42\n",
    "\n",
    "REMOVE_LB_TEST_OVERLAPPING_DATA = False\n",
    "REMOVE_SOME_DATA_AT_THE_BEGINNING = True\n",
    "\n",
    "EVALUATION = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b36e4655",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:03:10.802044Z",
     "iopub.status.busy": "2022-01-26T12:03:10.801381Z",
     "iopub.status.idle": "2022-01-26T12:03:10.805744Z",
     "shell.execute_reply": "2022-01-26T12:03:10.806225Z",
     "shell.execute_reply.started": "2022-01-26T10:27:08.560340Z"
    },
    "papermill": {
     "duration": 0.03017,
     "end_time": "2022-01-26T12:03:10.806403",
     "exception": false,
     "start_time": "2022-01-26T12:03:10.776233",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def fix_all_seeds(seed):\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "\n",
    "fix_all_seeds(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "08157fb5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:03:10.854688Z",
     "iopub.status.busy": "2022-01-26T12:03:10.854070Z",
     "iopub.status.idle": "2022-01-26T12:04:26.158410Z",
     "shell.execute_reply": "2022-01-26T12:04:26.157705Z",
     "shell.execute_reply.started": "2022-01-26T10:27:08.568219Z"
    },
    "papermill": {
     "duration": 75.329488,
     "end_time": "2022-01-26T12:04:26.158647",
     "exception": false,
     "start_time": "2022-01-26T12:03:10.829159",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 38.9 s, sys: 3.49 s, total: 42.4 s\n",
      "Wall time: 1min 15s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>Asset_ID</th>\n",
       "      <th>Count</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>VWAP</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1514764860</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "      <td>2376.580078</td>\n",
       "      <td>2399.500000</td>\n",
       "      <td>2357.139893</td>\n",
       "      <td>2374.590088</td>\n",
       "      <td>19.233006</td>\n",
       "      <td>2373.116455</td>\n",
       "      <td>-0.004218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1514764860</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>8.530000</td>\n",
       "      <td>8.530000</td>\n",
       "      <td>8.530000</td>\n",
       "      <td>8.530000</td>\n",
       "      <td>78.379997</td>\n",
       "      <td>8.530000</td>\n",
       "      <td>-0.014399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1514764860</td>\n",
       "      <td>1</td>\n",
       "      <td>229</td>\n",
       "      <td>13835.194336</td>\n",
       "      <td>14013.799805</td>\n",
       "      <td>13666.110352</td>\n",
       "      <td>13850.175781</td>\n",
       "      <td>31.550062</td>\n",
       "      <td>13827.062500</td>\n",
       "      <td>-0.014643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1514764860</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>7.659600</td>\n",
       "      <td>7.659600</td>\n",
       "      <td>7.656700</td>\n",
       "      <td>7.657600</td>\n",
       "      <td>6626.713379</td>\n",
       "      <td>7.657713</td>\n",
       "      <td>-0.013922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1514764860</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>25.920000</td>\n",
       "      <td>25.920000</td>\n",
       "      <td>25.874001</td>\n",
       "      <td>25.877001</td>\n",
       "      <td>121.087311</td>\n",
       "      <td>25.891363</td>\n",
       "      <td>-0.008264</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    timestamp  Asset_ID  Count          Open          High           Low  \\\n",
       "0  1514764860         2     40   2376.580078   2399.500000   2357.139893   \n",
       "1  1514764860         0      5      8.530000      8.530000      8.530000   \n",
       "2  1514764860         1    229  13835.194336  14013.799805  13666.110352   \n",
       "3  1514764860         5     32      7.659600      7.659600      7.656700   \n",
       "4  1514764860         7      5     25.920000     25.920000     25.874001   \n",
       "\n",
       "          Close       Volume          VWAP    Target  \n",
       "0   2374.590088    19.233006   2373.116455 -0.004218  \n",
       "1      8.530000    78.379997      8.530000 -0.014399  \n",
       "2  13850.175781    31.550062  13827.062500 -0.014643  \n",
       "3      7.657600  6626.713379      7.657713 -0.013922  \n",
       "4     25.877001   121.087311     25.891363 -0.008264  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_df = pd.read_csv(TRAIN_CSV, dtype={\n",
    "                'timestamp': np.int32, 'Asset_ID': np.int8,\n",
    "                'Open': np.float32, 'High': np.float32,\n",
    "                'Low': np.float32, 'Close': np.float32,\n",
    "                'Count': np.int32, 'Volume': np.float32,\n",
    "                'VWAP': np.float32})\n",
    "\n",
    "train_supplemental_df = pd.read_csv(TRAIN_SUPPLEMENTAL_CSV, dtype={\n",
    "                'timestamp': np.int32, 'Asset_ID': np.int8,\n",
    "                'Open': np.float32, 'High': np.float32,\n",
    "                'Low': np.float32, 'Close': np.float32,\n",
    "                'Count': np.int32, 'Volume': np.float32,\n",
    "                'VWAP': np.float32})\n",
    "\n",
    "\n",
    "gc.collect()\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f427f5d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:26.220937Z",
     "iopub.status.busy": "2022-01-26T12:04:26.220267Z",
     "iopub.status.idle": "2022-01-26T12:04:26.235530Z",
     "shell.execute_reply": "2022-01-26T12:04:26.236015Z",
     "shell.execute_reply.started": "2022-01-26T10:28:13.967065Z"
    },
    "papermill": {
     "duration": 0.053608,
     "end_time": "2022-01-26T12:04:26.236173",
     "exception": false,
     "start_time": "2022-01-26T12:04:26.182565",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 24236806 entries, 0 to 24236805\n",
      "Data columns (total 10 columns):\n",
      " #   Column     Dtype  \n",
      "---  ------     -----  \n",
      " 0   timestamp  int32  \n",
      " 1   Asset_ID   int8   \n",
      " 2   Count      int32  \n",
      " 3   Open       float32\n",
      " 4   High       float32\n",
      " 5   Low        float32\n",
      " 6   Close      float32\n",
      " 7   Volume     float32\n",
      " 8   VWAP       float32\n",
      " 9   Target     float64\n",
      "dtypes: float32(6), float64(1), int32(2), int8(1)\n",
      "memory usage: 947.7 MB\n"
     ]
    }
   ],
   "source": [
    "train_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7124ce0b",
   "metadata": {
    "papermill": {
     "duration": 0.023218,
     "end_time": "2022-01-26T12:04:26.282782",
     "exception": false,
     "start_time": "2022-01-26T12:04:26.259564",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Keep only values _before_ the LB test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "993b21be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:26.343370Z",
     "iopub.status.busy": "2022-01-26T12:04:26.342420Z",
     "iopub.status.idle": "2022-01-26T12:04:28.986275Z",
     "shell.execute_reply": "2022-01-26T12:04:28.986770Z",
     "shell.execute_reply.started": "2022-01-26T10:28:13.993763Z"
    },
    "papermill": {
     "duration": 2.680605,
     "end_time": "2022-01-26T12:04:28.986954",
     "exception": false,
     "start_time": "2022-01-26T12:04:26.306349",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Remove the future\n",
    "train_df['datetime'] = pd.to_datetime(train_df['timestamp'], unit='s')\n",
    "\n",
    "if EVALUATION:\n",
    "    test_df = train_df[train_df['datetime'] >= '2021-06-13 00:00:00'].copy()\n",
    "\n",
    "if REMOVE_LB_TEST_OVERLAPPING_DATA:\n",
    "    train_df = train_df[train_df['datetime'] < '2021-06-13 00:00:00']\n",
    "    \n",
    "if REMOVE_SOME_DATA_AT_THE_BEGINNING:\n",
    "    train_df = train_df[train_df['datetime'] >= '2018-07-01 00:00:00']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ee8cfcf6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:29.038304Z",
     "iopub.status.busy": "2022-01-26T12:04:29.037664Z",
     "iopub.status.idle": "2022-01-26T12:04:29.056592Z",
     "shell.execute_reply": "2022-01-26T12:04:29.056113Z",
     "shell.execute_reply.started": "2022-01-26T10:28:16.546013Z"
    },
    "papermill": {
     "duration": 0.046334,
     "end_time": "2022-01-26T12:04:29.056723",
     "exception": false,
     "start_time": "2022-01-26T12:04:29.010389",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Asset_ID</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Asset_Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4.304065</td>\n",
       "      <td>Binance Coin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>6.779922</td>\n",
       "      <td>Bitcoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>2.397895</td>\n",
       "      <td>Bitcoin Cash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>4.406719</td>\n",
       "      <td>Cardano</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4</td>\n",
       "      <td>3.555348</td>\n",
       "      <td>Dogecoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1.386294</td>\n",
       "      <td>EOS.IO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>5.894403</td>\n",
       "      <td>Ethereum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>Ethereum Classic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>8</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>IOTA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9</td>\n",
       "      <td>2.397895</td>\n",
       "      <td>Litecoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>10</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>Maker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>Monero</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>Stellar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>1.791759</td>\n",
       "      <td>TRON</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Asset_ID    Weight        Asset_Name\n",
       "1          0  4.304065      Binance Coin\n",
       "2          1  6.779922           Bitcoin\n",
       "0          2  2.397895      Bitcoin Cash\n",
       "10         3  4.406719           Cardano\n",
       "13         4  3.555348          Dogecoin\n",
       "3          5  1.386294            EOS.IO\n",
       "5          6  5.894403          Ethereum\n",
       "4          7  2.079442  Ethereum Classic\n",
       "11         8  1.098612              IOTA\n",
       "6          9  2.397895          Litecoin\n",
       "12        10  1.098612             Maker\n",
       "7         11  1.609438            Monero\n",
       "9         12  2.079442           Stellar\n",
       "8         13  1.791759              TRON"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n",
    "df_asset_details"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25423658",
   "metadata": {
    "papermill": {
     "duration": 0.023922,
     "end_time": "2022-01-26T12:04:29.104632",
     "exception": false,
     "start_time": "2022-01-26T12:04:29.080710",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80663a1d",
   "metadata": {
    "papermill": {
     "duration": 0.023232,
     "end_time": "2022-01-26T12:04:29.151854",
     "exception": false,
     "start_time": "2022-01-26T12:04:29.128622",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Concat train and train_supplemental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "024f37a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:29.206760Z",
     "iopub.status.busy": "2022-01-26T12:04:29.205474Z",
     "iopub.status.idle": "2022-01-26T12:04:29.808631Z",
     "shell.execute_reply": "2022-01-26T12:04:29.808033Z",
     "shell.execute_reply.started": "2022-01-26T10:28:16.565211Z"
    },
    "papermill": {
     "duration": 0.632853,
     "end_time": "2022-01-26T12:04:29.808763",
     "exception": false,
     "start_time": "2022-01-26T12:04:29.175910",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_df = pd.concat([train_df, train_supplemental_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d6845bf9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:29.862638Z",
     "iopub.status.busy": "2022-01-26T12:04:29.860572Z",
     "iopub.status.idle": "2022-01-26T12:04:31.920538Z",
     "shell.execute_reply": "2022-01-26T12:04:31.919896Z",
     "shell.execute_reply.started": "2022-01-26T10:28:17.112814Z"
    },
    "papermill": {
     "duration": 2.08774,
     "end_time": "2022-01-26T12:04:31.920683",
     "exception": false,
     "start_time": "2022-01-26T12:04:29.832943",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_df.sort_values(by='timestamp', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89ccc26a",
   "metadata": {
    "papermill": {
     "duration": 0.024032,
     "end_time": "2022-01-26T12:04:31.969641",
     "exception": false,
     "start_time": "2022-01-26T12:04:31.945609",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Forward filling missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6289c4e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:32.021324Z",
     "iopub.status.busy": "2022-01-26T12:04:32.020366Z",
     "iopub.status.idle": "2022-01-26T12:04:48.125876Z",
     "shell.execute_reply": "2022-01-26T12:04:48.126384Z",
     "shell.execute_reply.started": "2022-01-26T10:28:19.194094Z"
    },
    "papermill": {
     "duration": 16.133251,
     "end_time": "2022-01-26T12:04:48.126546",
     "exception": false,
     "start_time": "2022-01-26T12:04:31.993295",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def asset_ffill(grp):\n",
    "    new_index = range(grp.index.get_level_values(1)[0],grp.index.get_level_values(1)[-1]+60,60)\n",
    "    return grp.reset_index(level=0,drop=True).reindex(new_index, method='pad')\n",
    "\n",
    "train_df = train_df.set_index(['Asset_ID', 'timestamp'])\n",
    "train_df = train_df.groupby(level='Asset_ID').apply(asset_ffill).reset_index()\n",
    "train_df['datetime'] = pd.to_datetime(train_df['timestamp'], unit='s')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecb8a87c",
   "metadata": {
    "papermill": {
     "duration": 0.024278,
     "end_time": "2022-01-26T12:04:48.175198",
     "exception": false,
     "start_time": "2022-01-26T12:04:48.150920",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2887100e",
   "metadata": {
    "papermill": {
     "duration": 0.023864,
     "end_time": "2022-01-26T12:04:48.223030",
     "exception": false,
     "start_time": "2022-01-26T12:04:48.199166",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "50f288a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:04:48.275001Z",
     "iopub.status.busy": "2022-01-26T12:04:48.273953Z",
     "iopub.status.idle": "2022-01-26T12:08:54.912624Z",
     "shell.execute_reply": "2022-01-26T12:08:54.911965Z",
     "shell.execute_reply.started": "2022-01-26T10:28:34.432862Z"
    },
    "papermill": {
     "duration": 246.665916,
     "end_time": "2022-01-26T12:08:54.912817",
     "exception": false,
     "start_time": "2022-01-26T12:04:48.246901",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/arraylike.py:364: RuntimeWarning: invalid value encountered in log\n",
      "  result = getattr(ufunc, method)(*inputs, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 48s, sys: 18.3 s, total: 4min 6s\n",
      "Wall time: 4min 6s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Two new features from the competition tutorial\n",
    "def upper_shadow(df):\n",
    "    return df['High'] - np.maximum(df['Close'], df['Open'])\n",
    "\n",
    "def lower_shadow(df):\n",
    "    return np.minimum(df['Close'], df['Open']) - df['Low']\n",
    "\n",
    "# define function to compute log returns\n",
    "def log_return(series, periods=1):\n",
    "    return np.log(series).diff(periods=periods)\n",
    "\n",
    "# Consecutive price trends\n",
    "@jit(nopython=True)\n",
    "def create_up_down(price_array, window):\n",
    "    pastUD = np.zeros(len(price_array))\n",
    "    for i in range(window+1, len(price_array)):\n",
    "        pastUD[i] = window - 2*np.sum(price_array[i-window:i] < price_array[i-window-1:i-1])\n",
    "    return pastUD\n",
    "\n",
    "# A utility function to build features from the original df\n",
    "# It works for rows to, so we can reutilize it.\n",
    "def calc_features(df, row=False):\n",
    "    df_feat = df\n",
    "    df_feat[\"Upper_Shadow\"] = upper_shadow(df_feat)\n",
    "    df_feat[\"Lower_Shadow\"] = lower_shadow(df_feat)\n",
    "\n",
    "    ## Adding some more features\n",
    "    df_feat[\"Close/Open\"] = (df_feat[\"Close\"] / df_feat[\"Open\"]).astype(np.float32)\n",
    "    df_feat[\"Close-Open\"] = (df_feat[\"Close\"] - df_feat[\"Open\"]).astype(np.float32) \n",
    "    df_feat[\"High-Low\"] = (df_feat[\"High\"] - df_feat[\"Low\"]).astype(np.float32)\n",
    "    df_feat[\"High/Low\"] = (df_feat[\"High\"] / df_feat[\"Low\"]).astype(np.float32)\n",
    "    \n",
    "    df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1).astype(np.float32)\n",
    "\n",
    "    df_feat[\"High/Mean\"] = (df_feat[\"High\"] / df_feat[\"Mean\"]).astype(np.float32)\n",
    "#     df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n",
    "    df_feat[\"Volume/Count\"] = (df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1) ).astype(np.float32)\n",
    "\n",
    "    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n",
    "    ### to do: add cyclical features for seasonality\n",
    "    times = pd.to_datetime(df_feat[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n",
    "\n",
    "    df_feat[\"hour\"] = times.dt.hour  # .dt\n",
    "    df_feat[\"dayofweek\"] = times.dt.dayofweek \n",
    "    df_feat[\"day\"] = times.dt.day \n",
    "\n",
    "#     df_feat[\"Median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median(axis=1)\n",
    "#     df_feat[\"High/Median\"] = df_feat[\"High\"] / df_feat[\"Median\"]\n",
    "#     df_feat[\"Low/Median\"] = df_feat[\"Low\"] / df_feat[\"Median\"]\n",
    "    \n",
    "    # [log_return(btc.VWAP,periods=5), log_return(btc.VWAP,periods=1).abs()\n",
    "    \n",
    "    df_feat['Close/VWAP'] = (df_feat['Close'] / df_feat['VWAP']).astype(np.float32)\n",
    "    \n",
    "    # Lags and rollings\n",
    "    df_feat_close_grpby = df_feat.groupby('Asset_ID')['Close']\n",
    "    df_feat_vwap_grpby = df_feat.groupby('Asset_ID')['VWAP']\n",
    "    \n",
    "    for window in [15, 30, 60]:#\n",
    "        df_feat['sma{}'.format(window)] = df_feat_close_grpby.transform(lambda x: x.rolling(window).mean() / x - 1).astype(np.float32)\n",
    "        df_feat['log_return{}'.format(window)] = df_feat_close_grpby.transform(lambda x: log_return(x, window)).astype(np.float32)\n",
    "        df_feat['up_down{}'.format(window)] = df_feat_close_grpby.transform(lambda x: create_up_down(x.values, window)).astype(np.float32)\n",
    "        \n",
    "#     df_feat['up_down30'] = df_feat_close_grpby.transform(lambda x: create_up_down(x.values, 30)).astype(np.float32)\n",
    "\n",
    "    df_feat['sma_diff15-60'] = (df_feat['sma15'] - df_feat['sma60']).astype(np.float32)\n",
    "    \n",
    "    df_feat['ma_close/vwap15'.format(15)] = df_feat.groupby('Asset_ID')['Close/VWAP'].transform(lambda x: x.rolling(15).mean()).astype(np.float32)\n",
    "    \n",
    "    df_feat['log_return1'] = df_feat_close_grpby.transform(lambda x: log_return(x, 1)).astype(np.float32)\n",
    "    \n",
    "    df_feat['vwap_log_return1'] = df_feat_vwap_grpby.transform(lambda x: log_return(x, 1)).astype(np.float32)\n",
    "    df_feat['vwap_log_return5'] = df_feat_vwap_grpby.transform(lambda x: log_return(x, 5)).astype(np.float32)\n",
    "    df_feat['vwap_log_return15'] = df_feat_vwap_grpby.transform(lambda x: log_return(x, 15)).astype(np.float32)\n",
    "    \n",
    "    df_feat['volume_rmean5'] = df_feat.groupby('Asset_ID')['Volume'].transform(lambda x: x.rolling(5).mean()).astype(np.float32)\n",
    "    df_feat['volume_rmean15'] = df_feat.groupby('Asset_ID')['Volume'].transform(lambda x: x.rolling(15).mean()).astype(np.float32)\n",
    "    \n",
    "#     df_feat['close/open_rmean15'] = df_feat.groupby('Asset_ID')['Close/Open'].transform(lambda x: x.rolling(15).mean())\n",
    "    \n",
    "    # Grouping by timestamp (current market condition)\n",
    "    df_feat['time_sma15'] = df_feat.groupby('timestamp')['sma15'].transform('mean').astype(np.float32)\n",
    "#     df_feat['time_up_down30'] = df_feat.groupby('timestamp')['up_down30'].transform('mean').astype(np.float32)\n",
    "#     df_feat['time_sma1440'] = df_feat.groupby('timestamp')['sma1440'].transform('mean')\n",
    "#     df_feat['time_sma60'] = df_feat.groupby('timestamp')['sma60'].transform('mean') # Score got worse, need CV test\n",
    "\n",
    "#     df_feat['time_log_return15'] = df_feat.groupby('timestamp')['log_return15'].transform('mean') # Score slightly got worse\n",
    "    \n",
    "#     df_feat['time_volume_rmean15'] = df_feat.groupby('timestamp')['volume_rmean15'].transform('mean') # Score got worse\n",
    "#     df_feat['time_volume_rmean5'] = df_feat.groupby('timestamp')['volume_rmean5'].transform('mean')\n",
    "#     df_feat['time_ma_close/vwap_mean'] = df_feat.groupby('timestamp')['ma_close/vwap15'].transform('mean')vwap_log_return15\n",
    "\n",
    "#     df_feat['time_close/open_rmean15_mean'] = df_feat.groupby('timestamp')['close/open_rmean15'].transform('mean')\n",
    "\n",
    "    df_feat = df_feat.fillna(0)\n",
    "        \n",
    "    return df_feat\n",
    "\n",
    "train_df = calc_features(train_df)\n",
    "train_df = train_df.dropna(how=\"any\")\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fd1ebcd0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:08:54.965183Z",
     "iopub.status.busy": "2022-01-26T12:08:54.964576Z",
     "iopub.status.idle": "2022-01-26T12:08:54.967778Z",
     "shell.execute_reply": "2022-01-26T12:08:54.968323Z",
     "shell.execute_reply.started": "2022-01-26T10:32:54.958502Z"
    },
    "papermill": {
     "duration": 0.030903,
     "end_time": "2022-01-26T12:08:54.968470",
     "exception": false,
     "start_time": "2022-01-26T12:08:54.937567",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if EVALUATION:\n",
    "    test_df = calc_features(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2dd323ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:08:55.021635Z",
     "iopub.status.busy": "2022-01-26T12:08:55.021065Z",
     "iopub.status.idle": "2022-01-26T12:08:55.066973Z",
     "shell.execute_reply": "2022-01-26T12:08:55.066417Z",
     "shell.execute_reply.started": "2022-01-26T10:32:55.370287Z"
    },
    "papermill": {
     "duration": 0.074085,
     "end_time": "2022-01-26T12:08:55.067145",
     "exception": false,
     "start_time": "2022-01-26T12:08:54.993060",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Asset_ID</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>Count</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>VWAP</th>\n",
       "      <th>Target</th>\n",
       "      <th>...</th>\n",
       "      <th>up_down60</th>\n",
       "      <th>sma_diff15-60</th>\n",
       "      <th>ma_close/vwap15</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>vwap_log_return1</th>\n",
       "      <th>vwap_log_return5</th>\n",
       "      <th>vwap_log_return15</th>\n",
       "      <th>volume_rmean5</th>\n",
       "      <th>volume_rmean15</th>\n",
       "      <th>time_sma15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403260</td>\n",
       "      <td>29</td>\n",
       "      <td>14.6931</td>\n",
       "      <td>14.7220</td>\n",
       "      <td>14.6900</td>\n",
       "      <td>14.6900</td>\n",
       "      <td>604.219971</td>\n",
       "      <td>14.699802</td>\n",
       "      <td>0.003204</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403320</td>\n",
       "      <td>109</td>\n",
       "      <td>14.6900</td>\n",
       "      <td>14.7222</td>\n",
       "      <td>14.6501</td>\n",
       "      <td>14.6701</td>\n",
       "      <td>4506.430176</td>\n",
       "      <td>14.694633</td>\n",
       "      <td>-0.000143</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001355</td>\n",
       "      <td>-0.000352</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403380</td>\n",
       "      <td>26</td>\n",
       "      <td>14.6783</td>\n",
       "      <td>14.6918</td>\n",
       "      <td>14.6620</td>\n",
       "      <td>14.6780</td>\n",
       "      <td>591.239990</td>\n",
       "      <td>14.676436</td>\n",
       "      <td>-0.004589</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000538</td>\n",
       "      <td>-0.001239</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403440</td>\n",
       "      <td>36</td>\n",
       "      <td>14.6830</td>\n",
       "      <td>14.7300</td>\n",
       "      <td>14.6750</td>\n",
       "      <td>14.7076</td>\n",
       "      <td>823.119995</td>\n",
       "      <td>14.712523</td>\n",
       "      <td>-0.000531</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002014</td>\n",
       "      <td>0.002456</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403500</td>\n",
       "      <td>16</td>\n",
       "      <td>14.7080</td>\n",
       "      <td>14.7181</td>\n",
       "      <td>14.6896</td>\n",
       "      <td>14.6896</td>\n",
       "      <td>211.289993</td>\n",
       "      <td>14.706444</td>\n",
       "      <td>-0.004728</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001225</td>\n",
       "      <td>-0.000413</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1347.260010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403560</td>\n",
       "      <td>10</td>\n",
       "      <td>14.7128</td>\n",
       "      <td>14.7128</td>\n",
       "      <td>14.6900</td>\n",
       "      <td>14.7010</td>\n",
       "      <td>87.800003</td>\n",
       "      <td>14.705889</td>\n",
       "      <td>-0.006988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000776</td>\n",
       "      <td>-0.000038</td>\n",
       "      <td>0.000414</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1243.976074</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403620</td>\n",
       "      <td>11</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>14.7010</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>236.759995</td>\n",
       "      <td>14.705564</td>\n",
       "      <td>-0.005472</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000612</td>\n",
       "      <td>-0.000022</td>\n",
       "      <td>0.000744</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>390.041992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403680</td>\n",
       "      <td>89</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>14.7485</td>\n",
       "      <td>14.6940</td>\n",
       "      <td>14.6940</td>\n",
       "      <td>2551.590088</td>\n",
       "      <td>14.721313</td>\n",
       "      <td>-0.011141</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001088</td>\n",
       "      <td>0.001070</td>\n",
       "      <td>0.003053</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>782.112000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403740</td>\n",
       "      <td>68</td>\n",
       "      <td>14.6851</td>\n",
       "      <td>14.7499</td>\n",
       "      <td>14.6746</td>\n",
       "      <td>14.7204</td>\n",
       "      <td>1385.589966</td>\n",
       "      <td>14.705611</td>\n",
       "      <td>-0.011512</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001795</td>\n",
       "      <td>-0.001067</td>\n",
       "      <td>-0.000470</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>894.606018</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403800</td>\n",
       "      <td>42</td>\n",
       "      <td>14.7204</td>\n",
       "      <td>14.7428</td>\n",
       "      <td>14.6939</td>\n",
       "      <td>14.7413</td>\n",
       "      <td>1356.959961</td>\n",
       "      <td>14.717756</td>\n",
       "      <td>-0.006135</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001419</td>\n",
       "      <td>0.000826</td>\n",
       "      <td>0.000769</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1123.739990</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403860</td>\n",
       "      <td>98</td>\n",
       "      <td>14.7103</td>\n",
       "      <td>14.7159</td>\n",
       "      <td>14.6538</td>\n",
       "      <td>14.6698</td>\n",
       "      <td>1924.369995</td>\n",
       "      <td>14.681911</td>\n",
       "      <td>-0.010817</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.004862</td>\n",
       "      <td>-0.002439</td>\n",
       "      <td>-0.001632</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1491.053955</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403920</td>\n",
       "      <td>18</td>\n",
       "      <td>14.6659</td>\n",
       "      <td>14.6990</td>\n",
       "      <td>14.6659</td>\n",
       "      <td>14.6990</td>\n",
       "      <td>535.000000</td>\n",
       "      <td>14.684669</td>\n",
       "      <td>-0.012168</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001988</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>-0.001422</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1550.702026</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>1530403980</td>\n",
       "      <td>31</td>\n",
       "      <td>14.6898</td>\n",
       "      <td>14.7272</td>\n",
       "      <td>14.6898</td>\n",
       "      <td>14.7193</td>\n",
       "      <td>419.880005</td>\n",
       "      <td>14.700133</td>\n",
       "      <td>-0.013218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001380</td>\n",
       "      <td>0.001053</td>\n",
       "      <td>-0.001440</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1124.359985</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404040</td>\n",
       "      <td>8</td>\n",
       "      <td>14.7036</td>\n",
       "      <td>14.7036</td>\n",
       "      <td>14.6895</td>\n",
       "      <td>14.6993</td>\n",
       "      <td>173.419998</td>\n",
       "      <td>14.696194</td>\n",
       "      <td>-0.014511</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001360</td>\n",
       "      <td>-0.000268</td>\n",
       "      <td>-0.000641</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>881.925964</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404100</td>\n",
       "      <td>54</td>\n",
       "      <td>14.6989</td>\n",
       "      <td>14.7187</td>\n",
       "      <td>14.6508</td>\n",
       "      <td>14.6508</td>\n",
       "      <td>4473.419922</td>\n",
       "      <td>14.674799</td>\n",
       "      <td>-0.016194</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999803</td>\n",
       "      <td>-0.003305</td>\n",
       "      <td>-0.001457</td>\n",
       "      <td>-0.002923</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1505.218018</td>\n",
       "      <td>1325.406006</td>\n",
       "      <td>-0.001562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404160</td>\n",
       "      <td>60</td>\n",
       "      <td>14.6508</td>\n",
       "      <td>14.6686</td>\n",
       "      <td>14.6100</td>\n",
       "      <td>14.6659</td>\n",
       "      <td>2114.120117</td>\n",
       "      <td>14.635762</td>\n",
       "      <td>-0.019175</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999984</td>\n",
       "      <td>0.001030</td>\n",
       "      <td>-0.002664</td>\n",
       "      <td>-0.003148</td>\n",
       "      <td>-0.004366</td>\n",
       "      <td>1543.167969</td>\n",
       "      <td>1426.066040</td>\n",
       "      <td>0.000154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404220</td>\n",
       "      <td>52</td>\n",
       "      <td>14.6659</td>\n",
       "      <td>14.7185</td>\n",
       "      <td>14.6600</td>\n",
       "      <td>14.7171</td>\n",
       "      <td>830.119995</td>\n",
       "      <td>14.691285</td>\n",
       "      <td>-0.017089</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000213</td>\n",
       "      <td>0.003485</td>\n",
       "      <td>0.003786</td>\n",
       "      <td>0.000450</td>\n",
       "      <td>-0.000228</td>\n",
       "      <td>1602.192017</td>\n",
       "      <td>1180.978638</td>\n",
       "      <td>-0.001918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404280</td>\n",
       "      <td>20</td>\n",
       "      <td>14.7171</td>\n",
       "      <td>14.7171</td>\n",
       "      <td>14.6659</td>\n",
       "      <td>14.6759</td>\n",
       "      <td>120.080002</td>\n",
       "      <td>14.680056</td>\n",
       "      <td>-0.013538</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000187</td>\n",
       "      <td>-0.002803</td>\n",
       "      <td>-0.000765</td>\n",
       "      <td>-0.001367</td>\n",
       "      <td>0.000247</td>\n",
       "      <td>1542.232056</td>\n",
       "      <td>1149.567993</td>\n",
       "      <td>-0.000882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404340</td>\n",
       "      <td>29</td>\n",
       "      <td>14.6603</td>\n",
       "      <td>14.6953</td>\n",
       "      <td>14.6400</td>\n",
       "      <td>14.6401</td>\n",
       "      <td>360.609985</td>\n",
       "      <td>14.655682</td>\n",
       "      <td>-0.014971</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000138</td>\n",
       "      <td>-0.002443</td>\n",
       "      <td>-0.001662</td>\n",
       "      <td>-0.002760</td>\n",
       "      <td>-0.003871</td>\n",
       "      <td>1579.670044</td>\n",
       "      <td>1118.734009</td>\n",
       "      <td>0.000597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404400</td>\n",
       "      <td>17</td>\n",
       "      <td>14.6529</td>\n",
       "      <td>14.6907</td>\n",
       "      <td>14.6357</td>\n",
       "      <td>14.6818</td>\n",
       "      <td>214.410004</td>\n",
       "      <td>14.666153</td>\n",
       "      <td>-0.013300</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000286</td>\n",
       "      <td>0.002844</td>\n",
       "      <td>0.000714</td>\n",
       "      <td>-0.000589</td>\n",
       "      <td>-0.002743</td>\n",
       "      <td>727.868042</td>\n",
       "      <td>1118.942017</td>\n",
       "      <td>0.001748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404460</td>\n",
       "      <td>41</td>\n",
       "      <td>14.6817</td>\n",
       "      <td>14.6817</td>\n",
       "      <td>14.6001</td>\n",
       "      <td>14.6315</td>\n",
       "      <td>1408.349976</td>\n",
       "      <td>14.618641</td>\n",
       "      <td>-0.012384</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000367</td>\n",
       "      <td>-0.003432</td>\n",
       "      <td>-0.003245</td>\n",
       "      <td>-0.001171</td>\n",
       "      <td>-0.005950</td>\n",
       "      <td>586.713989</td>\n",
       "      <td>1206.978638</td>\n",
       "      <td>0.003298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404520</td>\n",
       "      <td>32</td>\n",
       "      <td>14.6311</td>\n",
       "      <td>14.6311</td>\n",
       "      <td>14.5922</td>\n",
       "      <td>14.6072</td>\n",
       "      <td>674.650024</td>\n",
       "      <td>14.599182</td>\n",
       "      <td>-0.011263</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000383</td>\n",
       "      <td>-0.001662</td>\n",
       "      <td>-0.001332</td>\n",
       "      <td>-0.006289</td>\n",
       "      <td>-0.007261</td>\n",
       "      <td>555.619995</td>\n",
       "      <td>1236.171387</td>\n",
       "      <td>0.003685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404580</td>\n",
       "      <td>36</td>\n",
       "      <td>14.6069</td>\n",
       "      <td>14.6319</td>\n",
       "      <td>14.5870</td>\n",
       "      <td>14.6136</td>\n",
       "      <td>1454.680054</td>\n",
       "      <td>14.599094</td>\n",
       "      <td>-0.009748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000573</td>\n",
       "      <td>0.000438</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.005530</td>\n",
       "      <td>-0.008337</td>\n",
       "      <td>822.540039</td>\n",
       "      <td>1163.043945</td>\n",
       "      <td>0.003462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404640</td>\n",
       "      <td>75</td>\n",
       "      <td>14.5907</td>\n",
       "      <td>14.6314</td>\n",
       "      <td>14.5500</td>\n",
       "      <td>14.5564</td>\n",
       "      <td>3152.679932</td>\n",
       "      <td>14.570982</td>\n",
       "      <td>-0.010884</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000439</td>\n",
       "      <td>-0.003922</td>\n",
       "      <td>-0.001927</td>\n",
       "      <td>-0.005796</td>\n",
       "      <td>-0.009197</td>\n",
       "      <td>1380.953979</td>\n",
       "      <td>1280.849976</td>\n",
       "      <td>0.003559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404700</td>\n",
       "      <td>53</td>\n",
       "      <td>14.5859</td>\n",
       "      <td>14.5864</td>\n",
       "      <td>14.5330</td>\n",
       "      <td>14.5716</td>\n",
       "      <td>2293.840088</td>\n",
       "      <td>14.544219</td>\n",
       "      <td>-0.009589</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000458</td>\n",
       "      <td>0.001044</td>\n",
       "      <td>-0.001838</td>\n",
       "      <td>-0.008349</td>\n",
       "      <td>-0.011861</td>\n",
       "      <td>1796.839966</td>\n",
       "      <td>1343.308716</td>\n",
       "      <td>0.002576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404760</td>\n",
       "      <td>37</td>\n",
       "      <td>14.5716</td>\n",
       "      <td>14.6100</td>\n",
       "      <td>14.5386</td>\n",
       "      <td>14.5798</td>\n",
       "      <td>1536.020020</td>\n",
       "      <td>14.567747</td>\n",
       "      <td>-0.006217</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000568</td>\n",
       "      <td>0.000562</td>\n",
       "      <td>0.001616</td>\n",
       "      <td>-0.003488</td>\n",
       "      <td>-0.007806</td>\n",
       "      <td>1822.374023</td>\n",
       "      <td>1317.418701</td>\n",
       "      <td>0.001269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404820</td>\n",
       "      <td>88</td>\n",
       "      <td>14.6097</td>\n",
       "      <td>14.6161</td>\n",
       "      <td>14.5294</td>\n",
       "      <td>14.5400</td>\n",
       "      <td>2327.419922</td>\n",
       "      <td>14.562181</td>\n",
       "      <td>-0.005866</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000402</td>\n",
       "      <td>-0.002733</td>\n",
       "      <td>-0.000382</td>\n",
       "      <td>-0.002537</td>\n",
       "      <td>-0.008376</td>\n",
       "      <td>2152.927979</td>\n",
       "      <td>1436.913330</td>\n",
       "      <td>0.000519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404880</td>\n",
       "      <td>71</td>\n",
       "      <td>14.5399</td>\n",
       "      <td>14.5497</td>\n",
       "      <td>14.5002</td>\n",
       "      <td>14.5402</td>\n",
       "      <td>3171.739990</td>\n",
       "      <td>14.510673</td>\n",
       "      <td>-0.003923</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000450</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>-0.003544</td>\n",
       "      <td>-0.006075</td>\n",
       "      <td>-0.012972</td>\n",
       "      <td>2496.340088</td>\n",
       "      <td>1620.370728</td>\n",
       "      <td>0.000728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>1530404940</td>\n",
       "      <td>44</td>\n",
       "      <td>14.5361</td>\n",
       "      <td>14.5366</td>\n",
       "      <td>14.5029</td>\n",
       "      <td>14.5050</td>\n",
       "      <td>1876.199951</td>\n",
       "      <td>14.528194</td>\n",
       "      <td>0.000575</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000330</td>\n",
       "      <td>-0.002424</td>\n",
       "      <td>0.001207</td>\n",
       "      <td>-0.002941</td>\n",
       "      <td>-0.011497</td>\n",
       "      <td>2241.043945</td>\n",
       "      <td>1733.889282</td>\n",
       "      <td>0.000318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>1530405000</td>\n",
       "      <td>363</td>\n",
       "      <td>14.5050</td>\n",
       "      <td>14.5308</td>\n",
       "      <td>14.3770</td>\n",
       "      <td>14.4382</td>\n",
       "      <td>14766.299805</td>\n",
       "      <td>14.437519</td>\n",
       "      <td>0.002724</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000442</td>\n",
       "      <td>-0.004616</td>\n",
       "      <td>-0.006261</td>\n",
       "      <td>-0.007363</td>\n",
       "      <td>-0.016301</td>\n",
       "      <td>4735.536133</td>\n",
       "      <td>2420.081299</td>\n",
       "      <td>0.004508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Asset_ID   timestamp  Count     Open     High      Low    Close  \\\n",
       "0          0  1530403260     29  14.6931  14.7220  14.6900  14.6900   \n",
       "1          0  1530403320    109  14.6900  14.7222  14.6501  14.6701   \n",
       "2          0  1530403380     26  14.6783  14.6918  14.6620  14.6780   \n",
       "3          0  1530403440     36  14.6830  14.7300  14.6750  14.7076   \n",
       "4          0  1530403500     16  14.7080  14.7181  14.6896  14.6896   \n",
       "5          0  1530403560     10  14.7128  14.7128  14.6900  14.7010   \n",
       "6          0  1530403620     11  14.7100  14.7100  14.7010  14.7100   \n",
       "7          0  1530403680     89  14.7100  14.7485  14.6940  14.6940   \n",
       "8          0  1530403740     68  14.6851  14.7499  14.6746  14.7204   \n",
       "9          0  1530403800     42  14.7204  14.7428  14.6939  14.7413   \n",
       "10         0  1530403860     98  14.7103  14.7159  14.6538  14.6698   \n",
       "11         0  1530403920     18  14.6659  14.6990  14.6659  14.6990   \n",
       "12         0  1530403980     31  14.6898  14.7272  14.6898  14.7193   \n",
       "13         0  1530404040      8  14.7036  14.7036  14.6895  14.6993   \n",
       "14         0  1530404100     54  14.6989  14.7187  14.6508  14.6508   \n",
       "15         0  1530404160     60  14.6508  14.6686  14.6100  14.6659   \n",
       "16         0  1530404220     52  14.6659  14.7185  14.6600  14.7171   \n",
       "17         0  1530404280     20  14.7171  14.7171  14.6659  14.6759   \n",
       "18         0  1530404340     29  14.6603  14.6953  14.6400  14.6401   \n",
       "19         0  1530404400     17  14.6529  14.6907  14.6357  14.6818   \n",
       "20         0  1530404460     41  14.6817  14.6817  14.6001  14.6315   \n",
       "21         0  1530404520     32  14.6311  14.6311  14.5922  14.6072   \n",
       "22         0  1530404580     36  14.6069  14.6319  14.5870  14.6136   \n",
       "23         0  1530404640     75  14.5907  14.6314  14.5500  14.5564   \n",
       "24         0  1530404700     53  14.5859  14.5864  14.5330  14.5716   \n",
       "25         0  1530404760     37  14.5716  14.6100  14.5386  14.5798   \n",
       "26         0  1530404820     88  14.6097  14.6161  14.5294  14.5400   \n",
       "27         0  1530404880     71  14.5399  14.5497  14.5002  14.5402   \n",
       "28         0  1530404940     44  14.5361  14.5366  14.5029  14.5050   \n",
       "29         0  1530405000    363  14.5050  14.5308  14.3770  14.4382   \n",
       "\n",
       "          Volume       VWAP    Target  ... up_down60  sma_diff15-60  \\\n",
       "0     604.219971  14.699802  0.003204  ...       0.0            0.0   \n",
       "1    4506.430176  14.694633 -0.000143  ...       0.0            0.0   \n",
       "2     591.239990  14.676436 -0.004589  ...       0.0            0.0   \n",
       "3     823.119995  14.712523 -0.000531  ...       0.0            0.0   \n",
       "4     211.289993  14.706444 -0.004728  ...       0.0            0.0   \n",
       "5      87.800003  14.705889 -0.006988  ...       0.0            0.0   \n",
       "6     236.759995  14.705564 -0.005472  ...       0.0            0.0   \n",
       "7    2551.590088  14.721313 -0.011141  ...       0.0            0.0   \n",
       "8    1385.589966  14.705611 -0.011512  ...       0.0            0.0   \n",
       "9    1356.959961  14.717756 -0.006135  ...       0.0            0.0   \n",
       "10   1924.369995  14.681911 -0.010817  ...       0.0            0.0   \n",
       "11    535.000000  14.684669 -0.012168  ...       0.0            0.0   \n",
       "12    419.880005  14.700133 -0.013218  ...       0.0            0.0   \n",
       "13    173.419998  14.696194 -0.014511  ...       0.0            0.0   \n",
       "14   4473.419922  14.674799 -0.016194  ...       0.0            0.0   \n",
       "15   2114.120117  14.635762 -0.019175  ...       0.0            0.0   \n",
       "16    830.119995  14.691285 -0.017089  ...       0.0            0.0   \n",
       "17    120.080002  14.680056 -0.013538  ...       0.0            0.0   \n",
       "18    360.609985  14.655682 -0.014971  ...       0.0            0.0   \n",
       "19    214.410004  14.666153 -0.013300  ...       0.0            0.0   \n",
       "20   1408.349976  14.618641 -0.012384  ...       0.0            0.0   \n",
       "21    674.650024  14.599182 -0.011263  ...       0.0            0.0   \n",
       "22   1454.680054  14.599094 -0.009748  ...       0.0            0.0   \n",
       "23   3152.679932  14.570982 -0.010884  ...       0.0            0.0   \n",
       "24   2293.840088  14.544219 -0.009589  ...       0.0            0.0   \n",
       "25   1536.020020  14.567747 -0.006217  ...       0.0            0.0   \n",
       "26   2327.419922  14.562181 -0.005866  ...       0.0            0.0   \n",
       "27   3171.739990  14.510673 -0.003923  ...       0.0            0.0   \n",
       "28   1876.199951  14.528194  0.000575  ...       0.0            0.0   \n",
       "29  14766.299805  14.437519  0.002724  ...       0.0            0.0   \n",
       "\n",
       "    ma_close/vwap15  log_return1  vwap_log_return1  vwap_log_return5  \\\n",
       "0          0.000000     0.000000          0.000000          0.000000   \n",
       "1          0.000000    -0.001355         -0.000352          0.000000   \n",
       "2          0.000000     0.000538         -0.001239          0.000000   \n",
       "3          0.000000     0.002014          0.002456          0.000000   \n",
       "4          0.000000    -0.001225         -0.000413          0.000000   \n",
       "5          0.000000     0.000776         -0.000038          0.000414   \n",
       "6          0.000000     0.000612         -0.000022          0.000744   \n",
       "7          0.000000    -0.001088          0.001070          0.003053   \n",
       "8          0.000000     0.001795         -0.001067         -0.000470   \n",
       "9          0.000000     0.001419          0.000826          0.000769   \n",
       "10         0.000000    -0.004862         -0.002439         -0.001632   \n",
       "11         0.000000     0.001988          0.000188         -0.001422   \n",
       "12         0.000000     0.001380          0.001053         -0.001440   \n",
       "13         0.000000    -0.001360         -0.000268         -0.000641   \n",
       "14         0.999803    -0.003305         -0.001457         -0.002923   \n",
       "15         0.999984     0.001030         -0.002664         -0.003148   \n",
       "16         1.000213     0.003485          0.003786          0.000450   \n",
       "17         1.000187    -0.002803         -0.000765         -0.001367   \n",
       "18         1.000138    -0.002443         -0.001662         -0.002760   \n",
       "19         1.000286     0.002844          0.000714         -0.000589   \n",
       "20         1.000367    -0.003432         -0.003245         -0.001171   \n",
       "21         1.000383    -0.001662         -0.001332         -0.006289   \n",
       "22         1.000573     0.000438         -0.000006         -0.005530   \n",
       "23         1.000439    -0.003922         -0.001927         -0.005796   \n",
       "24         1.000458     0.001044         -0.001838         -0.008349   \n",
       "25         1.000568     0.000562          0.001616         -0.003488   \n",
       "26         1.000402    -0.002733         -0.000382         -0.002537   \n",
       "27         1.000450     0.000014         -0.003544         -0.006075   \n",
       "28         1.000330    -0.002424          0.001207         -0.002941   \n",
       "29         1.000442    -0.004616         -0.006261         -0.007363   \n",
       "\n",
       "    vwap_log_return15  volume_rmean5  volume_rmean15  time_sma15  \n",
       "0            0.000000       0.000000        0.000000    0.000000  \n",
       "1            0.000000       0.000000        0.000000    0.000000  \n",
       "2            0.000000       0.000000        0.000000    0.000000  \n",
       "3            0.000000       0.000000        0.000000    0.000000  \n",
       "4            0.000000    1347.260010        0.000000    0.000000  \n",
       "5            0.000000    1243.976074        0.000000    0.000000  \n",
       "6            0.000000     390.041992        0.000000    0.000000  \n",
       "7            0.000000     782.112000        0.000000    0.000000  \n",
       "8            0.000000     894.606018        0.000000    0.000000  \n",
       "9            0.000000    1123.739990        0.000000    0.000000  \n",
       "10           0.000000    1491.053955        0.000000    0.000000  \n",
       "11           0.000000    1550.702026        0.000000    0.000000  \n",
       "12           0.000000    1124.359985        0.000000    0.000000  \n",
       "13           0.000000     881.925964        0.000000    0.000000  \n",
       "14           0.000000    1505.218018     1325.406006   -0.001562  \n",
       "15          -0.004366    1543.167969     1426.066040    0.000154  \n",
       "16          -0.000228    1602.192017     1180.978638   -0.001918  \n",
       "17           0.000247    1542.232056     1149.567993   -0.000882  \n",
       "18          -0.003871    1579.670044     1118.734009    0.000597  \n",
       "19          -0.002743     727.868042     1118.942017    0.001748  \n",
       "20          -0.005950     586.713989     1206.978638    0.003298  \n",
       "21          -0.007261     555.619995     1236.171387    0.003685  \n",
       "22          -0.008337     822.540039     1163.043945    0.003462  \n",
       "23          -0.009197    1380.953979     1280.849976    0.003559  \n",
       "24          -0.011861    1796.839966     1343.308716    0.002576  \n",
       "25          -0.007806    1822.374023     1317.418701    0.001269  \n",
       "26          -0.008376    2152.927979     1436.913330    0.000519  \n",
       "27          -0.012972    2496.340088     1620.370728    0.000728  \n",
       "28          -0.011497    2241.043945     1733.889282    0.000318  \n",
       "29          -0.016301    4735.536133     2420.081299    0.004508  \n",
       "\n",
       "[30 rows x 42 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4b183aef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:08:55.127454Z",
     "iopub.status.busy": "2022-01-26T12:08:55.126702Z",
     "iopub.status.idle": "2022-01-26T12:08:55.129560Z",
     "shell.execute_reply": "2022-01-26T12:08:55.130283Z",
     "shell.execute_reply.started": "2022-01-26T10:32:55.897392Z"
    },
    "papermill": {
     "duration": 0.036969,
     "end_time": "2022-01-26T12:08:55.130448",
     "exception": false,
     "start_time": "2022-01-26T12:08:55.093479",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          2018-07-01 00:01:00\n",
       "1          2018-07-01 00:02:00\n",
       "2          2018-07-01 00:03:00\n",
       "3          2018-07-01 00:04:00\n",
       "4          2018-07-01 00:05:00\n",
       "                   ...        \n",
       "25573784   2022-01-09 23:56:00\n",
       "25573785   2022-01-09 23:57:00\n",
       "25573786   2022-01-09 23:58:00\n",
       "25573787   2022-01-09 23:59:00\n",
       "25573788   2022-01-10 00:00:00\n",
       "Name: datetime, Length: 25573789, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['datetime']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8cddd75",
   "metadata": {
    "papermill": {
     "duration": 0.026488,
     "end_time": "2022-01-26T12:08:55.183814",
     "exception": false,
     "start_time": "2022-01-26T12:08:55.157326",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Prepare weights for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7986bf61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:08:55.247823Z",
     "iopub.status.busy": "2022-01-26T12:08:55.244373Z",
     "iopub.status.idle": "2022-01-26T12:08:55.250586Z",
     "shell.execute_reply": "2022-01-26T12:08:55.249951Z",
     "shell.execute_reply.started": "2022-01-26T10:32:59.526029Z"
    },
    "papermill": {
     "duration": 0.040619,
     "end_time": "2022-01-26T12:08:55.250724",
     "exception": false,
     "start_time": "2022-01-26T12:08:55.210105",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/38641691/weighted-correlation-coefficient-with-pandas\n",
    "def wmean(x, w):\n",
    "    return np.sum(x * w) / np.sum(w)\n",
    "\n",
    "def wcov(x, y, w):\n",
    "    return np.sum(w * (x - wmean(x, w)) * (y - wmean(y, w))) / np.sum(w)\n",
    "\n",
    "def wcorr(x, y, w):\n",
    "    return wcov(x, y, w) / np.sqrt(wcov(x, x, w) * wcov(y, y, w))\n",
    "\n",
    "def eval_wcorr(preds, train_data):\n",
    "    w = train_data.add_w.values.flatten()\n",
    "    y_true = train_data.get_label()\n",
    "    return 'eval_wcorr', wcorr(preds, y_true, w), True\n",
    "\n",
    "#create dictionnary of weights\n",
    "dict_weights = {}\n",
    "for i in range(df_asset_details.shape[0]):\n",
    "    dict_weights[df_asset_details.iloc[i,0]] = df_asset_details.iloc[i,1]\n",
    "\n",
    "#     train_df['weights'] = train_df.Asset_ID.map(dict_weights).astype('float32')\n",
    "if EVALUATION:\n",
    "    test_df['weights'] = test_df.Asset_ID.map(dict_weights).astype('float32')\n",
    "    test_df['preds'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6416ac6d",
   "metadata": {
    "papermill": {
     "duration": 0.02646,
     "end_time": "2022-01-26T12:08:55.304915",
     "exception": false,
     "start_time": "2022-01-26T12:08:55.278455",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Loop over all assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a650944c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:08:55.367707Z",
     "iopub.status.busy": "2022-01-26T12:08:55.367057Z",
     "iopub.status.idle": "2022-01-26T12:19:43.256220Z",
     "shell.execute_reply": "2022-01-26T12:19:43.256751Z",
     "shell.execute_reply.started": "2022-01-26T10:33:01.291533Z"
    },
    "papermill": {
     "duration": 647.925384,
     "end_time": "2022-01-26T12:19:43.257001",
     "exception": false,
     "start_time": "2022-01-26T12:08:55.331617",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model for Binance Coin     (ID=0 )\n",
      "Training model for Bitcoin          (ID=1 )\n",
      "Training model for Bitcoin Cash     (ID=2 )\n",
      "Training model for Cardano          (ID=3 )\n",
      "Training model for Dogecoin         (ID=4 )\n",
      "Training model for EOS.IO           (ID=5 )\n",
      "Training model for Ethereum         (ID=6 )\n",
      "Training model for Ethereum Classic (ID=7 )\n",
      "Training model for IOTA             (ID=8 )\n",
      "Training model for Litecoin         (ID=9 )\n",
      "Training model for Maker            (ID=10)\n",
      "Training model for Monero           (ID=11)\n",
      "Training model for Stellar          (ID=12)\n",
      "Training model for TRON             (ID=13)\n",
      "CPU times: user 41min 42s, sys: 8.01 s, total: 41min 50s\n",
      "Wall time: 10min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "params = {'n_estimators': 600,\n",
    "        'objective': 'regression',\n",
    "        'metric': 'None',\n",
    "        'boosting_type': 'gbdt',\n",
    "        'max_depth': -1, \n",
    "        'learning_rate': 0.01,\n",
    "        'seed': SEED,\n",
    "        'verbose': -1,\n",
    "        }\n",
    "\n",
    "\n",
    "models = {}\n",
    "\n",
    "for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n",
    "    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n",
    "    \n",
    "    asset_df = train_df[train_df[\"Asset_ID\"] == asset_id]\n",
    "    \n",
    "        \n",
    "    features = train_df.columns.difference(['timestamp', 'Open', 'High', 'Low', 'Close', \n",
    "                                        'VWAP', 'Mean', 'Asset_ID', 'datetime', 'Target'])  \n",
    "\n",
    "    X = asset_df[features]\n",
    "    y = asset_df['Target']\n",
    "    \n",
    "    model = LGBMRegressor(**params)\n",
    "    model.fit(X, y)\n",
    "    \n",
    "    models[asset_id] = model\n",
    "    \n",
    "    # Evaluation\n",
    "    if EVALUATION:\n",
    "        x_test = test_df.loc[test_df[\"Asset_ID\"] == asset_id, features]\n",
    "        preds = model.predict(x_test)\n",
    "        test_df.loc[test_df[\"Asset_ID\"] == asset_id, 'preds'] = preds\n",
    "        score = np.corrcoef(test_df.loc[test_df['Asset_ID'] == asset_id, 'preds'],\n",
    "                            test_df.loc[test_df['Asset_ID'] == asset_id, 'Target'])[0,1]\n",
    "        print(f'Test score for {asset_name}: ', f\"{score:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "85ac61a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:19:43.327416Z",
     "iopub.status.busy": "2022-01-26T12:19:43.326578Z",
     "iopub.status.idle": "2022-01-26T12:19:43.328442Z",
     "shell.execute_reply": "2022-01-26T12:19:43.327933Z",
     "shell.execute_reply.started": "2022-01-26T10:44:06.496438Z"
    },
    "papermill": {
     "duration": 0.039827,
     "end_time": "2022-01-26T12:19:43.328567",
     "exception": false,
     "start_time": "2022-01-26T12:19:43.288740",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if EVALUATION:\n",
    "    final_score = wcorr(test_df['preds'], test_df['Target'], test_df['weights'])\n",
    "    print(f'Final test score: {final_score:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6a93a975",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:19:43.394908Z",
     "iopub.status.busy": "2022-01-26T12:19:43.394287Z",
     "iopub.status.idle": "2022-01-26T12:19:43.408290Z",
     "shell.execute_reply": "2022-01-26T12:19:43.408772Z",
     "shell.execute_reply.started": "2022-01-26T10:44:06.505482Z"
    },
    "papermill": {
     "duration": 0.048686,
     "end_time": "2022-01-26T12:19:43.408960",
     "exception": false,
     "start_time": "2022-01-26T12:19:43.360274",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-6.7466836125097135e-06"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the model interface\n",
    "x = train_df.iloc[1]\n",
    "x = x[features]\n",
    "y_pred = models[0].predict([x])\n",
    "y_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1b4ca66",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-11-02T20:57:49.349459Z",
     "iopub.status.idle": "2021-11-02T20:57:49.349757Z",
     "shell.execute_reply": "2021-11-02T20:57:49.349613Z",
     "shell.execute_reply.started": "2021-11-02T20:57:49.349596Z"
    },
    "papermill": {
     "duration": 0.031884,
     "end_time": "2022-01-26T12:19:43.474656",
     "exception": false,
     "start_time": "2022-01-26T12:19:43.442772",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Predict & submit\n",
    "\n",
    "References: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n",
    "\n",
    "Something that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n",
    "\n",
    "```python\n",
    "import pdb; pdb.set_trace()\n",
    "```\n",
    "\n",
    "See [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2514e04a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-26T12:19:43.550748Z",
     "iopub.status.busy": "2022-01-26T12:19:43.549725Z",
     "iopub.status.idle": "2022-01-26T12:19:44.076615Z",
     "shell.execute_reply": "2022-01-26T12:19:44.077719Z",
     "shell.execute_reply.started": "2022-01-26T10:44:06.535284Z"
    },
    "papermill": {
     "duration": 0.571591,
     "end_time": "2022-01-26T12:19:44.077918",
     "exception": false,
     "start_time": "2022-01-26T12:19:43.506327",
     "status": "completed"
    },
    "tags": [],
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4it [00:00,  7.76it/s]\n"
     ]
    }
   ],
   "source": [
    "all_df_test = []\n",
    "\n",
    "# gresearch_crypto.make_env.__called__ = False\n",
    "\n",
    "env = gresearch_crypto.make_env()\n",
    "iter_test = env.iter_test()\n",
    "\n",
    "history = pd.DataFrame()\n",
    "\n",
    "for i, (df_test, df_pred) in tqdm(enumerate(iter_test)):\n",
    "    df_test_opt = df_test.astype({\n",
    "                'timestamp': np.int32, 'Asset_ID': np.int8,\n",
    "                'Open': np.float32, 'High': np.float32,\n",
    "                'Low': np.float32, 'Close': np.float32,\n",
    "                'Count': np.int32, 'Volume': np.float32,\n",
    "                'VWAP': np.float32})\n",
    "    \n",
    "    history = pd.concat([history, df_test_opt])\n",
    "    history = history.iloc[-856:] # Keeping only last needed rows\n",
    "    df_test_feat = calc_features(history.copy()) \n",
    "    \n",
    "    df_test_feat = df_test_feat.iloc[-len(df_test):] # Getting latest test data\n",
    "    try:\n",
    "        df_pred['Target'] = df_test_feat.apply(lambda row: models[row['Asset_ID']].predict([row[features]])[0], axis=1)\n",
    "    except:\n",
    "        df_pred['Target'] = 0\n",
    "\n",
    "    # Send submissions\n",
    "    env.predict(df_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1006.912975,
   "end_time": "2022-01-26T12:19:45.606584",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-01-26T12:02:58.693609",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}