cleaning_over_times.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 420240 entries, 0 to 420239\n",
      "Data columns (total 19 columns):\n",
      " #   Column               Non-Null Count   Dtype  \n",
      "---  ------               --------------   -----  \n",
      " 0   id                   420240 non-null  int64  \n",
      " 1    user                420240 non-null  object \n",
      " 2    lat                 420240 non-null  float64\n",
      " 3    long                420240 non-null  float64\n",
      " 4    tags                316730 non-null  object \n",
      " 5    title               381911 non-null  object \n",
      " 6    date_taken_minute   420239 non-null  float64\n",
      " 7    date_taken_hour     420240 non-null  int64  \n",
      " 8    date_taken_day      420240 non-null  int64  \n",
      " 9    date_taken_month    420240 non-null  int64  \n",
      " 10   date_taken_year     420240 non-null  int64  \n",
      " 11   date_upload_minute  420228 non-null  object \n",
      " 12   date_upload_hour    420238 non-null  object \n",
      " 13   date_upload_day     420238 non-null  float64\n",
      " 14   date_upload_month   420240 non-null  int64  \n",
      " 15   date_upload_year    420239 non-null  float64\n",
      " 16  Unnamed: 16          142 non-null     float64\n",
      " 17  Unnamed: 17          0 non-null       float64\n",
      " 18  Unnamed: 18          2 non-null       float64\n",
      "dtypes: float64(8), int64(6), object(5)\n",
      "memory usage: 60.9+ MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rsaguesriv\\AppData\\Local\\Temp\\ipykernel_4796\\2209637920.py:3: DtypeWarning: Columns (11,12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_table(\"data/flickr_data2.csv\", sep=\",\")\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_table(\"data/flickr_data2.csv\", sep=\",\")\n",
    "\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 420240 entries, 0 to 420239\n",
      "Data columns (total 19 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   id                  420240 non-null  int64  \n",
      " 1   user                420240 non-null  object \n",
      " 2   lat                 420240 non-null  float64\n",
      " 3   long                420240 non-null  float64\n",
      " 4   tags                316730 non-null  object \n",
      " 5   title               381911 non-null  object \n",
      " 6   date_taken_minute   420239 non-null  float64\n",
      " 7   date_taken_hour     420240 non-null  int64  \n",
      " 8   date_taken_day      420240 non-null  int64  \n",
      " 9   date_taken_month    420240 non-null  int64  \n",
      " 10  date_taken_year     420240 non-null  int64  \n",
      " 11  date_upload_minute  420228 non-null  object \n",
      " 12  date_upload_hour    420238 non-null  object \n",
      " 13  date_upload_day     420238 non-null  float64\n",
      " 14  date_upload_month   420240 non-null  int64  \n",
      " 15  date_upload_year    420239 non-null  float64\n",
      " 16  Unnamed: 16         142 non-null     float64\n",
      " 17  Unnamed: 17         0 non-null       float64\n",
      " 18  Unnamed: 18         2 non-null       float64\n",
      "dtypes: float64(8), int64(6), object(5)\n",
      "memory usage: 60.9+ MB\n"
     ]
    }
   ],
   "source": [
    "#columns have spaces\n",
    "cols = ['id', 'user', 'lat', 'long', 'tags', 'title', 'date_taken_minute',\n",
    "       'date_taken_hour', 'date_taken_day', 'date_taken_month',\n",
    "       'date_taken_year', 'date_upload_minute', 'date_upload_hour',\n",
    "       'date_upload_day', 'date_upload_month', 'date_upload_year',\n",
    "       'Unnamed: 16', 'Unnamed: 17', 'Unnamed: 18']\n",
    "df.columns = cols\n",
    "\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rsaguesriv\\AppData\\Local\\Temp\\ipykernel_4796\\1794550704.py:4: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  df[types_cols].map(type).astype(str).apply(pd.value_counts).fillna(0, downcast='infer')\n",
      "C:\\Users\\rsaguesriv\\AppData\\Local\\Temp\\ipykernel_4796\\1794550704.py:4: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  df[types_cols].map(type).astype(str).apply(pd.value_counts).fillna(0, downcast='infer')\n",
      "C:\\Users\\rsaguesriv\\AppData\\Local\\Temp\\ipykernel_4796\\1794550704.py:4: FutureWarning: The 'downcast' keyword in fillna is deprecated and will be removed in a future version. Use res.infer_objects(copy=False) to infer non-object dtype, or pd.to_numeric with the 'downcast' keyword to downcast numeric results.\n",
      "  df[types_cols].map(type).astype(str).apply(pd.value_counts).fillna(0, downcast='infer')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_minute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>&lt;class 'float'&gt;</th>\n",
       "      <td>2</td>\n",
       "      <td>59800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>&lt;class 'int'&gt;</th>\n",
       "      <td>387472</td>\n",
       "      <td>65536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>&lt;class 'str'&gt;</th>\n",
       "      <td>32766</td>\n",
       "      <td>294904</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 date_upload_hour  date_upload_minute\n",
       "<class 'float'>                 2               59800\n",
       "<class 'int'>              387472               65536\n",
       "<class 'str'>               32766              294904"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Some cols have dtype as object when expected numeric types\n",
    "types_cols = ['date_upload_hour', 'date_upload_minute']\n",
    "\n",
    "df[types_cols].map(type).astype(str).apply(pd.value_counts).fillna(0, downcast='infer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique value in date_upload_hour: [20 17 12 10 21 18 16 14 9 4 1 23 15 7 22 19 3 11 2 0 13 8 6 5 47 32 46 49\n",
      " 44 28 '18' '23' '17' '15' '14' '13' '7' '3' '2' '0' '22' '19' '9' '16'\n",
      " '11' '10' '4' '21' '20' '8' '12' '5' '1' '6' '28' nan '.!' '26' 26 38 27\n",
      " 36 54 42 33 24 53 29 45 25]\n",
      "\n",
      "Unique value in date_upload_minute: [23 52 33 38 1 0 29 28 21 15 45 41 36 35 50 11 4 3 42 18 19 22 37 27 26 25\n",
      " 24 59 58 57 54 48 10 5 12 8 56 16 13 55 6 47 40 34 2 43 14 30 39 17 32 31\n",
      " 53 51 49 44 20 9 7 46 nan '47' '20' '13' '9' '24' '1' '38' '12' '41' '36'\n",
      " '34' '23' '39' '52' '21' '0' '59' '58' '54' '57' '56' '55' '53' '51' '50'\n",
      " '14' '8' '43' '46' '45' '42' '22' '32' '27' '11' '28' '26' '25' '19' '18'\n",
      " '17' '16' '15' '10' '7' '6' '5' '4' '3' '2' '49' '48' '44' '35' '40' '31'\n",
      " '30' '29' ' Stone' ')' '37' '33' ' Rhône' '-)'\n",
      " '-) #babyfoot #tournee #chik #laurent #foot #football #soccer #ligue1 #lyon #ol #webstagram #tournoi'\n",
      " ') mais on a gagner le coeur du public c ca la plus belle recompense :D'\n",
      " ') #night #nightlife #lyon #friends #france #velove #hs @sugardaddy'\n",
      " ' Sky' ') I miss you ^^' '-) #VTC'\n",
      " ' 302 n°63 + Stadler Tango n°106 RhôneExpress  -  Lyon Part-Dieu'\n",
      " ') #noel #fete' ') #France #França' ') 🌹🌹🌹' ' le tadorne (2015 07 09)'\n",
      " '-)  http://trouver-sa-voie-changer-sa-vie.fr/2015/10/changement-87-mollo-lapero/ #changerdevie'\n",
      " ' ST jean' '-)   (A16-0651)'\n",
      " '-) #IDRACalumni #gyroo #Idrac #MaisonArtilion #urban360 #Lyon 🔋⭕️🏃👍'\n",
      " ' Lets #Burn those #Bridges.  Http://www.JoshuaHideki.com' ' )'\n",
      " ' Eglise Saint-nizier' ' La croix rousse' ' Basilique de Fourvière'\n",
      " ' Opéra de Lyon' '-) Pablo et Kathleen'\n",
      " '-) #friends #e #birthdaycard #graphicdesign #idid #photooftheday #creativity #love #seeyousoon #48'\n",
      " ' 1 place des minimes Lyon 5°' ' a smile is its sword.'\n",
      " ' le chantier la nuit (1978)' ' Lyon' ' 40' '2018']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Visualise those strings\n",
    "\n",
    "for col in types_cols:\n",
    "    print(f\"Unique value in {col}: {df[col].unique()}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>137622</th>\n",
       "      <td>8715425964</td>\n",
       "      <td>29713277@N02</td>\n",
       "      <td>45.75256</td>\n",
       "      <td>4.788547</td>\n",
       "      <td>lyon,zup,propriétéprivée,défensedentrer</td>\n",
       "      <td>:</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.!</td>\n",
       "      <td>18.0</td>\n",
       "      <td>21</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142855</th>\n",
       "      <td>8715425964</td>\n",
       "      <td>29713277@N02</td>\n",
       "      <td>45.75256</td>\n",
       "      <td>4.788547</td>\n",
       "      <td>lyon,zup,propriétéprivée,défensedentrer</td>\n",
       "      <td>:</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.!</td>\n",
       "      <td>18.0</td>\n",
       "      <td>21</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                id          user       lat      long  \\\n",
       "137622  8715425964  29713277@N02  45.75256  4.788547   \n",
       "142855  8715425964  29713277@N02  45.75256  4.788547   \n",
       "\n",
       "                                           tags title  date_taken_minute  \\\n",
       "137622  lyon,zup,propriétéprivée,défensedentrer     :                5.0   \n",
       "142855  lyon,zup,propriétéprivée,défensedentrer     :                5.0   \n",
       "\n",
       "        date_taken_hour  date_taken_day  date_taken_month  date_taken_year  \\\n",
       "137622             2013              28                12               25   \n",
       "142855             2013              28                12               25   \n",
       "\n",
       "       date_upload_minute date_upload_hour  date_upload_day  \\\n",
       "137622                NaN               .!             18.0   \n",
       "142855                NaN               .!             18.0   \n",
       "\n",
       "        date_upload_month  date_upload_year  Unnamed: 16  Unnamed: 17  \\\n",
       "137622                 21               6.0         12.0          NaN   \n",
       "142855                 21               6.0         12.0          NaN   \n",
       "\n",
       "        Unnamed: 18  \n",
       "137622          NaN  \n",
       "142855          NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#observing one of those value\n",
    "\n",
    "df[df['date_upload_minute'] ==  '-) #IDRACalumni #gyroo #Idrac #MaisonArtilion #urban360 #Lyon 🔋⭕️🏃👍'] #seems a parsing error, values are shifted to the rigth one colomn because of \";\"\n",
    "df[df['date_upload_minute'] ==  ' 1 place des minimes Lyon 5°']\n",
    "df[df['date_upload_minute'] ==  ' Rhône']\n",
    "df[df['date_upload_hour'] ==  '.!'] #deux ;;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>85950</th>\n",
       "      <td>6674970791</td>\n",
       "      <td>29713277@N02</td>\n",
       "      <td>45.753948</td>\n",
       "      <td>4.788145</td>\n",
       "      <td>portrait,throughtheleaves,autraversdesfeuillages</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>47</td>\n",
       "      <td>21.0</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90872</th>\n",
       "      <td>6674970791</td>\n",
       "      <td>29713277@N02</td>\n",
       "      <td>45.753948</td>\n",
       "      <td>4.788145</td>\n",
       "      <td>portrait,throughtheleaves,autraversdesfeuillages</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>47</td>\n",
       "      <td>21.0</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98808</th>\n",
       "      <td>7386785280</td>\n",
       "      <td>37290448@N04</td>\n",
       "      <td>45.771315</td>\n",
       "      <td>4.835829</td>\n",
       "      <td>city,plants,plant,france,brick,green,stone,wal...</td>\n",
       "      <td>Plant</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>Stone</td>\n",
       "      <td>32</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98816</th>\n",
       "      <td>7387000024</td>\n",
       "      <td>38586649@N00</td>\n",
       "      <td>45.779196</td>\n",
       "      <td>4.853596</td>\n",
       "      <td>uploaded:by=flicksquare,foursquare:venue=4b851...</td>\n",
       "      <td>Beautiful weather in #Lyon</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>46</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>)</td>\n",
       "      <td>46</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102366</th>\n",
       "      <td>7583546064</td>\n",
       "      <td>37290448@N04</td>\n",
       "      <td>45.772827</td>\n",
       "      <td>4.841065</td>\n",
       "      <td>city,blackandwhite,france,water,river,french,c...</td>\n",
       "      <td>Bridge</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>38</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>Rhône</td>\n",
       "      <td>5</td>\n",
       "      <td>18.0</td>\n",
       "      <td>16</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390775</th>\n",
       "      <td>33105832478</td>\n",
       "      <td>53987060@N02</td>\n",
       "      <td>45.771063</td>\n",
       "      <td>4.829993</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Looking from the Roman arenas to the Basilica ...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>57</td>\n",
       "      <td>11</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "      <td>-)</td>\n",
       "      <td>45</td>\n",
       "      <td>13.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391474</th>\n",
       "      <td>31807499637</td>\n",
       "      <td>53987060@N02</td>\n",
       "      <td>45.762097</td>\n",
       "      <td>4.821281</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Inside the Basilica of Notre-Dame de Fourvière...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>55</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>-)</td>\n",
       "      <td>25</td>\n",
       "      <td>8.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391596</th>\n",
       "      <td>31807499637</td>\n",
       "      <td>53987060@N02</td>\n",
       "      <td>45.762097</td>\n",
       "      <td>4.821281</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Inside the Basilica of Notre-Dame de Fourvière...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>55</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>-)</td>\n",
       "      <td>25</td>\n",
       "      <td>8.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395508</th>\n",
       "      <td>47103124882</td>\n",
       "      <td>15917364@N02</td>\n",
       "      <td>45.776985</td>\n",
       "      <td>4.833829</td>\n",
       "      <td>lyon,rhône,croixrousse,escaliers,rue,personne,...</td>\n",
       "      <td>Lyon - En solitaire à la Croix-Rousse</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>54</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>12.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395546</th>\n",
       "      <td>47103124882</td>\n",
       "      <td>15917364@N02</td>\n",
       "      <td>45.776985</td>\n",
       "      <td>4.833829</td>\n",
       "      <td>lyon,rhône,croixrousse,escaliers,rue,personne,...</td>\n",
       "      <td>Lyon - En solitaire à la Croix-Rousse</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>54</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>12.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>142 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id          user        lat      long  \\\n",
       "85950    6674970791  29713277@N02  45.753948  4.788145   \n",
       "90872    6674970791  29713277@N02  45.753948  4.788145   \n",
       "98808    7386785280  37290448@N04  45.771315  4.835829   \n",
       "98816    7387000024  38586649@N00  45.779196  4.853596   \n",
       "102366   7583546064  37290448@N04  45.772827  4.841065   \n",
       "...             ...           ...        ...       ...   \n",
       "390775  33105832478  53987060@N02  45.771063  4.829993   \n",
       "391474  31807499637  53987060@N02  45.762097  4.821281   \n",
       "391596  31807499637  53987060@N02  45.762097  4.821281   \n",
       "395508  47103124882  15917364@N02  45.776985  4.833829   \n",
       "395546  47103124882  15917364@N02  45.776985  4.833829   \n",
       "\n",
       "                                                     tags  \\\n",
       "85950    portrait,throughtheleaves,autraversdesfeuillages   \n",
       "90872    portrait,throughtheleaves,autraversdesfeuillages   \n",
       "98808   city,plants,plant,france,brick,green,stone,wal...   \n",
       "98816   uploaded:by=flicksquare,foursquare:venue=4b851...   \n",
       "102366  city,blackandwhite,france,water,river,french,c...   \n",
       "...                                                   ...   \n",
       "390775                                                NaN   \n",
       "391474                                                NaN   \n",
       "391596                                                NaN   \n",
       "395508  lyon,rhône,croixrousse,escaliers,rue,personne,...   \n",
       "395546  lyon,rhône,croixrousse,escaliers,rue,personne,...   \n",
       "\n",
       "                                                    title  date_taken_minute  \\\n",
       "85950                                                 NaN             2012.0   \n",
       "90872                                                 NaN             2012.0   \n",
       "98808                                               Plant             2012.0   \n",
       "98816                         Beautiful weather in #Lyon              2012.0   \n",
       "102366                                             Bridge             2012.0   \n",
       "...                                                   ...                ...   \n",
       "390775  Looking from the Roman arenas to the Basilica ...             2019.0   \n",
       "391474  Inside the Basilica of Notre-Dame de Fourvière...             2019.0   \n",
       "391596  Inside the Basilica of Notre-Dame de Fourvière...             2019.0   \n",
       "395508              Lyon - En solitaire à la Croix-Rousse             2019.0   \n",
       "395546              Lyon - En solitaire à la Croix-Rousse             2019.0   \n",
       "\n",
       "        date_taken_hour  date_taken_day  date_taken_month  date_taken_year  \\\n",
       "85950                29              12                10                9   \n",
       "90872                29              12                10                9   \n",
       "98808                17               9                 6                6   \n",
       "98816                46              11                17                6   \n",
       "102366               38              11                 7                6   \n",
       "...                 ...             ...               ...              ...   \n",
       "390775               57              11                16               11   \n",
       "391474               55              12                15               11   \n",
       "391596               55              12                15               11   \n",
       "395508               54              17                18                2   \n",
       "395546               54              17                18                2   \n",
       "\n",
       "       date_upload_minute date_upload_hour  date_upload_day  \\\n",
       "85950                 NaN               47             21.0   \n",
       "90872                 NaN               47             21.0   \n",
       "98808               Stone               32             17.0   \n",
       "98816                   )               46             17.0   \n",
       "102366              Rhône                5             18.0   \n",
       "...                   ...              ...              ...   \n",
       "390775                 -)               45             13.0   \n",
       "391474                 -)               25              8.0   \n",
       "391596                 -)               25              8.0   \n",
       "395508                NaN               29             12.0   \n",
       "395546                NaN               29             12.0   \n",
       "\n",
       "        date_upload_month  date_upload_year  Unnamed: 16  Unnamed: 17  \\\n",
       "85950                  10               1.0       2011.0          NaN   \n",
       "90872                  10               1.0       2011.0          NaN   \n",
       "98808                  17               6.0       2012.0          NaN   \n",
       "98816                  17               6.0       2012.0          NaN   \n",
       "102366                 16               7.0       2012.0          NaN   \n",
       "...                   ...               ...          ...          ...   \n",
       "390775                  4               2.0       2018.0          NaN   \n",
       "391474                 15               1.0       2018.0          NaN   \n",
       "391596                 15               1.0       2018.0          NaN   \n",
       "395508                 20               2.0       2019.0          NaN   \n",
       "395546                 20               2.0       2019.0          NaN   \n",
       "\n",
       "        Unnamed: 18  \n",
       "85950           NaN  \n",
       "90872           NaN  \n",
       "98808           NaN  \n",
       "98816           NaN  \n",
       "102366          NaN  \n",
       "...             ...  \n",
       "390775          NaN  \n",
       "391474          NaN  \n",
       "391596          NaN  \n",
       "395508          NaN  \n",
       "395546          NaN  \n",
       "\n",
       "[142 rows x 19 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['Unnamed: 16'].isna() == False]\n",
    "# df.loc[df['Unnamed: 18'].isna() == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 187598 entries, 0 to 420239\n",
      "Data columns (total 19 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   id                  187598 non-null  int64  \n",
      " 1   user                187598 non-null  object \n",
      " 2   lat                 187598 non-null  float64\n",
      " 3   long                187598 non-null  float64\n",
      " 4   tags                140629 non-null  object \n",
      " 5   title               170633 non-null  object \n",
      " 6   date_taken_minute   187597 non-null  float64\n",
      " 7   date_taken_hour     187598 non-null  int64  \n",
      " 8   date_taken_day      187598 non-null  int64  \n",
      " 9   date_taken_month    187598 non-null  int64  \n",
      " 10  date_taken_year     187598 non-null  int64  \n",
      " 11  date_upload_minute  187594 non-null  object \n",
      " 12  date_upload_hour    187597 non-null  object \n",
      " 13  date_upload_day     187597 non-null  float64\n",
      " 14  date_upload_month   187598 non-null  int64  \n",
      " 15  date_upload_year    187597 non-null  float64\n",
      " 16  Unnamed: 16         53 non-null      float64\n",
      " 17  Unnamed: 17         0 non-null       float64\n",
      " 18  Unnamed: 18         1 non-null       float64\n",
      "dtypes: float64(8), int64(6), object(5)\n",
      "memory usage: 28.6+ MB\n"
     ]
    }
   ],
   "source": [
    "#First lets tackle the duplicates\n",
    "df.duplicated()[df.duplicated() == True].sum() #232642\n",
    "df = df.drop_duplicates()\n",
    "df.duplicated()[df.duplicated() == True].sum() #0\n",
    "df.info()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique value in date_upload_hour: [20 17 12 10 21 18 16 14 9 4 1 23 15 7 22 19 3 11 2 0 13 8 6 5 47 32 46 49\n",
      " 44 28 '18' '23' '17' '15' '14' '13' '7' '3' '2' '0' '22' '19' '9' '16'\n",
      " '11' '10' '4' '21' '20' '8' '12' '5' '1' '6' '28' '26' 26 38 27 36 54 42\n",
      " 33 24 53 29 45 25]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, user, lat, long, tags, title, date_taken_minute, date_taken_hour, date_taken_day, date_taken_month, date_taken_year, date_upload_minute, date_upload_hour, date_upload_day, date_upload_month, date_upload_year, Unnamed: 16, Unnamed: 17, Unnamed: 18]\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(df[df['date_upload_hour'] ==  '.!'].index.item()) #One row\n",
    "df = df.drop(df.loc[df['Unnamed: 18'].isna() == False].index.item()) #One row\n",
    "\n",
    "col = \"date_upload_hour\"\n",
    "print(f\"Unique value in {col}: {df[col].unique()}\\n\")\n",
    "df.loc[df['Unnamed: 18'].isna() == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique value in date_upload_minute: [23 52 33 38 1 0 29 28 21 15 45 41 36 35 50 11 4 3 42 18 19 22 37 27 26 25\n",
      " 24 59 58 57 54 48 10 5 12 8 56 16 13 55 6 47 40 34 2 43 14 30 39 17 32 31\n",
      " 53 51 49 44 20 9 7 46 '47' '20' '13' '9' '24' '1' '38' '12' '41' '36'\n",
      " '34' '23' '39' '52' '21' '0' '59' '58' '54' '57' '56' '55' '53' '51' '50'\n",
      " '14' '8' '43' '46' '45' '42' '22' '32' '27' '11' '28' '26' '25' '19' '18'\n",
      " '17' '16' '15' '10' '7' '6' '5' '4' '3' '2' '49' '48' '44' '35' '40' '31'\n",
      " '30' '29' '37' '33' ' 40']\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, user, lat, long, tags, title, date_taken_minute, date_taken_hour, date_taken_day, date_taken_month, date_taken_year, date_upload_minute, date_upload_hour, date_upload_day, date_upload_month, date_upload_year, Unnamed: 16, Unnamed: 17, Unnamed: 18]\n",
       "Index: []"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.loc[df['Unnamed: 16'].isna() == True]\n",
    "col = \"date_upload_minute\"\n",
    "print(f\"Unique value in {col}: {df[col].unique()}\\n\")\n",
    "df.loc[df['Unnamed: 18'].isna() == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique value in date_upload_minute: [23 52 33 38  1  0 29 28 21 15 45 41 36 35 50 11  4  3 42 18 19 22 37 27\n",
      " 26 25 24 59 58 57 54 48 10  5 12  8 56 16 13 55  6 47 40 34  2 43 14 30\n",
      " 39 17 32 31 53 51 49 44 20  9  7 46]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, user, lat, long, tags, title, date_taken_minute, date_taken_hour, date_taken_day, date_taken_month, date_taken_year, date_upload_minute, date_upload_hour, date_upload_day, date_upload_month, date_upload_year, Unnamed: 16, Unnamed: 17, Unnamed: 18]\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"date_upload_minute\"] = df[\"date_upload_minute\"].map(lambda a: int(a))\n",
    "df[\"date_upload_hour\"] = df[\"date_upload_hour\"].map(lambda a: int(a))\n",
    "print(f\"Unique value in {col}: {df[col].unique()}\\n\")\n",
    "df.loc[df['Unnamed: 18'].isna() == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 187545 entries, 0 to 420239\n",
      "Data columns (total 19 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   id                  187545 non-null  int64  \n",
      " 1   user                187545 non-null  object \n",
      " 2   lat                 187545 non-null  float64\n",
      " 3   long                187545 non-null  float64\n",
      " 4   tags                140591 non-null  object \n",
      " 5   title               170583 non-null  object \n",
      " 6   date_taken_minute   187545 non-null  float64\n",
      " 7   date_taken_hour     187545 non-null  int64  \n",
      " 8   date_taken_day      187545 non-null  int64  \n",
      " 9   date_taken_month    187545 non-null  int64  \n",
      " 10  date_taken_year     187545 non-null  int64  \n",
      " 11  date_upload_minute  187545 non-null  int64  \n",
      " 12  date_upload_hour    187545 non-null  int64  \n",
      " 13  date_upload_day     187545 non-null  float64\n",
      " 14  date_upload_month   187545 non-null  int64  \n",
      " 15  date_upload_year    187544 non-null  float64\n",
      " 16  Unnamed: 16         0 non-null       float64\n",
      " 17  Unnamed: 17         0 non-null       float64\n",
      " 18  Unnamed: 18         0 non-null       float64\n",
      "dtypes: float64(8), int64(8), object(3)\n",
      "memory usage: 28.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>59.0</td>\n",
       "      <td>23</td>\n",
       "      <td>31</td>\n",
       "      <td>2011</td>\n",
       "      <td>2238</td>\n",
       "      <td>59</td>\n",
       "      <td>23</td>\n",
       "      <td>31.0</td>\n",
       "      <td>2011</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     date_taken_minute  date_taken_hour  date_taken_day  date_taken_month  \\\n",
       "min                0.0                0               1                 1   \n",
       "max               59.0               23              31              2011   \n",
       "\n",
       "     date_taken_year  date_upload_minute  date_upload_hour  date_upload_day  \\\n",
       "min               11                   0                 0              1.0   \n",
       "max             2238                  59                23             31.0   \n",
       "\n",
       "     date_upload_month  date_upload_year  Unnamed: 16  Unnamed: 17  \\\n",
       "min                  1            2009.0          NaN          NaN   \n",
       "max               2011            2019.0          NaN          NaN   \n",
       "\n",
       "     Unnamed: 18  \n",
       "min          NaN  \n",
       "max          NaN  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_cols = ['date_taken_minute',\n",
    "       'date_taken_hour', 'date_taken_day', 'date_taken_month',\n",
    "       'date_taken_year', 'date_upload_minute', 'date_upload_hour',\n",
    "       'date_upload_day', 'date_upload_month', 'date_upload_year',\n",
    "       'Unnamed: 16', 'Unnamed: 17', 'Unnamed: 18']\n",
    "\n",
    "df[date_cols].agg(['min', 'max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               id          user        lat     long  \\\n",
      "42366  5464485473  35635047@N03  45.765517  4.76651   \n",
      "\n",
      "                            tags                                       title  \\\n",
      "42366  lundimatin,lyondefi38nuit  une lundi matin comme tout les autre ;-(25   \n",
      "\n",
      "       date_taken_minute  date_taken_hour  date_taken_day  date_taken_month  \\\n",
      "42366                6.0               21               2              2011   \n",
      "\n",
      "       date_taken_year  date_upload_minute  date_upload_hour  date_upload_day  \\\n",
      "42366               11                  15                21              2.0   \n",
      "\n",
      "       date_upload_month  date_upload_year  Unnamed: 16  Unnamed: 17  \\\n",
      "42366               2011               NaN          NaN          NaN   \n",
      "\n",
      "       Unnamed: 18  \n",
      "42366          NaN  \n",
      "                id          user        lat      long tags title  \\\n",
      "106207  7690629716  83991262@N03  45.750606  4.835368  NaN   NaN   \n",
      "\n",
      "        date_taken_minute  date_taken_hour  date_taken_day  date_taken_month  \\\n",
      "106207               46.0                5              16                 2   \n",
      "\n",
      "        date_taken_year  date_upload_minute  date_upload_hour  \\\n",
      "106207             2238                  57                13   \n",
      "\n",
      "        date_upload_day  date_upload_month  date_upload_year  Unnamed: 16  \\\n",
      "106207              1.0                  8            2012.0          NaN   \n",
      "\n",
      "        Unnamed: 17  Unnamed: 18  \n",
      "106207          NaN          NaN  \n"
     ]
    }
   ],
   "source": [
    "print(df.loc[df['date_taken_month'] > 12])\n",
    "df = df.drop(df.loc[df['date_taken_month'] > 12].index.item()) #One row\n",
    "\n",
    "print(df.loc[df['date_taken_year'] > 2025])\n",
    "df = df.drop(df.loc[df['date_taken_year'] > 2025].index.item()) #One row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_taken_minute</th>\n",
       "      <th>date_taken_hour</th>\n",
       "      <th>date_taken_day</th>\n",
       "      <th>date_taken_month</th>\n",
       "      <th>date_taken_year</th>\n",
       "      <th>date_upload_minute</th>\n",
       "      <th>date_upload_hour</th>\n",
       "      <th>date_upload_day</th>\n",
       "      <th>date_upload_month</th>\n",
       "      <th>date_upload_year</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "      <th>Unnamed: 17</th>\n",
       "      <th>Unnamed: 18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1926</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>59.0</td>\n",
       "      <td>23</td>\n",
       "      <td>31</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>59</td>\n",
       "      <td>23</td>\n",
       "      <td>31.0</td>\n",
       "      <td>12</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     date_taken_minute  date_taken_hour  date_taken_day  date_taken_month  \\\n",
       "min                0.0                0               1                 1   \n",
       "max               59.0               23              31                12   \n",
       "\n",
       "     date_taken_year  date_upload_minute  date_upload_hour  date_upload_day  \\\n",
       "min             1926                   0                 0              1.0   \n",
       "max             2019                  59                23             31.0   \n",
       "\n",
       "     date_upload_month  date_upload_year  Unnamed: 16  Unnamed: 17  \\\n",
       "min                  1            2009.0          NaN          NaN   \n",
       "max                 12            2019.0          NaN          NaN   \n",
       "\n",
       "     Unnamed: 18  \n",
       "min          NaN  \n",
       "max          NaN  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[date_cols].agg(['min', 'max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                         0\n",
       "user                       0\n",
       "lat                        0\n",
       "long                       0\n",
       "tags                   46953\n",
       "title                  16961\n",
       "date_taken_minute          0\n",
       "date_taken_hour            0\n",
       "date_taken_day             0\n",
       "date_taken_month           0\n",
       "date_taken_year            0\n",
       "date_upload_minute         0\n",
       "date_upload_hour           0\n",
       "date_upload_day            0\n",
       "date_upload_month          0\n",
       "date_upload_year           0\n",
       "Unnamed: 16           187543\n",
       "Unnamed: 17           187543\n",
       "Unnamed: 18           187543\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'pandas' has no attribute 'to_csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[16], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m pd\u001b[39m.\u001b[39;49mto_csv(\u001b[39m\"\u001b[39m\u001b[39mcleaned-data.csv\u001b[39m\u001b[39m\"\u001b[39m, index\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: module 'pandas' has no attribute 'to_csv'"
     ]
    }
   ],
   "source": [
    "df.to_csv(\"data/cleaned-data.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}