my_model_by random sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "qkb0cp6pmflO"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 243
    },
    "id": "QcgPgc0xmflQ",
    "outputId": "1eb21927-2985-4344-e764-b56ff1ab47ad"
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"train.csv\")\n",
    "test_data = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 337
    },
    "id": "R53ifcrbmflQ",
    "outputId": "5496c5ac-8966-430f-bd93-b01ac0b03d21"
   },
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dXOiB-9emflR",
    "outputId": "a0da247e-ecbd-4175-ca1e-a0114eabfc87"
   },
   "outputs": [],
   "source": [
    "train_data.shape\n",
    "#we have over 2 million data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aOFSHE0LmflR",
    "outputId": "e8a88737-aaff-4970-f16a-7827fc8f317b"
   },
   "outputs": [],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "dx0u1HVxmflR"
   },
   "outputs": [],
   "source": [
    "#create a backup\n",
    "backup = train_data\n",
    "backup_test = test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49
    },
    "id": "dd-bM7yDmflR",
    "outputId": "2860a4a4-3be7-4368-bef5-dcd2425d0f92"
   },
   "outputs": [],
   "source": [
    "#FIRST TASK IS TO CLEAN THE DESCRIPTION AND BULLET POINTS COLUMNS AND JOIN THEM\n",
    "\n",
    "#TITLE\n",
    "train_data[train_data[\"TITLE\"].isna() == True ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "a_1E2CFSmflS"
   },
   "outputs": [],
   "source": [
    "#Drop these\n",
    "train_data = train_data.dropna(subset=[\"TITLE\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "id": "b9lITFg4mflS",
    "outputId": "c89382c8-eac9-4e69-8ec1-8043362e7983"
   },
   "outputs": [],
   "source": [
    "#check for NULL in description and bullet points both\n",
    "\n",
    "duplicate_desc = train_data[\"DESCRIPTION\"].isna()\n",
    "duplicate_bullet = train_data[\"BULLET_POINTS\"].isna()\n",
    "both_duplicate = duplicate_bullet & duplicate_desc\n",
    "train_data[both_duplicate]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "115tfaSdmflS"
   },
   "outputs": [],
   "source": [
    "#dropping these\n",
    "train_data = train_data.dropna(subset=['DESCRIPTION', 'BULLET_POINTS'], how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q_dhYN6bmflS",
    "outputId": "cf252f7a-075f-4803-99b2-6ef945dd9e60"
   },
   "outputs": [],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 658
    },
    "id": "WScTGFyOmflS",
    "outputId": "d95e603b-51b3-4dc6-84cb-76fb8c04f885"
   },
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "Z-yalmHUmflT"
   },
   "outputs": [],
   "source": [
    "#checking the remaining missing values\n",
    "\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].fillna(\"\")\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].fillna(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 658
    },
    "id": "q0uFcjGNmflT",
    "outputId": "57d3f82d-26a1-477d-d0a9-9fd9ec730480"
   },
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "3p2DZvUemflT",
    "outputId": "6d0319ce-d180-4e42-e229-b7917c8191b9"
   },
   "outputs": [],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "8fF1KTqsmflT"
   },
   "outputs": [],
   "source": [
    "#the max length product is an outlier \n",
    "train_data = train_data[train_data[\"PRODUCT_LENGTH\"] < 10e+4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WZ0Piha2mflT",
    "outputId": "152129aa-e338-4c1a-abab-ea31b93c3fad"
   },
   "outputs": [],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "IKkMQLMKmflT"
   },
   "outputs": [],
   "source": [
    "train_data = train_data[train_data[\"PRODUCT_LENGTH\"] > 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "vAdczyv5mflU",
    "outputId": "a3938319-d3e7-4358-e91f-55d612fee7d5"
   },
   "outputs": [],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['TITLE'] = train_data['TITLE'].str[:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "t2fqufIjmflU"
   },
   "outputs": [],
   "source": [
    "#now we will be cleaning the textual data like in title, description and bullet points\n",
    "\n",
    "#Escaping out HTML characters\n",
    "import re\n",
    "CLEANR = re.compile('<.*?>') \n",
    "\n",
    "def cleanhtml(raw_html):\n",
    "  cleantext = re.sub(CLEANR, '', raw_html)\n",
    "  return cleantext\n",
    "\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(cleanhtml)\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(cleanhtml)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(cleanhtml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "oSHu1iR5mflU"
   },
   "outputs": [],
   "source": [
    "#Encode from UTF-8 to ascii\n",
    "def encode_decode(html):\n",
    "    encode =html.encode('ascii','ignore')\n",
    "\n",
    "    #decode from ascii to UTF-8\n",
    "    decode=encode.decode(encoding='UTF-8')\n",
    "    return decode\n",
    "\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(encode_decode)\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(encode_decode)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(encode_decode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "VWnSshasmflU"
   },
   "outputs": [],
   "source": [
    "#library for regular expressions\n",
    "import re\n",
    "\n",
    "def regexx(tweet):\n",
    "    # remove hyperlinks\n",
    "    tweet = re.sub(r'https?:\\/\\/.\\S+', \"\", tweet)\n",
    "\n",
    "    # remove hashtags\n",
    "    # only removing the hash # sign from the word\n",
    "    tweet = re.sub(r'#', '', tweet)\n",
    "\n",
    "    # remove old style retweet text \"RT\"\n",
    "    tweet = re.sub(r'^RT[\\s]+', '', tweet)\n",
    "\n",
    "    return tweet\n",
    "\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(regexx)\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(regexx)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(regexx)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "q-kEFG4FmflU"
   },
   "outputs": [],
   "source": [
    "#dictionary consisting of the contraction and the actual value\n",
    "def apos(tweet):\n",
    "    Apos_dict={\"'s\":\" is\",\"n't\":\" not\",\"'m\":\" am\",\"'ll\":\" will\",\n",
    "            \"'d\":\" would\",\"'ve\":\" have\",\"'re\":\" are\"}\n",
    "\n",
    "    #replace the contractions\n",
    "    for key,value in Apos_dict.items():\n",
    "        if key in tweet:\n",
    "            tweet=tweet.replace(key,value)\n",
    "\n",
    "    return tweet\n",
    "\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(apos)\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(apos)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(apos)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "YDFMvzeKmflV"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "#separate the words\n",
    "def split(tweet):\n",
    "    tweet = \" \".join([s for s in re.split(\"([A-Z][a-z]+[^A-Z]*)\",tweet) if s])\n",
    "    return tweet\n",
    "\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(split)\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(split)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(split)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "HDIEG1bumflV"
   },
   "outputs": [],
   "source": [
    "#convert to lower case\n",
    "def lower(tweet):\n",
    "    tweet=tweet.lower()\n",
    "    return tweet\n",
    "\n",
    "train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(lower)\n",
    "train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(lower)\n",
    "train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(lower)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "OBirrF_xmflV"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "def stopw(tweet):\n",
    "    #download the stopwords from nltk using\n",
    "    nltk.download('stopwords')\n",
    "    #import stopwords\n",
    "    from nltk.corpus import stopwords\n",
    "\n",
    "    #import english stopwords list from nltk\n",
    "    stopwords_eng = stopwords.words('english')\n",
    "\n",
    "    tweet_tokens=tweet.split()\n",
    "    tweet_list=[]\n",
    "    #remove stopwords\n",
    "    for word in tweet_tokens:\n",
    "        if word not in stopwords_eng:\n",
    "            tweet_list.append(word)\n",
    "\n",
    "    return tweet_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xDRhrfx4mflV",
    "outputId": "a1107074-3ac5-4911-8445-8f70426c255a"
   },
   "outputs": [],
   "source": [
    "# train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(stopw)\n",
    "#pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_ZplFJvXpXCA",
    "outputId": "68e0f28c-5a8b-44bc-cc25-889daf9d8122"
   },
   "outputs": [],
   "source": [
    "# train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(stopw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "m1sddlxc1L69"
   },
   "outputs": [],
   "source": [
    "#for string operations\n",
    "import string\t\n",
    "def removepunc(tweet_list):\t\n",
    "  clean_tweet=[]\n",
    "  #remove punctuations\n",
    "  for word in tweet_list:\n",
    "    if word not in string.punctuation:\n",
    "      clean_tweet.append(word)\n",
    "\n",
    "  return clean_tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "Yio6oobJ1ijZ"
   },
   "outputs": [],
   "source": [
    "# train_data[\"BULLET_POINTS\"] = train_data[\"BULLET_POINTS\"].apply(removepunc)\n",
    "# train_data[\"DESCRIPTION\"] = train_data[\"DESCRIPTION\"].apply(removepunc)\n",
    "# train_data[\"TITLE\"] = train_data[\"TITLE\"].apply(removepunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "EzTqr2kt2aDH"
   },
   "outputs": [],
   "source": [
    "#make a new column called text\n",
    "train_data[\"text\"] = train_data[\"TITLE\"] + train_data[\"BULLET_POINTS\"] + train_data[\"DESCRIPTION\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_duplicates = lambda x: ' '.join(list(set(x.split())))\n",
    "train_data['text'] = train_data['text'].apply(remove_duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_symbols = lambda x: re.sub(r'[^\\w\\s]', '', x)\n",
    "train_data['text'] = train_data['text'].apply(remove_symbols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[\"text\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "#stop words\n",
    "stop_words = set(stopwords.words('english'))\n",
    "remove_stop_words = lambda x: ' '.join([word for word in x.split() if word.lower() not in stop_words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['text'] = train_data['text'].apply(remove_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.drop(['TITLE', 'DESCRIPTION', 'BULLET_POINTS'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "uZy3L3OZ3aZK"
   },
   "outputs": [],
   "source": [
    "train_data = train_data.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 815
    },
    "id": "UxZw9KIz4aBw",
    "outputId": "38ffff35-6e12-4eee-8fa8-b3d0ab9caf33"
   },
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_data.to_csv(\"cleaned_train.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "xoREaoWNmflW"
   },
   "outputs": [],
   "source": [
    "backup = train_data\n",
    "backup_test = test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 624
    },
    "id": "poXbBpFtmflW",
    "outputId": "b1742969-4e39-46fa-b83a-67f042f240ea"
   },
   "outputs": [],
   "source": [
    "#time to clean the test data (we cant hamper any rows ie cannot delete any of them)\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data[test_data[\"TITLE\"].isna()== True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = pd.read_csv(\"sample_submission.csv\")\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data[test_data[\"TITLE\"].isna() == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].fillna(\"\")\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].fillna(\"\")\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].fillna(\"\")\n",
    "\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(cleanhtml)\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(cleanhtml)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(cleanhtml)\n",
    "\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(encode_decode)\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(encode_decode)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(encode_decode)\n",
    "\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(regexx)\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(regexx)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(regexx)\n",
    "\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(apos)\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(apos)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(apos)\n",
    "\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(split)\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(split)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(split)\n",
    "\n",
    "test_data[\"DESCRIPTION\"] = test_data[\"DESCRIPTION\"].apply(lower)\n",
    "test_data[\"BULLET_POINTS\"] = test_data[\"BULLET_POINTS\"].apply(lower)\n",
    "test_data[\"TITLE\"] = test_data[\"TITLE\"].apply(lower)\n",
    "\n",
    "test_data[\"text\"] = test_data[\"TITLE\"] + test_data[\"BULLET_POINTS\"] + test_data[\"DESCRIPTION\"]\n",
    "\n",
    "test_data['text'] = test_data['text'].apply(remove_duplicates)\n",
    "\n",
    "test_data['text'] = test_data['text'].apply(remove_symbols)\n",
    "\n",
    "test_data['text'] = test_data['text'].apply(remove_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = test_data.drop(['TITLE', 'DESCRIPTION', 'BULLET_POINTS'], axis=1)\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_data.to_csv(\"cleaned_test.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "id": "SIWPstarmflW",
    "outputId": "b8318cf2-cdf3-4793-ee8c-1001bb18ec50"
   },
   "outputs": [],
   "source": [
    "sample = pd.read_csv(\"sample_submission.csv\")\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import make_column_transformer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train_data.drop(columns=[\"PRODUCT_ID\",\"PRODUCT_LENGTH\"])\n",
    "y = train_data[\"PRODUCT_LENGTH\"]\n",
    "X_test = test_data[[\"PRODUCT_TYPE_ID\",\"text\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = sample[\"PRODUCT_LENGTH\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.astype('float16')\n",
    "y[np.isinf(y)] = 0\n",
    "y.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = y_test.astype(\"float16\")\n",
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression,Ridge,Lasso\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor,AdaBoostRegressor,ExtraTreesRegressor\n",
    "from sklearn.svm import SVR\n",
    "# from xgboost import XGBRegressor\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import r2_score,mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "step1 = ColumnTransformer(transformers=[\n",
    "    ('col_tnf',OneHotEncoder(),[1])\n",
    "],remainder='passthrough')\n",
    "\n",
    "step2 = LinearRegression()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('step1',step1),\n",
    "    ('step2',step2)\n",
    "])\n",
    "\n",
    "pipe.fit(X=X,y=y.astype(\"float16\"))\n",
    "\n",
    "# y_pred = pipe.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y_pred = pipe.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y_pred.to_csv(\"predicted_submission.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"cleaned_train.csv\")\n",
    "test_data = pd.read_csv(\"cleaned_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reducing size\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split the dataset into training and validation sets\n",
    "train_data, val_data = train_test_split(train_data, test_size=0.2, random_state=42)\n",
    "\n",
    "# Perform random sampling on the training set\n",
    "sample_size = 100000  # Set the desired sample size\n",
    "train_data_sample = train_data.sample(n=sample_size, random_state=42)\n",
    "\n",
    "# Save the sampled training data to a CSV file\n",
    "train_data_sample.to_csv('sample.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_reduced = pd.read_csv(\"sample.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_reduced.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.histplot(train_data_reduced['PRODUCT_LENGTH'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#transforming our textual data with tfid vectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# Load your data\n",
    "text_column = train_data_reduced[\"text\"]\n",
    "\n",
    "# Replace missing values with empty strings\n",
    "text_column = np.where(text_column.isnull(), '', text_column)\n",
    "\n",
    "# Create a TF-IDF vectorizer\n",
    "vectorizer = TfidfVectorizer()\n",
    "\n",
    "# Fit the vectorizer on your text data\n",
    "vectorizer.fit(text_column)\n",
    "\n",
    "# Transform your text data into numerical values\n",
    "text_column_numerical = vectorizer.transform(text_column)\n",
    "\n",
    "\n",
    "#SAME FOR TEST DATA\n",
    "\n",
    "# Load your data\n",
    "text_column_test = test_data[\"text\"]\n",
    "\n",
    "# Replace missing values with empty strings_test\n",
    "text_column_test = np.where(text_column_test.isnull(), '', text_column_test)\n",
    "\n",
    "# Transform your text data into numerical values\n",
    "text_column_numerical_test = vectorizer.transform(text_column_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_column_numerical, text_column_numerical_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from scipy.sparse import hstack, csc_matrix\n",
    "import sklearn.metrics as metrics\n",
    "# Load your data\n",
    "# text_column_numerical = TF-IDF values for your text column\n",
    "# product_type_id_column = product_type_id column\n",
    "# product_length_column = product_length column\n",
    "product_type_id_column = train_data_reduced[\"PRODUCT_TYPE_ID\"]\n",
    "product_length_column = train_data_reduced[\"PRODUCT_LENGTH\"]\n",
    "\n",
    "product_type_id_column_test = test_data[\"PRODUCT_TYPE_ID\"]\n",
    "\n",
    "# Convert the text_column_numerical array into a CSC matrix\n",
    "text_column_numerical = csc_matrix(text_column_numerical)\n",
    "text_column_numerical_test = csc_matrix(text_column_numerical_test)\n",
    "\n",
    "\n",
    "# Convert the product_type_id_column array into a CSC matrix\n",
    "product_type_id_column = csc_matrix(product_type_id_column.to_numpy().reshape(-1, 1))\n",
    "product_type_id_column_test = csc_matrix(product_type_id_column_test.to_numpy().reshape(-1, 1))\n",
    "\n",
    "\n",
    "# Combine your input features into a single matrix\n",
    "X = hstack([text_column_numerical, product_type_id_column])\n",
    "\n",
    "X_test = hstack([text_column_numerical_test,product_type_id_column_test])\n",
    "\n",
    "# Split your data into training and test sets\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, product_length_column, test_size=0.2)\n",
    "X_train = X\n",
    "y_train = product_length_column\n",
    "y_test = product_type_id_column_test\n",
    "# Create a linear regression model\n",
    "model = LinearRegression()\n",
    "\n",
    "# Train the model on the training data\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "y_pred = model.predict(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "score = max( 0 , 100*(1-metrics.mean_absolute_percentage_error(y_test.toarray(),y_pred)))\n",
    "mse = mean_squared_error(y_test.toarray(), y_pred)\n",
    "print(f'Mean squared error: {mse}')\n",
    "print(f'Score: {score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(test_data[\"PRODUCT_ID\"])\n",
    "\n",
    "# Add a new column with the y_pred values\n",
    "df[\"y_pred\"] = y_pred\n",
    "\n",
    "# Set the column names\n",
    "df.columns = [\"PRODUCT_ID\", \"PRODUCT_LENGTH\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = pd.read_csv(\"sample_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"predicted_submission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  },
  "vscode": {
   "interpreter": {
    "hash": "939a19b7adb6e51f20fcd25b61225be92c16aa52f1a82911cccc0b7f4e8698e9"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}