Behavioural_Economics_Pattern_Recognition_Data_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PATTERN RECOGNITION AND DATA ANALYSIS FOR BEHAVIOURAL ECONOMICS \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "df = pd.read_csv('./pagestatistcs.csv')\n",
    "df.set_index(df['date'])\n",
    "df_globalControlFalse = df[df['globalControl'] ==  False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.express as px\n",
    "X_X='price'\n",
    "Y_Y='amount'\n",
    "df1 = df[[Y_Y, X_X]]\n",
    "df1[Y_Y][df1[Y_Y] != 0] = 1\n",
    "df143=df1[df1[Y_Y] == 1]\n",
    "fig = px.histogram(df1, x=X_X,y=Y_Y)\n",
    "fig.show()\n",
    "dd=df143[X_X]\n",
    "print(dd.value_counts().plot(kind='bar', figsize=(20,20)))\n",
    "df123=df1[df1[X_X] == 29.99]\n",
    "total_occurences=df1[X_X][df1[X_X] == 29.99].value_counts().tolist()\n",
    "print(\"Total Number of occurences of the price mentioned: \",total_occurences[0])\n",
    "x211=df123[Y_Y][df123[Y_Y] == 0].value_counts().tolist()\n",
    "print(x211)\n",
    "num_price_occurences_in_Y=df123[Y_Y][df123[Y_Y] == 1].value_counts().tolist()\n",
    "print(num_price_occurences_in_Y)\n",
    "Ration=num_price_occurences_in_Y[0]*100/(total_occurences[0])\n",
    "print(\"Conversion Rate i.e. number of price occurences in cartadd/total number of occurences\", Ration)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explanatory data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Between incentive columns and cart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df[['cart:add', 'globalControl','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2']]\n",
    "columnName='2.4.2'\n",
    "df1[columnName][df1[columnName] != 0] = 1\n",
    "df1['cart:add'][df1['cart:add'] != 0] = 1\n",
    "print(\"Instances where globalControl is False\")\n",
    "temp_df = df1[df1['globalControl'] == False]\n",
    "l1=temp_df['cart:add'][temp_df[columnName] == 1]\n",
    "x11=temp_df['cart:add'][temp_df[columnName] == 1].value_counts()\n",
    "x21=temp_df['cart:add'][temp_df[columnName] == 1].value_counts().tolist()\n",
    "print(x11)\n",
    "print(x21)\n",
    "print(\"Taking values where cart:add is one:\",x21[1])\n",
    "Ratio=(x21[1]/((x21[0])+(x21[1])))*100\n",
    "print(\"Calculating the conversion rate i.e for cart addition:\",Ratio)\n",
    "dfj = pd.DataFrame(l1)",
    "sd=dfj.plot.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Between Incentives and checkout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df[['checkout', 'globalControl','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2']]\n",
    "columnName='2.9.1'\n",
    "df2[columnName][df2[columnName] != 0] = 1\n",
    "df2['checkout'][df2['checkout'] != 0] = 1\n",
    "print(\"Instances where globalControl is False\")\n",
    "temp_df_2 = df2[df2['globalControl'] == False]\n",
    "l2=temp_df_2['checkout'][temp_df_2[columnName] == 1]\n",
    "x12=temp_df_2['checkout'][temp_df_2[columnName] == 1].value_counts()\n",
    "x22=temp_df_2['checkout'][temp_df_2[columnName] == 1].value_counts().tolist()\n",
    "print(x12)\n",
    "print(x22)\n",
    "print(\"Taking values where checkout is one:\",x22[0])\n",
    "Ratio2=(x22[0]/((x22[0])+(x22[1])))*100\n",
    "print(\"Calculating the conversion rate i.e for cart addition:\",Ratio2)\n",
    "dfj = pd.DataFrame(l2)\n",
    "sd=dfj.plot.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Between incentives and Amount\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = df[['amount', 'globalControl','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2']]\n",
    "columnName='2.91.1'\n",
    "df3[columnName][df3[columnName] != 0] = 1\n",
    "df3['amount'][df3['amount'] != 0] = 1\n",
    "print(\"Instances where globalControl is False\")\n",
    "temp_df_3 = df3[df3['globalControl'] == False]\n",
    "l3=temp_df_3['amount'][temp_df_3[columnName] == 1]\n",
    "x13=temp_df_3['amount'][temp_df_3[columnName] == 1].value_counts()\n",
    "x23=temp_df_3['amount'][temp_df_3[columnName] == 1].value_counts().tolist()\n",
    "print(x13)\n",
    "print(x23)\n",
    "print(\"Taking values where checkout is one:\",x23[1])\n",
    "Ratio3=(x23[1]/((x23[0])+(x23[1])))*100\n",
    "print(\"Calculating the conversion rate i.e for cart addition:\",Ratio3)\n",
    "dfj = pd.DataFrame(l3)\n",
    "sd=dfj.plot.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chi-squared test of independance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "from scipy.stats import chi2_contingency\n",
    "df_cs = pd.read_csv('./pagestatistcs.csv')\n",
    "#df_cs[df_cs['globalControl'] == False]\n",
    "def chisq_of_df_cols(df, c1, c2):\n",
    "    groupsizes = df.groupby([c1, c2]).size()\n",
    "    ctsum = groupsizes.unstack(c1)\n",
    "    # fillna(0) is necessary to remove any NAs which will cause exceptions\n",
    "    return(chi2_contingency(ctsum.fillna(0)))\n",
    "df_cs['incentive_present'] = df[['2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2']].sum(axis=1)\n",
    "df_cs['incentive_present'][df_cs['incentive_present'] >= 1] = 1\n",
    "columnName='2.4.2'\n",
    "df_cs[columnName][df_cs[columnName] != 0] = 1\n",
    "chisquared_value22 = chisq_of_df_cols(df_cs, 'amount', columnName)\n",
    "print('test statistics: ', chisquared_value22[0])\n",
    "print('P-value: ', chisquared_value22[1])\n",
    "cst=pd.DataFrame(chisquared_value22)\n",
    "print(cst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I created a new column, called 'incentive_present' which is a binary column telling whether any of the individual incentive was shown or not. Then i ran the chi-squared test of independance two test whether incentive_present and cart:add are independant or not. Since the p-value of the test is less than the critical value ie, 0.05, we can reject the null hypothesis H0, hence, both variables are dependant on each other. P value of 0.000 is expressed as P<0.001 (  https://www.researchgate.net/post/What_should_p_value_be_reported_when_SPSS_report_it_0000 )."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above chi-squared test of independance shows us that since the p value is less than critical value, both variables are dependant on each other.\n",
    "For example, '2.9.5','revenue','tempSessionId','date' incentives has p-value above the critical value thus we cannot reject the null hypothesis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have established that there is a relationship between incentives and cart:add, we want to do a comprehensive analysis between two variables to see the direction of relationship.\n",
    "For checking the direction, I will conduct a statistical test based on collinearity coefficient(Pearson) test. It will tell us the strength and direction of the relationship between our columns and cart:add/amount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regression analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr\n",
    "column_Name='2.24.1'\n",
    "corr, _ = pearsonr(df[column_Name],df['cart:add'] )\n",
    "print('Pearsons correlation: %.3f' % corr)\n",
    "from sklearn import linear_model\n",
    "reg = linear_model.LinearRegression()\n",
    "reg.fit(df[[column_Name]], df['cart:add'])\n",
    "print(\"Linear Regression Coefficient: \",reg.coef_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Pearson correlation coefficient, r, can take a range of values from +1 to -1. A value of 0 indicates that there is no association between the two variables. A value greater than 0 indicates a positive association; that is, as the value of one variable increases, so does the value of the other variable. A value less than 0 indicates a negative association; that is, as the value of one variable increases, the value of the other variable decreases. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although the correlation coefficient is low, but it tells us that the coefficient is poistive which shows us that there is a positive relation between incentive_present and cart:add\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "dfr = pd.read_csv('./pagestatistcs.csv')\n",
    "print(dfr.isnull().values.any())\n",
    "dfr['cart:add'][dfr['cart:add'] != 0] = 1\n",
    "dfr1=dfr[dfr['cart:add'] == 1]\n",
    "dd=dfr1['price'].value_counts().rename_axis('unique_values').reset_index(name='counts')\n",
    "dfj = pd.DataFrame(dd)\n",
    "X =dfj.iloc[:, 0].values.reshape(-1, 1)  # values converts it into a numpy array\n",
    "Y = dfj.iloc[:, 1].values.reshape(-1, 1)  # -1 means that calculate the dimension of rows, but have 1 column\n",
    "linear_regressor = LinearRegression()  # create object for the class\n",
    "lr= linear_regressor.fit(X, Y)  # perform linear regression\n",
    "Y_pred = linear_regressor.predict(X)  # make predictions\n",
    "plt.scatter(X, Y)\n",
    "plt.plot(X, Y_pred, color='red')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting feature importance using xgboost classifier\n",
    "from numpy import loadtxt\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost import plot_importance\n",
    "from matplotlib import pyplot\n",
    "print(\"Instances where globalControl is False\")\n",
    "#temp_df = df[df['globalControl'] == False]\n",
    "temp_df1=df\n",
    "temp_df1['incentive_present'] = temp_df1[['2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2']].sum(axis=1)\n",
    "temp_df1['incentive_present'][temp_df1['incentive_present'] >= 1] = 1\n",
    "# split data into X and y\n",
    "X = temp_df1[['price', 'totalSaved', 'saleProducts','totalRevenue','nonSaleProducts','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2', 'incentive_present']]\n",
    "y = temp_df1['amount']\n",
    "# fit model no training data\n",
    "model = XGBClassifier()\n",
    "model.fit(X, y)\n",
    "# plot feature importance\n",
    "plot_importance(model)\n",
    "pyplot.show()\n",
    "# trainign data accuracy\n",
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = model.predict(X)\n",
    "predictions = [round(value) for value in y_pred]\n",
    "# evaluate predictions\n",
    "accuracy = accuracy_score(y, predictions)\n",
    "print(\"Accuracy: %.2f%%\" % (accuracy * 100.0))\n",
    "from sklearn.metrics import confusion_matrix\n",
    "cf= confusion_matrix(y, predictions)\n",
    "print(cf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splitting data into X and y\n",
    "X = df_globalControlFalse[['price', 'totalSaved', 'saleProducts', 'nonSaleProducts','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2', 'incentive_present', 'cart:add']]\n",
    "y = df_globalControlFalse['checkout']\n",
    "# fitting model no training data\n",
    "model = XGBClassifier()\n",
    "model.fit(X, y)\n",
    "# plotting feature importance\n",
    "plot_importance(model)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_globalControlFalse[['price', 'totalSaved', 'saleProducts', 'nonSaleProducts','2.1.1', '2.6.2', '2.4.4' ,'2.24.1', '2.9.1', '2.91.1', '2.9.5', '2.4.2', 'incentive_present', 'cart:add']]\n",
    "y = df_globalControlFalse['amount']\n",
    "# fit model no training data\n",
    "model = XGBClassifier()\n",
    "model.fit(X, y)\n",
    "# plot feature importance\n",
    "plot_importance(model)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Relation of Price with cart:add, checkout and amount "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.express as px\n",
    "df = pd.read_csv('./pagestatistcs.csv')\n",
    "X_X='price'\n",
    "Y_Y='amount'\n",
    "df1 = df[[Y_Y, X_X]]\n",
    "df1[Y_Y][df1[Y_Y] != 0] = 1\n",
    "df143=df1[df1[Y_Y] == 1]\n",
    "fig = px.histogram(df1, x=X_X,y=Y_Y)\n",
    "fig.show()\n",
    "dd=df143[X_X]\n",
    "print(dd.value_counts().plot(kind='bar', figsize=(20,20)))\n",
    "df123=df1[df1[X_X] == 29.99]\n",
    "total_occurences=df1[X_X][df1[X_X] == 29.99].value_counts().tolist()\n",
    "print(\"Total Number of occurences of the price mentioned: \",total_occurences[0])\n",
    "x211=df123[Y_Y][df123[Y_Y] == 0].value_counts().tolist()\n",
    "print(x211)\n",
    "num_price_occurences_in_Y=df123[Y_Y][df123[Y_Y] == 1].value_counts().tolist()\n",
    "print(num_price_occurences_in_Y)\n",
    "Ration=num_price_occurences_in_Y[0]*100/(total_occurences[0])\n",
    "print(\"Conversion Rate i.e. number of price occurences in cartadd/total number of occurences\", Ration)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CONCLUSIONS:\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Incentives overall have a positive correlation with cart:add, checkout and amount\n",
    "2. Incentives: '2.9.1' and '2.91.1' have the best conversion rate when they are shown\n",
    "3. Incentive: '2.1.1', although, is shown 244K times but only 10-15K is actually converted into cart:add,amount\n",
    "4. Prices with X7.99 have a much better conversion rate than X9.99 except at 17.99 and 19.99 mainly because the price is already small\n",
    "5. All other incentives have a similiar low conversion rate\n",
    "6. Cart:add>checkout>amount although conversion rates of individual incentives with them are checkout>cart:add>amount\n",
    "7. Checkout should be reviewed since it stays 1 if it's clicked once by a tempSessionID\n",
    "8. Revenue, ID, date etc are not correlated to cart:add,checkout and amount "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}