tps-september2024-s4e9-ml.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":76728,"databundleVersionId":9057646,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\n\nfrom sklearn.model_selection import train_test_split\n\n%pip install category_encoders\nimport category_encoders as ce\n\nimport sklearn\nimport os\nimport re\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:41:15.269080Z","iopub.execute_input":"2024-09-24T13:41:15.269563Z","iopub.status.idle":"2024-09-24T13:41:35.231858Z","shell.execute_reply.started":"2024-09-24T13:41:15.269520Z","shell.execute_reply":"2024-09-24T13:41:35.230683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## load dataset ","metadata":{}},{"cell_type":"code","source":"# Loading in Kaggle's sample submission file for uploading\nsubmission = pd.read_csv('/kaggle/input/playground-series-s4e9/sample_submission.csv')\n# Applying predicted 'price' values to 'submission'\n# submission['price'] = y_pred\n# Exporting 'submission' as a .csv for scoring\n# submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:46:07.768058Z","iopub.execute_input":"2024-09-24T13:46:07.768966Z","iopub.status.idle":"2024-09-24T13:46:07.858103Z","shell.execute_reply.started":"2024-09-24T13:46:07.768919Z","shell.execute_reply":"2024-09-24T13:46:07.857092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e9/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e9/test.csv')\nprint(train.info())\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:46:08.865319Z","iopub.execute_input":"2024-09-24T13:46:08.865762Z","iopub.status.idle":"2024-09-24T13:46:10.669741Z","shell.execute_reply.started":"2024-09-24T13:46:08.865717Z","shell.execute_reply":"2024-09-24T13:46:10.668589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def nullValuesCols(df):\n    columns_with_null = df.columns[df.isnull().any()]\n    print(f'Total null values in the dataset {df.isnull().sum().sum()}')\n    print(f'columns {columns_with_null}')\n    for col in columns_with_null:\n        print(f' -- Null value for column {col}: {df[col].isnull().sum()}')\n        \n    print('\\n\\n')\n    \n    \ndef uniqueVlaues(df):   \n    object_cols = df.select_dtypes(include = ['object']).columns\n    print('All the unique values in the dataset')\n    for cols in object_cols:\n        if df[f'{cols}'].nunique() > 10:\n            print(f\" -- {cols} has: {df[cols].nunique()}, too many to list\")\n        else:\n            plt.figure(figsize=(10, 3))\n            sns.histplot(df[cols], kde = False) #, bins = 5)\n            plt.title(f\" {cols} has:\\n {df[cols].unique()}\")\n            plt.xlabel(f'{cols}')\n            plt.ylabel('Frequency')\n            plt.show()\n\n        print('\\n\\n')      \n    \n    \ndef dataExploration(df):\n\n    if df.isnull().sum().sum() >0:\n        nullValuesCols(df)\n        \n    has_objectColumns = any(df.dtypes == 'object')\n    if has_objectColumns:\n        uniqueVlaues(df)\n\n    \n    \n    return \n\ndataExploration(train)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:46:10.671346Z","iopub.execute_input":"2024-09-24T13:46:10.671687Z","iopub.status.idle":"2024-09-24T13:46:13.984167Z","shell.execute_reply.started":"2024-09-24T13:46:10.671651Z","shell.execute_reply":"2024-09-24T13:46:13.983130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- perfom anova on the large grouped varibale and if there is difference between them the perfoom one the encpding https://www.kaggle.com/code/arashnic/an-overview-of-categorical-encoding-methods \n- https://www.tutorialspoint.com/correlation-between-categorical-and-continuous-variables#:~:text=Now%2C%20if%20you%20want%20to,the%20target%20column%20is%20categorical. to check anova ","metadata":{}},{"cell_type":"code","source":"\ndef fill_null_with_median_or_mode(df):\n    print('Before filling the null values:', df.isnull().sum().sum())\n    \n    # Loop through each column in the DataFrame\n    for column in df.columns:\n        if df[column].dtype.name in ['float64', 'int64']:\n            # For numerical columns, use median\n            median_value = df[column].median()\n            df[column] = df[column].fillna(median_value)\n        elif df[column].dtype == 'object':\n            # For object (categorical) columns, use mode\n            mode_value = df[column].mode().iloc[0]  # Get the first mode if there are multiple\n            df[column] = df[column].fillna(mode_value)\n    \n    print('After filling the null values:', df.isnull().sum().sum())\n    \n    return df\n\n\ndef feature_engineering(dataframe):\n    df = dataframe.copy()\n    ## extract important features \n    df['horse_power'] = df['engine'].str.extract(r'(\\d+\\.\\d+)HP')\n    df['horse_power'] = df['horse_power'].astype(float)\n    \n    df['liters'] = df['engine'].str.extract(r'(\\d+\\.\\d+)L')\n    df['liters'] = df['liters'].astype(float)\n    \n    df['cylinders'] = df['engine'].str.extract(r'(\\d+) Cylinder')\n    df['cylinders'] = df['cylinders'].astype(float)\n    \n    df['turbo'] = df['engine'].str.extract(r'(turbo\\w*)' , flags=re.IGNORECASE)\n    df['turbo'] = np.where(df['turbo'] == 'Turbo', 1, 0)\n    \n    df['clean_title'] = np.where(df['clean_title'] == 'Yes', 1, 0)\n    df['accident'] = np.where(df['accident'] == 'At least 1 accident or damage reported', 1, 0)\n    \n    ## prepare new features indicating car features\n    df['car_age'] = 2024 - df['model_year']\n    df['Power_to_Weight_Ratio'] = df['horse_power'] / df['liters']\n    df['milage_per_year'] = np.where(df['car_age'] == 0, df['milage'], df['milage'] / df['car_age'])\n    \n    df.drop(columns = ['engine', 'model_year'], inplace = True)\n    \n    df = fill_null_with_median_or_mode(df)\n    \n    return df\n    \ntrain_df = feature_engineering(train)\ntest_df = feature_engineering(test)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:46:41.907368Z","iopub.execute_input":"2024-09-24T13:46:41.907956Z","iopub.status.idle":"2024-09-24T13:46:48.001519Z","shell.execute_reply.started":"2024-09-24T13:46:41.907903Z","shell.execute_reply":"2024-09-24T13:46:48.000367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-23T10:18:20.346945Z","iopub.execute_input":"2024-09-23T10:18:20.347513Z","iopub.status.idle":"2024-09-23T10:18:20.374634Z","shell.execute_reply.started":"2024-09-23T10:18:20.347433Z","shell.execute_reply":"2024-09-23T10:18:20.372873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n- Target Encoding - Mean Likelihood Encoding ,\"The Right Way !\"\nMean encoding means replacing the category with the mean target value for that category. We start by grouping each category alone, and for each group, we calculate the mean of the target in the corresponding observations. Then we assign that mean to that category. Thus, we encoded the category with the mean of the target. Here’s a detailed illustration of mean encoding.\n- P-value ≤ α: The differences between some of the means are statistically significant\nIf the p-value is less than or equal to the significance level, you reject the null hypothesis and conclude that not all population means are equal. Use your specialized knowledge to determine whether the differences are practically significant. For more information, go to Statistical and practical significance.\n- P-value > α: The differences between the means are not statistically significant\nIf the p-value is greater than the significance level, you do not have enough evidence to reject the null hypothesis that the population means are all equal. Verify that your test has enough power to detect a difference that is practically significant. For more information, go to Increase the power of a hypothesis test.\n- Why Target Encoding in This Case?\nIf you find that the means of different groups (e.g., races in your example) are significantly different with respect to a continuous target variable, it suggests that the categorical variable (e.g., race) has a meaningful relationship with the target. In such cases, target encoding can help translate this relationship into a numerical form that a machine learning model can understand.]]\n\nhttps://mode.com/blog/violin-plot-examples","metadata":{}},{"cell_type":"code","source":"def anovaVisualisation(df, colName):\n    df = pd.DataFrame(df)\n    groups = df.groupby(colName[0])\n    group_values = []\n\n    for name, group in groups:\n        group_values.append(group[colName[1]])\n    \n    f_stat, p_value = stats.f_oneway(*group_values)\n    if p_value <= 0.05:\n        text = f\"{colName[0]} --> ANOVA F-statistic: {f_stat}, \\n  p-value: {p_value}\\n Perform mean target encoding\"\n    else:\n        text = f\"{colName[0]} --> ANOVA F-statistic: {f_stat}, \\n p-value: {p_value}\\n Perform frequency encoding\"\n\n    \n    # Visualization: violin plot to visualize the distribution of the target variable across the groups\n    plt.figure(figsize=(20, 8))\n    sns.violinplot(x = df[colName[0]], y = df[colName[1]], data = df);\n    plt.title(f\"{text}\")\n    plt.xlabel(colName[0])\n    plt.ylabel(colName[1])\n    plt.yscale('log')\n    _ = plt.xticks(rotation=45, ha='right')\n    plt.show()\n\n# anovaVisualisation(train[['fuel_type', 'price']], ['fuel_type', 'price'])","metadata":{"execution":{"iopub.status.busy":"2024-09-23T10:18:20.376859Z","iopub.execute_input":"2024-09-23T10:18:20.377442Z","iopub.status.idle":"2024-09-23T10:18:20.403240Z","shell.execute_reply.started":"2024-09-23T10:18:20.377373Z","shell.execute_reply":"2024-09-23T10:18:20.401586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_cols = train_df.select_dtypes(include = ['object']).columns\n\nfor col in object_cols:\n    anovaVisualisation(train_df[[col, 'price']], [col, 'price'])\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-23T10:18:20.406031Z","iopub.execute_input":"2024-09-23T10:18:20.406696Z","iopub.status.idle":"2024-09-23T10:20:28.093500Z","shell.execute_reply.started":"2024-09-23T10:18:20.406633Z","shell.execute_reply":"2024-09-23T10:20:28.092070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## correlation between the features ","metadata":{}},{"cell_type":"code","source":"def featureCorr(df):\n    x = df.drop(['id','price'], axis =1)\n    y = df['price']\n    print(\"perform target encoding\")\n    objectCols = df.select_dtypes(include = ['object']).columns\n    _encoder = ce.TargetEncoder(cols = objectCols)\n    _encoder.fit(x, y)\n\n    # Transform both training and testing sets\n    en_df = _encoder.transform(x)\n    en_df['price'] = df['price']\n    \n    corr = en_df.corr()\n    plt.figure(figsize=(15, 8))\n    sns.heatmap(corr, xticklabels = corr.columns.values, yticklabels = corr.columns.values,\n                cmap = \"YlGnBu\", annot = True, fmt='.2g')\n    plt.title(\"pearson Correlation Heatmap\", fontsize=16)\n    plt.show();\n    \n\nfeatureCorr(train_df)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-23T10:55:03.543564Z","iopub.execute_input":"2024-09-23T10:55:03.545390Z","iopub.status.idle":"2024-09-23T10:55:08.497383Z","shell.execute_reply.started":"2024-09-23T10:55:03.545321Z","shell.execute_reply":"2024-09-23T10:55:08.496012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model training section ","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\n\n\ndef encoding_scaling(datasets, _type = None):\n    df = datasets[0]\n    test = datasets[1].drop(['id'], axis = 1)\n    x = df.drop(['id', 'price'], axis = 1)\n    y = df['price']\n    \n    # Split into training and testing sets\n    X_train, X_val, y_train, y_val = train_test_split(x, y, test_size = 0.3, random_state = 42)\n\n    \n    object_cols = df.select_dtypes(include = ['object']).columns\n    if _type == 'target_encoding':\n        encoder = ce.TargetEncoder(cols = object_cols)\n    elif _type == 'frequency_encoding':\n        encoder = ce.CountEncoder(cols = object_cols)\n    else:\n        encoder = ce.TargetEncoder(cols = object_cols)\n        \n    encoder.fit(X_train, y_train)\n    \n    X_train = encoder.transform(X_train)\n    X_val = encoder.transform(X_val)\n    test = encoder.transform(test)\n    \n    scaler = MinMaxScaler()\n    X_train = pd.DataFrame(scaler.fit_transform(X_train), columns = x.columns)\n    X_val = pd.DataFrame(scaler.fit_transform(X_val), columns = x.columns)\n    test = pd.DataFrame(scaler.fit_transform(test), columns = test.columns)\n    \n    \n    return (X_train, X_val, y_train, y_val), test\n\ntrain, test = encoding_scaling([train_df, test_df])","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:47:52.726368Z","iopub.execute_input":"2024-09-24T13:47:52.727671Z","iopub.status.idle":"2024-09-24T13:47:54.809593Z","shell.execute_reply.started":"2024-09-24T13:47:52.727585Z","shell.execute_reply":"2024-09-24T13:47:54.808153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\ndef model_prediction_regression(model, x_train, x_test, y_train, y_test):\n    # Fit the model\n    model.fit(x_train, y_train)\n    \n    # Predictions for both training and testing sets\n    y_train_pred = model.predict(x_train)\n    y_test_pred = model.predict(x_test)\n    \n    # Evaluation Metrics\n    mse_train = mean_squared_error(y_train, y_train_pred)\n    mse_test = mean_squared_error(y_test, y_test_pred)\n    \n    mae_train = mean_absolute_error(y_train, y_train_pred)\n    mae_test = mean_absolute_error(y_test, y_test_pred)\n    \n    r2_train = r2_score(y_train, y_train_pred)\n    r2_test = r2_score(y_test, y_test_pred)\n    \n    rmse_train = np.sqrt(mse_train)\n    rmse_test = np.sqrt(mse_test)\n    \n    # Printing Evaluation Metrics\n    print(f\"Training Metrics for {model} model:\")\n    print(f\"Mean Squared Error (MSE): {mse_train}\")\n    print(f\"Mean Absolute Error (MAE): {mae_train}\")\n    print(f\"R-Squared (R²): {r2_train}\")\n    print(f\"Root Mean Squared Error (RMSE): {rmse_train}\")\n    \n    print(\"\\nTesting Metrics for {model} model:\")\n    print(f\"Mean Squared Error (MSE): {mse_test}\")\n    print(f\"Mean Absolute Error (MAE): {mae_test}\")\n    print(f\"R-Squared (R²): {r2_test}\")\n    print(f\"Root Mean Squared Error (RMSE): {rmse_test}\")\n    \n    # Visualizing predictions vs true values\n    plt.figure(figsize=(8, 6))\n    plt.scatter(y_test, y_test_pred, color=\"blue\", label=\"Predicted vs Actual\")\n    plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=2, label=\"Ideal Fit\")\n    plt.title(f\"Predictions vs True Values for {model}\")\n    plt.xlabel(\"True Values\")\n    plt.ylabel(\"Predictions\")\n    plt.legend()\n    plt.show()\n    \n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:51:09.411195Z","iopub.execute_input":"2024-09-24T13:51:09.412401Z","iopub.status.idle":"2024-09-24T13:51:09.426124Z","shell.execute_reply.started":"2024-09-24T13:51:09.412348Z","shell.execute_reply":"2024-09-24T13:51:09.424683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\n# Example: Using Linear Regression\nLinear_model = LinearRegression()\nmodel = model_prediction_regression(Linear_model, *train)\nsubmission['price'] = model.predict(test)\nsubmission.to_csv('LinearSubmission.csv', index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:52:29.493498Z","iopub.execute_input":"2024-09-24T13:52:29.494067Z","iopub.status.idle":"2024-09-24T13:52:31.526509Z","shell.execute_reply.started":"2024-09-24T13:52:29.494020Z","shell.execute_reply":"2024-09-24T13:52:31.525383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\n# Create an instance of the LightGBM Regressor with the RMSE metric. \nlgbm_model = LGBMRegressor(metric='rmse') \n  \n# Train the model using the training data. \nmodel = model_prediction_regression(lgbm_model, *train)\nsubmission['price'] = model.predict(test)\nsubmission.to_csv('LgbmSubmission.csv', index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:53:08.340193Z","iopub.execute_input":"2024-09-24T13:53:08.340933Z","iopub.status.idle":"2024-09-24T13:53:14.587102Z","shell.execute_reply.started":"2024-09-24T13:53:08.340869Z","shell.execute_reply":"2024-09-24T13:53:14.585917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xg \n\nxgb_model = xg.XGBRegressor(objective ='reg:linear', n_estimators = 10, seed = 123) \n\n# Train the model using the training data. \nmodel = model_prediction_regression(xgb_model,*train)\nsubmission['price'] = model.predict(test)\nsubmission.to_csv('xgbSubmission.csv', index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:53:39.631587Z","iopub.execute_input":"2024-09-24T13:53:39.632120Z","iopub.status.idle":"2024-09-24T13:53:42.493687Z","shell.execute_reply.started":"2024-09-24T13:53:39.632075Z","shell.execute_reply":"2024-09-24T13:53:42.492521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n \n# Create a random forest regression model\nrf_model = RandomForestRegressor(n_estimators=100)\n\n# Train the model using the training data. \nmodel = model_prediction_regression(rf_model,*train)\nsubmission['price'] = model.predict(test)\nsubmission.to_csv('rfSubmission.csv', index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T13:54:13.866057Z","iopub.execute_input":"2024-09-24T13:54:13.866540Z","iopub.status.idle":"2024-09-24T13:56:32.913753Z","shell.execute_reply.started":"2024-09-24T13:54:13.866496Z","shell.execute_reply":"2024-09-24T13:56:32.912531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## combine all the results ","metadata":{}},{"cell_type":"code","source":"col = []\n\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        path = os.path.join(dirname, filename)\n        col.append(filename)\n        submission[f'{filename}'] = pd.read_csv(path)['price']\n\n\nsubmission['price'] = submission[col].mean(axis=1)\nsubmission[['id', 'price']].to_csv('All_Submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T14:13:23.277112Z","iopub.execute_input":"2024-09-24T14:13:23.278594Z","iopub.status.idle":"2024-09-24T14:13:23.965715Z","shell.execute_reply.started":"2024-09-24T14:13:23.278533Z","shell.execute_reply":"2024-09-24T14:13:23.963919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}