05_Feature_Selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNXnxBbCMrqbdoDfDXDbfko",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/satwiksps/Machine_Learning/blob/main/Feature_Selection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "UWtyM9sg-cfk"
      },
      "outputs": [],
      "source": [
        "from pandas import read_csv\n",
        "url = 'https://raw.githubusercontent.com/erojaso/MLMasteryEndToEnd/master/data/pima-indians-diabetes.data.csv' #Load CSV using Pandas\n",
        "column_names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class'] #Define column names\n",
        "data = read_csv(url, names=column_names) #Load CSV using Pandas\n",
        "array = data.values #Convert to NumPy array\n",
        "Input = array[:,0:8] #Split into column\n",
        "Output = array[:,8] #Split into output column"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from numpy import set_printoptions"
      ],
      "metadata": {
        "id": "a7AjMT7b_c9L"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Univariate Selection"
      ],
      "metadata": {
        "id": "lCfLIgfsmnzg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_selection import SelectKBest\n",
        "from sklearn.feature_selection import chi2\n",
        "test = SelectKBest(score_func=chi2, k=4)\n",
        "fit = test.fit(Input,Output)\n",
        "\n",
        "# summarize scores\n",
        "set_printoptions(precision=3)\n",
        "print(fit.scores_)\n",
        "features = fit.transform(Input)\n",
        "\n",
        "# summarize selected features\n",
        "print(features[0:5,:])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s3rruTAW_fme",
        "outputId": "48b0e2d2-c9e4-4345-8714-36cfbfe1f1a5"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[ 111.52  1411.887   17.605   53.108 2175.565  127.669    5.393  181.304]\n",
            "[[148.    0.   33.6  50. ]\n",
            " [ 85.    0.   26.6  31. ]\n",
            " [183.    0.   23.3  32. ]\n",
            " [ 89.   94.   28.1  21. ]\n",
            " [137.  168.   43.1  33. ]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Recursive Feature Elimination"
      ],
      "metadata": {
        "id": "lE8PNe22mjwL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_selection import RFE\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "model = LogisticRegression(solver='liblinear')\n",
        "rfe = RFE(estimator=model, n_features_to_select=3)\n",
        "fit = rfe.fit(Input,Output)\n",
        "print(\"Num Features: %d\" % fit.n_features_)\n",
        "print(\"Selected Features: %s\" % fit.support_)\n",
        "print(\"Feature Ranking: %s\" % fit.ranking_)"
      ],
      "metadata": {
        "id": "IgATso0Xjfui",
        "outputId": "2dc366f2-1e1a-45aa-ac96-16c516093a6c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Num Features: 3\n",
            "Selected Features: [ True False False False False  True  True False]\n",
            "Feature Ranking: [1 2 3 5 6 1 1 4]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Principal Component Analysis"
      ],
      "metadata": {
        "id": "7ZpfGzJpmviD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA\n",
        "pca = PCA(n_components=3)\n",
        "fit = pca.fit(Input)\n",
        "print(\"Explained Variance: %s\" % fit.explained_variance_ratio_)\n",
        "print(fit.components_)"
      ],
      "metadata": {
        "id": "J137rNYqmwtc",
        "outputId": "55fbae93-96c7-490a-a918-3a6c31c251d6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Explained Variance: [0.889 0.062 0.026]\n",
            "[[-2.022e-03  9.781e-02  1.609e-02  6.076e-02  9.931e-01  1.401e-02\n",
            "   5.372e-04 -3.565e-03]\n",
            " [-2.265e-02 -9.722e-01 -1.419e-01  5.786e-02  9.463e-02 -4.697e-02\n",
            "  -8.168e-04 -1.402e-01]\n",
            " [-2.246e-02  1.434e-01 -9.225e-01 -3.070e-01  2.098e-02 -1.324e-01\n",
            "  -6.400e-04 -1.255e-01]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Bagged decision trees\n",
        "Random Forest and Extra Trees"
      ],
      "metadata": {
        "id": "ZTZsGkwrwJeh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "model = ExtraTreesClassifier(n_estimators=100)\n",
        "model.fit(Input,Output)\n",
        "print(model.feature_importances_)"
      ],
      "metadata": {
        "id": "NPjcnTV6vJEV",
        "outputId": "38543542-9104-4764-df7a-eaaf506d65cb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.11  0.237 0.097 0.079 0.077 0.141 0.116 0.143]\n"
          ]
        }
      ]
    }
  ]
}