XGB_RELIEF_BrutForce.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copie de XGB_RELIEF_BrutForce",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "pip install ITMO_FS"
      ],
      "metadata": {
        "id": "pziwqVWgnlQG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ec2ec080-9222-4514-c641-ab6c995f0161"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ITMO_FS\n",
            "  Downloading ITMO_FS-0.3.3-py3-none-any.whl (70 kB)\n",
            "\u001b[?25l\r\u001b[K     |████▋                           | 10 kB 20.9 MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 20 kB 16.7 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 30 kB 11.0 MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 40 kB 9.6 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 51 kB 4.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 61 kB 5.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 70 kB 3.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: imblearn in /usr/local/lib/python3.7/dist-packages (from ITMO_FS) (0.0)\n",
            "Collecting qpsolvers\n",
            "  Downloading qpsolvers-1.9.1-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 5.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from ITMO_FS) (1.21.6)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from ITMO_FS) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from ITMO_FS) (1.0.2)\n",
            "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.7/dist-packages (from imblearn->ITMO_FS) (0.8.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn->imblearn->ITMO_FS) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->ITMO_FS) (3.1.0)\n",
            "Collecting quadprog>=0.1.8\n",
            "  Downloading quadprog-0.1.11.tar.gz (121 kB)\n",
            "\u001b[K     |████████████████████████████████| 121 kB 11.5 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: quadprog\n",
            "  Building wheel for quadprog (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for quadprog: filename=quadprog-0.1.11-cp37-cp37m-linux_x86_64.whl size=290748 sha256=9efe23f58dc53824e1bc351b0147dc29e492c42a1921084bb906104af11118c8\n",
            "  Stored in directory: /root/.cache/pip/wheels/4a/4e/d7/41034ea11aeef1266df3cae546116cb6094e955c41ae3e2589\n",
            "Successfully built quadprog\n",
            "Installing collected packages: quadprog, qpsolvers, ITMO-FS\n",
            "Successfully installed ITMO-FS-0.3.3 qpsolvers-1.9.1 quadprog-0.1.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ba0Ulyu5ycf2"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from imblearn.over_sampling import RandomOverSampler\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import RocCurveDisplay\n",
        "from sklearn.metrics import auc\n",
        "from ITMO_FS.filters.univariate import reliefF_measure"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"data.csv\")\n"
      ],
      "metadata": {
        "id": "y8W60GrryeUN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lis = []\n",
        "for i in range(len(df)) :\n",
        "  if df['diagnosis'][i] == 'M' :\n",
        "    lis.append(1)\n",
        "  else :\n",
        "    lis.append(0)\n",
        "Y_ = np.array(lis)"
      ],
      "metadata": {
        "id": "Yb-Pge068El8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Y_"
      ],
      "metadata": {
        "id": "VN7kmXSn8eze",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2e8d9075-7d9b-4179-f019-6001d83ce85f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1,\n",
              "       0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1,\n",
              "       0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1,\n",
              "       1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0,\n",
              "       0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1,\n",
              "       1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1,\n",
              "       0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0,\n",
              "       0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1,\n",
              "       1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,\n",
              "       1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1,\n",
              "       0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0,\n",
              "       0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
              "       0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n",
              "       0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0,\n",
              "       0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clmns = ['radius_mean', 'texture_mean', 'perimeter_mean',\n",
        "       'area_mean', 'smoothness_mean', 'compactness_mean', 'concavity_mean',\n",
        "       'concave points_mean', 'symmetry_mean', 'fractal_dimension_mean',\n",
        "       'radius_se', 'texture_se', 'perimeter_se', 'area_se', 'smoothness_se',\n",
        "       'compactness_se', 'concavity_se', 'concave points_se', 'symmetry_se',\n",
        "       'fractal_dimension_se', 'radius_worst', 'texture_worst',\n",
        "       'perimeter_worst', 'area_worst', 'smoothness_worst',\n",
        "       'compactness_worst', 'concavity_worst', 'concave points_worst',\n",
        "       'symmetry_worst', 'fractal_dimension_worst']"
      ],
      "metadata": {
        "id": "ugeMxSqn8Dxd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def accuracy(m) :\n",
        "  return (m[0][0]+m[1][1])/(m[0][0]+m[0][1]+m[1][0]+m[1][1])\n",
        "\n",
        "def sensibility(m) :\n",
        "  return m[1][1]/(m[1][1]+m[1][0])\n",
        "\n",
        "def specificity(m) :\n",
        "  return m[0][0]/(m[0][0]+m[0][1])\n",
        "\n",
        "def mcc(m) :\n",
        "  a = m[0][0]*m[1][1] - m[0][1]*m[1][0]\n",
        "  b = np.sqrt((m[1][1]+m[0][1])*(m[1][1]+m[1][0])*(m[0][0]+m[0][1])*(m[0][0]+m[1][0]))\n",
        "  return a/b\n",
        "\n",
        "def auc(model, testX, testY) :\n",
        "  y_pred_proba = model.predict_proba(testX)[::,1]\n",
        "  return metrics.roc_auc_score(testY, y_pred_proba)\n",
        "\n",
        "def most_k_imp(scrs_, k) :\n",
        "  scrs = scrs_.copy()\n",
        "  keys = scrs.keys()\n",
        "  max_scr_key = 0\n",
        "  for ke in keys :\n",
        "    max_scr_key = ke\n",
        "    break\n",
        "  top_scrs = []\n",
        "  for i in range(k) :\n",
        "    for ke in keys:\n",
        "      if scrs[ke] > scrs[max_scr_key] :\n",
        "        max_scr_key = ke\n",
        "    top_scrs.append(max_scr_key)\n",
        "    scrs[max_scr_key] = 0\n",
        "  return top_scrs"
      ],
      "metadata": {
        "id": "oadv3cAhyl2V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def cross_val(X, Y, algorithm, features=-1, cross_validation='cv', k=5, balancing_method='under_sampling') :\n",
        "  if (features != -1) :\n",
        "    X_ = pd.DataFrame({feature : df[feature] for feature in features})\n",
        "    X_ = X_.to_numpy()\n",
        "    Y_ = df[\"diagnosis\"]\n",
        "    Y_ = Y_.to_numpy()\n",
        "  else :\n",
        "    X_ = X\n",
        "    Y_ = Y\n",
        "  if cross_validation == 'cv' :\n",
        "    splt = KFold(n_splits=k, random_state=1, shuffle=True)\n",
        "    splits = splt.split(X_)\n",
        "  elif cross_validation == 'skf' :\n",
        "    splt = StratifiedKFold(n_splits=k)\n",
        "    splits = splt.split(X_, Y_)\n",
        "  model_predictions = []\n",
        "  actual_labels = []\n",
        "  scores = []\n",
        "  if algorithm == 'Gradient_Boosting' :\n",
        "    model = GradientBoostingClassifier(random_state=42)\n",
        "  elif algorithm == 'XGradient_Boosting' :\n",
        "    model = XGBClassifier()\n",
        "  elif algorithm == 'Logistic_Regression' :\n",
        "    model = LogisticRegression(random_state=0,  max_iter=100)\n",
        "  elif algorithm == 'Random_Forest' :\n",
        "    model = RandomForestClassifier(max_depth=5, random_state=0)\n",
        "  elif algorithm == 'Gaussian_nb' :\n",
        "    model = GaussianNB()\n",
        "  elif algorithm == 'K_Nearest_Neighbors' :\n",
        "    model = KNeighborsClassifier(n_neighbors=7)\n",
        "  tprs = []\n",
        "  aucs = []\n",
        "  mean_fpr = np.linspace(0, 1, 100)\n",
        "  fig, ax = plt.subplots()\n",
        "  n = 0\n",
        "  for train_index, test_index in splits :\n",
        "    X_train, Y_train = X_[train_index], Y_[train_index]\n",
        "    X_test, Y_test = X_[test_index], Y_[test_index]\n",
        "    if (balancing_method != None and cross_validation == 'cv') :\n",
        "      if balancing_method == \"over_sampling\" :\n",
        "        balancing = RandomOverSampler(random_state=0)\n",
        "      elif balancing_method == \"smote\" :\n",
        "        balancing = SMOTE()\n",
        "      elif balancing_method == \"under_sampling\" :\n",
        "        balancing = RandomUnderSampler(random_state=0)\n",
        "      X_train, Y_train = balancing.fit_resample(X_train, Y_train)\n",
        "    model = model.fit(X_train, Y_train)\n",
        "    #ROC Plot\n",
        "    viz = RocCurveDisplay.from_estimator(model, X_test, Y_test, name=\"ROC fold {}\".format(n), alpha=0.3, lw=1, ax=ax)\n",
        "    interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)\n",
        "    interp_tpr[0] = 0.0\n",
        "    tprs.append(interp_tpr)\n",
        "    aucs.append(viz.roc_auc)\n",
        "    model_predictions.append(model.predict(X_test))\n",
        "    actual_labels.append(Y_test)\n",
        "    scores.append(model.score(X_test, Y_test))\n",
        "    n += 1\n",
        "  performance_metrics = [{} for i in range(k+1)]\n",
        "  for i in range(k) :\n",
        "    performance_metrics[i]['confusion_matrix'] = confusion_matrix(model_predictions[i], actual_labels[i])\n",
        "    conf_mat = performance_metrics[i]['confusion_matrix']\n",
        "    performance_metrics[i]['accuracy'] = accuracy(conf_mat)\n",
        "    performance_metrics[i]['sensibility'] = sensibility(conf_mat)\n",
        "    performance_metrics[i]['specificity'] = specificity(conf_mat)\n",
        "    performance_metrics[i]['mcc'] = mcc(conf_mat)\n",
        "  performance_metrics[k]['accuracy_average'] = np.mean([performance_metrics[i]['accuracy'] for i in range(k)])\n",
        "  #performance_metrics[k]['accuracy_deviation'] = np.std([performance_metrics[i]['accuracy'] for i in range(k)])\n",
        "  performance_metrics[k]['sensibility_average'] = np.mean([performance_metrics[i]['sensibility'] for i in range(k)])\n",
        "  #performance_metrics[k]['sensibility_deviation'] = np.std([performance_metrics[i]['sensibility'] for i in range(k)])\n",
        "  performance_metrics[k]['specificity_average'] = np.mean([performance_metrics[i]['specificity'] for i in range(k)])\n",
        "  #performance_metrics[k]['specificity_deviation'] = np.std([performance_metrics[i]['specificity'] for i in range(k)])\n",
        "  performance_metrics[k]['mcc_average'] = np.mean([performance_metrics[i]['mcc'] for i in range(k)])\n",
        "  #performance_metrics[k]['mcc_deviation'] = np.std([performance_metrics[i]['mcc'] for i in range(k)])\n",
        "  return performance_metrics"
      ],
      "metadata": {
        "id": "NjPKXo8GdqKZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def cross_val(X, Y, algorithm, features=-1, cross_validation='cv', k=5, balancing_method='under_sampling') :\n",
        "  if (features != -1) :\n",
        "    X_ = pd.DataFrame({feature : df[feature] for feature in features})\n",
        "    X_ = X_.to_numpy()\n",
        "    Y_ = df[\"diagnosis\"]\n",
        "    Y_ = Y_.to_numpy()\n",
        "  else :\n",
        "    X_ = X\n",
        "    Y_ = Y\n",
        "  if cross_validation == 'cv' :\n",
        "    splt = KFold(n_splits=k, random_state=1, shuffle=True)\n",
        "    splits = splt.split(X_)\n",
        "  elif cross_validation == 'skf' :\n",
        "    splt = StratifiedKFold(n_splits=k)\n",
        "    splits = splt.split(X_, Y_)\n",
        "  model_predictions = []\n",
        "  actual_labels = []\n",
        "  scores = []\n",
        "  if algorithm == 'Gradient_Boosting' :\n",
        "    model = GradientBoostingClassifier(random_state=42)\n",
        "  elif algorithm == 'XGradient_Boosting' :\n",
        "    model = XGBClassifier()\n",
        "  elif algorithm == 'Logistic_Regression' :\n",
        "    model = LogisticRegression(random_state=0,  max_iter=100)\n",
        "  elif algorithm == 'Random_Forest' :\n",
        "    model = RandomForestClassifier(max_depth=5, random_state=0)\n",
        "  elif algorithm == 'Gaussian_nb' :\n",
        "    model = GaussianNB()\n",
        "  elif algorithm == 'K_Nearest_Neighbors' :\n",
        "    model = KNeighborsClassifier(n_neighbors=7)\n",
        "  n = 0\n",
        "  performance_metrics = [{} for i in range(k+1)]\n",
        "  for train_index, test_index in splits :\n",
        "    X_train, Y_train = X_[train_index], Y_[train_index]\n",
        "    X_test, Y_test = X_[test_index], Y_[test_index]\n",
        "    if (balancing_method != None and cross_validation == 'cv') :\n",
        "      if balancing_method == \"over_sampling\" :\n",
        "        balancing = RandomOverSampler(random_state=0)\n",
        "      elif balancing_method == \"smote\" :\n",
        "        balancing = SMOTE()\n",
        "      elif balancing_method == \"under_sampling\" :\n",
        "        balancing = RandomUnderSampler(random_state=0)\n",
        "      X_train, Y_train = balancing.fit_resample(X_train, Y_train)\n",
        "    model = model.fit(X_train, Y_train)\n",
        "    model_predictions.append(model.predict(X_test))\n",
        "    actual_labels.append(Y_test)\n",
        "    scores.append(model.score(X_test, Y_test))\n",
        "    performance_metrics[n]['auc'] = auc(model, X_test, Y_test)\n",
        "    n += 1\n",
        "  for i in range(k) :\n",
        "    performance_metrics[i]['confusion_matrix'] = confusion_matrix(model_predictions[i], actual_labels[i])\n",
        "    conf_mat = performance_metrics[i]['confusion_matrix']\n",
        "    performance_metrics[i]['accuracy'] = accuracy(conf_mat)\n",
        "    performance_metrics[i]['sensibility'] = sensibility(conf_mat)\n",
        "    performance_metrics[i]['specificity'] = specificity(conf_mat)\n",
        "    performance_metrics[i]['mcc'] = mcc(conf_mat)\n",
        "  performance_metrics[k]['accuracy_average'] = np.mean([performance_metrics[i]['accuracy'] for i in range(k)])\n",
        "  performance_metrics[k]['sensibility_average'] = np.mean([performance_metrics[i]['sensibility'] for i in range(k)])\n",
        "  performance_metrics[k]['specificity_average'] = np.mean([performance_metrics[i]['specificity'] for i in range(k)])\n",
        "  performance_metrics[k]['mcc_average'] = np.mean([performance_metrics[i]['mcc'] for i in range(k)])\n",
        "  performance_metrics[k]['auc_average'] = np.mean([performance_metrics[i]['auc'] for i in range(k)])\n",
        "  return performance_metrics"
      ],
      "metadata": {
        "id": "9UCfriPzn4Q0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from mlxtend.plotting import plot_confusion_matrix\n",
        "def plt_conf_mat(mat) :\n",
        "  fig, ax = plot_confusion_matrix(conf_mat=mat)\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "n6e4gouFLrRp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = df.copy()\n",
        "X.drop(['id','diagnosis', 'Unnamed: 32'],axis=1,inplace=True)\n",
        "X = X.to_numpy()\n",
        "Y = df[\"diagnosis\"]\n",
        "Y = Y.to_numpy()"
      ],
      "metadata": {
        "id": "pDTxoq2B4uaF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def top_k(scrs_, k) :\n",
        "  fs = [False for i in range(30)]\n",
        "  n = len(scrs_)\n",
        "  scrs = scrs_.copy()\n",
        "  for i in range(0, k) :\n",
        "    max_index = 0\n",
        "    for j in range(n) :\n",
        "      if scrs[j] > scrs[max_index] :\n",
        "        max_index = j\n",
        "    fs[max_index] = True\n",
        "    scrs[max_index] = -2\n",
        "  return fs"
      ],
      "metadata": {
        "id": "DtZrXX0u9QuI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def transform(X, fs) :\n",
        "  X_ = df.copy()\n",
        "  X_.drop(['id','diagnosis', 'Unnamed: 32'], axis=1, inplace=True)\n",
        "  for i in range(len(fs)) :\n",
        "    if fs[i] == False :\n",
        "      X_.drop([clmns[i]], axis=1, inplace=True)\n",
        "  X_ = X_.to_numpy()\n",
        "  return X_\n",
        "\n"
      ],
      "metadata": {
        "id": "rdt3gkbp7eEc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def prod(x) :\n",
        "  p = 1\n",
        "  for i in x:\n",
        "    p = p*i\n",
        "  return p\n",
        "def signa(x, n) :\n",
        "  a, b, c = sum(x[:n]), prod(x[:n]), sum([x[i]*x[i] for i in range(n)])\n",
        "  return (a*b*c-(a+b+c))/(a*b+a*c+b*c)\n",
        "def vars(bo) :\n",
        "  r = []\n",
        "  for i in range(30) :\n",
        "    if bo[i] == True :\n",
        "      r.append(i)\n",
        "  return r"
      ],
      "metadata": {
        "id": "0nnOniPe_Tf9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "algo = 'XGradient_Boosting'"
      ],
      "metadata": {
        "id": "qFSoX1TW0Qb7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_lists = []\n",
        "for n in range(1, len(df)-1) :\n",
        "  a = reliefF_measure(X, Y_, k_neighbors=n)\n",
        "  print(n, ' done')\n",
        "  scores_lists.append(a)"
      ],
      "metadata": {
        "id": "KXj9o_w5DYlt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 432
        },
        "outputId": "ea23768e-5b00-4309-d1e8-15ded128e7d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1  done\n",
            "2  done\n",
            "3  done\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-7e0b3291afb2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mscores_lists\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreliefF_measure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk_neighbors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' done'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0mscores_lists\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ITMO_FS/filters/univariate/measures.py\u001b[0m in \u001b[0;36mreliefF_measure\u001b[0;34m(X, y, k_neighbors)\u001b[0m\n\u001b[1;32m    375\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 377\u001b[0;31m                 \u001b[0mmisses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m__take_k\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdm_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk_neighbors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    378\u001b[0m                 \u001b[0mind_misses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmisses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m                 \u001b[0mm_c\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mind_misses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ITMO_FS/filters/univariate/measures.py\u001b[0m in \u001b[0;36m__take_k\u001b[0;34m(dm_i, k, r_index, choice_func)\u001b[0m\n\u001b[1;32m    305\u001b[0m     \u001b[0mdm_i\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdm_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0msamp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdm_i\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mr_index\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mchoice_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m             \u001b[0mhits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m             \u001b[0mk\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scores_df = pd.DataFrame()\n",
        "for k in range(1, len(df)-1) :\n",
        "  scores_df['neighbors_' + str(k)] = scores_lists[k - 1] "
      ],
      "metadata": {
        "id": "SmTTcS8_Mt66"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list = pd.read_csv('scores')"
      ],
      "metadata": {
        "id": "J0sFgjo7OmGI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list = scores_list.transpose()"
      ],
      "metadata": {
        "id": "Fcn4-GZtQtHE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bGTygCTLoLdH",
        "outputId": "752baa9b-143d-4af7-a999-ce4e0f084c23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RangeIndex(start=0, stop=30, step=1)"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list = scores_list.to_numpy()"
      ],
      "metadata": {
        "id": "I1BkToxyN1rq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list = [scores_list[i] for i in range(1, len(scores_list))]"
      ],
      "metadata": {
        "id": "wbmDhpBIpAtp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores_list[len(scores_list)-1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K_UuEYPi0cIL",
        "outputId": "e66a246c-74c4-4138-9dde-f4f9d955dadf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.03566496, -0.00918876,  0.03874905,  0.03502235, -0.0136165 ,\n",
              "        0.01506641,  0.04249711,  0.05734537, -0.01428939, -0.02160777,\n",
              "        0.01361124, -0.02274733,  0.01242359,  0.01632284, -0.01795566,\n",
              "       -0.01040459, -0.00507149, -0.00585945, -0.0149966 , -0.01273578,\n",
              "        0.04892028, -0.00786818,  0.04921943,  0.04026428, -0.01033924,\n",
              "        0.01505931,  0.02664337,  0.06111994, -0.00272811, -0.00670007])"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(scores_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZoDyj-J8nXFZ",
        "outputId": "42ebc8f5-ed6b-471c-e149-bafc2c23d346"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "567"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "signs = []\n",
        "min_acc = 0.97\n",
        "max_acc = 0\n",
        "models = []\n",
        "best_model = {'algo' : algo, 'k_vars' : 0, 'k_neighbors' : 0, 'acc' : 0, 'sen' : 0, 'spe' : 0, 'mcc' : 0, 'auc' : 0}\n",
        "k_neighbors = 1\n",
        "vars_ = []\n",
        "for scores in scores_list :\n",
        "  for k in range(1, 30) :\n",
        "    fs = top_k(scores, k)\n",
        "    sign = signa(vars(fs), k)\n",
        "    if sign in signs :\n",
        "      continue\n",
        "    signs.append(sign)\n",
        "    X_ = transform(X, fs)\n",
        "    perf = cross_val(X_, Y, algo, cross_validation='skf')\n",
        "    if perf[5]['accuracy_average'] >= min_acc :\n",
        "      mod = {'algo' : algo,\n",
        "             'k_vars' : k,\n",
        "             'k_neighbors' : k_neighbors,\n",
        "             'acc' : perf[5]['accuracy_average'],\n",
        "             'sen' : perf[5]['sensibility_average'],\n",
        "             'spe' : perf[5]['specificity_average'],\n",
        "             'auc' : perf[5]['auc_average'],\n",
        "             'mcc' : perf[5]['mcc_average']}\n",
        "      models.append(mod)\n",
        "      vars_.append(fs)\n",
        "      print(mod)\n",
        "    print(k_neighbors, k)\n",
        "  k_neighbors += 1"
      ],
      "metadata": {
        "id": "U1kuVduM1CBp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f5f0babd-a135-4ffd-b6bc-fb31e3b5b9d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1 1\n",
            "1 2\n",
            "1 3\n",
            "1 4\n",
            "1 5\n",
            "1 6\n",
            "1 7\n",
            "1 8\n",
            "1 9\n",
            "1 10\n",
            "1 11\n",
            "1 12\n",
            "1 13\n",
            "1 14\n",
            "1 15\n",
            "1 16\n",
            "1 17\n",
            "1 18\n",
            "1 19\n",
            "1 20\n",
            "1 21\n",
            "1 22\n",
            "1 23\n",
            "1 24\n",
            "1 25\n",
            "1 26\n",
            "1 27\n",
            "1 28\n",
            "1 29\n",
            "2 2\n",
            "2 5\n",
            "2 6\n",
            "2 7\n",
            "2 11\n",
            "2 12\n",
            "2 13\n",
            "2 14\n",
            "2 15\n",
            "2 17\n",
            "2 18\n",
            "2 19\n",
            "2 20\n",
            "2 21\n",
            "2 22\n",
            "2 24\n",
            "2 26\n",
            "3 5\n",
            "3 6\n",
            "3 7\n",
            "3 8\n",
            "3 9\n",
            "3 10\n",
            "3 11\n",
            "3 13\n",
            "3 14\n",
            "3 15\n",
            "3 16\n",
            "3 17\n",
            "3 18\n",
            "3 19\n",
            "3 20\n",
            "3 21\n",
            "3 24\n",
            "3 25\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 26, 'k_neighbors': 3, 'acc': 0.9701288619779536, 'sen': 0.9710514669675192, 'spe': 0.9700728419221569, 'auc': 0.9924962132831443, 'mcc': 0.936387325405404}\n",
            "3 26\n",
            "3 27\n",
            "3 28\n",
            "4 11\n",
            "4 14\n",
            "4 15\n",
            "4 19\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 23, 'k_neighbors': 4, 'acc': 0.971883247942866, 'sen': 0.9758075645284947, 'spe': 0.9702209352209351, 'auc': 0.9906559728742828, 'mcc': 0.9402349174496205}\n",
            "4 23\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 25, 'k_neighbors': 4, 'acc': 0.9701288619779536, 'sen': 0.9710514669675192, 'spe': 0.9700728419221569, 'auc': 0.9924319387470387, 'mcc': 0.936387325405404}\n",
            "4 25\n",
            "4 27\n",
            "4 28\n",
            "5 3\n",
            "5 4\n",
            "5 6\n",
            "5 11\n",
            "5 12\n",
            "5 13\n",
            "5 14\n",
            "5 15\n",
            "5 17\n",
            "5 28\n",
            "6 12\n",
            "6 13\n",
            "6 14\n",
            "6 18\n",
            "6 19\n",
            "6 20\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 24, 'k_neighbors': 6, 'acc': 0.9701288619779536, 'sen': 0.9710514669675192, 'spe': 0.9700728419221569, 'auc': 0.9919006502408074, 'mcc': 0.936387325405404}\n",
            "6 24\n",
            "6 26\n",
            "7 3\n",
            "7 4\n",
            "7 5\n",
            "7 6\n",
            "7 7\n",
            "7 14\n",
            "7 15\n",
            "7 16\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 24, 'k_neighbors': 7, 'acc': 0.9701288619779538, 'sen': 0.9756914205099116, 'spe': 0.9675192609439185, 'auc': 0.9912446253749889, 'mcc': 0.9364425518872805}\n",
            "7 24\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 25, 'k_neighbors': 7, 'acc': 0.9701288619779538, 'sen': 0.9711676109861023, 'spe': 0.9699967384213959, 'auc': 0.991966787806945, 'mcc': 0.9363561483538259}\n",
            "7 25\n",
            "7 26\n",
            "8 5\n",
            "8 6\n",
            "8 7\n",
            "8 15\n",
            "8 16\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 17, 'k_neighbors': 8, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.9907227170083377, 'mcc': 0.9363039016989904}\n",
            "8 17\n",
            "8 18\n",
            "8 19\n",
            "8 25\n",
            "9 8\n",
            "9 16\n",
            "9 17\n",
            "9 19\n",
            "9 20\n",
            "10 5\n",
            "10 11\n",
            "10 12\n",
            "10 15\n",
            "10 16\n",
            "10 18\n",
            "10 20\n",
            "10 21\n",
            "10 27\n",
            "11 7\n",
            "11 24\n",
            "12 10\n",
            "13 11\n",
            "13 17\n",
            "14 14\n",
            "14 19\n",
            "14 23\n",
            "14 24\n",
            "15 6\n",
            "15 13\n",
            "15 14\n",
            "15 15\n",
            "16 14\n",
            "16 26\n",
            "17 9\n",
            "17 10\n",
            "17 29\n",
            "18 12\n",
            "18 23\n",
            "19 13\n",
            "19 14\n",
            "19 22\n",
            "20 13\n",
            "20 23\n",
            "21 19\n",
            "23 4\n",
            "23 5\n",
            "23 8\n",
            "23 16\n",
            "23 20\n",
            "23 22\n",
            "24 9\n",
            "25 5\n",
            "26 7\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 24, 'k_neighbors': 26, 'acc': 0.9718987734823784, 'sen': 0.9713910885862106, 'spe': 0.972712781274425, 'auc': 0.9932932868529004, 'mcc': 0.940213438853742}\n",
            "26 24\n",
            "27 8\n",
            "27 11\n",
            "27 12\n",
            "28 6\n",
            "28 13\n",
            "28 23\n",
            "29 7\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 25, 'k_neighbors': 29, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.9934227457769875, 'mcc': 0.9363039016989904}\n",
            "29 25\n",
            "32 5\n",
            "32 6\n",
            "32 28\n",
            "33 2\n",
            "33 10\n",
            "33 15\n",
            "34 11\n",
            "34 24\n",
            "35 12\n",
            "36 21\n",
            "36 22\n",
            "36 25\n",
            "37 9\n",
            "37 21\n",
            "38 14\n",
            "39 8\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 20, 'k_neighbors': 39, 'acc': 0.9718987734823784, 'sen': 0.9803600464576074, 'spe': 0.9673422988877359, 'auc': 0.9923689206730477, 'mcc': 0.9400599704987259}\n",
            "39 20\n",
            "39 21\n",
            "39 23\n",
            "40 5\n",
            "40 10\n",
            "42 9\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 26, 'k_neighbors': 42, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.993094570870529, 'mcc': 0.9363039016989904}\n",
            "42 26\n",
            "43 11\n",
            "43 20\n",
            "44 3\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 21, 'k_neighbors': 45, 'acc': 0.9701443875174662, 'sen': 0.966861471861472, 'spe': 0.972560574272903, 'auc': 0.9940167057469711, 'mcc': 0.9364708394378933}\n",
            "45 21\n",
            "45 22\n",
            "46 12\n",
            "46 25\n",
            "46 27\n",
            "48 11\n",
            "49 15\n",
            "49 18\n",
            "52 10\n",
            "53 13\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 28, 'k_neighbors': 53, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.9935543926782053, 'mcc': 0.9363039016989904}\n",
            "53 28\n",
            "54 4\n",
            "54 5\n",
            "56 11\n",
            "57 26\n",
            "58 10\n",
            "59 2\n",
            "59 25\n",
            "59 27\n",
            "60 12\n",
            "61 6\n",
            "61 22\n",
            "64 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 19, 'k_neighbors': 64, 'acc': 0.9718987734823784, 'sen': 0.9757200929152148, 'spe': 0.9698958798659744, 'auc': 0.9922448125445177, 'mcc': 0.9400047440168494}\n",
            "64 19\n",
            "64 20\n",
            "65 13\n",
            "66 23\n",
            "70 3\n",
            "70 18\n",
            "71 4\n",
            "74 11\n",
            "74 19\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 18, 'k_neighbors': 75, 'acc': 0.9701288619779538, 'sen': 0.9671544715447155, 'spe': 0.9726027397260275, 'auc': 0.9942110457750811, 'mcc': 0.9365783629732182}\n",
            "75 18\n",
            "76 7\n",
            "77 20\n",
            "79 8\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 27, 'k_neighbors': 79, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.9937493609373729, 'mcc': 0.9363039016989904}\n",
            "79 27\n",
            "79 28\n",
            "82 3\n",
            "84 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 21, 'k_neighbors': 85, 'acc': 0.971914299021891, 'sen': 0.975714690867839, 'spe': 0.9700111069974083, 'auc': 0.9942144685511867, 'mcc': 0.9400550096848754}\n",
            "85 21\n",
            "87 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 19, 'k_neighbors': 90, 'acc': 0.9736376339077782, 'sen': 0.9720267131242741, 'spe': 0.9754525073018223, 'auc': 0.9942116740061385, 'mcc': 0.9442183205797747}\n",
            "90 19\n",
            "92 5\n",
            "92 18\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 20, 'k_neighbors': 94, 'acc': 0.9701133364384411, 'sen': 0.9669675852602682, 'spe': 0.9724058632784065, 'auc': 0.9942085545139918, 'mcc': 0.9363972915557008}\n",
            "94 20\n",
            "96 24\n",
            "97 25\n",
            "98 9\n",
            "99 6\n",
            "100 19\n",
            "101 2\n",
            "101 5\n",
            "101 23\n",
            "102 3\n",
            "102 14\n",
            "105 7\n",
            "105 10\n",
            "107 24\n",
            "109 25\n",
            "110 24\n",
            "113 6\n",
            "114 20\n",
            "117 15\n",
            "123 22\n",
            "127 14\n",
            "128 8\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 16, 'k_neighbors': 128, 'acc': 0.971883247942866, 'sen': 0.971904761904762, 'spe': 0.9728608745732032, 'auc': 0.9943442524223723, 'mcc': 0.9405233645585203}\n",
            "128 16\n",
            "129 23\n",
            "131 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 15, 'k_neighbors': 132, 'acc': 0.9701288619779536, 'sen': 0.9717765567765568, 'spe': 0.9703383520506808, 'auc': 0.9942781148562346, 'mcc': 0.9368533682518363}\n",
            "132 15\n",
            "133 14\n",
            "133 15\n",
            "133 16\n",
            "133 24\n",
            "134 15\n",
            "135 16\n",
            "136 18\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 21, 'k_neighbors': 136, 'acc': 0.9701443875174662, 'sen': 0.966861471861472, 'spe': 0.972560574272903, 'auc': 0.9946739870749042, 'mcc': 0.9364708394378933}\n",
            "136 21\n",
            "138 17\n",
            "139 9\n",
            "140 16\n",
            "140 18\n",
            "141 23\n",
            "142 14\n",
            "144 13\n",
            "145 14\n",
            "145 17\n",
            "145 18\n",
            "145 25\n",
            "157 13\n",
            "157 15\n",
            "158 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 20, 'k_neighbors': 158, 'acc': 0.9701288619779536, 'sen': 0.9629512735326688, 'spe': 0.9752209811586621, 'auc': 0.9947376117168124, 'mcc': 0.9367570495689744}\n",
            "158 20\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 22, 'k_neighbors': 162, 'acc': 0.9701443875174661, 'sen': 0.9670654926468881, 'spe': 0.9726984586361395, 'auc': 0.9940762793817166, 'mcc': 0.9366247196017289}\n",
            "162 22\n",
            "163 23\n",
            "166 22\n",
            "167 16\n",
            "179 19\n",
            "182 11\n",
            "187 20\n",
            "190 26\n",
            "208 18\n",
            "210 27\n",
            "210 28\n",
            "219 17\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 18, 'k_neighbors': 223, 'acc': 0.971883247942866, 'sen': 0.9672994060203364, 'spe': 0.9751002568125855, 'auc': 0.9948689553340717, 'mcc': 0.940235773418556}\n",
            "223 18\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 18, 'k_neighbors': 226, 'acc': 0.9701443875174661, 'sen': 0.962971175166297, 'spe': 0.9749902152641878, 'auc': 0.9945432716887025, 'mcc': 0.9366126422846701}\n",
            "226 18\n",
            "226 19\n",
            "233 19\n",
            "256 16\n",
            "273 21\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 19, 'k_neighbors': 275, 'acc': 0.9701443875174661, 'sen': 0.962971175166297, 'spe': 0.9749902152641878, 'auc': 0.9945435749726611, 'mcc': 0.9366126422846701}\n",
            "275 19\n",
            "276 20\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 19, 'k_neighbors': 277, 'acc': 0.9701599130569788, 'sen': 0.9672889702469962, 'spe': 0.9723985825355689, 'auc': 0.9948701901330465, 'mcc': 0.9365516226568891}\n",
            "277 19\n",
            "282 17\n",
            "287 21\n",
            "288 7\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 25, 'k_neighbors': 297, 'acc': 0.9701288619779536, 'sen': 0.9711796720957244, 'spe': 0.9698556384172822, 'auc': 0.9942749953640881, 'mcc': 0.9363039016989904}\n",
            "297 25\n",
            "306 18\n",
            "{'algo': 'XGradient_Boosting', 'k_vars': 24, 'k_neighbors': 310, 'acc': 0.9701443875174661, 'sen': 0.9670654926468881, 'spe': 0.9726984586361395, 'auc': 0.9938791231454186, 'mcc': 0.9366247196017289}\n",
            "310 24\n",
            "316 19\n",
            "325 17\n",
            "329 12\n",
            "334 6\n",
            "334 11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "av_acc = np.mean([models[i]['acc'] for i in range(30)])\n",
        "std_acc = np.std([models[i]['acc'] for i in range(30)])\n",
        "av_mcc = np.mean([models[i]['mcc'] for i in range(30)])\n",
        "std_mcc = np.std([models[i]['mcc'] for i in range(30)])\n",
        "av_auc = np.mean([models[i]['auc'] for i in range(30)])\n",
        "std_auc = np.std([models[i]['auc'] for i in range(30)])"
      ],
      "metadata": {
        "id": "LcboO9QuW6h_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "x = np.arange(30)\n",
        "width = 0.40\n",
        "  \n",
        "# plot data in grouped manner of bar type\n",
        "\n",
        "plt.bar(x-0.2, [models[i]['mcc']for i in range(30)], width)\n"
      ],
      "metadata": {
        "id": "PhrSUyiS5gSt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "outputId": "aadbe11f-6e72-4b3b-c238-48e61becb472"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<BarContainer object of 30 artists>"
            ]
          },
          "metadata": {},
          "execution_count": 120
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANCUlEQVR4nO3cf6zdd13H8eeLlokZk0F6JWQtdGpRG0IYuZkaiC4CpsOk07gsazIDyWT+Yc3MiHH+yJwzJvxQ9J+JlkBEIpQKiE2omURn/BE3d8d+QNsMyxyudW7dGOpiZFbe/nG+lbPrvb3ftqe7PW+ej6Tp+X7Pp9/z+eS7Pnv2Pfd8U1VIknp4wXpPQJI0O0Zdkhox6pLUiFGXpEaMuiQ1snG9XnjTpk21devW9Xp5SZpL995775NVtbDa8+sW9a1bt7K0tLReLy9JcynJl0/1vJdfJKkRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqZF1+0ap9M1q682fec72I+/60XWaiTqay6ifzl+KsWNnPe75eO1zcczz8bXPxTHX87VPR6d1+9/vuLFny8svktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqRGjLkmNGHVJasSoS1IjRl2SGjHqktSIUZekRkZFPcmOJA8lOZLk5hWef2WSO5Pcl+TBJG+d/VQlSWtZM+pJNgC3A1cC24FdSbYvG/YrwL6qugy4FvjdWU9UkrS2Me/ULweOVNXDVfUssBe4atmYAr5tePwS4F9mN0VJ0lhjon4J8OjU9tFh37RbgeuSHAUOAD+70oGS3JBkKcnS8ePHz2C6kqRTmdUHpbuAP6iqzcBbgY8k+X/Hrqo9VbVYVYsLCwszemlJ0kljon4M2DK1vXnYN+16YB9AVf098CJg0ywmKEkab0zU7wG2Jbk0yQVMPgjdv2zMPwNvAkjyvUyi7vUVSXqerRn1qjoB7AbuAA4z+SmXg0luS7JzGPZO4B1JHgA+Bry9qupcTVqStLKNYwZV1QEmH4BO77tl6vEh4A2znZok6XT5jVJJasSoS1IjRl2SGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqRGjLkmNGHVJasSoS1IjRl2SGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaGRX1JDuSPJTkSJKbVxlzTZJDSQ4m+ehspylJGmPjWgOSbABuB94CHAXuSbK/qg5NjdkG/CLwhqp6Osm3n6sJS5JWN+ad+uXAkap6uKqeBfYCVy0b8w7g9qp6GqCqnpjtNCVJY4yJ+iXAo1PbR4d9014NvDrJ3yW5K8mOlQ6U5IYkS0mWjh8/fmYzliStalYflG4EtgFXALuADyS5ePmgqtpTVYtVtbiwsDCjl5YknTQm6seALVPbm4d9044C+6vqv6vqn4AvMom8JOl5NCbq9wDbklya5ALgWmD/sjGfZvIunSSbmFyOeXiG85QkjbBm1KvqBLAbuAM4DOyrqoNJbkuycxh2B/BUkkPAncDPV9VT52rSkqSVrfkjjQBVdQA4sGzfLVOPC7hp+CVJWid+o1SSGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqRGjLkmNGHVJasSoS1IjRl2SGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGRkU9yY4kDyU5kuTmU4z7iSSVZHF2U5QkjbVm1JNsAG4HrgS2A7uSbF9h3EXAjcDds56kJGmcMe/ULweOVNXDVfUssBe4aoVxvw68G/ivGc5PknQaxkT9EuDRqe2jw77/k+T1wJaq+sypDpTkhiRLSZaOHz9+2pOVJJ3aWX9QmuQFwPuAd641tqr2VNViVS0uLCyc7UtLkpYZE/VjwJap7c3DvpMuAl4D/FWSR4DvB/b7YakkPf/GRP0eYFuSS5NcAFwL7D/5ZFX9W1VtqqqtVbUVuAvYWVVL52TGkqRVrRn1qjoB7AbuAA4D+6rqYJLbkuw81xOUJI23ccygqjoAHFi275ZVxl5x9tOSJJ0Jv1EqSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqRGjLkmNGHVJasSoS1IjRl2SGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI6OinmRHkoeSHEly8wrP35TkUJIHk/xFklfNfqqSpLWsGfUkG4DbgSuB7cCuJNuXDbsPWKyq1wKfAN4z64lKktY25p365cCRqnq4qp4F9gJXTQ+oqjur6j+HzbuAzbOdpiRpjDFRvwR4dGr76LBvNdcDf7bSE0luSLKUZOn48ePjZylJGmWmH5QmuQ5YBN670vNVtaeqFqtqcWFhYZYvLUkCNo4YcwzYMrW9edj3HEneDPwy8ENV9bXZTE+SdDrGvFO/B9iW5NIkFwDXAvunByS5DPh9YGdVPTH7aUqSxlgz6lV1AtgN3AEcBvZV1cEktyXZOQx7L/Bi4I+T3J9k/yqHkySdQ2Muv1BVB4ADy/bdMvX4zTOelyTpDPiNUklqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0Zdkhox6pLUiFGXpEaMuiQ1YtQlqRGjLkmNGHVJasSoS1IjRl2SGjHqktSIUZekRoy6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIaMeqS1IhRl6RGjLokNWLUJakRoy5JjRh1SWrEqEtSI0ZdkhoZFfUkO5I8lORIkptXeP5bknx8eP7uJFtnPVFJ0trWjHqSDcDtwJXAdmBXku3Lhl0PPF1V3wX8NvDuWU9UkrS2Me/ULweOVNXDVfUssBe4atmYq4APD48/AbwpSWY3TUnSGKmqUw9IrgZ2VNVPDds/CXxfVe2eGvOFYczRYftLw5gnlx3rBuCGYfO7gYdmtI5NwJNrjpofruf81mk9ndYC3xzreVVVLaz2Bzae2/k8V1XtAfbM+rhJlqpqcdbHXS+u5/zWaT2d1gKuB8ZdfjkGbJna3jzsW3FMko3AS4CnTmcikqSzNybq9wDbklya5ALgWmD/sjH7gbcNj68G/rLWuq4jSZq5NS+/VNWJJLuBO4ANwIeq6mCS24ClqtoPfBD4SJIjwFeYhP/5NPNLOuvM9ZzfOq2n01rA9az9QakkaX74jVJJasSoS1Ijcx/1tW5hMG+SPJLk80nuT7K03vM5XUk+lOSJ4bsLJ/e9LMlnk/zj8PtL13OOY62ylluTHBvOz/1J3rqeczwdSbYkuTPJoSQHk9w47J/X87PaeubuHCV5UZJ/SPLAsJZfG/ZfOtx65chwK5YL1jzWPF9TH25h8EXgLcBRJj+ps6uqDq3rxM5CkkeAxeVf3JoXSX4QeAb4w6p6zbDvPcBXqupdwz+8L62qX1jPeY6xylpuBZ6pqt9cz7mdiSSvAF5RVZ9LchFwL/BjwNuZz/Oz2nquYc7O0fAN/Aur6pkkLwT+FrgRuAn4VFXtTfJ7wANV9f5THWve36mPuYWBnkdV9ddMfgJq2vRtJD7M5C/eeW+Vtcytqnqsqj43PP4P4DBwCfN7flZbz9ypiWeGzRcOvwr4YSa3XoGR52beo34J8OjU9lHm9KROKeDPk9w73Fahg5dX1WPD438FXr6ek5mB3UkeHC7PzMWliuWGO6leBtxNg/OzbD0wh+coyYYk9wNPAJ8FvgR8tapODENG9W3eo97RG6vq9UzuivkzwyWANoYvpc3vNT94P/CdwOuAx4DfWt/pnL4kLwY+CfxcVf379HPzeH5WWM9cnqOq+p+qeh2Tb+1fDnzPmRxn3qM+5hYGc6Wqjg2/PwH8CZOTO+8eH65/nrwO+sQ6z+eMVdXjw1++rwMfYM7Oz3C99pPAH1XVp4bdc3t+VlrPvJ+jqvoqcCfwA8DFw61XYGTf5j3qY25hMDeSXDh84EOSC4EfAb5w6j81F6ZvI/E24E/XcS5n5WT8Bj/OHJ2f4cO4DwKHq+p9U0/N5flZbT3zeI6SLCS5eHj8rUx++OMwk7hfPQwbdW7m+qdfAIYfV/odvnELg99Y5ymdsSTfweTdOUxu4fDReVtPko8BVzC5ZejjwK8Cnwb2Aa8EvgxcU1Xn/QeQq6zlCib/W1/AI8BPT12PPq8leSPwN8Dnga8Pu3+JyXXoeTw/q61nF3N2jpK8lskHoRuYvNneV1W3DU3YC7wMuA+4rqq+dspjzXvUJUnfMO+XXyRJU4y6JDVi1CWpEaMuSY0YdUlqxKhLUiNGXZIa+V/M+90jTEo2cwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max([models[i]['mcc']for i in range(30)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zW_9usVAgDTd",
        "outputId": "b154ed90-9de7-497f-84b2-5bead937f62d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9442183205797747"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_models = []\n",
        "max_acc = 0\n",
        "index = 0\n",
        "i = 0\n",
        "for mdl in models :\n",
        "  if mdl['acc'] > max_acc :\n",
        "    max_acc = mdl['acc']\n",
        "    index = i\n",
        "  i += 1\n"
      ],
      "metadata": {
        "id": "M-ZFpBa6-sW1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vars_occ = [sum([x[i] for x in vars_]) for i in range(30)]\n",
        "print(vars_occ)\n"
      ],
      "metadata": {
        "id": "15BdVu0HW_uN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d8262a17-330a-451b-872d-d8600709959b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[29, 19, 30, 29, 20, 30, 30, 30, 13, 15, 25, 0, 19, 21, 2, 14, 6, 12, 6, 9, 30, 30, 30, 30, 26, 30, 30, 30, 28, 26]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print([vars_[index][i]*vars_occ[i] for i in range(30)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ePuFl7qleJQM",
        "outputId": "a48dc63d-ac64-409a-fdb3-ab5ea9f0f0dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[29, 0, 30, 29, 20, 30, 30, 30, 0, 0, 25, 0, 0, 21, 0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 26, 30, 30, 30, 28, 26]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(vars_[index])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jd2kuSKCTa2l",
        "outputId": "467f0852-9226-45a4-8645-8b8152ecd604"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[True, False, True, True, True, True, True, True, False, False, True, False, False, True, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "perfs = []\n",
        "for i in range(30) :\n",
        "  new_fs = vars_[index].copy()\n",
        "  if new_fs[i] == True :\n",
        "    new_fs[i] = False\n",
        "  else :\n",
        "    new_fs[i] = True\n",
        "  perf = cross_val(transform(X, new_fs), Y, algo, cross_validation='skf')\n",
        "  if perf[5]['accuracy_average'] >= models[index]['acc'] :\n",
        "    perfs.append({'feature' : i, 'state' : vars_[index][i], 'acc' : perf[5]['accuracy_average']})"
      ],
      "metadata": {
        "id": "l3122nyCeefM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "perfs"
      ],
      "metadata": {
        "id": "B7XCuxmKIZBA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_fs = vars_[index].copy()\n",
        "new_fs[22] = False\n",
        "perf = cross_val(transform(X, new_fs), Y, algo, cross_validation='skf')\n",
        "perf[5]\n"
      ],
      "metadata": {
        "id": "XYCwSzgUKYAc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "best_so_far = new_fs.copy()\n",
        "best_model = cross_val(transform(X, best_so_far), Y, algo, cross_validation='skf')\n",
        "best_model[5]['accuracy_average']"
      ],
      "metadata": {
        "id": "V8CNsmPFNWy4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "perfs = []\n",
        "for i in range(30) :\n",
        "  new_fs = best_so_far.copy()\n",
        "  if new_fs[i] == True :\n",
        "    new_fs[i] = False\n",
        "  else :\n",
        "    new_fs[i] = True\n",
        "  perf = cross_val(transform(X, new_fs), Y, algo, cross_validation='skf')\n",
        "  if perf[5]['accuracy_average'] >= best_model[5]['accuracy_average'] :\n",
        "    perfs.append({'feature' : i, 'state' : best_so_far[i], 'acc' : perf[5]['accuracy_average']})\n"
      ],
      "metadata": {
        "id": "RmfWwaojNGW9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "perfs"
      ],
      "metadata": {
        "id": "V-gyrXHCNyYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_fs = best_so_far.copy()\n",
        "new_fs[0] = True\n",
        "perf = cross_val(transform(X, new_fs), Y, algo, cross_validation='skf')\n",
        "perf[5]"
      ],
      "metadata": {
        "id": "jv6l4KQLRDbY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "explored_models = []\n",
        "accs = []\n",
        "def explor(fs, acc) :\n",
        "  if (sum(fs) < 10 or sum(fs) > 28) :\n",
        "    return\n",
        "  explored_models.append(fs)\n",
        "  perfs = []\n",
        "  for i in range(30) :\n",
        "    neighbor = fs.copy()\n",
        "    neighbor[i] = not fs[i]\n",
        "    if (neighbor not in explored_models) :\n",
        "      perf = cross_val(transform(X, neighbor), Y, algo, cross_validation='skf')\n",
        "      if perf[5]['accuracy_average'] >= acc :\n",
        "        accs.append(perf[5]['accuracy_average'])\n",
        "        explor(neighbor, perf[5]['accuracy_average'])\n",
        "        print(perf[5]['accuracy_average'])\n",
        "      "
      ],
      "metadata": {
        "id": "4FeBtRH3VGW2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "fss = [random.randint(0, 1) == 0 for i in range(30)]\n",
        "fss"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BEBotx47UeMD",
        "outputId": "b9675f01-b368-4a03-8db3-972bbfdea14a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[False,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " False,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " False,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " False,\n",
              " False,\n",
              " True,\n",
              " True]"
            ]
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "explor(fss, 0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "id": "81CbAgPoZl22",
        "outputId": "88250f51-2a6c-454b-b09b-32d3769f9817"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9806707033069397\n",
            "0.9806707033069397\n",
            "0.9806707033069397\n",
            "0.9806707033069397\n",
            "0.9806862288464524\n",
            "0.9824406148113647\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-113-7881ba875ba2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mexplor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-92-b2b8d7425d79>\u001b[0m in \u001b[0;36mexplor\u001b[0;34m(fs, acc)\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mneighbor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mneighbor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexplored_models\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m       \u001b[0mperf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_val\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mneighbor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malgo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcross_validation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'skf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0maccs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy_average'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-7a4b94d9f3a3>\u001b[0m in \u001b[0;36mcross_val\u001b[0;34m(X, Y, algorithm, features, cross_validation, k, balancing_method)\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mbalancing\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomUnderSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m       \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbalancing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_resample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m     \u001b[0mmodel_predictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0mactual_labels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/xgboost/sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, callbacks)\u001b[0m\n\u001b[1;32m    730\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    731\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mxgb_model\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 732\u001b[0;31m                               callbacks=callbacks)\n\u001b[0m\u001b[1;32m    733\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    734\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb_options\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"objective\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/xgboost/training.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[1;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/xgboost/training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/xgboost/core.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[1;32m   1107\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[0;32m-> 1109\u001b[0;31m                                                     dtrain.handle))\n\u001b[0m\u001b[1;32m   1110\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m             \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_acc = 0\n",
        "pos = 0\n",
        "for i in range(len(accs)) :\n",
        "  if accs[i] > max_acc :\n",
        "    max_acc = accs[i]\n",
        "    pos = i\n",
        "print(max_acc, pos)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c2lojBgDbG1q",
        "outputId": "4a292502-7390-4300-ea6d-ffc7fcce214a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9859493867411893 109\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(explored_models), len(accs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K_eWfM_ThrUi",
        "outputId": "9d194b65-c43b-4ef4-8f95-dd37c108bebc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "395 394\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "models_ = explored_models.copy()"
      ],
      "metadata": {
        "id": "yZuGVmZ6yM9y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "best_so_far = explored_models[327]\n",
        "perf_max = cross_val(transform(X, best_so_far), Y, algo, cross_validation='skf')\n",
        "print(best_so_far)\n",
        "perf_max[5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YlztZQ4vyS__",
        "outputId": "b478bda7-5dc9-459c-aff0-9d017b76dfa1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[True, True, True, True, True, False, False, True, True, False, False, False, True, True, False, False, False, False, False, True, False, True, False, True, False, True, True, False, False, True]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy_average': 0.987719298245614,\n",
              " 'auc_average': 0.9948676555456771,\n",
              " 'mcc_average': 0.9739726853024475,\n",
              " 'sensibility_average': 0.9952380952380953,\n",
              " 'specificity_average': 0.9835966103089391}"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print([int(vars_[index][i]) for i in range(30)])\n",
        "print([int(best_so_far[i]) for i in range(30)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LQgPp6p8-ukQ",
        "outputId": "40a77158-7b15-4ff8-f94a-b0819e1f5a65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
            "[1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1oexYyQwBrjR",
        "outputId": "83ccd796-e55f-40df-a7b0-f144f0d79697"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[29, 19, 30, 29, 20, 0, 0, 30, 13, 0, 0, 0, 19, 21, 0, 0, 0, 0, 0, 9, 0, 30, 0, 30, 0, 30, 30, 0, 0, 26]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_so_far"
      ],
      "metadata": {
        "id": "o9gKs0yWdeWf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bfs(best_so_far, perf_max[5]['accuracy_average'])"
      ],
      "metadata": {
        "id": "q3Kl0YjxzRTE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "3dTj58KeiDLK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "best_so_far = explored_models[256].copy()"
      ],
      "metadata": {
        "id": "57GhOKOuipoS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bfs(best_so_far, perf_max[5]['accuracy_average'])"
      ],
      "metadata": {
        "id": "-4fGev3wilKW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = plt.figure()\n",
        "ax = fig.add_axes([0,0,1,1])\n",
        "ax.bar([i for i in range(30)], vars_occ, width=0.4)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "CQtzOKJLGU0J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "_X = df.copy()\n",
        "_X.drop(['id','diagnosis', 'Unnamed: 32'],axis=1,inplace=True)\n",
        "corr = _X.corr()\n",
        "mask = np.triu(np.ones_like(corr, dtype=bool))\n",
        "f, ax = plt.subplots(figsize=(11, 9))\n",
        "cmap = sns.diverging_palette(230, 20, as_cmap=True)\n",
        "sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n",
        "            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})"
      ],
      "metadata": {
        "id": "PaY-SyzrH5DY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "_X"
      ],
      "metadata": {
        "id": "PhaAPx6fIhRM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "_vars = [False for i in range(30)]\n",
        "for i in range(30) :\n",
        "  if vars_occ[i] > 25 :\n",
        "    _vars[i] = True"
      ],
      "metadata": {
        "id": "3Cp7-TAqDLgF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sum(_vars)"
      ],
      "metadata": {
        "id": "unT4qhadFUwO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_ = transform(X, _vars)\n",
        "perf = cross_val(X_, Y, algo, cross_validation='skf')"
      ],
      "metadata": {
        "id": "qHGzHh2CFaNz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "perf[5]"
      ],
      "metadata": {
        "id": "8YY5iLnCF2PZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}