2_2_metrics.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/minyoung347/python101/blob/main/2_2_metrics.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j9s7BMDN1GcW"
      },
      "source": [
        "## 3-1 Accuracy(정확도)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "su6SjJSb1Gce",
        "outputId": "65aa7e71-b63a-49c4-9a44-0639fa913df1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.2.2\n"
          ]
        }
      ],
      "source": [
        "import sklearn\n",
        "\n",
        "print(sklearn.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "yHKhPKin1Gcj"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from sklearn.base import BaseEstimator\n",
        "\n",
        "class MyDummyClassifier(BaseEstimator):\n",
        "    # fit( ) 메소드는 아무것도 학습하지 않음.\n",
        "    def fit(self, X , y=None):\n",
        "        pass\n",
        "\n",
        "    # predict( ) 메소드는 단순히 Sex feature가 1 이면 0 , 그렇지 않으면 1 로 예측함.\n",
        "    def predict(self, X):\n",
        "        pred = np.zeros( ( X.shape[0], 1 ))\n",
        "        for i in range (X.shape[0]) :\n",
        "            if X['Sex'].iloc[i] == 1:\n",
        "                pred[i] = 0\n",
        "            else :\n",
        "                pred[i] = 1\n",
        "\n",
        "        return pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "lIUPOhQh1Gcm"
      },
      "outputs": [],
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# Null 처리 함수\n",
        "def fillna(df):\n",
        "    df['Age'].fillna(df['Age'].mean(), inplace=True)\n",
        "    df['Cabin'].fillna('N', inplace=True)\n",
        "    df['Embarked'].fillna('N', inplace=True)\n",
        "    df['Fare'].fillna(0, inplace=True)\n",
        "    return df\n",
        "\n",
        "# 머신러닝 알고리즘에 불필요한 피처 제거\n",
        "def drop_features(df):\n",
        "    df.drop(['PassengerId', 'Name', 'Ticket'], axis=1, inplace=True)\n",
        "    return df\n",
        "\n",
        "# 레이블 인코딩 수행.\n",
        "def format_features(df):\n",
        "    df['Cabin'] = df['Cabin'].str[:1]\n",
        "    features = ['Cabin', 'Sex', 'Embarked']\n",
        "    for feature in features:\n",
        "        le = LabelEncoder()\n",
        "        le = le.fit(df[feature])\n",
        "        df[feature] = le.transform(df[feature])\n",
        "    return df\n",
        "\n",
        "# 앞에서 설정한 데이터 전처리 함수 호출\n",
        "def transform_features(df):\n",
        "    df = fillna(df)\n",
        "    df = drop_features(df)\n",
        "    df = format_features(df)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c5_gHx0u1Gcp",
        "outputId": "ecffe916-e29d-47d4-be82-742224fb06e8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dummy Classifier의 정확도는: 0.7877\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "# 원본 데이터를 재로딩, 데이터 가공, 학습 데이터/테스트 데이터 분할.\n",
        "titanic_df = pd.read_csv('./titanic_train.csv')\n",
        "y_titanic_df = titanic_df['Survived']\n",
        "X_titanic_df= titanic_df.drop('Survived', axis=1)\n",
        "X_titanic_df = transform_features(X_titanic_df)\n",
        "X_train, X_test, y_train, y_test=train_test_split(X_titanic_df, y_titanic_df,\n",
        "                                                  test_size=0.2, random_state=0)\n",
        "\n",
        "# 위에서 생성한 Dummy Classifier를 이용해 학습/예측/평가 수행.\n",
        "myclf = MyDummyClassifier()\n",
        "myclf.fit(X_train, y_train)\n",
        "\n",
        "mypredictions = myclf.predict(X_test)\n",
        "print('Dummy Classifier의 정확도는: {0:.4f}'.format(accuracy_score(y_test, mypredictions)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3e7_fp5u1Gcr",
        "outputId": "3060ee30-b77b-4a83-ecb9-59c445b6e821"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[ 0.  0.  5. ...  0.  0.  0.]\n",
            " [ 0.  0.  0. ... 10.  0.  0.]\n",
            " [ 0.  0.  0. ... 16.  9.  0.]\n",
            " ...\n",
            " [ 0.  0.  1. ...  6.  0.  0.]\n",
            " [ 0.  0.  2. ... 12.  0.  0.]\n",
            " [ 0.  0. 10. ... 12.  1.  0.]]\n",
            "### digits.data.shape: (1797, 64)\n",
            "[0 1 2 ... 8 9 8]\n",
            "### digits.target.shape: (1797,)\n"
          ]
        }
      ],
      "source": [
        "from sklearn.datasets import load_digits\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.base import BaseEstimator\n",
        "from sklearn.metrics import accuracy_score\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "class MyFakeClassifier(BaseEstimator):\n",
        "    def fit(self,X,y):\n",
        "        pass\n",
        "\n",
        "    # 입력값으로 들어오는 X 데이터 셋의 크기만큼 모두 0값으로 만들어서 반환\n",
        "    def predict(self,X):\n",
        "        return np.zeros( (len(X), 1) , dtype=bool)\n",
        "\n",
        "# 사이킷런의 내장 데이터 셋인 load_digits( )를 이용하여 MNIST 데이터 로딩\n",
        "digits = load_digits()\n",
        "\n",
        "print(digits.data)\n",
        "print(\"### digits.data.shape:\", digits.data.shape)\n",
        "print(digits.target)\n",
        "print(\"### digits.target.shape:\", digits.target.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ttQuv6S91Gcu",
        "outputId": "2ad99470-4662-43e0-dff7-19473330cd18"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([False, False, False, ..., False, False, False])"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "digits.target == 7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T2-OpeVz1Gcy"
      },
      "outputs": [],
      "source": [
        "# digits번호가 7번이면 True이고 이를 astype(int)로 1로 변환, 7번이 아니면 False이고 0으로 변환.\n",
        "y = (digits.target == 7).astype(int)\n",
        "X_train, X_test, y_train, y_test = train_test_split( digits.data, y, random_state=11)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0uoQX5zt1Gc9",
        "outputId": "ad4a6b76-4a2e-475b-9e2e-76c3da43c483"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "레이블 테스트 세트 크기 : (450,)\n",
            "테스트 세트 레이블 0 과 1의 분포도\n",
            "0    405\n",
            "1     45\n",
            "dtype: int64\n",
            "모든 예측을 0으로 하여도 정확도는:0.900\n"
          ]
        }
      ],
      "source": [
        "# 불균형한 레이블 데이터 분포도 확인.\n",
        "print('레이블 테스트 세트 크기 :', y_test.shape)\n",
        "print('테스트 세트 레이블 0 과 1의 분포도')\n",
        "print(pd.Series(y_test).value_counts())\n",
        "\n",
        "# Dummy Classifier로 학습/예측/정확도 평가\n",
        "fakeclf = MyFakeClassifier()\n",
        "fakeclf.fit(X_train , y_train)\n",
        "fakepred = fakeclf.predict(X_test)\n",
        "print('모든 예측을 0으로 하여도 정확도는:{:.3f}'.format(accuracy_score(y_test , fakepred)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tAXIxFqI1GdA"
      },
      "source": [
        "## Confusion Matrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I-e1ony_1GdB",
        "outputId": "f6e2855c-a2aa-4ad8-9db2-000ed9b91e38"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[405,   0],\n",
              "       [ 45,   0]], dtype=int64)"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "# 앞절의 예측 결과인 fakepred와 실제 결과인 y_test의 Confusion Matrix출력\n",
        "confusion_matrix(y_test , fakepred)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fPBrsfem1GdM"
      },
      "source": [
        "## 정밀도(Precision) 과 재현율(Recall)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U40TtAgz1GdN"
      },
      "source": [
        "**MyFakeClassifier의 예측 결과로 정밀도와 재현율 측정**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VV3LTuPS1GdN",
        "outputId": "acf86419-3349-4b74-ae95-7644070e3740"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "정밀도: 0.0\n",
            "재현율: 0.0\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\yong\\anaconda3\\envs\\pymldgrev2\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import accuracy_score, precision_score , recall_score\n",
        "\n",
        "print(\"정밀도:\", precision_score(y_test, fakepred))\n",
        "print(\"재현율:\", recall_score(y_test, fakepred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8RP0A7YZ1GdO"
      },
      "source": [
        "**오차행렬, 정확도, 정밀도, 재현율을 한꺼번에 계산하는 함수 생성**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m3he1mXo1GdP"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import accuracy_score, precision_score , recall_score , confusion_matrix\n",
        "\n",
        "def get_clf_eval(y_test , pred):\n",
        "    confusion = confusion_matrix( y_test, pred)\n",
        "    accuracy = accuracy_score(y_test , pred)\n",
        "    precision = precision_score(y_test , pred)\n",
        "    recall = recall_score(y_test , pred)\n",
        "    print('오차 행렬')\n",
        "    print(confusion)\n",
        "    print('정확도: {0:.4f}, 정밀도: {1:.4f}, 재현율: {2:.4f}'.format(accuracy , precision ,recall))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a50-RVVo1GdP",
        "outputId": "410c0f9d-95d5-4af0-ef87-c649a7c6d41f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "오차 행렬\n",
            "[[108  10]\n",
            " [ 14  47]]\n",
            "정확도: 0.8659, 정밀도: 0.8246, 재현율: 0.7705\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# 원본 데이터를 재로딩, 데이터 가공, 학습데이터/테스트 데이터 분할.\n",
        "titanic_df = pd.read_csv('./titanic_train.csv')\n",
        "y_titanic_df = titanic_df['Survived']\n",
        "X_titanic_df= titanic_df.drop('Survived', axis=1)\n",
        "X_titanic_df = transform_features(X_titanic_df)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X_titanic_df, y_titanic_df, \\\n",
        "                                                    test_size=0.20, random_state=11)\n",
        "\n",
        "lr_clf = LogisticRegression(solver='liblinear')\n",
        "\n",
        "lr_clf.fit(X_train , y_train)\n",
        "pred = lr_clf.predict(X_test)\n",
        "get_clf_eval(y_test , pred)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zCJDUTmO1GdQ"
      },
      "source": [
        "### Precision/Recall Trade-off"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UiAN2z7L1GdR"
      },
      "source": [
        "**predict_proba( ) 메소드 확인**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cQ1Pc-xm1GdR",
        "outputId": "b9cb2a1f-2c07-4791-aa03-0880b2bf988e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "pred_proba()결과 Shape : (179, 2)\n",
            "pred_proba array에서 앞 3개만 샘플로 추출 \n",
            ": [[0.44935228 0.55064772]\n",
            " [0.86335513 0.13664487]\n",
            " [0.86429645 0.13570355]]\n",
            "두개의 class 중에서 더 큰 확률을 클래스 값으로 예측 \n",
            " [[0.44935228 0.55064772 1.        ]\n",
            " [0.86335513 0.13664487 0.        ]\n",
            " [0.86429645 0.13570355 0.        ]]\n"
          ]
        }
      ],
      "source": [
        "pred_proba = lr_clf.predict_proba(X_test)\n",
        "pred  = lr_clf.predict(X_test)\n",
        "print('pred_proba()결과 Shape : {0}'.format(pred_proba.shape))\n",
        "print('pred_proba array에서 앞 3개만 샘플로 추출 \\n:', pred_proba[:3])\n",
        "\n",
        "# 예측 확률 array 와 예측 결과값 array 를 concatenate 하여 예측 확률과 결과값을 한눈에 확인\n",
        "pred_proba_result = np.concatenate([pred_proba , pred.reshape(-1,1)],axis=1)\n",
        "print('두개의 class 중에서 더 큰 확률을 클래스 값으로 예측 \\n',pred_proba_result[:3])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MTGaqGqV1GdS"
      },
      "source": [
        "**Binarizer 활용**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cx4cejdT1GdS",
        "outputId": "c80a1a30-33dd-45cc-e5d3-593b8fa57c57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0. 0. 1.]\n",
            " [1. 0. 0.]\n",
            " [0. 0. 1.]]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.preprocessing import Binarizer\n",
        "\n",
        "X = [[ 1, -1,  2],\n",
        "     [ 2,  0,  0],\n",
        "     [ 0,  1.1, 1.2]]\n",
        "\n",
        "# threshold 기준값보다 같거나 작으면 0을, 크면 1을 반환\n",
        "binarizer = Binarizer(threshold=1.1)\n",
        "print(binarizer.fit_transform(X))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ee4HNCW01GdT"
      },
      "source": [
        "**분류 결정 임계값 0.5 기반에서 Binarizer를 이용하여 예측값 변환**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S81KqbNX1GdT",
        "outputId": "08ab98cf-54d7-42b2-f7c7-f5f8aeaf92c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "오차 행렬\n",
            "[[108  10]\n",
            " [ 14  47]]\n",
            "정확도: 0.8659, 정밀도: 0.8246, 재현율: 0.7705\n"
          ]
        }
      ],
      "source": [
        "from sklearn.preprocessing import Binarizer\n",
        "\n",
        "#Binarizer의 threshold 설정값. 분류 결정 임곗값임.\n",
        "custom_threshold = 0.5\n",
        "\n",
        "# predict_proba( ) 반환값의 두번째 컬럼 , 즉 Positive 클래스 컬럼 하나만 추출하여 Binarizer를 적용\n",
        "pred_proba_1 = pred_proba[:,1].reshape(-1,1)\n",
        "\n",
        "binarizer = Binarizer(threshold=custom_threshold).fit(pred_proba_1)\n",
        "custom_predict = binarizer.transform(pred_proba_1)\n",
        "\n",
        "get_clf_eval(y_test, custom_predict)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TFzDWZyV1GdU"
      },
      "source": [
        "**분류 결정 임계값 0.4 기반에서 Binarizer를 이용하여 예측값 변환**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IyyhthNP1GdV",
        "outputId": "ea353ece-bff8-40e7-8507-41dbc671a816"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "오차 행렬\n",
            "[[97 21]\n",
            " [11 50]]\n",
            "정확도: 0.8212, 정밀도: 0.7042, 재현율: 0.8197\n"
          ]
        }
      ],
      "source": [
        "# Binarizer의 threshold 설정값을 0.4로 설정. 즉 분류 결정 임곗값을 0.5에서 0.4로 낮춤\n",
        "custom_threshold = 0.4\n",
        "pred_proba_1 = pred_proba[:,1].reshape(-1,1)\n",
        "binarizer = Binarizer(threshold=custom_threshold).fit(pred_proba_1)\n",
        "custom_predict = binarizer.transform(pred_proba_1)\n",
        "\n",
        "get_clf_eval(y_test , custom_predict)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GYP6waHi1GdV"
      },
      "source": [
        "**여러개의 분류 결정 임곗값을 변경하면서  Binarizer를 이용하여 예측값 변환**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IwVBUays1GdV",
        "outputId": "aeda511d-6f76-453f-c930-92218f3eedf9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "임곗값: 0.4\n",
            "오차 행렬\n",
            "[[97 21]\n",
            " [11 50]]\n",
            "정확도: 0.8212, 정밀도: 0.7042, 재현율: 0.8197\n",
            "임곗값: 0.45\n",
            "오차 행렬\n",
            "[[105  13]\n",
            " [ 13  48]]\n",
            "정확도: 0.8547, 정밀도: 0.7869, 재현율: 0.7869\n",
            "임곗값: 0.5\n",
            "오차 행렬\n",
            "[[108  10]\n",
            " [ 14  47]]\n",
            "정확도: 0.8659, 정밀도: 0.8246, 재현율: 0.7705\n",
            "임곗값: 0.55\n",
            "오차 행렬\n",
            "[[111   7]\n",
            " [ 16  45]]\n",
            "정확도: 0.8715, 정밀도: 0.8654, 재현율: 0.7377\n",
            "임곗값: 0.6\n",
            "오차 행렬\n",
            "[[113   5]\n",
            " [ 17  44]]\n",
            "정확도: 0.8771, 정밀도: 0.8980, 재현율: 0.7213\n"
          ]
        }
      ],
      "source": [
        "# 테스트를 수행할 모든 임곗값을 리스트 객체로 저장.\n",
        "thresholds = [0.4, 0.45, 0.50, 0.55, 0.60]\n",
        "\n",
        "def get_eval_by_threshold(y_test , pred_proba_c1, thresholds):\n",
        "    # thresholds list객체내의 값을 차례로 iteration하면서 Evaluation 수행.\n",
        "    for custom_threshold in thresholds:\n",
        "        binarizer = Binarizer(threshold=custom_threshold).fit(pred_proba_c1)\n",
        "        custom_predict = binarizer.transform(pred_proba_c1)\n",
        "        print('임곗값:',custom_threshold)\n",
        "        get_clf_eval(y_test , custom_predict)\n",
        "\n",
        "get_eval_by_threshold(y_test ,pred_proba[:,1].reshape(-1,1), thresholds )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HIb608mt1GdW"
      },
      "source": [
        "**precision_recall_curve( ) 를 이용하여 임곗값에 따른 정밀도-재현율 값 추출**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vWVVQRwC1GdW",
        "outputId": "798ef6eb-a60b-4269-b0f3-7b64e29999d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "반환된 분류 결정 임곗값 배열의 Shape: (147,)\n",
            "반환된 precisions 배열의 Shape: (148,)\n",
            "반환된 recalls 배열의 Shape: (148,)\n",
            "thresholds 5 sample: [0.11573101 0.11636721 0.11819211 0.12102773 0.12349478]\n",
            "precisions 5 sample: [0.37888199 0.375      0.37735849 0.37974684 0.38216561]\n",
            "recalls 5 sample: [1.         0.98360656 0.98360656 0.98360656 0.98360656]\n",
            "샘플 추출을 위한 임계값 배열의 index 10개: [  0  15  30  45  60  75  90 105 120 135]\n",
            "샘플용 10개의 임곗값:  [0.12 0.13 0.15 0.17 0.26 0.38 0.49 0.63 0.76 0.9 ]\n",
            "샘플 임계값별 정밀도:  [0.379 0.424 0.455 0.519 0.618 0.676 0.797 0.93  0.964 1.   ]\n",
            "샘플 임계값별 재현율:  [1.    0.967 0.902 0.902 0.902 0.82  0.77  0.656 0.443 0.213]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import precision_recall_curve\n",
        "\n",
        "# 레이블 값이 1일때의 예측 확률을 추출\n",
        "pred_proba_class1 = lr_clf.predict_proba(X_test)[:, 1]\n",
        "\n",
        "# 실제값 데이터 셋과 레이블 값이 1일 때의 예측 확률을 precision_recall_curve 인자로 입력\n",
        "precisions, recalls, thresholds = precision_recall_curve(y_test, pred_proba_class1 )\n",
        "print('반환된 분류 결정 임곗값 배열의 Shape:', thresholds.shape)\n",
        "print('반환된 precisions 배열의 Shape:', precisions.shape)\n",
        "print('반환된 recalls 배열의 Shape:', recalls.shape)\n",
        "\n",
        "print(\"thresholds 5 sample:\", thresholds[:5])\n",
        "print(\"precisions 5 sample:\", precisions[:5])\n",
        "print(\"recalls 5 sample:\", recalls[:5])\n",
        "\n",
        "#반환된 임계값 배열 로우가 147건이므로 샘플로 10건만 추출하되, 임곗값을 15 Step으로 추출.\n",
        "thr_index = np.arange(0, thresholds.shape[0], 15)\n",
        "print('샘플 추출을 위한 임계값 배열의 index 10개:', thr_index)\n",
        "print('샘플용 10개의 임곗값: ', np.round(thresholds[thr_index], 2))\n",
        "\n",
        "# 15 step 단위로 추출된 임계값에 따른 정밀도와 재현율 값\n",
        "print('샘플 임계값별 정밀도: ', np.round(precisions[thr_index], 3))\n",
        "print('샘플 임계값별 재현율: ', np.round(recalls[thr_index], 3))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3YwfCh6W1GdX"
      },
      "source": [
        "**임곗값의 변경에 따른 정밀도-재현율 변화 곡선을 그림**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PhVFH_Sh1GdX",
        "outputId": "c3d4973b-92e5-4dd5-9fad-67567153ea00"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAAFzCAYAAAAnoZDUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABP5klEQVR4nO3dd5hU5fnG8e+zfZddFthd6tJ7710UsGEDCwaNvcSSaIymaZKfMdEUTTQmUZNYwRIjGjWoWBJlRREUUIr0DkvvywLLtvf3xxlwgS2zsLNnZvb+XNdcM3POmTP3y8zw7Gnva845REREJPLF+B1AREREaoaKuoiISJRQURcREYkSKuoiIiJRQkVdREQkSqioi4iIRIk4vwNUV4MGDVyHDh38jnFS9u/fT7169fyOcVIivQ2Rnh/UhnChNoSHSG9DZfnnzp27wzmXFcx6Iq6oN2nShDlz5vgd46Tk5OQwcuRIv2OclEhvQ6TnB7UhXKgN4SHS21BZfjNbF+x6tPtdREQkSqioi4iIRAkVdRERkSihoi4iIhIlVNRFRESihIq6iIhIlFBRFxERiRIq6iIiIlFCRV1ERCRKhKyom9mzZrbNzL6uYL6Z2V/MbKWZLTCzfqHKIiIiUheEckt9IjCmkvnnAB0Dt5uAv4Uwi4iISNQLWd/vzrnpZtamkkXGAc875xwwy8wamFkz59zmUGWqUN5m2Dwf2p4KCSm1/vYiIlIzDhWX8MWaXRSXuiPT+rZsQIOUBLbsLWDJlrzjXjOgdUPSkuLJ3X2AFdvyj5s/pG0GyQmxrN95gFU7jp8/rH0GiXGxrN6ez7pdB+jVIp2M1MSabViQzKupIVq5V9Tfds71KGfe28DvnXOfBp5/CPzUOXfcaC1mdhPe1jxZWVn9J0+eXKM5m2zJoevSP/H5oCc4mNKiRtddnvz8fFJTU0P+PqEU6W2I9PygNoQLtSE8HG7DO6sLeXV50VHz7hmUROdGsczYWMRTCwuPe+2vhiXRun4sH64v4oXFx89/6NRkGqfEMHV1IZOPWTfAX0alUD/R+PeKQt5aVcRd/RPplVW9bebKPoNRo0bNdc4NCGY9ETFKm3PuSeBJgM6dO7saH4lnWQEshcG9ukB2/5pddzkifTQhiPw2RHp+UBvChdoQHg634ZlVn9O9eSEPXPjNtmSHxqmkJcXTa38h547Yf9xrOzdNIyUhju77DnHxyAPHze/arD5J8bF07VfA5XsOHje/R4t04mNj6NTnINfnFdAuK5X05PgTyn+y/CzqG4GWZZ5nB6bVvuQG3n3BHl/eXkREasbE6waxI/8QTeonHTevUb0EGtVLqPC1WWmJZKVVvNu8Sf2kctd7WPMGyTRvkFy9wDXMz0vapgBXB86CHwLs9eV4OkBSundfsNeXtxcRkZoRG2OVFt5oF7ItdTN7GRgJZJpZLvBLIB7AOfd3YCpwLrASOABcF6osVTpc1FfngCv1Hmd2hGa9fYskIiLV88PJ8+nUJJWbT2vvdxTfhPLs98urmO+A74Xq/asluREkpMKXk7wbQL3G8OMV/uYSEZGg7C9yvDlvIzef2s7vKL6KiBPlQi4+Ce5YAAd2es9nPAqL3vQzkYiIVMO8bcWUlDrO6t7U7yi+UlE/rF6GdwOolwWlxf7mERGRoH25rYQm9RPp1SLd7yi+Ut/v5YmJg9Ljr0UUEZHwU1BUwsIdJZzVrSkxMeZ3HF9pS708sfHeCXOznwE75gvSpAe0HORPLhGp0z5evp0ZK3eQkhBLamIcKQlx1EuM5fSuTUhNjCOvoAhXCvWT47Bj/++KYnkFRQxsEsf5vZr5HcV3KurlSc/27t+56/h5DVrBDxbWbh4RqbN27S8kNsZIT45n/6FinpuxhqKSo3sC/fSno0hNjOOFmev4w/vLSIyLISstkQYp8TRITuBvV/YjLSmer9bvPtLRSjRpnJbEd3olMrhdht9RfBddn2xN6XsldDwbXMnR0z/4P1gz3Z9MIlKnrNqezzOfruHfc3P57sgO3HFGR8Z0b8ryB86h1MGBwmL2Hyphf2HxkeuyT+2YRWJcDFvzCtiRX8jeg0XsOVBIYlwszjn++fl6cpZv54yujenarD5dmtanQ+PI7h62uKSUldvzCWWX55FERb0iqVnHT4tPBvTFEZHQmblqJ09/spoPl24jIS6Gi/u24Nye3hndh48XxxqkJcWTlnR0V6Q9s9PpmV3xiWLj+rRg94Ei3v16Cy9/sQGA0V0ac3Ubb/5vpy6haf0kemWn06JhMhn1EkmIC+9Tr+as281lT87ijn6JjPI7TBhQUa+OOnSMSkRqT2mpO1Kwn/l0DfM27OEHZ3TkyiGtyazB0b5O6ZjJKR0zcc6xeW8By7bsIzkhloL1CzlUXMJb8zexeW/BUa+55bT23H1OFwqKSvjh5PlkN0pmfL9sOjZJq7FcJ+P9RVtIiIuha6NYv6OEBRX1ajHI3wqH8iExsndZiYj/9hwo5J9frOfFmet4+aYhtM6oxwMX9qBBSjxJ8aErUmZ2VD/lOeshMS6Wmfeczta8Ar7euJcteQXs2FdIn1YNAMg/VMySzXl8sHgL//h4NZmpiSQnxHDfBd05vWsTFm3ayx/eX0ZyfCzJ8bEkJcSSFBfLtwe3pEPjNNbvPMAnK7eTkhCYH1iuW/P6x+1xCJZzjg8WbWVEh0yS4o4fqKUuUlGvjpRG3v3St6H3Zf5mEZEat3nvQTbsOsigto34+8erKCwu5dyeTWmbmUpsDV4qtXbHfp6dsYZX5+RysKiEER0zOVjkncPTNN3ffssrGrQkMzWRj340kp35h3htbi7rdh2goLDkyLjhhcWl7N5fyKaiEg4WlXCwsJSCohJO79qYDo3TWLhxLz9/4+vj1vvvW4fRv3VDpszfxIPvLqVZehJN0pNoWj+JZulJXNS3RYVjky/alMfGPQe54/SOsH9Vzf5DRCgV9eoY8UP45GHY58+4MyISOs45fvb6Qj5fs4vP7h7N6u35TJ6TyyP/XU69hFi6t0jnvJ7NuGZYGwA27TlI40pG9KrIvoIixvx5OiWljnF9WnDDKW3p2qx+DbcmdDJSE8vtW71vq4b857ZTKnzdGd0a8/nPTudgYaDoF5VQUFhCxybeXs+s1EQGtW3Elr0FLN6Ux0dLtnGwqITzApepTZyxhje+2kjHJmmc16sZozo35oPFW4kxOL1rYxbOUVEHFfXqSajn9RGfv83vJCJSw974aiPTlm3n3vO70SAlgd9f3IsbR7RjQe5eFubuYcHGvazYtg/w/gA4/eGPSU6IpUfDUha5lWSmJtCnZUM6N02jtNSxv7CYtKR4ikpKeffrLXyxZicPXNiTtKR4Hp3Qh36tGtK4Do0mlhgXS5P6FR9SGNo+g6Htv7kkzTnH3oNFR8Ylb5CSQGpSHB8t3cZrc3MZ27s5917QjSFtG1W4JV8XqahXV2pj77i6iESNbXkF/OqtxQxo3ZBrA1viMTFGpyZpdGqSxvj+2UctX1LquG9sNz5ZsYMPF29meu4yAH58dmc6N01jc14Bw3//Ef1bN2TznoNs2ltAu8x67D1QRHpKPGN6qJOUqpgZDVK+Gfv8wr4tuLBvCwqLS/lbzioe/XA59ZPjeODCnj6mDD8q6tWV2kRb6iIRYvLsDbTNqsfANo0qXMY5xy/e/JqCohIeGt8rqG5G42JjmDCwFRMGtiInJ4chw0ewc38hyYGT2+olxPL90zvy/My1dGmaxq/H9WB0l8Z1vgvTmpAQF8MdZ3SkTWYKQ9XZzHFU1KsrJQN26tiNSCT4yb8XcO2wNgxs04h5G/aweFMefVo2oGl6Eg1T4o90pTqobSOGtc+gXdaJXdWSFB9Li8CZ5ODtKr7rzE7cdWanGmmHHG9cnxZ+RwhLKurVldwACvb6nUJEgtAgJf5IT2PvLtzMP6avPjIvITaGJumJfPTDkdw4om6PwS3RQ0W9uhLTVdRFIkRpqTuyNX73OV24qF8L1mzfz5a8ArbkFZB3sIj42PDuMU2kOlTUqyspHYr2Q0mRN5qbiIQt5yAmUNTNjC5Nvf7ORaKV/kStrqRAv8q71/mbQ0SqVOocOjdN6hIV9epq3se7n/4HX2OISNVe+s6QI53FiNQFKurV1WoIDLsdFrwCWzSuukg469OyAS0bpfgdQ6TWqKifiBE/9HbD//dev5OISAX+M28jr8xer3G2pU5RUT8RyQ3htJ/Aqo9g5Yd+pxGRcjw+bSWvzc09cva7SF2gs99P1MAb4fN/wNs/gFbDqvXSLlu3wK6XQ5OrllTYhowOcNqPaz+QSBnLtuxj+dZ87h/X3e8oIrVKRf1ExSXCBY/Cuz+F9TOr9dL0ggI4tCY0uWpJuW04lAcL/gVDbtV48xJSK7fl8/aCTTRMSSAjNYFG9RLokJV6ZICUt+ZvIsbgnJ7qY13qFhX1k9F+NNw2u9ov+zwnh5EjR9Z8nlpUbhvmvQxv3uINeKOiLiFSXFLK9176kmVb9x01/WfnduGmU9uzbud+/vbxKoZ3yCRTo3dJHaOiLjUntbF3n78NMo4fb1mkJsTGGHee2YnE+Bh6tkhn1/5CduQfomVD7yz3+NgYvj2o1XEjq4nUBSrqUnNSm3j3GppWQsQ5r9vXMT2aHpmWmZpIpyZpR543b5DM/Rf28COeiO9U1KXmHC7q0//oXcd/stqeBkNuOfn1SFRwznHjpDkM75DJ9ae09TuOSFhSUZeak5IBXcfC7jWwd8PJrSt/G6z+GAbeoD72BYDXv9zIh0u3MbJLY7+jiIQtFXWpOTExMOGFmlnXojfh1Wtg01fQclDNrFMi0oZdB3jqk9W8MnsD/Vo14IpBrfyOJBK2VNQlPLUZ4d2vma6iXocVl5Qy/u+fsWt/IRf3zeauszoRoxFaRCqkoi7hqV4GNO7uFfVTf+R3GqlFc9ft5rW5udw/rjtxsTE8fGkf2jeuR7P0ZL+jiYQ9FXUJX21PhTnPwv9+BS36QYv+UL+536kkBJxzfLx8O3/LWcXna3bRICWe64a3oVOTNE7pmOl3PJGIoaIu4avPt73e+j77C5QWe9PSmkHzft8U+eZ9IbmBrzHlaAVFJfxu6hLMjPvGet20Tlu6jYNFJTRM8Xp/a1QvgQYp3gmQW/YWcMOk2SzalEez9CT+7/xuXDawJfUS9d+TSHXpVyPhq1kvuPljKCrwhrndOBc2fendL3vnm+UyOngFvkV/r+A37QnxSf7lruMe/d8KJs1cx+gyZ6k/+uEK5m/Yc9RyvVs24M7ukJWWSNP6SVwztA0X9m1BQpzGmRI5USrqEv7ik6DlQO922MHdsGmeV+A3fuld/nb42viYOGjS45ut+Rb9IbMTxMT6Er8u+XrjXp76ZDUTBrTkwfG9jkx/+uoB7Nx/iF35hew6UMju/YXUT46HPSuIjTGeuXZgJWsVkWCpqEtkSm4I7Ud5t8PyNgWKfKDQL3zNOyYPkJAKzfoECn0/EguKwDnQsJw1pqiklJ+8toBG9RL42bldj5qXlZZIVtrx/bDn5KyorXgidYKKukSP+s29W9cLvOelpbBz5Te77DfOhc//DiWFDAVY+DM4/V7od7WfqaPG9n2HKC4t5f5xPUhPUYdBIn5QUZfoFRMDWZ28W+/LvGnFh2DrIpbn/ItOW9+Gr19XUT9Jew4UsudAEW0y6zH1+yOIi9UxcRG/qKhL3RKXCC36salFHp0StsHmBX4nikjOOebn7uWFmet4e8EmhrTLYNL1g1TQRXymoi51V4NWsPQdbzd9jIpRsN5duJnHc1by9cY86iXEMr5/NlcOae13LBFBRV3qsgatoKQQ8reoU5sqrNyWT8tGySTGxbJ25wGKih33j+vOhX1bkJak4+ci4UJFXequBm28+93rVNTLKC4p5ddvL6awuJRTOmby0qz1zFy9kz9f1odxfVpw44i23HJaO0xXDoiEHRV1qbsaBcbk3rUKWg/1N0uY2Lz3IP9bso24mBien72Of83eQIsGyfz47M4M7+B11xqv4+YiYUtFXequBq0hJh526Frpw17+YgN//WgF0388ivN6NWVfQTEjOmYRq5HRRCKCirrUXbFx0KidinrAgcJiXpm9npGdsmjZKIWWjVL8jiQi1aT9aFK3ZXaEnXW7qBeXlDLps7Wc+lAOW/MOcd3wtn5HEpETpC11qdsyO8LSt+HBNtV7XdNecPGTkNY0JLFqU4wZL3+xnvZZ9fjHVf3o37qR35FE5ASpqEvd1u8abxQ4VxL8a0qLYf4r8I/T4LKXIHtA6PKFQGmp492vt/D0p6uZeO0g0lPi+ddNQ0hPjtcZ7SIRTkVd6rZGbeGc31f/dQNvhJcvh+fOgfP/BH2vrPlsNcw5R87y7fzx/WUs2pRHx8apbM47SHpKPA1SEvyOJyI1IKTH1M1sjJktM7OVZnZ3OfNbmdk0M/vKzBaY2bmhzCNSY5p0h5tyoNVQ+M/34N2fQkmR36kq9dLn67nuudnsKyjmkW/15r0fnEqXpvX9jiUiNShkW+pmFgs8DpwJ5AKzzWyKc25xmcV+AUx2zv3NzLoBU4E2ocokUqNSGsGVr8N/74VZj8PWRXDpJKiX4Xeyci3I3cOQdo14/vrBJMTpHFmRaBTK3e+DgJXOudUAZvYvYBxQtqg74PCmQjqwKYR5RGpebByM+S007QFv/QD+0B6smgXTYmDsX6HP5SGJeNhD43tTUFSigi4SxUJZ1FsAG8o8zwUGH7PMfcAHZnY7UA84I4R5REKnz7e9XfJL3gLnqvfaef+Eha8eVdQPFBZz9qPT2ZlfSGpiHKlJcTxxRT+6NK3PZyt38PpXG9m74xBzC5eRmhhHWlI85/VqRnpyPNvyCth1oPDI9AOFxRQUldI2sx5J8bE13HARCSfmqvsfULArNhsPjHHO3Rh4fhUw2Dl3W5ll7gpkeNjMhgLPAD2cc6XHrOsm4CaArKys/pMnTw5J5tqSn59Pamqq3zFOSqS3IZzyd1jxFM02f8CM4S9RGuudsDY9t4hnvy5kRIs4zKCg2PGtzglkJscwY2MR/15RxMGiUgpKjMO/4D+cmkxWSgxvryrktRVHH99PioVHRqaQEh9eZ7eH0+dwotSG8BDpbags/6hRo+Y654K6zCaUW+obgZZlnmcHppV1AzAGwDk308ySgExgW9mFnHNPAk8CdO7c2Y0cOTJEkWtHTk4OaoO/wip/swJ4+W1ObZsA7UYCsH3OBkYc2sTz1w867jKzkcDP8dpw6qmncaCohPyCYjJTE4iLjaFV93xGb9lHfkExeQVF5B8qpluz+pzVPfyuqQ+rz+EEqQ3hIdLbUFP5Q1nUZwMdzawtXjG/DPj2McusB04HJppZVyAJ2B7CTCLhp80pXh/0K/4L2YMAuLRXBpf2zoQqrhuPiTFv93ziNz/ldlmptMuK3C0WETlxISvqzrliM7sNeB+IBZ51zi0ys18Dc5xzU4AfAk+Z2Z14J81d60J1PEAkXCWmQsvBMPMx71ZWw7bQfpS3Bd/2VEhu6EtEEYkMIe18xjk3Fe8ytbLT7i3zeDEwPJQZRCLCeX+E5e+zJa+AiZ+tZUTHTIa3TYfcubBgMsx51jtLvnlfaDcK2o/CSsP7ungRqX3qUU4kHDTuisvqwh1PzmJ5Qj9uvXQUpMR780qKIHcOrJ4Gq6bBp3+CT/7IKTFJsPnUI0WerC5V7q4Xkeimoi4SJt77egufr9nF/Rf2IP1wQQeIjYfWQ73bqJ9BwV5Y8wlbPnmRFrtWwIoPvOXSmnm76duNgnanRcVgMyJSPSrqImHisWkr6dQklcsHtqx8waR06Ho+K7am0mLkSNizHlbneFvxy9+H+S97yzXu9s1WfJPu1esUJ6EeJKadaFNExCcq6iJhoKikFOdgbO/mxMVWs8e3Bq2g39XerbQUtizwdtWvzoHZT3td2FZXfArctQSSG1T/tSLiGxV1ER8553jm0zX0btmAqXeM4KQv/oiJgeZ9vNspd0LRQVg/E3avDX4dWxZ6J+btzVVRF4kwKuoiPlq8OY8H3llCvYRYJt8ylO7N02v2DeKTof3o6r1m7adeUT+wo2aziEjIaWQHER9NmbeJuBjjjG5NmLN2t99xPPWyvPv9KuoikUZb6iI+KS11TJm/idM6ZfHny/r6HecbKZne/YGd/uYQkWpTURfxyb6CYga0acS5PcLs0rPkhhATB3nHDtUgIuFORV3EJ+kp8fz18jDaQj8sJgaa9vQ6vBGRiKJj6iI+2LjnIMu37vM7RsVaD/eKevEhv5OISDWoqIvUsvxDxTzw9mLO/+un7D0Ypv23tx4GJYdg45d+JxGRatDud5FaUlBUwouz1vFEzip27S/k9tEdSE+Or/qFfmg11Ltf9IbXTW11WWAX/om8VkROmIq6SC0oLXWMfexTlm/NZ0THTH50Vmd6t2zgd6yKpTTyivIX//BuJ6JBK68DnD5XQFxizeYTkXKpqIuEUM6ybZzWKYuYGOPmU9vTrEESw9pn+h0rOJf/C7YtObHXHtwDn/8d3r4Tpv8Rht/hdWMbn1yjEUXkaCrqIiHy4qx1/OLNr3nmmgGc3rUJl/TP9jtS9aRne7cT1XO81wf9x3+Ad38CnzwMw74PA67zBowRkRqnE+VEQmDmqp3cN2URIztncVqnLL/j+MPM66L2+nfh2ne88d4/+Dk82hM+eQQOhfHZ/yIRSlvqIjVs/c4D3PrSXNpk1uMvl/et/qhr0ajNKd5t/ecw/SH48Fcw48+0bnoOrCw5sXXWbw6Nu9ZsTpEIp6IuUoNKSx3f++eXOAdPXz2A+kk6+/sorQbDlf+GjXNh+h9pu+xlWPvyia9vwA1wxn2QVL/GIopEsiqLupmlAD8EWjnnvmNmHYHOzrm3Q55OJMKYwZ1ndsQw2mTquHGFWvSHy1/m86kvM7hHuxNbx+L/wKwnYPn7cMGj0PHMGo0oEomC2VJ/DpgLBC5cZSPwKqCiLnXaXa/MY39hMb2yG9ArO52eLdJpkJLA6C5N/I4WMQ6mNPO23k9Eq8HQ/SKYchu8NB56XQZjfuddjidSRwVT1Ns75yaY2eUAzrkDZmYhziUStjbsOkB8bAzJCbF8tWEP7y/aemTe9cPb8ovzuhITo59IrWg5EG6e7l029+kjsOpDOPcP0O1Cb7eJSB0TTFEvNLNkwAGYWXtAHUJLnfWDV+aRGBfDP78zBIC9B4r4etNeFuTuJTUxVgW9tsUlwuifQ7ex8J/b4NVrocv50OMSr2e7NiOgXobfKUVqRTBF/ZfAe0BLM3sJGA5cG8pQIuHqUHEJubsP0K9VwyPT0lPiGd4hk+EdIqRTmWjVtCfc+CHMfAym/RaWBo4Q9r4cLvq7v9lEakmVRd05918z+xIYAhhwh3NuR8iTiYShpz9Zw9a8Q0wY2NLvKFKe2Dg45QfQ9yrYvw0+egBW/g9KS70hZUWiXJXfcjM7FegO7APygG6BaSJ1yoZdB3jso5Wc1a0JIzs39juOVKZehncNe5fzYf922LrQ70QitSKY3e8/LvM4CRiEdzb86JAkEgkjxSWl7D9UQnpKPFvzCih1jv87v5vfsSRY7QP/Ta38EJr19jeLSC0IZvf7BWWfm1lL4NFQBZK6yznHR0u3kRgXyykdM/l89U5mrNrJwDYNqZcYR1piHPWT42lSPynkOb5cv5v/zNvEOws2c3aPpvz2op70b92QN747nJaNUkL6/lKD0ppAk55eUR9xl99pRELuRHqUywXUN6PUGOccn67cwcMfLGfehj2M7tKYUzpm8ua8jbz8xYajlm3VKIXpPxkFwI2T5jA/dw9piXHUS4yjXmIsXZrW576x3QF4fuZa9h4oIjUpjtRE79Y0PYm+gZPctu87RGZqAmWv0PzHx6t4YdY6cncfJDEuhjO6NeHs7k0BMDO6NVfPZRGnw2iY+bjX13ximt9pREIqmB7l/krgcja8Y/B9gC9DmEnqkK837uU37yxh5uqdNE9P4ncX92R8YDSz317Uk+uGt2XvwSLyDxWTX1BMfOw3BXh4hwyy0hLYV1DM/kPF5B8qJq+g6Mj8V2ZvYNGmvKPeb3iHDF660bsU7eK/zWBw2wz+eOk3u2U37y2gXVYqd57RibO6NyFN3bxGvvanw4w/w5pPoMu5fqcRCalgttTnlHlcDLzsnJsRojxSxyzZnMeyrfu474JuXD64FYlxsUfmmRmdmlS8ZXXd8LaVrvud74+gsLj0SMHPP1RMfJnBVXq2SOe1ubkMa59B64wU+rduxC8v6Ib6VooyrYZAfIrXMY2KukS5YI6pT6qNIFI3HCgs5rGPVtKsQTJXDWnNJf2yGdOjaci2iBPiYkiIS6BhvYTj5j14SS8+X72LuybPp01GCv+76zSNqBaN4hK9DmhWfuh3EpGQq7Com9lCvtntftQswDnneoUslUSlklLH9176kmnLtnPtsDYAxMSYb7u405Li+dm5XXkiZyV/v7K/Cno063AGrHgfdq2GRic4gIxIBKhsS/38WkshdcLv313CtGXbeeDCHlw5pLXfcQC4pH82lwSO4UsUO3xp2+ocFXWJahUWdefcutoMItHt8WkreeqTNVw7rE3YFHSpQzLaQ2oTWPcZDLje7zQiIRNMj3JDzGy2meWbWaGZlZhZXlWvEykpdZSWekdwkuNjuaRfNr84T1dDig/MoPVwWDsDXHlHFUWiQzAHER8DLgdWAMnAjcDjoQwlkW/uut2c/9dPef2rjQBcN7wND3+rt45bi39aD4N9m2D3Wr+TiIRMUP/DOudWArHOuRLn3HPAmNDGkkj3vZe+ZM+BQhokeyfB6TIx8V2bU7z7tZ/6m0MkhIIp6gfMLAGYZ2YPmdmdQb5O6qi9B4rYklfAdcPbcEa3Jn7HEfFkdoaGbeF/98G2pX6nEQmJYIrzVYHlbgP2Ay2BS0IZSiLbnHW7AGiXmepzEpEyYmLgyn9DTCw8PxZ2rvI7kUiNC6ZHuf7AO865POBXIc4jEaigqISPlm7jrfmb2FdQTHpyPG0z6zGsQ4bf0USOltEerp4CE8+FSRfAdVOhYRu/U4nUmGCK+gXAn8xsOvAK8J5zrji0sSQSzFq9k1dmb+CDRVvYX1gCwP3juvOtgS3Zvb+IlIQTGS9IJMQad4Gr/wMTzw8U9nchXX0VSHQIppvY68wsHjgH7yz4x83sv865G0OeTsLajJU7+GjpNi7o3ZyxvZszuF0GsTHeCXFN02OreLWIj5r2hKvegOfHwcTzoNMx5/62GQFd1f+WRJ6gNqWcc0Vm9i5et7HJwIV4l7ZJHfPrtxbTqUkqTYGbTm3H7aM7khCn8yYlArXo5x1j//eNMP/lb6YX7vd6nlNRlwgUzNCr5wATgJFADvA08K2QppKw9OKsdTw7Yw03n9qOpiloWFKJfC0HwQ8WHD3tjVu8TmpEIlAwm1hXA28CnZ1z1zrnpuqYet1zsLCE301dwoiOmfxkTBe/44iETnwyFB3wO4XICQnmmPrltRFEwtt/l2xlf2EJ3x3Z4chxc5GoFJ8CRQf9TiFyQnQwVKrknOPxj1bSokEyg9s28juOSGgd3lJXH/ESgXTNkVRoYe5eOjdNIyEuhl+O7UaT+knEaCtdol16NuBg+zLv8jeRCKItdSnXe19vYdzjn/LCLG8E3mHtM2mfpR7ipA7oeJZ3v+wdf3OInIAKt9TNbCHeJWzHzQKcc65XyFKJr75av5sfvPIVvbIb8K0B6pRD6pj6zaF5X1j2Loz4od9pRKqlst3vukizjtl/qJhnP13Dk9NX0zgtiaevGaDL1qRu6nweTPsN7NsKaRqUSCJHhbvfnXPrKrsFs3IzG2Nmy8xspZndXcEy3zKzxWa2yMz+eaINkZN3oLCEv328iiHtM3jpxsFkpib6HUnEH13OBRwsf9fvJCLVUtnu931Uvvu9fmUrNrNY4HHgTCAXmG1mU5xzi8ss0xG4BxjunNttZo1PoA1yEgqKSnhuxlquG96GrLREcn48ksZpSX7HEvFX426QWB+2LfE7iUi1VFjUnXNpJ7nuQcBK59xqADP7FzAOWFxmme8Ajzvndgfec9tJvqdU018+XMETOavolZ3O8A6ZKugiAGaQ3BAO7vY7iUi1mAvyWszAVvSR//Gdc+urWH48MObwwC9mdhUw2Dl3W5ll3gSWA8OBWOA+59x75azrJuAmgKysrP6TJ08OKnO4ys/PJzXVvzPJi0sdC3eUMD23mHnbSjilRRw39Kzerna/23CyIj0/qA2h1n/OXRQmNGBhr3srXS6c2xAstcF/leUfNWrUXOfcgGDWE0zf72OBh4HmwDagNbAE6B502srfvyNev/LZwHQz6+mc21N2Iefck8CTAJ07d3YjR46sgbf2T05ODn624a8fruDPXy4nMzWRm05rze2jO5KaWL0uC/xuw8mK9PygNoTchtZQkFdlvrBuQ5DUBv/VVP5g/ie/HxgC/M8519fMRgFXBvG6jUDLMs+zA9PKygU+d84VAWvMbDlekZ8dxPqlGrbmFbAt7xDdmtfn4v7ZdG1Wn9M6ZxEfq64KRMqV3BB2rfY7hUi1BPM/epFzbicQY2YxzrlpQDC7AWYDHc2srZklAJcBU45Z5k28rXTMLBPoBOhXFAJvfrWRCx77lJ37D9GiQTJndGuigi5SmWa9Yfda2LrI7yQiQQvmf/U9ZpYKTAdeMrM/A/urelFgJLfbgPfxdtdPds4tMrNfB3bpE5i308wWA9OAHwf+gJAaNmv1Ttpn1dOJcCLB6nsVxCXDrCf8TiIStGCK+jjgAHAn8B6wCrggmJUHhmnt5Jxr75z7TWDavc65KYHHzjl3l3Oum3Oup3PuXyfWDKlMcUkpc9buZki7DL+jiESOlEbQ53JY8Crkb/c7jUhQginqjYEE51yxc24S8BRwspe7SS1avDmPfYeKGayiLlI9g2+FkkMw51m/k4gEJZii/ipQWuZ5SWCaRIhZq70jGkM0bKpI9WR1gg5nwuynofiQ32lEqhRMUY9zzhUefhJ4nBC6SFLTrhzSmlduGkLj+jqeLlJtQ26F/dvg69f9TiJSpWCK+vYyJ7ZhZuOAHaGLJDXlqemrWbI5j5SEOO16FzlR7UdDVheY9TgE2VmXiF+CKeq3AD8zsw1mth74KXBzaGPJyVq1PZ8H31vKy19U2vGfiFTFzNta37IQ1s3wO41Ipaos6s65Vc65IUBXoJtzbphzbmXoo8mJWpC7h1tfnEtyfCzfP72j33FEIl+vCZDcCGbq8jYJb1UWdTNrYmbPAK865/LNrJuZ3VAL2aSanHP85p3FXPj4DPYcKOKxK/pp+FSRmhCfDAOuh2VTNXKbhLVgdr9PxOskpnng+XLgByHKIyfBzDhUXMrlg1rxvx+exmmdsvyOJBI9Bt/iXbv++nd0JryErWCKeqZzbjKBy9oCPcWVhDSVBK201PHjV+czd503ROSvxnbnNxf1pH5SvM/JRKJMahaMe8I7tv7R/X6nESlXMEV9v5llAA7AzIYAe0OaSoI2Z91uXp2by8xV3gUJZuZzIpEo1nkMDLwRPvsrrJrmdxqR4wRT1O/CG4ilvZnNAJ4Hbg9pKgna1IWbSYqP4brhbf2OIlI3nHk/ZHaGN26BA7v8TiNylGDOfv8SOA0YhncpW3fUTWzY2Lz3IK0apVCvmuOhi8gJSkiBS56Gg7vgvXv8TiNylAqLupnFmtnlZvYjoLNzbhHQBvgYeKyW8kklVm7bx2erdpLdMMXvKCJ1S7Ne0PdKWPIWMSWFVS8vUksq27x7BmgJfAH81cw2Af2Be5xzb9ZCNgnYV1DExj0Hyd11kNzdBzhYVMqtI9vTMCWBoe0y+L/zu/kdUaTu6XwuzHmW9L1fA2f5nUYEqLyoDwB6OedKzSwJ2AK013jnobF7fyGz1+4id/dBtuYVcPc5XTAz7nl94XG9wjWql8Atp7UjIzWRJ68e4FNikTquzQiITyFj5xy/k4gcUVlRL3TOHb6MrcDMVqug17yd+Yd48pPVvDBzHQcKvSsFUxJi+d7oDtRPiuf0Lo1pnZFCdsNksht69xn1EnSWu4jf4pOg3Ugy1s32+oTXb1LCQGVFvYuZLQg8Nryz3xcEHjvnXK+Qp6sD5qzbzZPTVzO2d3OuHtqatpmpNEyJP1K0z+jWhDNo4nNKESlXp7NJXjYVvpwE/a/1O41IpUW9a62lqGMKSxyzVu9kSLsMzurWhI9+OJK2mfX8jiUi1dXzUnZ/8gwN37oDcufAuX/wupQV8UmFRd05t642g9QVzjmeWniI+dO+4JOfjKJJ/SQVdJFIlVCP+b3vY6SbCdP/AJvmwbcmQUZ7v5NJHRVM5zNSg16ctY7ZW0q468xONKmf5HccETlZFgujfwHffhX2boAnR8KSt/1OJXWUinot2pl/iCdyVtG5YQw3n9rO7zgiUpM6nQW3fAIZHeCVK+CDX0BJkd+ppI5RUQ8h5xy79nsdU5SWOs5+dDqb9xZwXrt4nb0uEo0atILr3/umf/j/3ed3IqljKjymbmYLCQziUh6d/V61uybPZ866XUz/8ShiYowHLuxJq0YpbFv+pd/RRCRU4hLhvIdh+zJYP9PvNFLHVHb2+/mB++8F7l8I3F8RujjRo6CohLfmb+Ls7k0pKXXExRpjejQFYNtyn8OJSOhldYaFr+oadqlVVZ79bmZnOuf6lpl1t5l9Cdwd6nCR7KOl2ygudVw6IJu4WB3lEKlzMjpAwV44sBPqZfqdRuqIYIb2MjMb7pybEXgyDB2Lr9KbX22keXoSIzpm+R1FRPyQ0cG7z50DLfrV3HqTG0JsfM2tT6JKMEX9BuBZM0vH601uN3B9SFNFgbU799MzO53YGO12E6mTMjt69y9PqNn11m/hDf3aeljNrleiQpVF3Tk3F+gdKOo45/aGPFUUePHGwWzeU+B3DBHxS8M28K0XYP+2mltnaSl8/jeYeB6MvAdG/BBiYmtu/RLxqizqZpYIXII3lnrc4UuxnHO/DmmyCPX56p10a16fxmlJNE5T5zIidVq3sTW/zj6Xw9t3wbTfwJrpcPFTUL9Zzb+PRKRgjo3/BxgHFAP7y9zkGIXFpdw4aQ6/f3ep31FEJFolpsHFT8K4J2DjXPj7cFjxX79TSZgI5ph6tnNuTMiTRIEFuXvYd6iYUzvp5DgRCSEz6HsFZA+E166Dl8bD0Nvg9F9CXILf6cRHwWypf2ZmPUOeJAos35oPQMfGqT4nEZE6IasT3Pg/GHADzHwM3rrD70Tis2C21E8BrjWzNcAhNJ56uVZvz+d37y6hY+NUWjZK8TuOiNQV8clw/iOQ0sgbKa7LedD1/KpfJ1EpmKJ+TshTRIEZK3eQFB/LM9cMJF6dzYhIbTv1J7D8fW9rveVgSNVhwLqoyurjnFsX6F3uIF5f8IdvUsZVQ9sw++dn0CpDW+ki4oO4BLjoH3AoD97+gdc9rdQ5VRZ1MxtrZiuANcDHwFrg3RDnEhGR6mrSzRvbfenbsOAVv9OID4LZT3w/MARY7pxrC5wOzAppqgj08AfLuOf1BX7HEJG6buht0GoovH0nrPrI7zRSy4Ip6kXOuZ1AjJnFOOemAQNCnCviTJ6zgbyDxX7HEJG6LiYWLp0EDdvCPyfA4il+J5JaFExR32NmqcB04CUz+zPqfOY4O/ILaZtZz+8YIiKQ1gSuewea9YFXr4GvXvQ7kdSSYIr6OOAAcCfwHrAKuCCUocJZaaljW97Rfbpv33eIUp2UIiLhJLkhXP0mtBsJ//kezHzc70RSC4IZ0OXwVnkpMCm0ccLbtGXbuO652QD8/NyuXDGkFcnxsXz7qVnEmjGgTUOfE4qIlJFQDy7/F7z+HXj/Z3BwN4z6udcjnUQlXVAdJOccE2esPfL8N1OXcN5fPsU5+PW4Hvz3rtMY2bmxfwFFRMoTlwjjn4M+V3id02xZ6HciCaFgOp+p8w4WlnDDpNl8tmono7s0Jis1kYv6tWBrXgExMcbQ9hl+RxQRqVhMLAy4Hua9BPs2QzN1CBqtVNSDMGPlDj5btZN7zunCTae2w7TrSkQiTXLg8ODBPb7GkNAKZjz14cB9QOvA8of7fm8X2mjhY/eBQgDO7dlMBV1EIlNSA+/+4G5fY0hoBbOl/gzeme9zgZLQxglPp3XO4sUbBtO4fqLfUURETkxSunevoh7Vginqe51zdbpb2MZpSTROS/I7hojIiYuNg+RGkLfR7yQSQsGc/T7NzP5gZkPNrN/hW8iThYkDhcU8+N5SVm3P9zuKiMjJadEPNs71O4WEUDBb6oMD92W7hnXA6JqPE34e+WA5z8xYw9B2GbTPSvU7jojIiWs5GKb9Fgr2frM7XqJKMJ3PjKqNIOFq/a4DdGqcxqmdNDaxiES4loMAB7lzoMPpfqeREAhm6NV0M3vEzOYEbg+bWZ35E6/UQUyMzngXkSjQoj9YDGz4wu8kEiLBHFN/FtgHfCtwywOeC2blZjbGzJaZ2Uozu7uS5S4xM2dmYTf6m3OOWPW7JyLRIDHNG+Rl8ZtQWicvZop6wZSr9s65XzrnVgduvwKqvEbdzGKBx4FzgG7A5WbWrZzl0oA7gM+rF712NK6fSPP0ZL9jiIjUjGG3wfalsOgNv5NICART1A+a2SmHnwQ6ozkYxOsGASsDfwgUAv/CG/HtWPcDDwIF5czz3e8u7sWTV4fdDgQRkRPT7SLI6go5v9fWehQyV8WQoWbWB290tnS83uR2Adc65+ZX8brxwBjn3I2B51cBg51zt5VZph/wc+fcJWaWA/zIOTennHXdBNwEkJWV1X/y5MlBNzAc5efnk5oa2WfSR3obIj0/qA3hIhLbkLn9M3osepAlXe5ka9OREdmGY0V6GyrLP2rUqLnOueC2Lp1zQd2A+kD9aiw/Hni6zPOrgMfKPI8BcoA2gec5wICq1tupUydXm37y6nz34LtLanSd06ZNq9H1+SHS2xDp+Z1TG8JFRLahpMS5J4Y79+c+zhUXRmYbjhHpbagsPzDHBVl7K7ykzcyudM69aGZ3HTP98B8Dj1Tx98JGoGWZ59mBaYelAT2AnMA6mwJTzGysK2dr3S9frt9NxyaR+9efiMhxYmJg9C/g5Qkw/Y9gQ/1OJDWksmPq9QL3aRXcqjIb6Ghmbc0sAbgMmHJ4pnNur3Mu0znXxjnXBpgFhFVB372/kLU795PdMMXvKCIiNavzGOj9bZj+EOl7FvudRmpIhVvqzrl/BO5/dSIrds4Vm9ltwPtALPCsc26Rmf0ab1fClMrX4L/Xv9pIUYljXJ/mfkcREal55z4E62fSdckjcPbl3wzPKhErmKFXHwIewDvj/T2gF3Cnc+7Fql7rnJsKTD1m2r0VLDsyiLwh99X63dz7n0VsyStg+75D9M5Op3vzOtPXjojUJYlpcMkzJDxzJrx9J4x/DjS8dEQLpu/3s5xzPzGzi4C1wMXAdKDKoh4p8g8V89b8TbyzYDOfrtxBZmoiL944iDlrd3N+r2Z+xxMRCZ3s/qxt823aLXoBOpwBfa/0O5GchGCK+uFlzgNedc7ttSj7S27j7oPc8/pCAG4b1YFbRrYnNTGOLk3r+5xMRCT01re6iHalq+GDX0DXCzTYSwQLpvOZt81sKdAf+NDMsgjTjmJOVKcmqfzivK688d1h/OjszqQmBvO3johIlLBYOPu3cHA3fPZXv9PISaiyqDvn7gaG4V1DXgTsp/ye4SKWmXHjiHb0baWTRESkjmreB7pfDDMfh/xtfqeRE1RhUTez0YH7i4GRwLjA4zF4RV5ERKLJ6F9ASSFM/4PfSeQEVbaf+TTgI+CCcuY54PWQJBIREX9ktId+V8Oc56D/ddDkuDG4JMxVdp36LwP319VeHBER8dWpP4HF/4Gnz4DzH4Hel/mdSKqhymPqZvZbM2tQ5nlDM3sgpKlERMQf9ZvBzZ9As97wxs3wxq1wKN/vVBKkYM5+P8c5t+fwE+fcbuDckCUSERF/pbeAa96C0+6G+S/Dk6fB5gV+p5IgBFPUY80s8fATM0sGEitZXkREIl1sHIy6xyvuhfu93fFfPAVVDNct/gqmqL+Ed336DWZ2A/BfvPHVRUQk2rUdAbd8Cu1Og6k/glevgdISv1NJBYK5Tv1BvL7fuwZu9zvnHgp1MBERCRP1MuHyV+C0n3on0a37zO9EUoFgttQBlgDvOed+BHxiZsEMvSoiItEiJgaGfR9iE2HZ1KqXF18Ec/b7d4DXgH8EJrUA3gxhJhERCUeJqdBuJCx9R8fWw1QwW+rfA4YDeQDOuRVA41CGEhGRMNXlXNizDrYu8juJlCOYon7IOVd4+ImZxeH1KCciInVNp3MAg2m/hdU5UBRV43tFvGCGI/vYzH4GJJvZmcB3gbdCG0tERMJSWhMYdBPMeQaWvQNxSdBqKLQfBe1GQZMe3vF38UUwRf2nwI3AQuBmYCrwdChDiYhIGDv3ITj9Xlg3A1ZNg9XT4L/3evNSMr3j7oeLfHoLX6PWNZUWdTOLBRY557oAT9VOJBERCXuJqdDpbO8GkLfJ2x2/Oscr9F+/5k3P6PhNgW9zCiTV9ytxnVBpUXfOlZjZMjNr5ZxbX1uhREQkwtRvDn2+7d2cg22Lv9mK//IF+OJJsFjIHvhNkW/RD2Lj/U4eVYLZ/d4QWGRmXwD7D090zo0NWSoREYlcZtCku3cbdhsUH4INnweKfA7k/B5yfgcJad6W/pm/gvRsv1NHhWCK+v+FPIWIiESvuERoe6p345dwYBesme5txS+YDCs+gLMe8MZyN/M7bUSrsKibWRJwC9AB7yS5Z5xzxbUVTEREolRKI+h+oXcb/gOYcju89X1Y9AaM/Ss0aOlzwMhV2XUHk4ABeAX9HODhWkkkIiJ1R6O2cPUUOO9hyJ0NTwyBOc+qx7oTVFlR7+acu9I59w9gPDCiljKJiEhdEhMDA2+EWz+D7AHw9p3w/FjYvdbvZBGnsqJedPiBdruLiEjINWwNV70JF/wZNn4FTwzzdslL0Co7Ua63meUFHhtej3J5gcfOOaeLDUVEpGaZQf9rocMZ8O8b4bXroXA/9L3S72QRocKi7pyLrc0gIiIiR6Rnw5WvwytXwn++B4UHYPBNfqcKe+qgV0REwlNCClz+MnQ5H979MXzyiN+Jwp6KuoiIhK+4RLh0IvS8FD78FXx4v86Mr0Qwnc+IiIj4JzYeLvoHxKfAJ3/0+p0/5U6/U4UlbamLiEj4i4n1zopvNxJm6zr2iqioi4hIZDCD7hfD3vWw9Wu/04QlFXUREYkcnc8BDJZO9TtJWFJRFxGRyJHa2Ot1bpmKenlU1EVEJLJ0Phc2z4O8TX4nCTsq6iIiElnanubd587xN0cYUlEXEZHI0qQ7WKy3tS5HUVEXEZHIEp8EjbvC5vl+Jwk7KuoiIhJ5mvWBTfN0vfox1KOciIhEnma9Yd6L8JumgDGitAQ+DYxDltYELp0Ezfv4mdAXKuoiIhJ5eo6H/dug+BAAGzdsoFXLlt68RW/CpAvg269A62H+ZfSBirqIiESelEYw+hdHnq7OyaHVyJHek8E3w/MXwgsXw4QXoeMZvkT0g46pi4hIdEnPhuvfg8yO8PJlsOgNvxPVGhV1ERGJPvUy4dq3vd7nXrsevnze70S1QkVdRESiU1I6XPk6tD8dptwOC171O1HIqaiLiEj0SkiBy/4JrYbBW3fAjpV+JwopFXUREYlucQlwydMQlwivXgtFBX4nChkVdRERiX7pLeCif8DWhfD+z/xOEzIq6iIiUjd0OguGfR/mPBO1Z8SrqIuISN1x+r2QPRDe/B7M+2fUdTMb0qJuZmPMbJmZrTSzu8uZf5eZLTazBWb2oZm1DmUeERGp42LjvQ5pmveFN2/1Lnc7uMfvVDUmZEXdzGKBx4FzgG7A5WbW7ZjFvgIGOOd6Aa8BD4Uqj4iICABpTeGaKd5W+5Ip8PdTYN1nfqeqEaHcUh8ErHTOrXbOFQL/AsaVXcA5N805dyDwdBaQHcI8IiIinphYGPFDuP4DiImDiefBR7+BkmK/k52UUBb1FsCGMs9zA9MqcgPwbgjziIiIHC27P9zyCfS+HKY/BM+NgV1r/E51wsyF6CQBMxsPjHHO3Rh4fhUw2Dl3WznLXgncBpzmnDtUzvybgJsAsrKy+k+ePDkkmWtLfn4+qampfsc4KZHehkjPD2pDuFAbwkNNtCFr2yd0XvY3SmPi+GLQExTH196/SWX5R40aNdc5NyCoFTnnQnIDhgLvl3l+D3BPOcudASwBGgez3k6dOrlIN23aNL8jnLRIb0Ok53dObQgXakN4qLE2bJrn3H0NnHv7hzWzviBVlh+Y44KsvaHc/T4b6Ghmbc0sAbgMmFJ2ATPrC/wDGOuc2xbCLCIiIlVr1hsG3uhdy755gd9pqi1kRd05V4y3S/19vC3xyc65RWb2azMbG1jsD0Aq8KqZzTOzKRWsTkREpHaM+hkkN4KpP46469jjQrly59xUYOox0+4t87jujFwvIiKRIbkhnHEfTLkNFrwCvS/zO1HQ1KOciIjIsfpc4XVQ88kjfiepFhV1ERGRY8XEQMvBsG+L30mqRUVdRESkPEkN4NBeKC3xO0nQVNRFRETKk9zAuy/Y62uM6lBRFxERKU9yQ+/+4G5/c1SDirqIiEh56mV69xHUbayKuoiISHlaD4ekdO+ytgihoi4iIlKe+GToeak3PGuEjLmuoi4iIlKRPldAcQF8/W+/kwRFRV1ERKQizftC4+7w1Yt+JwmKirqIiEhFzKD3BNj0JeRt8jtNlVTURUREKtOst3e/c6W/OYKgoi4iIlKZRu29exV1ERGRCFe/BcQlwc5Vfiepkoq6iIhIZWJioFE72LXa7yRVUlEXERGpioq6iIhIlEhvCXtzwTm/k1RKRV1ERKQqDVpCYX7YD+6ioi4iIlKV9Gzvfm+uvzmqoKIuIiJSlfSW3r2KuoiISIRr2Ma737bI1xhVUVEXERGpSkojaNYHVn7kd5JKqaiLiIgEo8MZsOHzsB6GVUVdREQkGB3OAFcCaz72O0mFVNRFRESCkT0QEtNh5f/8TlIhFXUREZFgxMZBm1Ng7ad+J6mQirqIiEiwWvT1uost2Ot3knLF+R2gJhQVFZGbm0tBQYHfUYKSnp7OkiVL/I5xRFJSEtnZ2cTHx/sdRUQkvDXr691vXgBtR/ibpRxRUdRzc3NJS0ujTZs2mJnfcaq0b98+0tLS/I4BgHOOnTt3kpubS9u2bf2OIyIS3pr19u43zw/Loh4Vu98LCgrIyMiIiIIebsyMjIyMiNnLISLiq9Qsb3z1zfP8TlKuqCjqgAr6SdC/nYhINbQa4p0BX3TQ7yTHiZqiHo3mzJnD97///Qrnb9q0ifHjx9diIhERod813mhti97wO8lxouKYeqQoKSkhNjY26OUHDBjAgAEDKpzfvHlzXnvttZqIJiIiwWp7KmR2hi+egj7f9jvNUbSlXkPWrl1Lly5duOKKK+jatSvjx4/nwIEDtGnThp/+9Kf069ePV199lQ8++IDTTz+dfv36cemll5Kfnw/A7NmzGTZsGL1792bQoEHs27ePnJwczj//fAA+/vhj+vTpQ58+fejbty/79u1j7dq19OjRA/DOK7juuuvo2bMnffv2Zdq0aQBMnDiRiy++mDFjxtCxY0d+8pOf+PMPJCISLcxg4I2w6UvYONfvNEeJyi31Cf+Yedy083s146qhbThYWMK1z31x3Pzx/bO5dEBLdu0v5NYXj/6QXrl5aFDvu2zZMp555hmGDx/O9ddfzxNPPAFARkYGX375JTt27ODiiy9mypQpNG3alAcffJBHHnmEu+++mwkTJvDKK68wcOBA8vLySE5OPmrdf/zjH3n88ccZPnw4+fn5JCUlHTX/8ccfx8xYuHAhS5cu5ayzzmL58uUAzJs3j6+++orExEQ6d+7M7bffTsuWLYNqk4iIlKP3ZfDhr+CLp+Gi/n6nOUJb6jWoZcuWDB8+HIArr7ySTz/1eh2aMGECALNmzWLx4sWcddZZ9OnTh0mTJrFu3TqWLVtGs2bNGDhwIAD169cnLu7ov7eGDx/OXXfdxV/+8hf27Nlz3PxPP/2UK6+8EoAuXbrQunXrI0X99NNPJz09naSkJLp168a6detC948gIlIXJNWHXt/yjquHUUc0UbmlXtmWdXJCbKXzG9VLCHrL/FjHnkV++Hm9evUA75rwM888kyeffPKo69QXLlxY5brvvvtuzjvvPKZOncrw4cN5//33j9tar0hiYuKRx7GxsRQXFwf1OhERqUTfq2DOs/D1v2HA9X6nAbSlXqPWr1/PzJnerv9//vOfnHLKKUfNHzJkCDNmzGDVqlUA7N+/n+XLl9O5c2c2b97M7NmzAa9zmmML76pVq+jZsyc//elPGThwIEuXLj1q/ogRI3jppZcAWL58OevXr6dz584haaeIiADN+0Lj7vDlC34nOUJFvQZ17tyZxx9/nK5du7J7925uvfXWo+ZnZWUxceJErr/+enr16sXQoUNZunQpCQkJvPLKK9x+++307t2bM88887jOYB599FF69OhBr169iI+P55xzzjlq/ne/+11KS0vp2bMnEyZMYOLEiUdtoYuISA0zg35XeSfMbV3kdxogSne/+yUuLo4XX3zxqGlr16496vno0aP5+OOPj+smduDAgcyaNeuoaSNHjmTkyJEA/PWvfz3u/dq0acPXX38NeP23P/fcc8ctc+2113Lttdceef72228H2xwREalKz2/BB/8HX70IY37ndxptqYuIiJywehnQ5TyY/y8oLvQ7jYp6TSm71SwiInVI36vg4C5YNtXvJCrqIiIiJ6X9KG+Ql69erHrZEFNRFxERORkxsV53sas+hL0b/Y3i67uLiIhEgz5XgCuF+f/0NYaKuoiIyMlq1BZaDoal7/gaQ0U9jE2cOJHbbrsNgPvuu48//vGPPicSEZEKdTgTNn0F+dt9i6CiHgLOOUpLS/2OISIitanjGd79qo98i6CiXkPWrl1L586dufrqq+nRowf3338/AwcOpFevXvzyl788stzzzz/P0KFD6d27N1dddRUAb731FoMHD6Zv376cccYZbN261a9miIjIiWraG1IyYeX/fIsQfT3KvXs3bKl6gJRqadoTzvl9lYutWLGCSZMmkZeXx2uvvcYXX3yBc46xY8cyffp0MjIyeOCBB/jggw9o06YNu3btAuCUU05h1qxZmBlPP/00Dz30EA8//HDNtkFEREIrJgY6nO4V9dJS73kti76i7qPWrVszZMgQfvSjH/HBBx/Qt29fAPLz81mxYgXz58/n0ksvJSMjA4BGjRoBkJuby4QJE9i8eTOFhYW0bdvWtzaIiMhJ6HI+HMqHgj2Q0qjW3z6kRd3MxgB/BmKBp51zvz9mfiLwPNAf2AlMcM6tPak3DWKLOlTKDrF6zz33cPPNNx81v7z+2wFuv/127rrrLsaOHUtOTg733XdfqKOKiEgodBvr3XwSsn0DZhYLPA6cA3QDLjezbscsdgOw2znXAfgT8GCo8tSms88+m2effZb8/HwANm7cyLZt2xg9ejSvvvoqO3fuBDiy+33v3r20aNECgEmTJvkTWkREIl4ot9QHASudc6sBzOxfwDhgcZllxgH3BR6/BjxmZuaccyHMFXJnnXUWS5YsYejQoQCkpqby4osv0r17d37+859z7rnnEh8fT9++fZk4cSL33Xcfl156KQ0bNmT06NGsWbPG5xaIiEgkCmVRbwFsKPM8Fxhc0TLOuWIz2wtkADtCmCskjh3Q5Y477uCOO+44brlrrrmGiy+++KihV8eNG8e4ceOOW7bssKnaJS8iIlWxUG0Um9l4YIxz7sbA86uAwc6528os83VgmdzA81WBZXYcs66bgJsAsrKy+k+ePPmo90pPT6dDhw4haUcolJSUEBsb63eMo6xcuZK9e/cGvXx+fj6pqakhTBRakZ4f1IZwoTaEh0hvQ2X5R40aNdc5NyCY9YRyS30j0LLM8+zAtPKWyTWzOCAd74S5ozjnngSeBOjcubMbOXLkUfOXLFly1JZvuNu3b1/Y5U1KSjpytn4wcnJyOPZziCSRnh/UhnChNoSHSG9DTeUP5UV0s4GOZtbWzBKAy4ApxywzBbgm8Hg88FGkH08XERHxS8i21APHyG8D3se7pO1Z59wiM/s1MMc5NwV4BnjBzFYCu/AK/4m+H2ZWE9HrHP0dJSISHUJ6nbpzbiow9Zhp95Z5XABcerLvk5SUxM6dO8nIyFBhrybnHDt37iQpKcnvKCIicpKioke57OxscnNz2b7dv5FxqqOgoCCsimhSUhLZ2dl+xxARkZMUFUU9Pj4+orpWzcnJqdZJaSIiIsHQKG0iIiJRQkVdREQkSqioi4iIRImQ9SgXKma2D1jmd46TlEkEdoV7jEhvQ6TnB7UhXKgN4SHS21BZ/tbOuaxgVhKJJ8otC7a7vHBlZnPUBn9Fen5QG8KF2hAeIr0NNZVfu99FRESihIq6iIhIlIjEov6k3wFqgNrgv0jPD2pDuFAbwkOkt6FG8kfciXIiIiJSvkjcUhcREZFyhFVRN7MxZrbMzFaa2d3lzE80s1cC8z83szaB6VeY2bwyt1Iz6xOG+U81sy/NrNjMxpeZPuqY/AVmdmGthv8mS1VtuMvMFpvZAjP70MxaB6ZHUhtuMbOFgZyfmlm3wPSw+B4FslTahjLLXWJmzswGBJ5HTBvM7Foz214m642B6WHxXQrmMzCzbwV+D4vM7J+BaWGRP5Clqs/gT2VyLjezPYHpkdSGVmY2zcy+Cvy/dG5geiT9FloH/j9dYGY5ZpYdmF79z8E5FxY3vOFZVwHtgARgPtDtmGW+C/w98Pgy4JVy1tMTWBWm+dsAvYDngfEVrKcR3jC0KWHahlGHswG3VvAZhHsb6pd5PBZ4L1y+R8G2IbBcGjAdmAUMiLQ2ANcCj1WxHl++S0Hm7wh8BTQMPG8cLvmr8z0qs/zteENkR1Qb8I5F3xp43A1YW856wv238CpwTeDxaOCFE/0cwmlLfRCw0jm32jlXCPwLGHfMMuOASYHHrwGnmx031urlgdfWtirzO+fWOucWAKWVrGc88K5z7kDoolYomDZMK5NtFlDe8G7h3oa8Mk/rAeWdWOLX9wiC+y0A3A88CBRUsJ5IaENV/PouBZP/O8DjzrndAM65beWsJ6x/C8e4HHi5nOnh3gYH1A88Tgc2lbOecP8tdAM+CjyeVs58CPJzCKei3gLYUOZ5bmBaucs454qBvUDGMctMoPwvZqgFkz8Yl+FPfqh+G24A3i1neti3wcy+Z2argIeA75ezHr++RxBEG8ysH9DSOfdOJesJ6zYEXBLY5fiambUsZ75f36Vg8ncCOpnZDDObZWZjyllP2P8WwNv9C7Tlm8JSVri34T7gSjPLBabi7XE4Vrj/FuYDFwceXwSkmdmxtS2ozyGcivpJM7PBwAHn3Nd+ZzkRZtYMbzfR+35nqYqZXQkMAP5wzPSIaINz7nHnXHvgp8Avys4L9++RmcUAjwA/rGSZsG5DwFtAG+dcL+C/fLMXDoiI71Ic3i74kXhbgk+ZWYPDMyMgf1mXAa8550rKToyQNlwOTHTOZQPnAi8EfiNAxPwWfgScZmZfAacBG4Ejn0V1PodwKuobgbJ/qWcHppW7jJnF4e1q2Vlmvp9/UQaTvyrfAt5wzhXVWKrqCaoNZnYG8HNgrHPu0DGzI6INZfwLuPCYaX5+j6DqNqQBPYAcM1sLDAGmHD5ZLiDc24BzbmeZ78/TQP9j1uHndymY71EuMMU5V+ScWwMsxyvyh0XSb6Gi70sktOEGYDKAc24mkITXj/phkfBb2OScu9g51xfv/1acc3vKLBL85+DHiQMVnEwQB6zG2wV0+GSC7scs8z2OPlFucpl5MYF/qHbhmr/MshMp50Q5vGPUo8L8M+iLd9JHxwrWEQlt6Fjm8QXAnHD5HlX3uxRYPocyJ8pFShuAZmUeXwTMCpfvUpD5xwCTAo8z8XaxZoRD/up8j4AuwFoC/ZaEy2dQjc/hXeDawOOueMfUD/fBEim/hUwgJvD4N8CvT/Rz8KWRlTT+XLy/dlcBPw9M+zXeFiF4f4G9CqwEvij7QeHtAptV25mrmX8g3l/3+/H2MCwq89o2gS9fTJi34X/AVmBe4DYlAtvwZ2BRIP+0sj+wcPgeBdOGY5bN4eiiHhFtAH4X+BzmBz6HLuH0XQoiv+EdBlkMLAQuC6f8wX6P8I5J/76c10ZEG/BOMpsR+B7NA84q89pI+S2MB1YElnkaSDzRz0E9yomIiESJcDqmLiIiIidBRV1ERCRKqKiLiIhECRV1ERGRKKGiLiIiEiVU1EXChJlllBmNaYuZbQw83mNmi0PwfveZ2Y+q+Zr8CqZPtDIjD55EphpZj0hdpaIuEiac18NaH+dcH+DvwJ8Cj/tQ+SBAwJFeFkWkDlNRF4kMsWb2VGDc7g/MLBkgMPbyo2Y2B7jDzPqb2cdmNtfM3g/0GY2ZfT8w7vcCMys7WlW3wDpWm9mRgW3M7C4z+zpw+8GxYczzWGCM6P8BjctZpouZfVHmeRszWxh4fK+ZzQ6s/8lyRlvEzNaaWWbg8QAzywk8rmdmz5rZF4ExtE9k9DeRqKSiLhIZOuIN89kd2ANcUmZegnNuAPAX4K94XRD3B57F63IS4G6gr/MGT7mlzGu7AGfjDQ/5SzOLN7P+wHXAYLx+5b9jZn2PyXMR0BmvN6+rgWHHBnbOLQUSzKxtYNIE4JXA48eccwOdcz2AZOD8avxb/Bz4yDk3CBgF/MHM6lXj9SJRS0VdJDKscc7NCzyei9d15GGHC2VnvIFe/mtm8/BGnzs83v0C4KXA6HrFZV77jnPukHNuB7ANaAKcgjd4xH7nXD7wOjDimDynAi8750qcc5sof8hO8AbamBB4XLaojzKzzwNb7qOB7lW0v6yzgLsDbczB6z66VTVeLxK1dAxOJDKUHQ2vBG/r9rD9gXvDG09gaDmvPw+vEF8A/NzMelaw3pr+P+EV4FUzex1wzrkVZpYEPIHXX/0GM7sPrzAfq5hvNjzKzjfgEufcshrOKhLxtKUuEj2WAVlmNhQgsCu9e2Bs6ZbOuWl448enA6mVrOcT4EIzSwns1r4oMK2s6cAEM4sNHLcfVd6KnHOr8P5Y+D++2Uo/XKB3mFkq3mAW5VnLN8Oxlj3c8D5w++Hj8OUcGhCps7SlLhIlnHOFgcvB/mJm6Xi/70fxRn56MTDNgL845/aUc27a4fV8aWYT8UZCBHjaOffVMYu9gbfbfDGwHphZSbRXgD/gDT1J4L2fAr4GtgCzK3jdr4BnzOx+vN3sh90faNeCwB8sa6jeMXmRqKVR2kRERKKEdr+LiIhECRV1ERGRKKGiLiIiEiVU1EVERKKEirqIiEiUUFEXERGJEirqIiIiUUJFXUREJEr8PzncG9Mvqmf1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "%matplotlib inline\n",
        "\n",
        "def precision_recall_curve_plot(y_test , pred_proba_c1):\n",
        "    # threshold ndarray와 이 threshold에 따른 정밀도, 재현율 ndarray 추출.\n",
        "    precisions, recalls, thresholds = precision_recall_curve( y_test, pred_proba_c1)\n",
        "\n",
        "    # X축을 threshold값으로, Y축은 정밀도, 재현율 값으로 각각 Plot 수행. 정밀도는 점선으로 표시\n",
        "    plt.figure(figsize=(8,6))\n",
        "    threshold_boundary = thresholds.shape[0]\n",
        "    plt.plot(thresholds, precisions[0:threshold_boundary], linestyle='--', label='precision')\n",
        "    plt.plot(thresholds, recalls[0:threshold_boundary],label='recall')\n",
        "\n",
        "    # threshold 값 X 축의 Scale을 0.1 단위로 변경\n",
        "    start, end = plt.xlim()\n",
        "    plt.xticks(np.round(np.arange(start, end, 0.1),2))\n",
        "\n",
        "    # x축, y축 label과 legend, 그리고 grid 설정\n",
        "    plt.xlabel('Threshold value'); plt.ylabel('Precision and Recall value')\n",
        "    plt.legend(); plt.grid()\n",
        "    plt.show()\n",
        "\n",
        "precision_recall_curve_plot( y_test, lr_clf.predict_proba(X_test)[:, 1] )\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWuBcHZW1GdY"
      },
      "source": [
        "### 3.4 F1 Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xFMHgtoB1GdY",
        "outputId": "d07fef2b-7efe-4e78-a414-d00f035e0ff2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "F1 스코어: 0.7966\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1 = f1_score(y_test , pred)\n",
        "print('F1 스코어: {0:.4f}'.format(f1))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "33GQCkbV1GdY",
        "outputId": "cf477fc1-84b7-4fa9-cb48-a974650ab3b1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "임곗값: 0.4\n",
            "오차 행렬\n",
            "[[97 21]\n",
            " [11 50]]\n",
            "정확도: 0.8212, 정밀도: 0.7042, 재현율: 0.8197, F1:0.7576\n",
            "임곗값: 0.45\n",
            "오차 행렬\n",
            "[[105  13]\n",
            " [ 13  48]]\n",
            "정확도: 0.8547, 정밀도: 0.7869, 재현율: 0.7869, F1:0.7869\n",
            "임곗값: 0.5\n",
            "오차 행렬\n",
            "[[108  10]\n",
            " [ 14  47]]\n",
            "정확도: 0.8659, 정밀도: 0.8246, 재현율: 0.7705, F1:0.7966\n",
            "임곗값: 0.55\n",
            "오차 행렬\n",
            "[[111   7]\n",
            " [ 16  45]]\n",
            "정확도: 0.8715, 정밀도: 0.8654, 재현율: 0.7377, F1:0.7965\n",
            "임곗값: 0.6\n",
            "오차 행렬\n",
            "[[113   5]\n",
            " [ 17  44]]\n",
            "정확도: 0.8771, 정밀도: 0.8980, 재현율: 0.7213, F1:0.8000\n"
          ]
        }
      ],
      "source": [
        "def get_clf_eval(y_test , pred):\n",
        "    confusion = confusion_matrix( y_test, pred)\n",
        "    accuracy = accuracy_score(y_test , pred)\n",
        "    precision = precision_score(y_test , pred)\n",
        "    recall = recall_score(y_test , pred)\n",
        "    # F1 스코어 추가\n",
        "    f1 = f1_score(y_test,pred)\n",
        "    print('오차 행렬')\n",
        "    print(confusion)\n",
        "    # f1 score print 추가\n",
        "    print('정확도: {0:.4f}, 정밀도: {1:.4f}, 재현율: {2:.4f}, F1:{3:.4f}'.format(accuracy, precision, recall, f1))\n",
        "\n",
        "thresholds = [0.4 , 0.45 , 0.50 , 0.55 , 0.60]\n",
        "pred_proba = lr_clf.predict_proba(X_test)\n",
        "get_eval_by_threshold(y_test, pred_proba[:,1].reshape(-1,1), thresholds)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ar9u-vm1GdZ"
      },
      "source": [
        "## 3-5 ROC Curve와 AUC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JUf9SvcE1GdZ",
        "outputId": "db2387c8-fa1c-475f-afa9-4b5da9fbe6e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "샘플 추출을 위한 임곗값 배열의 index: [ 1  6 11 16 21 26 31 36 41 46]\n",
            "샘플 index로 추출한 임곗값:  [0.94 0.73 0.62 0.52 0.44 0.28 0.15 0.14 0.13 0.12]\n",
            "샘플 임곗값별 FPR:  [0.    0.008 0.025 0.076 0.127 0.254 0.576 0.61  0.746 0.847]\n",
            "샘플 임곗값별 TPR:  [0.016 0.492 0.705 0.738 0.803 0.885 0.902 0.951 0.967 1.   ]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import roc_curve\n",
        "\n",
        "# 레이블 값이 1일때의 예측 확률을 추출\n",
        "pred_proba_class1 = lr_clf.predict_proba(X_test)[:, 1]\n",
        "\n",
        "fprs , tprs , thresholds = roc_curve(y_test, pred_proba_class1)\n",
        "# 반환된 임곗값 배열에서 샘플로 데이터를 추출하되, 임곗값을 5 Step으로 추출.\n",
        "# thresholds[0]은 max(예측확률)+1로 임의 설정됨. 이를 제외하기 위해 np.arange는 1부터 시작\n",
        "thr_index = np.arange(1, thresholds.shape[0], 5)\n",
        "print('샘플 추출을 위한 임곗값 배열의 index:', thr_index)\n",
        "print('샘플 index로 추출한 임곗값: ', np.round(thresholds[thr_index], 2))\n",
        "\n",
        "# 5 step 단위로 추출된 임계값에 따른 FPR, TPR 값\n",
        "print('샘플 임곗값별 FPR: ', np.round(fprs[thr_index], 3))\n",
        "print('샘플 임곗값별 TPR: ', np.round(tprs[thr_index], 3))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YMOZLqB21Gda",
        "outputId": "953307f3-0375-407e-8816-83bb06c51138"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzCElEQVR4nO3dd3gUhdbH8e9JKKEIKqIiXaQkdIgiIE0Q6aggVUioFlCugoWrryjXa7t2pYggeLGgWChXEK9Klw4BkiAttKAoIEVKICHn/WOXuOamkt3MbvZ8nicPuzOzM7/sDjk77YyoKsYYY4JXiNMBjDHGOMsKgTHGBDkrBMYYE+SsEBhjTJCzQmCMMUHOCoExxgQ5nxUCEXlfRH4TkdhMxouIvCUiu0Rki4g08lUWY4wxmfPlFsEMoEMW4zsC1d0/w4FJPsxijDEmEz4rBKq6DPg9i0m6A/9Wl9XA5SJSzld5jDHGZKyQg8suDxzweJ7oHvZL+glFZDiurQZKlCjRuFatWvkS0BgTvOJ/OcmF1MDvvJBy4jdSz52G1AtHVLVsRtM4WQhyTFWnAFMAIiMjdf369Q4nMsYUdHXGLaJLvXKMalfd6Si5drF1kIjwwbQpHDl8mFdf/Oe+zKZ3shAcBCp6PK/gHmaMMX6hRNFClCtdzOkYuXLw4EHuv/9+evfuTf/+/XnikVEAvPriPzN9jZOFYB4wUkRmAU2AE6r6P7uFjDHG29bt/Z1xc+Oy3PVz+nxKPibKO1Vl6tSpjBkzhuTkZDp37pzj1/qsEIjIJ0Br4CoRSQTGAYUBVHUysADoBOwCzgCDfJXFGGM8bdx3jPhfTtIu/BoKhUiG01S7ugSd6wXG+Su7d+9m2LBhLF68mDZt2vDee+9RrVq1HL/eZ4VAVftmM16BEb5avjHGZOetvg0oXiQgDpVmaevWrWzYsIEpU6YwdOhQRDIubpkJ/HfAGJOtpOQLbNx/DLv9iMveo6edjpBnsbGxbNy4kYEDB3LHHXeQkJBAmTJlLmleVgiMKeCSL6Qy8P21rN2T1WU9wadIoRAKhQRel53z58/z/PPP8/zzz3PNNdfQq1cvwsLCLrkIgBUCYwq8f369jbV7fufpLhHUKV/a6Th+4+rLilKkUGAVgjVr1jBkyBDi4uK45557eP311wkLC8vzfK0QGFOAfbkxkRk/7mXoLVUZfEtVp+OYPDh48CAtWrTgmmuu4T//+U+uzgrKjhUCYwqQP5KS+b85sZw65zr1cfnOIzSrVoYnOtrV+IFqx44d1KhRg/Lly/Ppp5/Stm1bSpUq5dVlBNZ2kTEmSzt+/YM5MT+z87dT/HIiiWbVyvB234YUCrX/6oHm+PHjDB8+nFq1arFs2TIA7rzzTq8XAbAtAmMKpH90r0PLGhm2lTEBYN68edx///0cOnSIRx99lBtvvNGny7NCYIwxfmTo0KFMmzaNunXrMnfuXCIjI32+TCsExhjjMM8mcZGRkVSuXJnHH3+cIkWK5MvyrRAYY4yDDhw4wH333UefPn0YMGAA9913X75nsCNIxhjjgNTUVCZNmkTt2rVZsmQJ586dcyyLbREYUwC8tyyBg8fP8tsfSU5HMTmwc+dOhg4dyrJly2jXrh1TpkyhalXnrvOwQmBMgDtxNpl/LthG0UIhFC0UQrnSYVS6srjTsUwW4uPj2bJlC++//z7R0dG5bhLnbVYIjAl07kZyj3eoZVcP+7HNmzcTExNDVFQU3bt3JyEhgSuuuMLpWIAVAsOfZyyYwKTY5+fPzp07x3PPPceLL75IuXLl6N27N2FhYX5TBMAKQdD7aM0+nvwq1ukYxgtCM7nBinHOqlWrGDJkCNu2bWPgwIG89tprXmkS521WCIJcwuHTFAkN4YE2Ob+bkfE/hUNDAuZuWsHi4MGDtGrVimuvvZYFCxbQsWNHpyNlygqBoUihEP7WrobTMYwpELZt20Z4eDjly5fns88+o23btlx22WVOx8qSFYIAdvD4WRbFHsrTHuLYgye8lseYYHbs2DFGjx7N9OnTWbZsGS1atOCOO+5wOlaOWCEIYFOXJzB95d48z6fmNf79bcUYf/fVV1/xwAMPcPjwYcaOHevzJnHeZoUggKVcUC4vXpilj7bJ03yKFwn1UiJjgs/gwYOZPn06DRo04Ouvv6ZRo0ZOR8o1KwQBLkSE0sUKOx3DmKDi2STu5ptvpnr16owZM4bChQPz/6IVAmOMyYV9+/Zx77330q9fPwYOHMjw4cOdjpRn1nTOGGNyIDU1lQkTJlCnTh1WrFhBcnKy05G8xrYIjDEmG9u3b2fo0KGsWLGC9u3b8+6771KlShWnY3mNFQJjjMnG9u3biYuLY8aMGQwcONDxJnHeZoXAGGMysGnTJmJiYhg0aBDdunUjISGByy+/3OlYPmHHCIwxxkNSUhJ///vfufHGG3nmmWdISnLd46GgFgGwQmCMMWlWrlxJgwYNeOGFFxg4cCAxMTF+2STO22zXkDHG4GoS16ZNG8qXL8+iRYto376905HyjW0RGGOCWnx8PADly5fniy++YOvWrUFVBMAKgTEmSP3+++9ER0dTu3Ztli1bBkDXrl0pWbKkw8nyn+0a8jPHTp/n0Mmc3YD89zPnfZzGmILpiy++YMSIERw9epQnn3ySm266yelIjrJC4Ge6T1jJ/t/P5Hj660oX/ANZxnhTdHQ0H3zwAY0aNeKbb76hQYMGTkdynBUCP3PibDKta5alz40VczT99WWDbzPWmNzybBLXrFkzwsPDGT16NIUK2Z9A8HEhEJEOwJtAKDBVVV9MN74S8AFwuXuaJ1R1gS8zBYIqZUrQoY7ddtAYb9izZw/Dhw/nnnvuISoqqkA0ifM2nxUCEQkFJgC3AYnAOhGZp6rxHpM9BXymqpNEJAJYAFTxVab8kpqqvPLtdo6cOpfr1545n+KDRMYEnwsXLjBhwgTGjh1LSEgI/fv3dzqS3/LlFsFNwC5VTQAQkVlAd8CzEChQyv24NPCzD/Pkm19OJjFxyW5KhRWiRNHcvcVlSxalYaXLfRPMmCCxbds2hgwZwqpVq+jYsSOTJ0+mUqVKTsfyW74sBOWBAx7PE4Em6aZ5BvhWRB4ESgDtMpqRiAwHhgMB9WE+1TmCXjnc12+M8Z5du3axfft2Zs6cSf/+/Qtckzhvc/pISV9ghqq+KiJNgZkiUkdVUz0nUtUpwBSAyMjIvNyr3StUldPnL2Q6/sw5271jTH7bsGEDmzdvZvDgwXTt2pU9e/ZQqlSp7F9ofFoIDgKeX4cruId5GgJ0AFDVVSISBlwF/ObDXHn2xnc7efP7ndlOFxpi30KM8bWzZ8/y7LPP8sorr1CxYkX69etHWFiYFYFc8GUhWAdUF5GquApAH6Bfumn2A22BGSISDoQBh32YySsOHDtD6WKFGdnmhkynKVIohPa1r8nHVMYEn2XLljF06FB27tzJkCFDeOWVV4KiSZy3+awQqGqKiIwEFuE6NfR9VY0TkfHAelWdB4wG3hORh3EdOI7Wiyf8+rnLwgoxrOX1TscwJmgdPHiQtm3bUrFiRb777jvatm3rdKSA5dNjBO5rAhakG/a0x+N4oLkvM+SWqvLx2v38firz9g3bfvkjHxMZYzxt3bqVunXrUr58eb766ivatGlDiRIlnI4V0Jw+WOx3Eo+d5cmvYrOdrmWNsvmQxhhz0ZEjR3j44Yf58MMPWbp0KS1btqRLly5OxyoQrBCkcyHVtWfqlbvrc0eD6zKdzg4EG5M/VJXZs2czcuRIjh07xrhx42jSJP2Z6CYvrBBkIjQECoVal25jnBYVFcXMmTOJjIzk+++/p27duk5HKnCsEBhj/I5nk7hWrVpRr149/va3v1mTOB+xr7zGGL+SkJBAu3btmDFjBgBDhgxhzJgxVgR8yAqBMcYvXLhwgTfeeIO6deuybt06QkLsz1N+CeoSu+u3P4g9ePIvww7/kfuOocaYvImPj2fw4MGsWbOGzp07M3nyZCpUqOB0rKAR1IVgzOwtxBw4nuG4K4oXyd8wxgSxPXv2sHv3bj7++GP69OljTeLyWVAXgnMpqTS/oQzP3fHXsxCKFgrhusuLOZTKmOCwbt06YmJiGDZsGJ07dyYhIYHLLrvM6VhBKagLAUDxIoWoepVdlWhMfjlz5gxPP/00r7/+OpUrV2bAgAGEhYVZEXBQUBSCtXt+5/kF20hN18Yo4fApKlxh3/yNyS9Llixh6NCh7N69m3vvvZeXXnrJmsT5gaAoBKsTjhJz4Dita5bFc89js2pluKthecdyGRNMEhMTue2226hcuTI//PADbdq0cTqScQuKQnDRtKgbrTWEMfls8+bN1K9fnwoVKjB37lxat25N8eLFnY5lPBS4QrDrt1OcOPvXzqE/Hz/rUBpjgtfhw4cZNWoUn3zyCUuWLKFVq1Z06tTJ6VgmAwWqEPx8/CztXlua4biwwiHYtoAxvqeqzJo1i4ceeogTJ07w7LPP0rRpU6djmSwUqEJw2n2v4BFtqtGkapm/jLvu8jBCbLeQMT43YMAAPvroI5o0acK0adOoXbu205FMNgpUIbgovFwpu1+AMfkoNTUVEUFEaNOmDY0bN+ahhx4iNDTU6WgmB6yZhzEmT3bt2kXbtm2ZPn064GoS9/DDD1sRCCBWCIwxlyQlJYVXXnmFunXrsmnTJooUsbYsgapA7hoyxvhWbGwsgwYNYv369XTv3p2JEydy3XWZ39HP+DcrBMaYXNu/fz/79u1j1qxZ9OrVy5rEBTgrBMaYHFmzZg2bN29m+PDhdOrUiYSEBEqWLOl0LOMFdozAGJOl06dP88gjj9C0aVNefvllzp1z3bPDikDBYYXAGJOpH374gXr16vH6669z3333sXHjRooWLep0LONltmvIGJOhxMREbr/9dqpWrcrSpUtp2bKl05GMj9gWgTHmLzZt2gRAhQoVmD9/Pps3b7YiUMBZITDGAPDrr7/Su3dvGjVqxNKlrp5dHTp0oFgxu2dHQWeFwJggp6p8+OGHREREMGfOHJ577jmaNWvmdCyTjwrEMYLOby1n52+nUPcdyELsnGZjcqxfv37MmjWLpk2bMm3aNMLDw52OZPJZgSgEcT+fJLLyFURWuZKwwiE0v+EqpyMZ49c8m8S1b9+epk2bMmLECOsPFKSyLQQi0hS4B2gBlAPOArHA18CHqnrCpwlzqPkNV/HwbTWcjmGM39uxYwfDhg1j4MCBDBkyhEGDBjkdyTgsy2MEIrIQGAosAjrgKgQRwFNAGDBXRLr5OqQxJu9SUlJ4+eWXqV+/Plu2bLGDwCZNdlsEA1T1SLphp4CN7p9XRcT2wxjj57Zs2cLgwYPZsGEDd955JxMmTKBcuXJOxzJ+IstCkEERuKRpjDHOSkxM5MCBA8yePZsePXpYkzjzFz49fVREOojIdhHZJSJPZDJNLxGJF5E4EfnYl3mMCSY//vgjkydPBkhrEtezZ08rAuZ/+KwQiEgoMAHoiOu4Ql8RiUg3TXVgLNBcVWsDf/NVHmOCxalTpxg1ahS33HILr776alqTuBIlSjiczPgrX24R3ATsUtUEVT0PzAK6p5tmGDBBVY8BqOpvPsxjTIH37bffUqdOHd5++21GjBhhTeJMjmR5jEBEtgKa0ShAVbVeFi8vDxzweJ4INEk3TQ33clYCocAzqvpNBjmGA8MBKlWqlFVkY4LWgQMH6Ny5M9WqVWPZsmXccsstTkcyASK7s4a65MPyqwOtgQrAMhGpq6rHPSdS1SnAFIDIyMiMCpMxQWvDhg00btyYihUrsmDBAlq0aEFYWJjTsUwAyXLXkKruy+onm3kfBCp6PK/gHuYpEZinqsmqugfYgaswGGOycejQIe6++24iIyPTmsTddtttVgRMrmV3QdkfInIyg58/RORkNvNeB1QXkaoiUgToA8xLN80cXFsDuK9HqAEkXMovYkywUFU++OADIiIimD9/Ps8//7w1iTN5kt11BJdd6oxVNUVERuK6KjkUeF9V40RkPLBeVee5x7UXkXjgAvCoqh691GUaEwz69OnDZ599RvPmzZk6dSq1atVyOpIJcLlqOiciV+NqLQGAqu7PanpVXQAsSDfsaY/HCjzi/jHGZMKzSVynTp1o0aIFDzzwACEh1kne5F2O1iIR6SYiO4E9wFJgL7DQh7mMMW4//fQTLVu2ZNq0aQBERUUxcuRIKwLGa3K6Jv0DuBnYoapVgbbAap+lMsaQnJzM888/T/369YmPj6dkyZJORzIFVE53DSWr6lERCRGREFVdLCJv+DKYMcEsJiaGQYMGERMTQ8+ePXn77be59tprnY5lCqicFoLjIlISWAZ8JCK/Aad9F8uY4Hbo0CEOHTrEF198wV133eV0HFPA5bQQdMd1Q5qHgf5AaWC8r0IZE4xWrFjBli1beOCBB+jQoQO7d++mePHiTscyQSCnxwiuBoqoaoqqfgC8B1zyqaXGmD/98ccfjBw5khYtWvDGG2+kNYmzImDyS04LwWwg1eP5BfcwY0weLFq0iDp16jBx4kRGjRplTeKMI3K6a6iQu4MoAKp63n21sDHmEh04cIAuXbpwww03sGLFCrs62Dgmp1sEhz3vTSwi3QG7M5kxuaSqrF27FoCKFSuycOFCNm3aZEXAOCqnheA+4O8ickBE9gOPA/f6LpYxBc8vv/xCjx49aNKkSVqTuHbt2lmTOOO4HO0aUtXdwM3uU0hR1VM+TWVMAaKqzJgxg0ceeYSkpCReeuklmjdv7nQsY9LkqBCIyDXA88B1qtrRfcvJpqo6zafpjCkAevXqxeeff06LFi2YOnUqNWrUcDqSMX+R011DM3B1Cr3O/XwHdn9hYzJ14cIFUlNdJ9p17dqViRMnsmTJEisCxi/ltBBcpaqf4T6FVFVTcJ1CaoxJZ9u2bbRo0SKtSdzAgQO5//77rUmc8Vs5XTNPi0gZ3PcvFpGbgRM+S2VMAEpOTua5556jQYMGbN++ndKlSzsdyZgcyel1BI/gurtYNfeN5ssCPX2WypgAs2nTJqKjo9myZQu9e/fmrbfe4uqrr3Y6ljE5ktOzhjaKSCugJiDAduAmXwYzJpD8+uuvHDlyhDlz5tC9e3en4xiTK1kWAhEJBXoB5YGF7ltNdgGmAMWAhr6PmLGn58by3/hf056LOJXEBKtly5axdetWRowYQYcOHdi1axfFihVzOpYxuZbdFsE0oCKwFnhbRH4GGgNjVXWOj7Nl6cfdRwkNEZpVK0NoiNClXjkn45ggcvLkSZ544gkmTZpEjRo1GDp0KEWLFrUiYAJWdoUgEqinqqkiEgYcAqr5yw3m61e4nJd71nc6hgkiCxYs4N577+Xnn3/mkUceYfz48dYkzgS87ArBeVW9eMpokogk+EsRMCa/HThwgO7du1OzZk0+//xzmjRp4nQkY7wiu0JQS0S2uB8LrrOGtrgfq6rW82k6YxymqqxZs4abb76ZihUr8u2339K8eXOKFLHmu6bgyK4QhOdLCmP80M8//8z999/PvHnzWLJkCa1ataJNmzZOxzLG67IsBKq6L7+CGOMvVJVp06YxZswYzp07xyuvvGJN4kyBltMLyowJGj179uTLL7+kVatWTJ06lRtuuMHpSMb4lBUCY3A1iRMRQkJCuOOOO2jfvj3Dhg2z/kAmKNhaboJebGwszZs3T2sSN2DAAO69914rAiZoZLmmi8h8EekqIoUzGHe9iIwXkcG+i2eM75w/f55nn32WRo0asXv3bq644gqnIxnjiOx2DQ3D1XDuDRH5HTgMhAFVgN3AO6o616cJjfGBDRs2EB0dTWxsLP369eONN96gbNmyTscyxhHZnTV0CHgMeExEqgDlgLPADlU94/t4xvjG0aNHOX78OPPnz6dLly5OxzHGUTk+WKyqe4G9ACISIiL9VfUjH+UyxusWL17M1q1beeihh2jfvj07d+60G8cbQ/bHCEqJyFgReUdE2ovLg0ACrq6kxvi9EydOcO+993LrrbcyadIkzp07B2BFwBi37E6LmInrHgRbgaHAYlw3pLlDVa3puvF78+fPJyIigqlTpzJmzBg2bNhgTeKMSSe7XUPXq2pdABGZCvwCVFLVJJ8nMyaPDhw4QI8ePahVqxZz5szhxhtvdDqSMX4puy2C5IsPVPUCkGhFwPgzVeXHH38ESGsSt379eisCxmQhu0JQX0ROisgfIvIHUM/j+cnsZi4iHURku4jsEpEnspiuh4ioiETm9hcw5qLExES6detG8+bNWbp0KQCtW7e2TqHGZCO700dDL3XG7ttcTgBuAxKBdSIyT1Xj0013GTAKWHOpyzLBLTU1lffee49HH32UlJQUXnvtNW655RanYxkTMLK7Z3EYcB9wA7AFeF9VU3I475uAXaqa4J7XLKA7EJ9uun8ALwGP5iK3MWl69OjBnDlzuPXWW3nvvfe4/vrrnY5kTEDJbtfQB7huV7kV6AS8mot5lwcOeDxPdA9LIyKNgIqq+nVWMxKR4SKyXkTWHz58OBcRTEGVkpJCamoq4CoE7733Ht99950VAWMuQXaFIEJV71HVd3GdNtrCWwsWkRDgNWB0dtOq6hRVjVTVSGsDYLZs2ULTpk157733ALjnnnsYOnQoIuJwMmMCU27OGsrpLqGLDgIVPZ5XcA+76DKgDrBERPYCNwPz7ICxycy5c+cYN24cjRs3Zt++fdYbyBgvye46ggYeZwcJUMz9/OI9i0tl8dp1QHURqYqrAPQB+l0cqaongKsuPheRJcAYVV2f69/CFHjr1q0jOjqa+Ph4BgwYwOuvv06ZMmWcjmVMgZBdIdisqg0vZcaqmiIiI4FFQCiuA81xIjIeWK+q8y5lviY4HTt2jFOnTrFgwQI6duzodBxjCpTsCoHmZeaqugBYkG7Y05lM2zovyzIFzw8//MDWrVsZNWoU7du3Z8eOHdYewhgfyK4QXC0ij2Q2UlVf83IeYzh+/DiPPvooU6dOJTw8nPvuu4+iRYtaETDGR7I7WBwKlMR1YDejH2O8au7cuURERPD+++/z2GOPWZM4Y/JBdlsEv6jq+HxJYoLe/v37ufvuuwkPD2fevHlERtoJZMbkh+y2COzEbONTqsry5csBqFSpEt999x3r1q2zImBMPsquELTNlxQmKO3fv5/OnTvTsmXLtCZxLVu2tCZxxuSzLAuBqv6eX0FM8EhNTWXixInUrl2bZcuW8dZbb1mTOGMclON7FhvjLXfddRdz587ltttuY8qUKVSpUsXpSMYENSsEJl+kpKQQEhJCSEgIvXv3pnv37kRHR1t/IGP8QHbHCIzJs82bN9OkSROmTJkCQN++fRk0aJAVAWP8hBUC4zNJSUk89dRTREZGkpiYyLXXXut0JGNMBmzXkPGJtWvXEhUVxU8//URUVBSvvfYaV155pdOxjDEZsEJgfOLkyZOcPXuWb775httvv93pOMaYLFghMF7z7bffEhcXx8MPP0y7du3Yvn27tYcwJgDYMQKTZ8eOHWPQoEHcfvvtTJs2jXPnzgFYETAmQFghMHny5ZdfEhERwcyZMxk7dizr16+3AmBMgLFdQ+aS7d+/nz59+lCnTh0WLFhAw4aXdA8jY4zDbIvA5IqqpvUFqlSpEj/88ANr1qyxImBMALNCYHJs3759dOzYkdatW6cVg1tuuYXChQs7nMwYkxdWCEy2UlNTeeedd6hduzYrVqzg7bffpkWLFk7HMsZ4iR0jMNm64447mD9/PrfffjvvvvsulStXdjqSMcaLrBCYDCUnJxMaGkpISAh9+/alZ8+eDBgwwPoDGVMA2a4h8z82btzITTfdxOTJkwFXk7iBAwdaETCmgLJCYNKcPXuWsWPHctNNN3Ho0CEqVqzodCRjTD6wXUMGgNWrVxMVFcWOHTsYPHgwr7zyCldccYXTsYwx+cAKgQHg9OnTJCcn89///pd27do5HccYk4+sEASxb775hri4OEaPHk3btm356aef7MbxxgShgDtGcPT0ef69ai/HzyQ7HSVgHT16lKioKDp27MgHH3zA+fPnAawIGBOkAq4Q/Hz8LE/PjePIqXNcWzrM6TgBRVX5/PPPiYiI4OOPP+app55i3bp1VgCMCXKiqk5nyJVSFWvqnm1bALiyRBE7pTEX9u3bR/Xq1alXrx7Tpk2jfv36TkcyxuQTEdmgqpEZjQu4LQJBKFOyKGVKFrUikAOqyg8//ABA5cqVWbJkCatXr7YiYIxJE3CFwOTcnj17aN++PW3btk1rEtesWTMKFbJzBIwxf7JCUABduHCBN998kzp16rBmzRomTZpkTeKMMZmyr4YFUPfu3fn666/p1KkTkydPtiuEjTFZskJQQHg2iRswYAB9+/alX79+dhzFGJMtn+4aEpEOIrJdRHaJyBMZjH9EROJFZIuIfC8i1t/4Eqxfv57IyEgmTZoEQO/evenfv78VAWNMjvisEIhIKDAB6AhEAH1FJCLdZJuASFWtB3wOvOyrPAXR2bNnefzxx2nSpAmHDx+2+wQYYy6JL7cIbgJ2qWqCqp4HZgHdPSdQ1cWqesb9dDVQwYd5CpRVq1ZRv359Xn75ZQYPHkx8fDxdunRxOpYxJgD58hhBeeCAx/NEoEkW0w8BFmY0QkSGA8MBiper5q18Ae3s2bOkpqby3Xff0bZtW6fjGGMCmF8cLBaRe4BIoFVG41V1CjAFoHTFWoF1KbQXLViwgLi4OB599FFuvfVWtm3bZjeON8bkmS93DR0EPM9brOAe9hci0g54Euimqud8mCdgHTlyhHvuuYfOnTvz0UcfpTWJsyJgjPEGXxaCdUB1EakqIkWAPsA8zwlEpCHwLq4i8JsPswQkVWXWrFmEh4fz2WefMW7cONauXWtN4owxXuWzXUOqmiIiI4FFQCjwvqrGich4YL2qzgP+BZQEZrtPddyvqt18lSnQ7N+/n6ioKOrXr8+0adOoW7eu05GMMQVQwHUfLV2xlp448JPTMXxGVfn+++/T7hK2evVqbrzxRkJDQx1OZowJZFl1H/WLg8XGZffu3QwbNozFixezZMkSWrVqxc033+x0LGMclZycTGJiIklJSU5HCQhhYWFUqFAhV8cQrRD4gYtN4p566ikKFy7Mu+++a03ijHFLTEzksssuo0qVKna1fDZUlaNHj5KYmEjVqlVz/DorBH6ga9euLFy4kC5dujBp0iQqVLDr6oy5KCkpyYpADokIZcqU4fDhw7l6nRUCh5w/f55ChQoREhJCdHQ0AwYMoE+fPrayG5MB+3+Rc5fyXtn9CBywdu1aGjduzMSJEwHo1asXffv2tZXdGOMIKwT56MyZM4wePZqmTZty7NgxqlWzdhnGBILQ0FAaNGhAnTp16Nq1K8ePH08bFxcXx6233krNmjWpXr06//jHP/A8G3PhwoVERkYSERFBw4YNGT16tAO/QdasEOSTFStWULduXV577TWGDRtGXFwcHTt2dDqWMSYHihUrRkxMDLGxsVx55ZVMmDABcPX86tatG0888QTbt29n8+bN/Pjjj2lb+7GxsYwcOZIPP/yQ+Ph41q9fzw033ODkr5IhO0aQTy7eOGbx4sW0bt3a6TjGBKRn58cR//NJr84z4rpSjOtaO8fTN23alC1btgDw8ccf07x5c9q3bw9A8eLFeeedd2jdujUjRozg5Zdf5sknn6RWrVqAa8vi/vvv92p+b7AtAh+aP38+L7/susVCmzZtiI+PtyJgTAC7cOEC33//Pd26uRogxMXF0bhx479MU61aNU6dOsXJkyeJjY39n/H+yLYIfODw4cOMGjWKTz75hAYNGvC3v/2NIkWKUKiQvd3G5EVuvrl709mzZ2nQoAEHDx4kPDyc2267zZEcvmJbBF6kqnz88ceEh4fz+eefM378eNasWWNN4owJcBePEezbtw9VTTtGEBERwYYNG/4ybUJCAiVLlqRUqVLUrl37f8b7JVUNqJ9SFWqqv9q7d68WKVJEmzRporGxsU7HMaZAiI+PdzqClihRIu3xxo0btVKlSpqcnKxnzpzRqlWr6n//+19VVT1z5ox27txZ33rrLVVV3bx5s1arVk23b9+uqqoXLlzQSZMm+TxvRu8ZrmafGf5dtS2CPEpNTWXRokUAVK5cmeXLl7Ny5Upq13ZmE9YY41sNGzakXr16fPLJJxQrVoy5c+fy3HPPUbNmTerWrcuNN97IyJEjAahXrx5vvPEGffv2JTw8nDp16pCQkODwb/C/rPtoHuzcuZNhw4axdOlSli5dSsuWLZ2OZEyBs23bNsLDw52OEVAyes+y6j5qWwSXICUlhX/961/Uq1ePmJgYpk2bZk3ijDEBy05juQRdunRh0aJFdO/enYkTJ3Ldddc5HckYYy6ZFYIcOnfuHIULFyYkJIShQ4cyePBg7r77busPZIwJeLZrKAdWr15No0aN0k4Z69mzJ7169bIiYIwpEKwQZOH06dM8/PDDNGvWjD/++IPq1as7HckYY7zOdg1lYvny5URFRbFnzx4eeOABXnjhBUqVKuV0LGOM8TorBJlISUmhcOHCdlqoMYbQ0FDq1q1LSkoKVatWZebMmVx++eV5nu+MGTNYv34977zzTt5D5oHtGvIwZ84cXnjhBcDVJC4uLs6KgDEm0zbUBYVtEQC//vorDz74ILNnz6ZRo0aMHj3amsQZ46cy6uDbq1cvHnjgAc6cOUOnTp3+Z3x0dDTR0dEcOXKEnj17/mXckiVLcrV8zzbUa9euZdSoUSQlJVGsWDGmT59OzZo1mTFjBvPmzePMmTPs3r2bO++8M60T8fTp03nhhRe4/PLLqV+/PkWLFgVg7969DB48mCNHjlC2bFmmT59OpUqViI6OplixYmzatInffvuN999/n3//+9+sWrWKJk2aMGPGjFzlz0hQbxGoKjNnziQiIoK5c+fyz3/+k9WrV1uTOGNMhtK3oa5VqxbLly9n06ZNjB8/nr///e9p08bExPDpp5+ydetWPv30Uw4cOMAvv/zCuHHjWLlyJStWrCA+Pj5t+gcffJCoqCi2bNlC//79eeihh9LGHTt2jFWrVvH666/TrVs3Hn74YeLi4ti6dSsxMTF5/r2C+ivv/v37GTp0KJGRkUybNi3t5hHGGP+V1Tf44sWLZzn+qquuyvUWAGTehvrEiRNERUWxc+dORITk5OS017Rt25bSpUsDri6l+/bt48iRI7Ru3ZqyZcsC0Lt3b3bs2AHAqlWr+PLLLwEYMGAAjz32WNq8unbtiohQt25drrnmGurWrQtA7dq12bt3Lw0aNMj17+Qp6LYIUlNTWbhwIeBqErdy5UqWLVtmRcAYk6nM2lD/3//9H23atCE2Npb58+eTlJSU9pqLu3zAdbA5JSXlkpd/cV4hISF/mW9ISEie5ps2nzzPIYDs2LGD1q1b06lTJ5YuXQpAZGQkoaGhDiczxgSC4sWL89Zbb/Hqq6+SkpLCiRMnKF++PECO9tU3adKEpUuXcvToUZKTk5k9e3bauGbNmjFr1iwAPvroo3ztXxYUhSAlJYWXXnqJevXqsXXrVqZPn25nAxljLolnG+rHHnuMsWPH0rBhwxx9My9XrhzPPPMMTZs2pXnz5n/pEPr2228zffp06tWrx8yZM3nzzTd9+Wv8RVC0ob799tv59ttvueuuu5gwYQLXXnutj9IZY7zN2lDnXm7bUBfYg8VJSUkULlyY0NBQhg8fzvDhw+nRo4fTsYwxxu8UyF1DK1eupEGDBmkHdHr06GFFwBhjMlGgCsGpU6d46KGHaNGiBUlJSbY5aUwBEWi7sJ10Ke9VgSkES5cupU6dOrzzzjuMHDmS2NjYtHN9jTGBKywsjKNHj1oxyAFV5ejRo4SFheXqdQXqGEHx4sVZvnw5zZs3dzqKMcZLKlSoQGJiIocPH3Y6SkAICwujQoUKuXpNQJ819OWXX/LTTz+lXdZ94cIFuybAGGMy4NjN60Wkg4hsF5FdIvJEBuOLisin7vFrRKRKTuZ76NAhevbsSY8ePfjqq684f/48gBUBY4y5BD4rBCISCkwAOgIRQF8RiUg32RDgmKreALwOvJTdfM+fPkF4eDj/+c9/eOGFF/jxxx+tSZwxxuSBL7cIbgJ2qWqCqp4HZgHd003THfjA/fhzoK1kcyPgpGO/UqdOHTZv3swTTzxB4cKFvR7cGGOCiS8PFpcHDng8TwSaZDaNqqaIyAmgDHDEcyIRGQ4Mdz89tWLFiu15bBJ3VfplOMAfMoB/5PCHDOAfOfwhA/hHDn/IAP6RwxsZKmc2IiDOGlLVKcAUb81PRNZndtAkv/hDBn/J4Q8Z/CWHP2Twlxz+kMFfcvg6gy93DR0EKno8r+AeluE0IlIIKA0c9WEmY4wx6fiyEKwDqotIVREpAvQB5qWbZh4Q5X7cE/hBA+18VmOMCXA+2zXk3uc/ElgEhALvq2qciIwH1qvqPGAaMFNEdgG/4yoW+cFru5nywB8ygH/k8IcM4B85/CED+EcOf8gA/pHDpxkC7oIyY4wx3lVgeg0ZY4y5NFYIjDEmyBWoQnCpLS1EpIqInBWRGPfPZB/naCkiG0UkRUR6pht3wSNH+oPr3szwiIjEi8gWEfleRCp7jPNKhhzmuE9EtrqXteLi1efe/Eyyy+AxXQ8RURGJ9HaGnOQQkWgROeyxvKEe4/JlvXBP08u9bsSJyMfezpCTHCLyuseydojIcW/nyEGGSiKyWEQ2uf+fdHIPz+/1orL7/+gWEVkiIhU8xnnnM1HVAvGD64D0buB6oAiwGYhIN80DwGT34z7Ap+7HVYDYfMxRBagH/BvomW7cqXzK0AYo7n58/8X3wlsZcpGjlMfjbsA33vxMcpLBPd1lwDJgNRDp0HoRDbyTyevza72oDmwCrnA/v9qJ9SLd9A/iOtkkv9+LKcD97scRwF6H1ovZQJT78a3ATG9/JgVpi8AnLS18kUNV96rqFiDVy8vOTYbFqnrG/XQ1rus8nMhx0uNpCcDbZy/kZL0A+AeuXldJXl5+bnP4Uk4yDAMmqOoxAFX9zaEcnvoCnziQQYFS7selgZ+9nCGnOSKAH9yPF2cwPs8KUiHIqKVF+cymUdUU4GJLC4Cq7k3ApSLSwsc5shImIutFZLWI3JFPGYYAC72cIcc5RGSEiOwGXgYe8hjljc8k2wwi0gioqKpfZ/D6/F4verh3AXwuIp4XZObXelEDqCEiK93L6uDlDDnNAbh2iwBV+fMPobdy5CTDM8A9IpIILMC1ZXJRfq4Xm4G73I/vBC4TkYt/t7zymQREi4l88AtQSVWPikhjYI6I1E73bTW/VFbVgyJyPfCDiGxV1d2+WpiI3ANEAq2cyqCqE4AJItIPeArXRYb58pmISAjwGq7dMunl93oxH/hEVc+JyL24tl5vdY/Lr8+kEK7dQ61xbSUuE5G6qno8HzN46gN8rqoXPIblV46+wAxVfVVEmuK65qkO+b9ejAHeEZFoXLsvDwIX3w+vvBcFaYvgkltaqOo5VT0KoKobcO2zq+HDHJlS1YPufxOAJUBDX2UQkXbAk0A3VT3n5Qw5zuFhFnCHe9ne+kyyy3AZUAdYIiJ7gZuBeSISmd/rhaoe9fgcpgKNPcbl13qRCMxT1WRV3QPswFUYnFov+pBut1A+vhdDgM/cy1oFhAFXObBe/Kyqd6lqQ1z/X3EXZu99Jt440OAPP7i+ySTg2oy8eNCldrppRvDXg8WfuR+XBULdj693fxBX+iqHx7Qz8DhYDFwBFHU/vgrYSRYH0fL4XjTEtQJXTzfcKxlykaO6x+OuuK4699pnkpvPwz39Ev48WJyv6wVQzuPxncBqB9aLDsAHHss6gGv3ab6uF+7pagF7cV/46sB7sRCIdj8Ox3WMQBxYL64CQtyP/wmM9+Z7oaoFpxC434xOuL7B7AaedA8bj+sbL7gq+mxgF7AWuN49vAcQB8QAG4GuPs5xI65vXqdxNdmLcw9vBmx1rwxbgSE+zPAd8Kv7d47B9S3QqxlymONNj/d+8cX/BN78TLLLkG7aJfxZCPJ7vXjBvbzN7veilgPrheDaVRbvXlYfJ9YL9/NngBfTvS4/34sIYKV7WTFAe4fWi564/sjvwLWlePGPv9feC2sxYYwxQa4gHSMwxhhzCawQGGNMkLNCYIwxQc4KgTHGBDkrBMYYE+SsEBi/kK6LYoy7w2NrETnhfr5NRMa5p/Uc/pOIvJJuXneIyNMZLKOWiKwSkXMiMuYSMoaIyFsiEiuujqnrRKTqpf/WGS7jR4/H/xJXB9B/iatL68AsXnediHzuftxA3J0yc7HcLuK6e6AJQnb6qPELInJKVUumG9YaGKOqXUSkBK7ztnvjagR2cXgxXN0yh6jqSvfrfsR1DvaRdPO7GqiM6+rlY6r6lwKSg4x9cZ1D3ktVU8XVDvi0uhu0eZuInMB1odKFbCf+6+uicV0LMTIXrxFc58Q31z+bEZogYVsEJiCo6mlgA3BDuuFncRWI8gAiUgM4l74IuKf9TVXXAcmXGKMc8Iuqprrnl3ixCIjIKXH10I8TV+/4su7h1UTkGxHZICLLRaSWe/g1IvKViGx2/zS7OB/3v/OAksAGEektIs9c3IoRkRtE5Dv36za6l1HFvaVSBNfFSL3dW0y9RWSnR54QcfW9L5vuvVFcF9N1ucT3xgQwKwTGXxTz2C30VfqR4uq2eDOuKzo9h1+BqxfOMveg5ri+2frCZ0BXd8ZXRcSzr0sJXO0xagNLgXHu4VOAB1W1Ma7mYRPdw98ClqpqfaAR6X4vVe0GnFXVBqr6abocH+FqFV0f19Wlv3i87jzwNK77S1x87YdAf/ck7YDNqno4g99vPZCXTpomQFn3UeMvzqpqgwyGtxCRTbju3fCiqsa5dxm1EJHNuIrAG6p6yD19OSCjP3J5pqqJIlITV0fQW4HvReRuVf3ene/iH+wPgS9FpCSuP9Sz5c/bXhR1/3srMNA93wu4WqJnS0QuA8qr6lfu1ya5h2f1sveBucAbwGBgeibT/QZcl5McpmCxQmD83XJVzWh3xXL3MYKqwGoR+UxVY4CzuLrKXhIRuZM/v80PVdX1nuPV1R10IbBQRH7Fdbzh+wxmpbi2uI9nUuDyjaoeEJFfReRWXDdC6Z/JpGG43j8TZGzXkAlo6mqV/CLwuHvQNtIdR8jl/L5y71JpkL4IiEgjEbnO/TgE1+1G97lHh+BqDgbQD1ihrv70e0TkbvdrRETqu6f5HtctQhGRUBHJUfFS1T+ARHHfhERc9+Eunm6yP3C11/Y0FdeWyuwsDj7XAGJzksMULFYITEEwGWgpIlVwHStoKBnsKxGRa8V1t6lHgKdEJFFESqWfLgtXA/NFJBbYAqQA77jHnQZuco+7FdcBW3B9+x7i3o0Vx5+3GRwFtBGRrbgOgkfkIscA4CER2QL8CFybbvxiIOLiwWL3sIsHnzPbLQSu+1hndJc2U8DZ6aOmwBGRN4H5qvpdPi7zf05/9SciEgm8rqoZHgwWkWuAj1W1bf4mM/7AtghMQfQ8kH53SdASkSeAL4CxWUxWCRidP4mMv7EtAmOMCXK2RWCMMUHOCoExxgQ5KwTGGBPkrBAYY0yQs0JgjDFB7v8BlRCX1W4cboYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "def roc_curve_plot(y_test , pred_proba_c1):\n",
        "    # 임곗값에 따른 FPR, TPR 값을 반환 받음.\n",
        "    fprs , tprs , thresholds = roc_curve(y_test ,pred_proba_c1)\n",
        "\n",
        "    # ROC Curve를 plot 곡선으로 그림.\n",
        "    plt.plot(fprs , tprs, label='ROC')\n",
        "    # 가운데 대각선 직선을 그림.\n",
        "    plt.plot([0, 1], [0, 1], 'k--', label='Random')\n",
        "\n",
        "    # FPR X 축의 Scale을 0.1 단위로 변경, X,Y 축명 설정등\n",
        "    start, end = plt.xlim()\n",
        "    plt.xticks(np.round(np.arange(start, end, 0.1),2))\n",
        "    plt.xlim(0,1); plt.ylim(0,1)\n",
        "    plt.xlabel('FPR( 1 - Specificity )'); plt.ylabel('TPR( Recall )')\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "roc_curve_plot(y_test, lr_clf.predict_proba(X_test)[:, 1] )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SAe6euJB1Gda",
        "outputId": "c464b9f5-a70e-4d39-9e3b-eb6ad3372ffa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ROC AUC 값: 0.8987\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import roc_auc_score\n",
        "pred_proba = lr_clf.predict_proba(X_test)[:, 1]\n",
        "roc_score = roc_auc_score(y_test, pred_proba)\n",
        "print('ROC AUC 값: {0:.4f}'.format(roc_score))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "68fpdmuN1Gdi"
      },
      "outputs": [],
      "source": [
        "def get_clf_eval(y_test, pred=None, pred_proba=None):\n",
        "    confusion = confusion_matrix( y_test, pred)\n",
        "    accuracy = accuracy_score(y_test , pred)\n",
        "    precision = precision_score(y_test , pred)\n",
        "    recall = recall_score(y_test , pred)\n",
        "    f1 = f1_score(y_test,pred)\n",
        "    # ROC-AUC 추가\n",
        "    roc_auc = roc_auc_score(y_test, pred_proba)\n",
        "    print('오차 행렬')\n",
        "    print(confusion)\n",
        "    # ROC-AUC print 추가\n",
        "    print('정확도: {0:.4f}, 정밀도: {1:.4f}, 재현율: {2:.4f},\\\n",
        "          F1: {3:.4f}, AUC:{4:.4f}'.format(accuracy, precision, recall, f1, roc_auc))\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.0"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}