F_divergence_Fairness.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Umj90sviWw60",
        "outputId": "2c9d350e-a4a4-4e90-e988-e404238a2175"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /gdrive\n",
            "/gdrive/My Drive/AdultDataset\n"
          ]
        }
      ],
      "source": [
        "# Imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.preprocessing import normalize, StandardScaler\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn.functional as F\n",
        "import csv\n",
        "\n",
        "!pip install -U -q PyDrive\n",
        "import os\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "from google.colab import drive\n",
        "\n",
        "\n",
        "drive.mount('/gdrive')\n",
        "%cd /gdrive/My\\ Drive/AdultDataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eKJH4E_mXDrP",
        "outputId": "977685bb-44b3-4a8a-ede1-5ff506443a12"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Index(['age', 'workclass', 'fnlwgt', 'education', 'educational-num',\n",
            "       'marital-status', 'occupation', 'relationship', 'race', 'gender',\n",
            "       'capital-gain', 'capital-loss', 'hours-per-week', 'native-country',\n",
            "       'income'],\n",
            "      dtype='object')\n",
            "Index(['age', 'workclass', 'fnlwgt', 'education', 'educational-num',\n",
            "       'marital-status', 'occupation', 'relationship', 'race', 'gender',\n",
            "       'capital-gain', 'capital-loss', 'hours-per-week', 'native-country',\n",
            "       'income'],\n",
            "      dtype='object')\n",
            "(32561, 1)\n",
            "(32561, 2)\n",
            "(16281, 1)\n",
            "(16281, 2)\n"
          ]
        }
      ],
      "source": [
        "# Prepare Data:\n",
        "def read_demographics_and_labels(data_name, number_of_sensitive_attributes=1):\n",
        "    data = pd.read_csv(data_name)\n",
        "\n",
        "    print(data.columns)\n",
        "\n",
        "    # Shuffle Data\n",
        "    # data = data.sample(frac=1)\n",
        "\n",
        "    data['y'] = 1\n",
        "    if data_name == \"adult.data\":\n",
        "      data['y'].values[data['income'].values == '<=50K'] = 0\n",
        "    else:\n",
        "      data['y'].values[data['income'].values == '<=50K.'] = 0\n",
        "\n",
        "    if number_of_sensitive_attributes == 2:\n",
        "      data['s0'] = 0\n",
        "      data['s0'].values[(data['gender'].values == 'Male') & (data['race'].values == 'White')] = 1\n",
        "\n",
        "      data['s1'] = 0\n",
        "      data['s1'].values[(data['gender'].values != 'Male') & (data['race'].values == 'White')] = 1\n",
        "\n",
        "      data['s2'] = 0\n",
        "      data['s2'].values[(data['gender'].values == 'Male') & (data['race'].values != 'White')] = 1\n",
        "\n",
        "      data['s3'] = 0\n",
        "      data['s3'].values[(data['gender'].values != 'Male') & (data['race'].values != 'White')] = 1\n",
        "\n",
        "      S = data[['s0', 's1', 's2', 's3']]\n",
        "\n",
        "    else:\n",
        "      data['s0'] = 0\n",
        "      data['s0'].values[(data['gender'].values == 'Male')] = 1\n",
        "\n",
        "      data['s1'] = 0\n",
        "      data['s1'].values[(data['gender'].values != 'Male')] = 1\n",
        "\n",
        "      S = data[['s0', 's1']]\n",
        "\n",
        "    S_matrix = S.to_numpy()\n",
        "    return data[['y']].to_numpy(), S_matrix\n",
        "\n",
        "def read_data(training_data_name, test_data_name):\n",
        "\n",
        "    training_data = pd.read_csv(training_data_name)\n",
        "\n",
        "    test_data = pd.read_csv(test_data_name)\n",
        "\n",
        "    X_train = training_data.to_numpy()\n",
        "    X_test = test_data.to_numpy()\n",
        "\n",
        "    X_train = normalize(X_train, axis=0)\n",
        "    X_test = normalize(X_test, axis=0)\n",
        "    # sc = StandardScaler()\n",
        "\n",
        "    # X_train = np.array(X_train)\n",
        "    # sc.fit(X_train)\n",
        "    # X_train = sc.transform(X_train)\n",
        "    # X_test = sc.transform(X_test)\n",
        "\n",
        "    intercept = X_train.shape[0] * [1]\n",
        "    intercept_numpy = np.array(intercept)\n",
        "    intercept_numpy = intercept_numpy[:, np.newaxis]\n",
        "    X_train = np.append(X_train, intercept_numpy, axis=1)\n",
        "\n",
        "    intercept = X_test.shape[0] * [1]\n",
        "    intercept_numpy = np.array(intercept)\n",
        "    intercept_numpy = intercept_numpy[:, np.newaxis]\n",
        "    X_test = np.append(X_test, intercept_numpy, axis=1)\n",
        "\n",
        "    return X_train, X_test\n",
        "\n",
        "\n",
        "y_train, S_Train = read_demographics_and_labels('adult.data')\n",
        "\n",
        "y_test, S_Test = read_demographics_and_labels('adult.test')\n",
        "\n",
        "print(y_train.shape)\n",
        "print(S_Train.shape)\n",
        "\n",
        "print(y_test.shape)\n",
        "print(S_Test.shape)\n",
        "\n",
        "# X_Train, X_Test = read_data('AdultTrain2Sensitive.csv', 'AdultTest2Sensitive.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eDu2ntI9GrBt",
        "outputId": "93d23bce-9aad-436b-c255-b8a309423acc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.29458554712693097\n",
            "0.46460489542704464\n",
            "0.036208961641227236\n",
            "0.20460059580479714\n",
            "0.33079450876815825\n",
            "0.15036347404667771\n"
          ]
        }
      ],
      "source": [
        "group00 = 0\n",
        "group01 = 0\n",
        "group10 = 0\n",
        "group11 = 0\n",
        "\n",
        "for i in range(y_train.shape[0]):\n",
        "  if y_train[i][0] == 0 and S_Train[i][0] == 0:\n",
        "    group00 += 1\n",
        "\n",
        "  elif y_train[i][0] == 0 and S_Train[i][0] == 1:\n",
        "    group01 += 1\n",
        "\n",
        "  elif y_train[i][0] == 1 and S_Train[i][0] == 0:\n",
        "    group10 += 1\n",
        "\n",
        "  elif y_train[i][0] == 1 and S_Train[i][0] == 1:\n",
        "    group11 += 1\n",
        "\n",
        "print(group00/32561)\n",
        "print(group01/32561)\n",
        "print(group10/32561)\n",
        "print(group11/32561)\n",
        "\n",
        "print((group00 + group10) / 32561)\n",
        "print((group10) / (group10 + group11))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7W6REDMzc88r"
      },
      "outputs": [],
      "source": [
        "class FERMI(torch.nn.Module):\n",
        "# class FERMI():\n",
        "\n",
        "  def __init__(self, X_train, X_test, Y_train, Y_test, S_train, S_test, batch_size=64, epochs=2000, lam=10):\n",
        "\n",
        "        super(FERMI, self).__init__()\n",
        "\n",
        "        self.X_train = X_train\n",
        "        self.Y_train = Y_train\n",
        "        self.X_test = X_test\n",
        "        self.Y_test = Y_test\n",
        "        self.S_train = S_train\n",
        "        self.S_test = S_test\n",
        "\n",
        "        self.batch_size = batch_size\n",
        "        self.epochs = epochs\n",
        "\n",
        "        self.n = X_train.shape[0]\n",
        "        self.d = X_train.shape[1]\n",
        "        self.m = Y_train.shape[1]\n",
        "        if self.m == 1:\n",
        "          self.m = 2\n",
        "\n",
        "        self.k = S_train.shape[1]\n",
        "\n",
        "        self.W = nn.Parameter(torch.zeros(self.k, self.m)) # k: Support of sensitive attributes, m: number of labels\n",
        "        self.theta = nn.Parameter(torch.zeros(self.d, 1))\n",
        "\n",
        "        sums = self.S_train.sum(axis=0) / self.n\n",
        "        print(sums)\n",
        "        self.p_s0 = sums[0]\n",
        "        self.p_s1 = sums[1]\n",
        "\n",
        "        print(sums.shape)\n",
        "\n",
        "        final_entries = []\n",
        "        for item in sums:\n",
        "          final_entries.append(1.0 / np.sqrt(item))\n",
        "\n",
        "        self.P_s = np.diag(sums)\n",
        "\n",
        "        self.P_s_sqrt_inv = torch.from_numpy(np.diag(final_entries)).double()\n",
        "        print(self.P_s_sqrt_inv)\n",
        "        self.lam = lam\n",
        "\n",
        "\n",
        "  def forward(self, X):\n",
        "    outputs = torch.mm(X.double(), self.theta.double())\n",
        "    logits = torch.sigmoid(outputs)\n",
        "    return logits\n",
        "\n",
        "\n",
        "  def grad_loss(self, X, Y):\n",
        "    outputs = torch.mm(X, self.theta.double())\n",
        "    probs = torch.sigmoid(outputs)\n",
        "    return torch.matmul(torch.t(X), probs - Y)\n",
        "\n",
        "  def fairness_regularizer(self, X, S, f_divergence):\n",
        "\n",
        "    current_batch_size = X.shape[0]\n",
        "    summation = 0\n",
        "\n",
        "    Y_hat = torch.sigmoid(torch.matmul(X, self.theta.double()))\n",
        "    Y_hat0 = 1 - Y_hat\n",
        "\n",
        "    p_y1 = torch.mean(Y_hat) # P(y = 1): Taking the average of Y_hat\n",
        "    p_y0 = 1 - p_y1\n",
        "    torch.mean(Y_hat)\n",
        "    p_s0 = torch.mean(S[:, 0])\n",
        "    p_s1 = torch.mean(S)\n",
        "    # print(Y_hat.shape)\n",
        "    # print(S[:, 0])\n",
        "    # print(S[:, 0].shape)\n",
        "    p_y1s0 = torch.mean(torch.mul(Y_hat, S[:, 0]))\n",
        "    p_y1s1 = torch.mean(torch.mul(Y_hat, S[:, 1]))\n",
        "    p_y0s0 = torch.mean(torch.mul(Y_hat0, S[:, 0]))\n",
        "    p_y0s1 = torch.mean(torch.mul(Y_hat0, S[:, 1]))\n",
        "\n",
        "    # print(p_y1s0)\n",
        "    # print(p_y0s1)\n",
        "    # print(p_y1s1)\n",
        "    # print(p_y0s0)\n",
        "\n",
        "    # print(self.W.shape)\n",
        "    reg = 0\n",
        "    if f_divergence == 'Chi2':\n",
        "      term1 = 2 * p_y1s1 * self.W.double()[1][1] - self.p_s1 * p_y1 * self.W.double()[1][1] * self.W.double()[1][1] + self.p_s1 * p_y1 - 2 * p_y1s1\n",
        "      term2 = 2 * p_y0s0 * self.W.double()[0][0] - self.p_s0 * p_y0 * self.W.double()[0][0] * self.W.double()[0][0] + self.p_s0 * p_y0 - 2 * p_y0s0\n",
        "      term3 = 2 * p_y1s0 * self.W.double()[1][0] - self.p_s0 * p_y1 * self.W.double()[1][0] * self.W.double()[1][0] + self.p_s0 * p_y1 - 2 * p_y1s0\n",
        "      term4 = 2 * p_y0s1 * self.W.double()[0][1] - self.p_s1 * p_y0 * self.W.double()[0][1] * self.W.double()[0][1] + self.p_s1 * p_y0 - 2 * p_y0s1\n",
        "      reg = term1 + term2 + term3 + term4\n",
        "\n",
        "    elif f_divergence == 'KL':\n",
        "      term1 = p_y1s1 * self.W.double()[1][1] - self.p_s1 * p_y1 * torch.exp(self.W.double()[1][1] - 1)\n",
        "      term2 = p_y0s0 * self.W.double()[0][0] - self.p_s0 * p_y0 * torch.exp(self.W.double()[0][0] - 1)\n",
        "      term3 = p_y1s0 * self.W.double()[1][0] - self.p_s0 * p_y1 * torch.exp(self.W.double()[1][0] - 1)\n",
        "      term4 = p_y0s1 * self.W.double()[0][1] - self.p_s1 * p_y0 * torch.exp(self.W.double()[0][1] - 1)\n",
        "      reg = term1 + term2 + term3 + term4\n",
        "    # print(reg)\n",
        "\n",
        "    return self.lam * reg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vsc2GiTtt3eH"
      },
      "outputs": [],
      "source": [
        "def fair_training(fermi, batch_size, epochs, initial_epochs = 300, initial_learning_rate = 1, lam=0.1, learning_rate_min = 0.01, learning_rate_max = 0.01, f_divergence='Chi2'):\n",
        "\n",
        "  X = fermi.X_train\n",
        "  S_Matrix = fermi.S_train\n",
        "  Y = fermi.Y_train\n",
        "  XTest = fermi.X_test\n",
        "  STest = fermi.S_test\n",
        "  YTest = fermi.Y_test\n",
        "\n",
        "  print(X.shape)\n",
        "  print(S_Matrix.shape)\n",
        "  print(Y.shape)\n",
        "\n",
        "  criterion=torch.nn.BCELoss()\n",
        "\n",
        "  minimizer = torch.optim.SGD([fermi.theta, fermi.W], lr=initial_learning_rate)\n",
        "  # maximizer = torch.optim.SGD([fermi.W], lr=learning_rate_max)\n",
        "\n",
        "  # minimizer_track = []\n",
        "   # maximizer_track = []\n",
        "\n",
        "  X_total = torch.from_numpy(X).double()\n",
        "  Y_total = torch.from_numpy(Y).double()\n",
        "\n",
        "  for ep in range(epochs + initial_epochs):\n",
        "\n",
        "      if ep % 100 == 99:\n",
        "        print(ep+1, \" epochs:\")\n",
        "        # Test:\n",
        "        pre_logits = np.dot(XTest, fermi.theta.detach().numpy())\n",
        "        output_logits = 1/(1 + np.exp(-pre_logits))\n",
        "        final_preds = output_logits > 0.5\n",
        "        print(final_preds.shape)\n",
        "\n",
        "        p = 0.3\n",
        "        t = p * torch.ones(16281,1)\n",
        "\n",
        "        random_numbers = torch.bernoulli(t)\n",
        "        print(random_numbers)\n",
        "        final_preds = random_numbers * final_preds\n",
        "        final_preds = final_preds.numpy()\n",
        "\n",
        "        test = YTest == 1\n",
        "        acc = final_preds == test\n",
        "        true_preds = acc.sum(axis=0)\n",
        "        print(\"Accuracy: \", true_preds[0] / output_logits.shape[0] * 100, \"%\")\n",
        "\n",
        "        final_preds = np.array(final_preds)\n",
        "        intersections = np.dot(final_preds.T, STest)\n",
        "        numbers = STest.sum(axis=0)\n",
        "\n",
        "        group1 = intersections[0][0] / numbers[0]\n",
        "        group2 = intersections[0][1] / numbers[1]\n",
        "        print(\"DP Violation: \", np.abs(group1 - group2))\n",
        "\n",
        "\n",
        "\n",
        "      number_of_iterations = X.shape[0] // batch_size\n",
        "      for i in range(number_of_iterations):\n",
        "\n",
        "\n",
        "          start = i * batch_size\n",
        "          end = (i+1) * batch_size\n",
        "\n",
        "          current_batch_X = X[start:end]\n",
        "          current_batch_Y = Y[start:end]\n",
        "          current_batch_S = S_Matrix[start:end]\n",
        "\n",
        "          XTorch = torch.from_numpy(current_batch_X).double()\n",
        "          logits = fermi(XTorch)\n",
        "          YTorch = torch.from_numpy(current_batch_Y).double()\n",
        "          STorch = torch.from_numpy(current_batch_S).double()\n",
        "\n",
        "          if ep < initial_epochs:\n",
        "            loss_min = criterion(logits, YTorch)\n",
        "          else:\n",
        "            loss_min = criterion(logits, YTorch) + fermi.fairness_regularizer(XTorch, STorch, f_divergence)\n",
        "          # loss_min = criterion(logits, YTorch)\n",
        "\n",
        "          minimizer.zero_grad()\n",
        "          loss_min.backward()\n",
        "\n",
        "          if ep >= initial_epochs:\n",
        "            fermi.theta.grad.data.mul_(learning_rate_min / initial_learning_rate) # You can have \\eta_w here\n",
        "            fermi.W.grad.data.mul_(-learning_rate_max / initial_learning_rate) # You can have \\eta_w here\n",
        "\n",
        "          minimizer.step()\n",
        "  return fermi.theta, fermi.W"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c3ZHFSoAulFA",
        "outputId": "a09e730d-a69b-48ef-e08a-1ffb44184831"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Index(['age', 'workclass', 'fnlwgt', 'education', 'educational-num',\n",
            "       'marital-status', 'occupation', 'relationship', 'race', 'gender',\n",
            "       'capital-gain', 'capital-loss', 'hours-per-week', 'native-country',\n",
            "       'income'],\n",
            "      dtype='object')\n",
            "Index(['age', 'workclass', 'fnlwgt', 'education', 'educational-num',\n",
            "       'marital-status', 'occupation', 'relationship', 'race', 'gender',\n",
            "       'capital-gain', 'capital-loss', 'hours-per-week', 'native-country',\n",
            "       'income'],\n",
            "      dtype='object')\n",
            "(32561, 1)\n",
            "(32561, 2)\n",
            "(16281, 1)\n",
            "(16281, 2)\n",
            "(32561, 60)\n",
            "(16281, 60)\n"
          ]
        }
      ],
      "source": [
        "Y_Train, S_Train = read_demographics_and_labels('adult.data', number_of_sensitive_attributes=1)\n",
        "Y_Test, S_Test = read_demographics_and_labels('adult.test', number_of_sensitive_attributes=1)\n",
        "\n",
        "print(Y_Train.shape)\n",
        "print(S_Train.shape)\n",
        "\n",
        "print(Y_Test.shape)\n",
        "print(S_Test.shape)\n",
        "\n",
        "X_Train, X_Test = read_data('AdultTrain2Sensitive.csv', 'AdultTest2Sensitive.csv')\n",
        "\n",
        "print(X_Train.shape)\n",
        "print(X_Test.shape)\n",
        "\n",
        "# fermi_instance = FERMI(X_Train, X_Test, Y_Train, Y_Test, S_Train, S_Test, lam=0.01)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "sgN1CHRyunD-",
        "outputId": "e7aeae0c-0e1d-4668-a704-7d66508920ec"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.66920549 0.33079451]\n",
            "(2,)\n",
            "tensor([[1.2224, 0.0000],\n",
            "        [0.0000, 1.7387]], dtype=torch.float64)\n",
            "(32561, 60)\n",
            "(32561, 2)\n",
            "(32561, 1)\n",
            "100  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  78.55782814323445 %\n",
            "DP Violation:  0.08179766429100663\n",
            "200  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.83919906639642 %\n",
            "DP Violation:  0.0883244105947711\n",
            "300  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [1.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.]])\n",
            "Accuracy:  77.66107732940237 %\n",
            "DP Violation:  0.09615190975141688\n",
            "400  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [1.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.]])\n",
            "Accuracy:  77.39696578834224 %\n",
            "DP Violation:  0.09098711680889032\n",
            "500  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [1.],\n",
            "        [1.]])\n",
            "Accuracy:  77.43381856151342 %\n",
            "DP Violation:  0.09475846437172404\n",
            "600  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [1.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.26798108224311 %\n",
            "DP Violation:  0.09310620351997194\n",
            "700  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.12056998955839 %\n",
            "DP Violation:  0.10139288484214753\n",
            "800  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [1.],\n",
            "        [1.]])\n",
            "Accuracy:  77.16356489159143 %\n",
            "DP Violation:  0.09908535559992294\n",
            "900  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [1.],\n",
            "        [0.]])\n",
            "Accuracy:  77.14513850500583 %\n",
            "DP Violation:  0.10323695145031446\n",
            "1000  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.13285424728211 %\n",
            "DP Violation:  0.0998204751116234\n",
            "1100  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.14513850500583 %\n",
            "DP Violation:  0.09484198446597589\n",
            "1200  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [1.],\n",
            "        [0.]])\n",
            "Accuracy:  76.95473251028807 %\n",
            "DP Violation:  0.1021218893988082\n",
            "1300  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.0345801854923 %\n",
            "DP Violation:  0.102127698606096\n",
            "1400  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.]])\n",
            "Accuracy:  77.01001167004485 %\n",
            "DP Violation:  0.09705131772185313\n",
            "1500  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [1.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  76.90559547939316 %\n",
            "DP Violation:  0.09659641602485118\n",
            "1600  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [1.],\n",
            "        [1.]])\n",
            "Accuracy:  77.37853940175665 %\n",
            "DP Violation:  0.10378148819660803\n",
            "1700  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.]])\n",
            "Accuracy:  76.86874270622198 %\n",
            "DP Violation:  0.10248899053303043\n",
            "1800  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  76.8134635464652 %\n",
            "DP Violation:  0.09899327456861541\n",
            "1900  epochs:\n",
            "(16281, 1)\n",
            "tensor([[0.],\n",
            "        [1.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  76.75204225784657 %\n",
            "DP Violation:  0.09834840160171057\n",
            "2000  epochs:\n",
            "(16281, 1)\n",
            "tensor([[1.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [0.],\n",
            "        [0.]])\n",
            "Accuracy:  77.17584914931516 %\n",
            "DP Violation:  0.09410441897225952\n"
          ]
        }
      ],
      "source": [
        "# Run FERMI\n",
        "fermi_instance = FERMI(X_Train, X_Test, Y_Train, Y_Test, S_Train, S_Test, lam=100)\n",
        "theta_star, W_star = fair_training(fermi_instance, batch_size = 2, epochs=2000, initial_epochs=2000, initial_learning_rate=1, learning_rate_min=0.01, learning_rate_max=0.01, f_divergence='Chi2')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTFngh6_dGHG",
        "outputId": "f342fa18-1176-4d1a-d017-3b49db4c529c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([16281, 1])\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        ...,\n",
            "        [1.],\n",
            "        [1.],\n",
            "        [0.]])\n"
          ]
        }
      ],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1CribuJdvLvH"
      },
      "outputs": [],
      "source": [
        "# Test:\n",
        "pre_logits = np.dot(X_Test, theta_star.detach().numpy())\n",
        "output_logits = 1/(1 + np.exp(-pre_logits))\n",
        "final_preds = output_logits > 0.5\n",
        "test = Y_Test == 1\n",
        "acc = final_preds == test\n",
        "true_preds = acc.sum(axis=0)\n",
        "print(true_preds / output_logits.shape[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oE6Wg0XG9VgT"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "-7-B6ZSX48Sf",
        "outputId": "127e15bd-1b71-4abf-a7b3-c57e44dd559a"
      },
      "outputs": [
        {
          "ename": "TypeError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-6cb11a814a05>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.05\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtheta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_Train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mW\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mS_Train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_Train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m500\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: Cannot interpret '1' as a data type"
          ]
        }
      ],
      "source": [
        "number_of_iterations = 5000\n",
        "step_size = 0.05\n",
        "theta = np.zeros((X_Train.shape[1], 1))\n",
        "W = np.zeros(S_Train.shape[1], Y_Train.shape[1])\n",
        "\n",
        "epochs = 500\n",
        "batch_size = 128\n",
        "for _ in range(epochs):\n",
        "      number_of_iterations = X_Train.shape[0] // batch_size\n",
        "      for i in range(number_of_iterations):\n",
        "\n",
        "          start = i * batch_size\n",
        "          end = (i+1) * batch_size\n",
        "\n",
        "          current_batch_X = X_Train[start:end]\n",
        "          current_batch_Y = Y_Train[start:end]\n",
        "          # current_batch_S = S_Matrix[start:end]\n",
        "\n",
        "          logits = np.dot(current_batch_X, theta)\n",
        "          probs = sigmoid(logits)\n",
        "          g1 = np.dot(current_batch_X.T, (probs - current_batch_Y))\n",
        "\n",
        "          theta -= step_size * g1\n",
        "\n",
        "# for i in range(number_of_iterations):\n",
        "#     if i % 100 == 0:\n",
        "#         print(i)\n",
        "#     logits = np.dot(X_Train, theta)\n",
        "#     probs = sigmoid(logits)\n",
        "#     g1 = np.dot(X_Train.T, (probs - Y_Train))\n",
        "#     # g2 = grad2()\n",
        "\n",
        "#     theta -= step_size * (g1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E0WM6VGK9W2X"
      },
      "outputs": [],
      "source": [
        "# Test:\n",
        "pre_logits = np.dot(X_Test, theta)\n",
        "output_logits = 1/(1 + np.exp(-pre_logits))\n",
        "final_preds = output_logits > 0.5\n",
        "test = Y_Test == 1\n",
        "acc = final_preds == test\n",
        "true_preds = acc.sum(axis=0)\n",
        "print(true_preds / output_logits.shape[0])\n",
        "\n",
        "i = 0\n",
        "# print(theta_star)\n",
        "\n",
        "# for item in final_preds:\n",
        "#  print(item)\n",
        "print(pre_logits)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}