Fed Learning - FL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 230718,
     "status": "ok",
     "timestamp": 1613280977791,
     "user": {
      "displayName": "Sadman Sakib",
      "photoUrl": "",
      "userId": "05491288056867041174"
     },
     "user_tz": 300
    },
    "id": "xGoKB6hv4tAv",
    "outputId": "2a49532c-fb67-4b69-ade9-051ace1d408f"
   },
   "outputs": [],
   "source": [
    "# =============================================================================\n",
    "# Created By  : SADMAN SAKIB\n",
    "# Created Date: March 2020\n",
    "# =============================================================================\n",
    "\"\"\"The Module Has Been Build for Experimental Research Simulations\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # ========= For CoLab =========\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n",
    "# !ls\n",
    "# import sys\n",
    "# root_path = '/content/drive/My Drive/Folder Name/'\n",
    "# sys.path.append(root_path)\n",
    "# # ========= For CoLab =========\n",
    "\n",
    "# =========  If CoLab Not Used =========\n",
    "root_path=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 233883,
     "status": "ok",
     "timestamp": 1613280980963,
     "user": {
      "displayName": "Sadman Sakib",
      "photoUrl": "",
      "userId": "05491288056867041174"
     },
     "user_tz": 300
    },
    "id": "z4LDRwwF5hrh"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from keras.utils import np_utils\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.layers import Flatten, BatchNormalization\n",
    "from tensorflow.keras.layers import Convolution2D, Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling2D, MaxPooling1D\n",
    "from keras import backend as K\n",
    "from keras import backend\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "import time\n",
    "import os\n",
    "import psutil\n",
    "import csv\n",
    "from itertools import repeat\n",
    "from PIL import Image\n",
    "from numpy import asarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 371022,
     "status": "ok",
     "timestamp": 1613281118105,
     "user": {
      "displayName": "Sadman Sakib",
      "photoUrl": "",
      "userId": "05491288056867041174"
     },
     "user_tz": 300
    },
    "id": "MNvzgGln5ind"
   },
   "outputs": [],
   "source": [
    "dfDS = pd.read_csv(root_path+'dataset.csv')\n",
    "\n",
    "X_full = dfDS.iloc[:, 1:len(dfDS.columns)].values\n",
    "Y_full = dfDS[\"label\"].values\n",
    "\n",
    "xTrain, xTest, yTrain, yTest = train_test_split(X_full, Y_full, test_size=0.10, random_state=123) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 377239,
     "status": "ok",
     "timestamp": 1613281124327,
     "user": {
      "displayName": "Sadman Sakib",
      "photoUrl": "",
      "userId": "05491288056867041174"
     },
     "user_tz": 300
    },
    "id": "-gBrWF105wK4",
    "outputId": "df4cd8f1-0a05-4f9f-8fc8-dbf1d71525d8"
   },
   "outputs": [],
   "source": [
    "algoName='CNN' #CNN, ANN, DNN\n",
    "\n",
    "xTrain = xTrain.astype('float32')\n",
    "xTest = xTest.astype('float32')\n",
    "xTrain = xTrain / 255.\n",
    "xTest = xTest / 255.\n",
    "\n",
    "if(algoName=='CNN'):\n",
    "    xTrain = np.expand_dims(xTrain, axis=2)\n",
    "    xTest = np.expand_dims(xTest, axis=2)\n",
    "\n",
    "outputClasses=len(set(Y_full))\n",
    "#One hot encoding\n",
    "yTrain = np.array(to_categorical(yTrain))\n",
    "yTest = np.array(to_categorical(yTest))\n",
    "print(\"xTrain\", xTrain.shape, \"yTrain\", yTrain.shape)\n",
    "print(\"xTest\", xTest.shape, \"yTest\", yTest.shape)\n",
    "\n",
    "# FOR TEST SPLIT\n",
    "xServer, xClients, yServer, yClients = train_test_split(xTrain, yTrain, test_size=0.80,random_state=523) \n",
    "\n",
    "def my_metrics(y_true, y_pred):\n",
    "    accuracy=accuracy_score(y_true, y_pred)\n",
    "    precision=precision_score(y_true, y_pred,average='weighted')\n",
    "    recall=recall_score(y_true, y_pred,average='weighted')\n",
    "    f1Score=f1_score(y_true, y_pred, average='weighted') \n",
    "    print(\"Accuracy  : {}\".format(accuracy))\n",
    "    print(\"Precision : {}\".format(precision))\n",
    "    print(\"Recall : {}\".format(recall))\n",
    "    print(\"f1Score : {}\".format(f1Score))\n",
    "    cm=confusion_matrix(y_true, y_pred)\n",
    "    print(cm)\n",
    "    return accuracy, precision, recall, f1Score\n",
    "\n",
    "verbose, epochs, batch_size = 0, 20, 64\n",
    "activationFun='relu'\n",
    "optimizerName='Adam'\n",
    "def createDeepModel():\n",
    "    model = Sequential()\n",
    "    \n",
    "    if(algoName=='CNN'):    \n",
    "        model.add(Conv1D(filters=10, kernel_size=3, activation=activationFun,input_shape = (X_full.shape[1], 1)))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(MaxPooling1D(pool_size=3))\n",
    "        model.add(BatchNormalization())\n",
    "        \n",
    "        model.add(Conv1D(filters=5, kernel_size=3, activation=activationFun))\n",
    "        model.add(Dropout(0.05))\n",
    "        model.add(MaxPooling1D(pool_size=3))\n",
    "        model.add(BatchNormalization())\n",
    "        \n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(100, activation=activationFun))\n",
    "        model.add(Dense(50, activation=activationFun))\n",
    "        model.add(Dense(30, activation=activationFun))\n",
    "        model.add(Dense(outputClasses, activation='softmax'))\n",
    "        model.compile(loss='mean_squared_error', optimizer=optimizerName, metrics=['accuracy'])\n",
    "        \n",
    "    elif(algoName=='ANN'):\n",
    "        model.add(Dense(200, input_dim=X_full.shape[1], activation=activationFun))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(outputClasses, activation='softmax'))\n",
    "        model.compile(loss='mean_squared_error', optimizer=optimizerName, metrics=['accuracy'])\n",
    "        \n",
    "    elif(algoName=='DNN'):\n",
    "        model.add(Dense(200, input_dim=X_full.shape[1], activation=activationFun))\n",
    "        model.add(Dense(100, activation=activationFun))\n",
    "        model.add(Dense(50, activation=activationFun))\n",
    "        model.add(Dense(25,  activation=activationFun))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(outputClasses, activation='softmax'))\n",
    "        model.compile(loss='mean_squared_error', optimizer=optimizerName, metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "def predictTestData(yPredict, yTest):\n",
    "    #Converting predictions to label\n",
    "    print(\"yPredict\",len(yPredict))\n",
    "    pred = list()\n",
    "    for i in range(len(yPredict)):\n",
    "        pred.append(np.argmax(yPredict[i]))\n",
    "    #Converting one hot encoded test label to label\n",
    "    test = list()\n",
    "    for i in range(len(yTest)):\n",
    "        test.append(np.argmax(yTest[i]))\n",
    "    return my_metrics(test, pred)\n",
    "\n",
    "def sumOfWeights(weights):\n",
    "    return sum(map(sum, weights))\n",
    "\n",
    "def getWeights(model):\n",
    "    allLayersWeights=deepModel.get_weights()\n",
    "    return allLayersWeights\n",
    "    \n",
    "# Initially train central deep model\n",
    "deepModel=createDeepModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TNqCb2fO50ax",
    "outputId": "62d85b61-4d85-46e0-ea30-eacee26b22fa"
   },
   "outputs": [],
   "source": [
    "numOfIterations=50\n",
    "numOfClients=10 # 10, 15, 20, 25, 30, 35, 40, 45, 50\n",
    "modelLocation=root_path+\"Models/\"+str(algoName)+\"_Sync_users_\"+str(numOfClients)+\"_\"+activationFun+\"_\"+optimizerName+\"_FL_Model.h5\"\n",
    "accList, precList, recallList, f1List = [], [], [], []\n",
    "\n",
    "deepModelAggWeights=[]\n",
    "firstClientFlag=True\n",
    "\n",
    "def updateServerModel(clientModel, clientModelWeight):\n",
    "    global firstClientFlag\n",
    "    for ind in range(len(clientModelWeight)):\n",
    "        if(firstClientFlag==True):\n",
    "            deepModelAggWeights.append(clientModelWeight[ind])            \n",
    "        else:\n",
    "            deepModelAggWeights[ind]=(deepModelAggWeights[ind]+clientModelWeight[ind])\n",
    "\n",
    "def updateClientsModels():\n",
    "    global clientsModelList\n",
    "    global deepModel\n",
    "    clientsModelList.clear()\n",
    "    for clientID in range(numOfClients):\n",
    "        m = keras.models.clone_model(deepModel)\n",
    "        m.set_weights(deepModel.get_weights())\n",
    "        clientsModelList.append(m)\n",
    "    \n",
    "# ----- 1. Train central model initially -----\n",
    "def trainInServer():\n",
    "    deepModel.fit(xServer, yServer, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "    # deepModel.fit(X_full, Y_full, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "    deepModel.save(modelLocation)\n",
    "trainInServer()\n",
    "# ------- 2. Separate clients data into lists ----------\n",
    "xClientsList=[]\n",
    "yClientsList=[]\n",
    "clientsModelList=[]\n",
    "clientDataInterval=len(xClients)//numOfClients\n",
    "lastLowerBound=0\n",
    "\n",
    "for clientID in range(numOfClients):\n",
    "    xClientsList.append(xClients[lastLowerBound : lastLowerBound+clientDataInterval])\n",
    "    yClientsList.append(yClients[lastLowerBound : lastLowerBound+clientDataInterval])\n",
    "    model=load_model(modelLocation)\n",
    "    clientsModelList.append(model)\n",
    "    lastLowerBound+=clientDataInterval\n",
    "# ------- 3. Update clients' model with intial server's deep-model ----------\n",
    "for clientID in range(numOfClients):\n",
    "    clientsModelList[clientID].fit(xClientsList[clientID], yClientsList[clientID], epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "        \n",
    "start_time = time.time()\n",
    "process = psutil.Process(os.getpid())\n",
    "for iterationNo in range(1,numOfIterations+1):\n",
    "    print(\"Iteration\",iterationNo)\n",
    "    for clientID in range(numOfClients):\n",
    "        print(\"clientID\",clientID)\n",
    "        clientsModelList[clientID].compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])\n",
    "        clientsModelList[clientID].fit(xClientsList[clientID], yClientsList[clientID], epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "        clientWeight=clientsModelList[clientID].get_weights()\n",
    "        # Find sum of all client's model\n",
    "        updateServerModel(clientsModelList[clientID], clientWeight)\n",
    "        firstClientFlag=False\n",
    "    #Avarage all clients model\n",
    "    for ind in range(len(deepModelAggWeights)):\n",
    "        deepModelAggWeights[ind]/=numOfClients\n",
    "\n",
    "    dw_last=deepModel.get_weights()\n",
    "\n",
    "    for ind in range(len(deepModelAggWeights)): \n",
    "        dw_last[ind]=deepModelAggWeights[ind]\n",
    "     \n",
    "    #Update server's model\n",
    "    deepModel.set_weights(dw_last) \n",
    "    print(\"Server's model updated\")\n",
    "    print(\"Saving model . . .\")\n",
    "    deepModel.save(modelLocation)\n",
    "    # Servers model is updated, now it can be used again by the clients\n",
    "    updateClientsModels()\n",
    "    firstClientFlag=True\n",
    "    deepModelAggWeights.clear()\n",
    "\n",
    "    yPredict = deepModel.predict(xTest)\n",
    "    acc, prec, recall, f1Score= predictTestData(yPredict, yTest)\n",
    "    accList.append(acc)\n",
    "    precList.append(prec)\n",
    "    recallList.append(recall)\n",
    "    f1List.append(f1Score)\n",
    "    print(\"Acc:\\n\", acc)\n",
    "    print(\"Prec:\\n\", prec)\n",
    "    print(\"Recall:\\n\", recall)\n",
    "    print(\"F1-Score:\\n\", f1Score)\n",
    "\n",
    "memoryTraining=process.memory_percent()\n",
    "timeTraining=time.time() - start_time\n",
    "print(\"---Memory---\",memoryTraining)\n",
    "print(\"--- %s seconds (TRAINING)---\" % (timeTraining))\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='auto')\n",
    "\n",
    "history = deepModel.fit(xServer, yServer, epochs=epochs, \n",
    "                        validation_data = (xTest,yTest))\n",
    "                        # callbacks=[early_stopping])\n",
    "\n",
    "learningAccs=history.history['val_accuracy']\n",
    "learningLoss=history.history['val_loss']\n",
    "\n",
    "# resultSaveLocation=root_path+'Results/'+algoName+'_Users_vs_TR_vs_Iterations_vs_AccLossMemTime'+'.csv'\n",
    "dfSave=pd.DataFrame(columns=['Clients', 'Iterations to converge', 'Accuracy', 'Loss', 'Memory', 'Time'])\n",
    "dfSaveIndex=0\n",
    "saveList = [numOfClients, len(learningLoss), learningAccs[len(learningAccs)-1], learningLoss[len(learningLoss)-1], memoryTraining, timeTraining]\n",
    "dfSave.loc[dfSaveIndex] = saveList\n",
    "\n",
    "yPredict = deepModel.predict(xTest)\n",
    "acc, prec, recall, f1Score= predictTestData(yPredict, yTest)\n",
    "\n",
    "print(\"Number of users:\", numOfClients)\n",
    "deepModel.save(modelLocation)\n",
    "print(\"Epochs:\", epochs)\n",
    "print(\"BatchSize:\", batch_size)\n",
    "print(\"Activation:\", activationFun, \"Optimizer:\", optimizerName)\n",
    "\n",
    "print(\"Iterations:\", numOfIterations)\n",
    "print(\"Memory:\", memoryTraining)\n",
    "print(\"Time:\", timeTraining)\n",
    "print(dfSave)\n",
    "\n",
    "df_performance_timeRounds = pd.DataFrame(\n",
    "    {'Accuracy': accList,\n",
    "     'Precision': precList,\n",
    "     'Recall': recallList,\n",
    "     'F1-Score': f1List \n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PG9fudXOEK_g"
   },
   "outputs": [],
   "source": [
    "df_performance_timeRounds"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyNMCew/Pgmz+rjcM2ttMax9",
   "collapsed_sections": [],
   "name": "SyncFL_newExp_CoLab.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}