Training_3_set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Import libraries\n",
    "import torch\n",
    "import pandas as pd\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "import numpy as np\n",
    "import random\n",
    "import time\n",
    "import datetime\n",
    "import os\n",
    "import sys\n",
    "import csv\n",
    "from torch import backends\n",
    "from beautifultable import BeautifulTable\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import precision_recall_fscore_support, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "##SETTINGS\n",
    "doTrain = True\n",
    "doEval = True\n",
    "doExtractBaso = False\n",
    "\n",
    "nfold = 1 #number of folds to train\n",
    "fold_offset = 23\n",
    "lr=0.01 #learning rate\n",
    "\n",
    "batch_size = 32\n",
    "val_split = 0.2 #trainset percentage allocated for devset\n",
    "test_val_split = 0.1 #trainset percentage allocated for test_val set (i.e. the test set of known patients)\n",
    "\n",
    "#cwd = os.getcwd()\n",
    "cwd = \"../subjects_inter/min-max/windows_20/tr-False/folds_inter/\"\n",
    "subject = 1 # serve per caricare le folds da cartelle diverse\n",
    "prefix_train = 'TrainFold'\n",
    "prefix_test = 'TestFold'\n",
    "\n",
    "spw=20 #samples per window\n",
    "nmuscles=10 #initial number of muscles acquired\n",
    "\n",
    "#Enable/Disable shuffle on trainset/testset\n",
    "shuffle_train = False\n",
    "shuffle_test= False\n",
    "\n",
    "#Delete electrogonio signals\n",
    "# 3 = Left Rectus Femuris; 5 = Left Goniometer; 9 = Right Rectus Femuris; 11 = Right Goniometer\n",
    "exclude_features = False\n",
    "#Only use electrogonio signals\n",
    "include_only_features = False\n",
    "#Features to selected/deselected for input to the networks\n",
    "features_select = [9, 10] #1 to 4\n",
    "\n",
    "#Select which models to run. Insert comma separated values into 'model_select' var.\n",
    "#List. 0:'FF', 1:'FC2', 2:'FC2DP', 3:'FC3', 4:'FC3dp', 5:'Conv1d', 6:'MultiConv1d' \n",
    "#e.g: model_select = [0,4,6] to select FF,FC3dp,MultiConv1d\n",
    "\n",
    "# Modelli del paper: 11 (FF2), 14 (FF4), 16 (FF5) --> Prova questi!\n",
    "# FF6 per testarlo potente dopo (17)\n",
    "model_lst = ['FF','FC2','FC2DP','FC3','FC3dp','Conv1d','MultiConv1d',\n",
    "             'MultiConv1d_2','MultiConv1d_3', 'MultiConv1d_4', 'MultiConv1d_5', \n",
    "             'FF2', 'CNN1', 'FF3', 'FF4', 'CNN2', 'FF5', 'FF6', 'CNN3', 'CNN1-FF5', 'CNN1-2','CNN1-1', 'CNN1-3', 'CNN_w60']\n",
    "model_select = [17] # 17 = FF6\n",
    "\n",
    "#Early stop settings\n",
    "maxepoch = 100\n",
    "maxpatience = 10\n",
    "\n",
    "use_cuda = True\n",
    "use_gputil = False\n",
    "cuda_device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CUDA\n",
    "\n",
    "if use_gputil and torch.cuda.is_available():\n",
    "    import GPUtil\n",
    "\n",
    "    # Get the first available GPU\n",
    "    os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "    try:\n",
    "        deviceIDs = GPUtil.getAvailable(order='memory', limit=1, maxLoad=100, maxMemory=20)  # return a list of available gpus\n",
    "    except:\n",
    "        print('GPU not compatible with NVIDIA-SMI')\n",
    "    else:\n",
    "        os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(deviceIDs[0])\n",
    "        \n",
    "    ttens = torch.tensor(np.array([[1, 2, 3], [4, 5, 6]]))\n",
    "    ttens = ttens.cuda()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is CUDA available? --> True\n",
      "Cuda Device: cuda:0\n"
     ]
    }
   ],
   "source": [
    "print('Is CUDA available? --> ' + str(torch.cuda.is_available()))\n",
    "print('Cuda Device: ' + str(cuda_device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Seeds\n",
    "def setSeeds(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "setSeeds(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prints header of beautifultable report for each fold\n",
    "def header(model_list,nmodel,nfold,traindataset,testdataset, testdataset_U):\n",
    "    print('+++++++++++++++++++++++++++++++++++++++++++++++++')\n",
    "    print('MODEL: '+model_list[nmodel])\n",
    "    print('Fold: '+str(nfold))\n",
    "    print('+++++++++++++++++++++++++++++++++++++++++++++++++\\n\\n')\n",
    "    shape = list(traindataset.x_data.shape)\n",
    "    print('Trainset fold'+str(i)+' shape: '+str(shape[0])+'x'+str((shape[1]+1)))\n",
    "    shape = list(testdataset.x_data.shape)\n",
    "    print('Testset (L) fold'+str(i)+' shape: '+str(shape[0])+'x'+str((shape[1]+1)))\n",
    "    shape = list(testdataset_U.x_data.shape)\n",
    "    print('Testset (U) fold' + str(i) + ' shape: ' + str(shape[0])+ 'x' + str((shape[1]+1)) +'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prints actual beautifultable for each fold\n",
    "def table(model_list,nmodel,accuracies,precisions,recalls,f1_scores,accuracies_dev):\n",
    "    table = BeautifulTable()\n",
    "    table.column_headers = [\"{}\".format(model_list[nmodel]), \"Avg\", \"Stdev\"]\n",
    "    table.append_row([\"Accuracy\",round(np.average(accuracies),3),round(np.std(accuracies),3)])\n",
    "    table.append_row([\"Precision\",round(np.average(precisions),3),round(np.std(precisions),3)])\n",
    "    table.append_row([\"Recall\",round(np.average(recalls),3),round(np.std(recalls),3)])\n",
    "    table.append_row([\"F1_score\",round(np.average(f1_scores),3),round(np.std(f1_scores),3)])\n",
    "    table.append_row([\"Accuracy_dev\",round(np.average(accuracies_dev),3),round(np.std(accuracies_dev),3)])    \n",
    "    print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saves best model state on disk for each fold\n",
    "def save_checkpoint (state, is_best, filename, logfile):\n",
    "    if is_best:\n",
    "        msg = \"=> Saving a new best. \"+'Epoch: '+str(state['epoch'])\n",
    "        print (msg)\n",
    "        logfile.write(msg + \"\\n\")\n",
    "        torch.save(state, filename)  \n",
    "    else:\n",
    "        msg = \"=> Validation accuracy did not improve. \"+'Epoch: '+str(state['epoch'])\n",
    "        print (msg)\n",
    "        logfile.write(msg + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Compute sklearn metrics: Recall, Precision, F1-score\n",
    "def pre_rec (loader, model, positiveLabel):\n",
    "    y_true = np.array([])\n",
    "    y_pred = np.array([])\n",
    "    with torch.no_grad():\n",
    "        for i,data in enumerate (loader,0):\n",
    "            inputs, labels = data\n",
    "            y_true = np.append(y_true,labels.cpu())\n",
    "            outputs = model(inputs)\n",
    "            outputs[outputs>=0.5] = 1\n",
    "            outputs[outputs<0.5] = 0\n",
    "            y_pred = np.append(y_pred,outputs.cpu())\n",
    "    y_true = np.where(y_true==positiveLabel,0,1)\n",
    "    y_pred = np.where(y_pred==positiveLabel,0,1)\n",
    "    precision, recall, f1_score, _ = precision_recall_fscore_support(y_true, y_pred, average='binary')\n",
    "    return round(precision*100,3), round(recall*100,3), round(f1_score*100,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculates model accuracy. Predicted vs Correct.\n",
    "def accuracy (loader, model):\n",
    "    total=0\n",
    "    correct=0\n",
    "    with torch.no_grad():\n",
    "        for i, data in enumerate(loader, 0):\n",
    "            inputs, labels = data    # Carica i dati\n",
    "            outputs = model(inputs)    # Tira fuori gli output\n",
    "            outputs[outputs>=0.5] = 1    # Se l'output è >= 0.5 --> outputs[i] = 1\n",
    "            outputs[outputs<0.5] = 0    # Se l'output è < 0.5 --> outputs[i] = 0\n",
    "            total += labels.size(0)     # Calcola i labels totali\n",
    "            correct += (outputs == labels).sum().item()    # Conta i corretti (quelli che matchano)\n",
    "    return round((100 * correct / total),3)    # Tira fuori la percentuale di accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Arrays to store metrics\n",
    "accs = np.empty([nfold,1])\n",
    "accs_test_val = np.empty([nfold,1])\n",
    "precisions_0_U = np.empty([nfold,1])\n",
    "recalls_0_U = np.empty([nfold,1])\n",
    "f1_scores_0_U = np.empty([nfold,1])\n",
    "precisions_1_U = np.empty([nfold,1])\n",
    "recalls_1_U = np.empty([nfold,1])\n",
    "f1_scores_1_U = np.empty([nfold,1])\n",
    "precisions_0_L = np.empty([nfold,1])\n",
    "recalls_0_L = np.empty([nfold,1])\n",
    "f1_scores_0_L = np.empty([nfold,1])\n",
    "precisions_1_L = np.empty([nfold,1])\n",
    "recalls_1_L = np.empty([nfold,1])\n",
    "f1_scores_1_L = np.empty([nfold,1])\n",
    "accs_dev = np.empty([nfold,1])\n",
    "times = np.empty([nfold,1])\n",
    "\n",
    "#Calculate avg metrics on folds\n",
    "def averages (vals):\n",
    "    avgs = []\n",
    "    for val in vals:\n",
    "        avgs.append(round(np.average(val),3))\n",
    "    return avgs\n",
    "\n",
    "#Calculate std metrics on folds\n",
    "def stds (vals):\n",
    "    stds = []\n",
    "    for val in vals:\n",
    "        stds.append(round(np.std(val),3))\n",
    "    return stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## -- Salva il basografico predetto su un file sfruttando l'accuracy\n",
    "\n",
    "def save_predicted_baso(loader, model, subject_predict, model_select, num_fold):\n",
    "    windows_length = 20\n",
    "    predicted_baso_windowed = []\n",
    "    with torch.no_grad():\n",
    "        \n",
    "        ## -- Estrae gli output di ciascuna window\n",
    "        for i, data, in enumerate(loader, 0):\n",
    "            inputs, labels = data\n",
    "            outputs = model(inputs)\n",
    "            outputs[outputs >= 0.5] = 1\n",
    "            outputs[outputs < 0.5] = 0\n",
    "            predicted_temp = outputs.tolist()\n",
    "            predicted_baso_windowed.extend(predicted_temp)\n",
    "        \n",
    "        predicted_baso_windowed = pd.DataFrame(predicted_baso_windowed)    # Converto in DataFrame\n",
    "        predicted_baso_windowed = predicted_baso_windowed.as_matrix()\n",
    "        \n",
    "        limit = len(predicted_baso_windowed)\n",
    "        predicted_baso = np.empty([limit*windows_length,1], dtype = int)\n",
    "        \n",
    "        count = 0\n",
    "        for j in range(0, limit):\n",
    "            predicted_baso[count:count+windows_length] = predicted_baso_windowed[j]\n",
    "            count += windows_length\n",
    "            \n",
    "        predicted_baso = pd.DataFrame(predicted_baso)\n",
    "        out_path = '../subjects_inter/min-max/windows_20/tr-False/folds_inter/Predicted/' \n",
    "        if not os.path.exists(out_path):\n",
    "            os.makedirs(out_path)\n",
    "        #np.savetxt(out_path + 's' + str(subject_predict) + '_predicted.csv', predicted_baso, delimiter = \",\")\n",
    "        predicted_baso = predicted_baso.to_csv(out_path + 's' + str(subject_predict) + '_predicted.csv', index = None, header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Shuffle\n",
    "def dev_shuffle (shuffle_train,shuffle_test,val_split,traindataset,testdataset):\n",
    "    train_size = len(traindataset)\n",
    "    test_size = len(testdataset)\n",
    "    train_indices = list(range(train_size))\n",
    "    test_indices = list(range(test_size))\n",
    "    split = int(np.floor(val_split * train_size))\n",
    "    if shuffle_train:\n",
    "        np.random.shuffle(train_indices)\n",
    "    if shuffle_test:\n",
    "        np.random.shuffle(test_indices) \n",
    "    train_indices, dev_indices = train_indices[split:], train_indices[:split]\n",
    "    # Samplers\n",
    "    tr_sampler = SubsetRandomSampler(train_indices)\n",
    "    d_sampler = SubsetRandomSampler(dev_indices)\n",
    "    te_sampler = SubsetRandomSampler(test_indices)\n",
    "    return tr_sampler,d_sampler,te_sampler\n",
    "\n",
    "# Questa funzione serve dividere i vari set una volta caricate le fold\n",
    "# In particolare, verrà usata solo per 'dev_loader', dato che lo split \n",
    "# nei vari set è già stato fatto quando sono state create le fold\n",
    "def data_split (shuffle_train,shuffle_test,val_split,test_val_split,traindataset,testdataset):\n",
    "    train_size = len(traindataset)\n",
    "    test_size = len(testdataset)\n",
    "    train_indices = list(range(train_size))\n",
    "    test_indices = list(range(test_size))\n",
    "    test_val_split = int(np.floor(test_val_split * train_size)) \n",
    "    dev_split = int(np.floor(val_split * (train_size-test_val_split) + test_val_split))\n",
    "    if shuffle_train:\n",
    "        np.random.shuffle(train_indices)\n",
    "    if shuffle_test:\n",
    "        np.random.shuffle(test_indices) \n",
    "    train_indices, dev_indices, test_val_indices = train_indices[dev_split:], train_indices[test_val_split:dev_split], train_indices[:test_val_split]\n",
    "    # Samplers\n",
    "    tr_sampler = SubsetRandomSampler(train_indices)\n",
    "    d_sampler = SubsetRandomSampler(dev_indices)\n",
    "    tv_sampler = SubsetRandomSampler(test_val_indices)                \n",
    "    te_sampler = SubsetRandomSampler(test_indices)\n",
    "    return tr_sampler,d_sampler,tv_sampler,te_sampler\n",
    "\n",
    "# Questa funzione serve se usiamo come input della rete 3 files:\n",
    "    # TRAINFOLD\n",
    "    # TESTFOLD_LEARNED\n",
    "    # TESTFOLD_UNLEARNED\n",
    "# L'unico file che deve essere effettivamente suddiviso, in questo caso, è il TRAINFOLD, dato che useremo\n",
    "# una percentuale 'val_split' (hyperparameter) per il validation, e una percentuale 1-'val_split'\n",
    "# per il vero e proprio training della rete.\n",
    "# La funzione da in output i sampler da utilizzare come parametri per il DataLoader:\n",
    "    # train_sampler\n",
    "    # dev_sampler\n",
    "def split_train ( val_split, traindataset ): \n",
    "    train_size = len(traindataset)    # lunghezza del TRAIN_DATASET\n",
    "    train_indices = list(range(train_size))    # crea una lista che va da 0 -> lunghezza di train_size totale\n",
    "    dev_size = int(np.floor(val_split * train_size))    # Dimensione del dev_set\n",
    "    split = int(np.floor((train_size - dev_size)))    # Valore di split dei 2 set\n",
    "\n",
    "    train_indices_bis, dev_indices = train_indices[:split], train_indices[split:]    # Indici splittati\n",
    "        \n",
    "    # Sampler\n",
    "    tr_sampler = SubsetRandomSampler(train_indices_bis)    # Train Sampler\n",
    "    d_sampler = SubsetRandomSampler(dev_indices)    # Dev Sampler\n",
    "    return tr_sampler, d_sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading fold 23\n",
      "Train dataset: 458378 windows;\n",
      "Test dataset (LEARNED): 50965 windows;\n",
      "Test dataset (UNLEARNED) 13593 windows.\n",
      "\n",
      "Each window is composed by 201 samples.\n",
      "The number of muscles is 10\n"
     ]
    }
   ],
   "source": [
    "#Loads and appends all folds all at once\n",
    "trainfolds = []    # Train set\n",
    "testfolds_L = []    # Test set (LEARNED)\n",
    "testfolds_U = []    # Test set (UNLEARNED)\n",
    "\n",
    "col_select = np.array([])\n",
    "\n",
    "#This is an hack to test smaller windows\n",
    "for i in range (spw*nmuscles,200):\n",
    "    col_select = np.append(col_select,i)\n",
    "    \n",
    "for i in range (0,spw*nmuscles,nmuscles):\n",
    "    for muscle in features_select:\n",
    "        col_select = np.append(col_select,muscle -1 + i)\n",
    "    cols=np.arange(0,spw*nmuscles+1)\n",
    "\n",
    "if exclude_features & (not include_only_features): #delete gonio\n",
    "    for j in range(fold_offset,fold_offset + nfold):\n",
    "        print(\"Loading fold \" + str(j))\n",
    "        traindata = pd.read_csv(os.path.join(cwd, prefix_train + '_' + str(j)+'.csv'),sep=',',header=None,dtype=np.float32,usecols=[i for i in cols if i not in col_select.astype(int)])\n",
    "        trainfolds.append(traindata)\n",
    "        testdata_L = pd.read_csv(os.path.join(cwd, prefix_test + '_L_' + str(j)+'.csv'),sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i not in col_select.astype(int)])\n",
    "        testfolds_L.append(testdata_L) \n",
    "        testdata_U = pd.read_csv(os.path.join(cwd, prefix_test + '_U_' + str(j) +'.csv'),sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i not in col_select.astype(int)])\n",
    "        testfolds_U.append(testdata_U)\n",
    "        print('Train dataset: ' + str(len(traindata)) + ' windows;')\n",
    "        print('Test dataset (LEARNED): ' + str(len(testdata_L)) + ' windows;')\n",
    "        print('Test dataset (UNLEARNED) ' + str(len(testdata_U)) + ' windows.')\n",
    "        \n",
    "elif include_only_features & (not exclude_features): #only gonio\n",
    "    for j in range(fold_offset, fold_offset + nfold):\n",
    "        print(\"Loading fold \" + str(j))\n",
    "        traindata = pd.read_csv(os.path.join(cwd, prefix_train + '_' + str(j)+'.csv'),sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i in col_select.astype(int)])\n",
    "        testdata_L = pd.read_csv(os.path.join(cwd, prefix_test + '_L_'+ str(j)+'.csv'),sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i in col_select.astype(int)])\n",
    "        testdata_U = pd.read_csv(os.path.join(cwd, prefix_test + '_U_' + str(j) +'.csv'),sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i in col_select.astype(int)])\n",
    "        trainfolds.append(traindata)\n",
    "        testfolds_L.append(testdata_L)\n",
    "        testfolds_U.append(testdata_U)\n",
    "        print('Train dataset: ' + str(len(traindata)) + ' windows;')\n",
    "        print('Test dataset (LEARNED): ' + str(len(testdata)) + ' windows;')\n",
    "        print('Test dataset (UNLEARNED) ' + str(len(testdata_U)) + ' windows.')\n",
    "        \n",
    "elif (not include_only_features) & (not exclude_features): \n",
    "    for j in range(fold_offset,fold_offset + nfold):\n",
    "        print(\"Loading fold \" + str(j))\n",
    "        traindata = pd.read_csv(os.path.join(cwd, prefix_train + '_' + str(j) + '.csv'),sep=',',header=None,dtype=np.float32)\n",
    "        testdata = pd.read_csv(os.path.join(cwd, prefix_test + '_L_' + str(j) +'.csv'),sep=',',header=None,dtype=np.float32)\n",
    "        testdata_U = pd.read_csv(os.path.join(cwd, prefix_test + '_U_' + str(j) +'.csv'),sep=',',header=None,dtype=np.float32)\n",
    "        trainfolds.append(traindata)\n",
    "        testfolds_L.append(testdata)\n",
    "        testfolds_U.append(testdata_U)    # Aggiunto testfold UNLEARNED \n",
    "        print('Train dataset: ' + str(len(traindata)) + ' windows;')\n",
    "        print('Test dataset (LEARNED): ' + str(len(testdata)) + ' windows;')\n",
    "        print('Test dataset (UNLEARNED) ' + str(len(testdata_U)) + ' windows.')\n",
    "else:\n",
    "    raise ValueError('use_gonio and del_gonio cannot be both True')\n",
    "\n",
    "nmuscles=int((len(traindata.columns)-1)/spw)\n",
    "print('\\nEach window is composed by ' + str(len(traindata.columns)) + ' samples.')\n",
    "print('The number of muscles is ' + str(nmuscles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## -- Carica il soggetto su cui vuoi predire il basografico\n",
    "\n",
    "''''nmuscles = 10\n",
    "subject_predict = 17\n",
    "directory_windows = '../subjects/min-max/windows_20/tr-False_sliding_1_c-False/'\n",
    "subj_prefix = 's'\n",
    "subj_suffix = '_norm_windows_20.csv'\n",
    "file = directory_windows + subj_prefix + str(subject_predict) + subj_suffix\n",
    "\n",
    "if doExtractBaso:\n",
    "    if exclude_features & (not include_only_features): #delete gonio\n",
    "        for j in range(fold_offset,fold_offset + nfold):\n",
    "            testfolds_U = []\n",
    "        \n",
    "        cols = [i for i in range(0,201)]\n",
    "        \n",
    "        col_select = []\n",
    "        for i in range(0,200,nmuscles):\n",
    "            col_select.append(8 + i)\n",
    "            col_select.append(9 + i)\n",
    "        \n",
    "        cols = np.asarray(cols)\n",
    "        col_select = np.asarray(col_select)\n",
    "        \n",
    "        print(\"Loading Subject \" + str(subject_predict))\n",
    "        testdata_U = pd.read_csv(file,sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i not in col_select.astype(int)])\n",
    "        testfolds_U.append(testdata_U)\n",
    "        #testdata_U = pd.read_csv(file,sep=',',header=None,dtype=np.float32)\n",
    "        #testfolds_U.append(testdata_U)\n",
    "        print('Subject ' + str(subject_predict) + ' loaded: ' + str(len(testdata_U)) + ' windows.')\n",
    "\n",
    "        nmuscles=int((len(traindata.columns)-1)/spw)\n",
    "        print('\\nEach window is composed by ' + str(len(testdata_U.columns)) + ' samples.')\n",
    "        print('The number of muscles is ' + str(nmuscles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "nmuscles=int((len(traindata.columns)-1)/spw) #used for layer dimensions and stride CNNs\n",
    "#trainfolds = None\n",
    "#traindata = None\n",
    "#import gc\n",
    "#gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import models\n",
    "from models import *\n",
    "models._spw = spw\n",
    "models._nmuscles = nmuscles\n",
    "models._batch_size = batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "print(models._nmuscles)\n",
    "\n",
    "#import models\n",
    "#from models import *\n",
    "#TEST DIMENSIONS\n",
    "#models.nmuscles = nmuscles\n",
    "def testdimensions():\n",
    "    model = Model23()\n",
    "    print(model)\n",
    "    x = torch.randn(32,1,480)\n",
    "    model.test_dim(x)\n",
    " \n",
    "#testdimensions()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fieldnames = ['Fold','Acc_L', 'Acc_U',\n",
    "              'R_0_U','R_1_U',\n",
    "              'R_0_L','R_1_L',\n",
    "              'Stop_epoch','Accuracy_dev'] #coloumn names report FOLD CSV\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "#TRAINING LOOP\n",
    "def train_test():\n",
    "    for k in model_select:\n",
    "        \n",
    "        table = BeautifulTable()\n",
    "        avgtable = BeautifulTable()\n",
    "        fieldnames1 = [model_lst[k],'Avg','Std_dev'] #column names report GLOBAL CSV\n",
    "        folder = os.path.join(cwd,'Report_'+str(model_lst[k]))\n",
    "        if not os.path.exists(folder):\n",
    "            os.mkdir(folder)\n",
    "\n",
    "        logfilepath = os.path.join(folder,'log.txt')\n",
    "        logfile = open(logfilepath,\"a\") \n",
    "\n",
    "        with open(os.path.join(folder,'Report_folds.csv'),'a') as f_fold, open(os.path.join(folder,'Report_global.csv'),'a') as f_global:\n",
    "            writer = csv.DictWriter(f_fold, fieldnames = fieldnames)\n",
    "            writer1  = csv.DictWriter(f_global, fieldnames = fieldnames1)\n",
    "            writer.writeheader()\n",
    "            writer1.writeheader()\n",
    "            t0 = 0\n",
    "            t1 = 0\n",
    "            \n",
    "            \n",
    "            \n",
    "            ## -- DEBUG 1\n",
    "            for i in range(1, nfold + 1):\n",
    "                \n",
    "                t0 = time.time()\n",
    "                setSeeds(0)\n",
    "                \n",
    "                class Traindataset(Dataset):\n",
    "                    def __init__(self):\n",
    "                        self.data=trainfolds[i-1]\n",
    "                        self.x_data=torch.from_numpy(np.asarray(self.data.iloc[:, 0:-1])) \n",
    "                        self.len=self.data.shape[0]\n",
    "                        self.y_data = torch.from_numpy(np.asarray(self.data.iloc[:, [-1]]))\n",
    "                        if (use_cuda):\n",
    "                            self.x_data = self.x_data.cuda()\n",
    "                            self.y_data = self.y_data.cuda()\n",
    "                    def __getitem__(self, index):\n",
    "                        return self.x_data[index], self.y_data[index]\n",
    "                    def __len__(self):\n",
    "                        return self.len\n",
    "                \n",
    "                # Classe per il Testset LEARNED\n",
    "                class Testdataset_L(Dataset):\n",
    "                    def __init__(self):\n",
    "                        self.data=testfolds_L[i-1]\n",
    "                        self.x_data=torch.from_numpy(np.asarray(self.data.iloc[:, 0:-1]))\n",
    "                        self.len=self.data.shape[0]\n",
    "                        self.y_data = torch.from_numpy(np.asarray(self.data.iloc[:, [-1]]))\n",
    "                        if (use_cuda):\n",
    "                            self.x_data = self.x_data.cuda()\n",
    "                            self.y_data = self.y_data.cuda()\n",
    "                    def __getitem__(self, index):\n",
    "                        return self.x_data[index], self.y_data[index]\n",
    "                    def __len__(self):\n",
    "                        return self.len\n",
    "                \n",
    "                # Classe per il Testset UNLEARNED\n",
    "                class Testdataset_U(Dataset):\n",
    "                    def __init__(self):\n",
    "                        self.data=testfolds_U[i-1]\n",
    "                        self.x_data=torch.from_numpy(np.asarray(self.data.iloc[:, 0:-1]))\n",
    "                        self.len=self.data.shape[0]\n",
    "                        self.y_data = torch.from_numpy(np.asarray(self.data.iloc[:, [-1]]))\n",
    "                        if (use_cuda):\n",
    "                            self.x_data = self.x_data.cuda()\n",
    "                            self.y_data = self.y_data.cuda()\n",
    "                    def __getitem__(self, index):\n",
    "                        return self.x_data[index], self.y_data[index]\n",
    "                    def __len__(self):\n",
    "                        return self.len\n",
    "\n",
    "                ## DEFINISCO I DATASET DA CARICARE    \n",
    "                traindataset = Traindataset()    # Train dataset\n",
    "                testdataset = Testdataset_L()    # Test dataset LEARNED\n",
    "                testdataset_U = Testdataset_U()    # Test dataset UNLEARNED\n",
    "                \n",
    "                ## -- 'fold_number' aggiunta per essere coerente nell'header\n",
    "                current_fold_number = i + fold_offset - 1\n",
    "                \n",
    "                header( model_lst, k, current_fold_number, traindataset, testdataset, testdataset_U)\n",
    "\n",
    "                #train_sampler,dev_sampler,test_sampler=dev_shuffle(shuffle_train,shuffle_test,val_split,traindataset,testdataset)\n",
    "                #train_sampler,dev_sampler,test_val_sampler,test_sampler=data_split(shuffle_train,shuffle_test,val_split,test_val_split,traindataset,testdataset)\n",
    "                \n",
    "                ## CREO I SAMPLER PER DIVIDERE IL TRAINDASTASET\n",
    "                tr_sampler, d_sampler = split_train(val_split, traindataset)\n",
    "                \n",
    "                ## LOADERS\n",
    "                # torch.utils.data.DataLoader deve avere come argomenti:\n",
    "                    # dataset -> quello che dobbiamo caricare\n",
    "                    # batch_size -> 'batch_size' definito negli hyperparameter della rete\n",
    "                    # drop_last = True -> Se la lunghezza del dataset non è divisibile per 'batch_size', \n",
    "                    # l'ultimo batch viene eliminato\n",
    "                # 'dev_loader e 'train_loader' hanno bisogno anche di\n",
    "                    # sampler = dev_sampler (una certa parte del trainset \n",
    "                    # deve essere usata come 'dev_set')\n",
    "                \n",
    "                # Training Set\n",
    "                train_loader = torch.utils.data.DataLoader(traindataset, batch_size = batch_size, \n",
    "                                                          sampler = tr_sampler, drop_last = True)\n",
    "                print('Train Set dimension: ' + str(len(train_loader)))\n",
    "                # Development Set\n",
    "                dev_loader = torch.utils.data.DataLoader(traindataset, batch_size=batch_size, \n",
    "                                                           sampler = d_sampler, drop_last=True)\n",
    "                print('Dev Set dimension: ' + str(len(dev_loader)))\n",
    "                # Testset LEARNED\n",
    "                test_val_loader = torch.utils.data.DataLoader(testdataset, batch_size=batch_size,\n",
    "                                                                drop_last=True)\n",
    "                print('Test Set (L) dimension: ' + str(len(test_val_loader)))\n",
    "                # Testset UNLEARNED\n",
    "                test_loader = torch.utils.data.DataLoader(testdataset_U, batch_size = batch_size, \n",
    "                                                          drop_last = True)\n",
    "                print('Test Set (U) dimension: ' + str(len(test_loader)))\n",
    "                \n",
    "                modelClass = \"Model\" + str(k)\n",
    "                model = eval(modelClass)()\n",
    "                \n",
    "                if (use_cuda):\n",
    "                    model = model.cuda()\n",
    "\n",
    "                if doTrain:\n",
    "                    \n",
    "                    # criterion = nn.BCELoss(size_average=True)\n",
    "                    criterion = nn.BCELoss(reduction = 'mean')    # Cambiato perché dava un warning (size_average era un vecchio metodo in Pytorch)\n",
    "                    optimizer = torch.optim.SGD(model.parameters(), lr)    \n",
    "                    msg = 'Accuracy on test set before training: '+str(accuracy(test_loader, model))+'\\n'\n",
    "                    print(msg)\n",
    "                    logfile.write(msg + \"\\n\")\n",
    "                    #EARLY STOP\n",
    "                    epoch = 0\n",
    "                    patience = 0\n",
    "                    best_acc_dev=0\n",
    "                    while (epoch<maxepoch and patience < maxpatience):\n",
    "                        running_loss = 0.0\n",
    "                        for l, data in enumerate(train_loader, 0):\n",
    "                            inputs, labels = data\n",
    "                            if use_cuda:\n",
    "                                inputs, labels = inputs.cuda(), labels.cuda()\n",
    "                            inputs, labels = Variable(inputs), Variable(labels)\n",
    "                            y_pred = model(inputs)\n",
    "                            if use_cuda:\n",
    "                                y_pred = y_pred.cuda()\n",
    "                            loss = criterion(y_pred, labels)\n",
    "                            optimizer.zero_grad()\n",
    "                            loss.backward()\n",
    "                            optimizer.step()\n",
    "                            running_loss += loss.item()\n",
    "                            #print accuracy ever l mini-batches\n",
    "                            if l % 2000 == 1999:\n",
    "                                msg = '[%d, %5d] loss: %.3f' %(epoch + 1, l + 1, running_loss / 999)\n",
    "                                print(msg)\n",
    "                                logfile.write(msg + \"\\n\")\n",
    "                                running_loss = 0.0\n",
    "                                #msg = 'Accuracy on dev set:' + str(accuracy(dev_loader))\n",
    "                                #print(msg)\n",
    "                                #logfile.write(msg + \"\\n\")        \n",
    "                        accdev = (accuracy(dev_loader, model))\n",
    "                        msg = 'Accuracy on dev set:' + str(accdev)\n",
    "                        print(msg)\n",
    "                        logfile.write(msg + \"\\n\")        \n",
    "                        is_best = bool(accdev > best_acc_dev)\n",
    "                        best_acc_dev = (max(accdev, best_acc_dev))\n",
    "                        save_checkpoint({\n",
    "                            'epoch': epoch + 1,\n",
    "                            'state_dict': model.state_dict(),\n",
    "                            'best_acc_dev': best_acc_dev\n",
    "                        }, is_best,os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'), logfile)\n",
    "                        if is_best:\n",
    "                            patience=0\n",
    "                        else:\n",
    "                            patience = patience+1\n",
    "                        epoch = epoch+1\n",
    "                        logfile.flush()\n",
    "                        \n",
    "                if doEval:\n",
    "                    if use_cuda:                        \n",
    "                        state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'))\n",
    "                    else:\n",
    "                        state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'), map_location=lambda storage, loc: storage)\n",
    "                    stop_epoch = state['epoch']\n",
    "                    model.load_state_dict(state['state_dict'])\n",
    "                    if not use_cuda:\n",
    "                        model.cpu()\n",
    "                    accuracy_dev = state['best_acc_dev']\n",
    "                    model.eval()\n",
    "                    \n",
    "                    acctest = (accuracy(test_loader, model))\n",
    "                    acctest_val = (accuracy(test_val_loader, model))\n",
    "                    accs[i-1] = acctest\n",
    "                    accs_test_val[i-1] = acctest_val\n",
    "                    \n",
    "                    precision_0_U,recall_0_U,f1_score_0_U = pre_rec(test_loader, model, 0.0)\n",
    "                    precisions_0_U[i-1] = precision_0_U\n",
    "                    recalls_0_U[i-1] = recall_0_U\n",
    "                    f1_scores_0_U[i-1] = f1_score_0_U\n",
    "                    \n",
    "                    precision_1_U,recall_1_U,f1_score_1_U = pre_rec(test_loader, model, 1.0)\n",
    "                    precisions_1_U[i-1] = precision_1_U\n",
    "                    recalls_1_U[i-1] = recall_1_U\n",
    "                    f1_scores_1_U[i-1] = f1_score_1_U\n",
    "                    \n",
    "                    precision_0_L,recall_0_L,f1_score_0_L = pre_rec(test_val_loader, model, 0.0)\n",
    "                    precisions_0_L[i-1] = precision_0_L\n",
    "                    recalls_0_L[i-1] = recall_0_L\n",
    "                    f1_scores_0_L[i-1] = f1_score_0_L\n",
    "                    \n",
    "                    precision_1_L,recall_1_L,f1_score_1_L = pre_rec(test_val_loader, model, 1.0)\n",
    "                    precisions_1_L[i-1] = precision_1_L\n",
    "                    recalls_1_L[i-1] = recall_1_L\n",
    "                    f1_scores_1_L[i-1] = f1_score_1_L\n",
    "                    \n",
    "                    accs_dev[i-1] = accuracy_dev\n",
    "                    \n",
    "                    writer.writerow({'Fold': current_fold_number,'Acc_L': acctest_val, 'Acc_U': acctest,\n",
    "                                     #'P_0_U': precision_0_U,'R_0_U': recall_0_U,'F1_0_U': f1_score_0_U,\n",
    "                                     'R_0_U': recall_0_U,\n",
    "                                     #'P_1_U': precision_1_U,'R_1_U': recall_1_U,'F1_1_U': f1_score_1_U,\n",
    "                                     'R_1_U': recall_1_U,\n",
    "                                     #'P_0_L': precision_0_L,'R_0_L': recall_0_L,'F1_0_L': f1_score_0_L,\n",
    "                                     'R_0_L': recall_0_L,\n",
    "                                     #'P_1_L': precision_1_L,'R_1_L': recall_1_L,'F1_1_L': f1_score_1_L,\n",
    "                                     'R_1_L': recall_1_L,\n",
    "                                     'Stop_epoch': stop_epoch,'Accuracy_dev': accuracy_dev})\n",
    "                    table.column_headers = fieldnames\n",
    "                    table.append_row([i,acctest_val,acctest,\n",
    "                                      #precision_0_U,recall_0_U,f1_score_0_U,\n",
    "                                      recall_0_U,\n",
    "                                      #precision_1_U,recall_1_U,f1_score_1_U,\n",
    "                                      recall_1_U,\n",
    "                                      #precision_0_L,recall_0_L,f1_score_0_L,\n",
    "                                      recall_0_L,\n",
    "                                      #precision_1_L,recall_1_L,f1_score_1_L,\n",
    "                                      recall_1_L,\n",
    "                                      stop_epoch,accuracy_dev])\n",
    "                    print(table)\n",
    "                    print('----------------------------------------------------------------------')\n",
    "                    logfile.write(str(table) + \"\\n----------------------------------------------------------------------\\n\")\n",
    "                    t1 = time.time()\n",
    "                    times[i-1] = int(t1-t0)\n",
    "                    \n",
    "                    ## -- Estrai il Basografico\n",
    "                \n",
    "                # fold_predicted += 1\n",
    "                if doExtractBaso:\n",
    "                    if use_cuda:                        \n",
    "                        state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'))\n",
    "                    else:\n",
    "                        state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'), map_location=lambda storage, loc: storage)\n",
    "                    stop_epoch = state['epoch']\n",
    "                    model.load_state_dict(state['state_dict'])\n",
    "                    if not use_cuda:\n",
    "                        model.cpu()\n",
    "                    accuracy_dev = state['best_acc_dev']\n",
    "                    model.eval()\n",
    "                    \n",
    "                    ## -- QUESTO DEVI CONTROLLARLO! SE ESCE CHE i È DIVERSO DALLA FOLD DEVI RIFARE TUTTO!\n",
    "                    print('F'+str(current_fold_number)+'best.pth.tar')\n",
    "                    predicted_baso = save_predicted_baso(test_loader, model, subject_predict, model_lst[k], current_fold_number)   \n",
    "            \n",
    "            duration = str(datetime.timedelta(seconds=np.sum(times)))\n",
    "            writer.writerow({})\n",
    "            writer.writerow({'Fold': 'Elapsed time: '+duration})\n",
    "            avg_acc_test_val = round(np.average(accs_test_val),3)\n",
    "            std_acc_test_val = round(np.std(accs_test_val),3)\n",
    "            \n",
    "            avg_acc_test_val,avg_a,avg_p_0_U,avg_r_0_U,avg_f_0_U,avg_p_1_U,avg_r_1_U,avg_f_1_U,avg_p_0_L,avg_r_0_L,avg_f_0_L,avg_p_1_L,avg_r_1_L,avg_f_1_L,avg_a_d=averages([accs_test_val,accs,precisions_0_U,recalls_0_U,f1_scores_0_U,precisions_1_U,recalls_1_U,f1_scores_1_U,precisions_0_L,recalls_0_L,f1_scores_0_L,precisions_1_L,recalls_1_L,f1_scores_1_L,accs_dev])\n",
    "            std_acc_test_val,std_a,std_p_0_U,std_r_0_U,std_f_0_U,std_p_1_U,std_r_1_U,std_f_1_U,std_p_0_L,std_r_0_L,std_f_0_L,std_p_1_L,std_r_1_L,std_f_1_L,std_a_d=stds([accs_test_val,accs,precisions_0_U,recalls_0_U,f1_scores_0_U,precisions_1_U,recalls_1_U,f1_scores_1_U,precisions_0_L,recalls_0_L,f1_scores_0_L,precisions_1_L,recalls_1_L,f1_scores_1_L,accs_dev])\n",
    "            \n",
    "            writer1.writerow({model_lst[k]: 'Acc_U','Avg': avg_a,'Std_dev': std_acc_test_val})\n",
    "            writer1.writerow({model_lst[k]: 'Acc_L','Avg': avg_acc_test_val,'Std_dev': std_a})\n",
    "            writer1.writerow({model_lst[k]: 'P_0_U','Avg': avg_p_0_U ,'Std_dev': std_p_0_U})\n",
    "            writer1.writerow({model_lst[k]: 'R_0_U','Avg': avg_r_0_U,'Std_dev': std_r_0_U})\n",
    "            writer1.writerow({model_lst[k]: 'F1_0_U','Avg': avg_f_0_U,'Std_dev': std_f_0_U})\n",
    "            writer1.writerow({model_lst[k]: 'P_1_U','Avg': avg_p_1_U,'Std_dev': std_p_1_U})\n",
    "            writer1.writerow({model_lst[k]: 'R_1_U','Avg': avg_r_1_U,'Std_dev': std_r_1_U})\n",
    "            writer1.writerow({model_lst[k]: 'F1_1_U','Avg': avg_f_1_U,'Std_dev': std_f_1_U})            \n",
    "            writer1.writerow({model_lst[k]: 'P_0_L','Avg': avg_p_0_L,'Std_dev': std_p_0_L})\n",
    "            writer1.writerow({model_lst[k]: 'R_0_L','Avg': avg_r_0_L,'Std_dev': std_r_0_L})\n",
    "            writer1.writerow({model_lst[k]: 'F1_0_L','Avg': avg_f_0_L,'Std_dev': std_f_0_L})\n",
    "            writer1.writerow({model_lst[k]: 'P_1_L','Avg': avg_p_1_L,'Std_dev': std_p_1_L})\n",
    "            writer1.writerow({model_lst[k]: 'R_1_L','Avg': avg_r_1_L,'Std_dev': std_r_1_L})\n",
    "            writer1.writerow({model_lst[k]: 'F1_1_L','Avg': avg_f_1_L,'Std_dev': std_f_1_L})                        \n",
    "            writer1.writerow({model_lst[k]: 'Acc_dev','Avg': avg_a_d,'Std_dev': std_a_d})\n",
    "            writer1.writerow({})\n",
    "            writer1.writerow({model_lst[k]: 'Elapsed time: '+duration})\n",
    "            avgtable.column_headers = fieldnames1\n",
    "            avgtable.append_row(['Acc_U',avg_a,std_a])\n",
    "            avgtable.append_row(['Acc_L',avg_acc_test_val,std_acc_test_val])\n",
    "            avgtable.append_row(['P_0_U',avg_p_0_U,std_p_0_U])\n",
    "            avgtable.append_row(['R_0_U',avg_r_0_U,std_r_0_U])\n",
    "            avgtable.append_row(['F1_0_U',avg_f_0_U,std_f_0_U])\n",
    "            avgtable.append_row(['P_1_U',avg_p_1_U,std_p_1_U])\n",
    "            avgtable.append_row(['R_1_U',avg_r_1_U,std_r_1_U])\n",
    "            avgtable.append_row(['F1_1_U',avg_f_1_U,std_f_1_U])                        \n",
    "            avgtable.append_row(['P_0_L',avg_p_0_L,std_p_0_L])\n",
    "            avgtable.append_row(['R_0_L',avg_r_0_L,std_r_0_L])\n",
    "            avgtable.append_row(['F1_0_L',avg_f_0_L,std_f_0_L])\n",
    "            avgtable.append_row(['P_1_L',avg_p_1_L,std_p_1_L])\n",
    "            avgtable.append_row(['R_1_L',avg_r_1_L,std_r_1_L])\n",
    "            avgtable.append_row(['F1_1_L',avg_f_1_L,std_f_1_L])            \n",
    "            avgtable.append_row(['Accuracy_dev',avg_a_d,std_a_d])\n",
    "            print(avgtable)\n",
    "            logfile.write(str(avgtable) + \"\\n\")\n",
    "            msg = 'Elapsed time: '+ duration + '\\n\\n'\n",
    "            print(msg)\n",
    "            logfile.write(msg )\n",
    "\n",
    "        logfile.close()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Subject 26\n",
      "Subject 26 loaded: 13593 windows.\n",
      "\n",
      "Each window is composed by 201 samples.\n",
      "The number of muscles is 10\n",
      "F23best.pth.tar\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Nabuconodosor II\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:18: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subject Predicted!\n"
     ]
    }
   ],
   "source": [
    "for k in model_select:\n",
    "## -- Carica il soggetto su cui vuoi predire il basografico\n",
    "\n",
    "    nmuscles = 10\n",
    "    window_length = 20\n",
    "    subject_predict = 26\n",
    "    fold_offset = 23\n",
    "    directory_windows = '../subjects_inter/min-max/windows_20/tr-False/'\n",
    "    subj_prefix = 's'\n",
    "    subj_suffix = '_norm_windows_20.csv'\n",
    "    folder = directory_windows + 'folds_inter/Report_' + model_lst[k] + '/'\n",
    "    file = directory_windows + subj_prefix + str(subject_predict) + subj_suffix\n",
    "    \n",
    "\n",
    "\n",
    "    for j in range(fold_offset,fold_offset + nfold):\n",
    "        testfolds_U = []\n",
    "        \n",
    "        cols = [i for i in range(0, nmuscles*window_length + 1)]\n",
    "        col_select = []\n",
    "        cols = np.asarray(cols)\n",
    "        col_select = np.asarray(col_select)\n",
    "        \n",
    "        print(\"Loading Subject \" + str(subject_predict))\n",
    "        testdata_U = pd.read_csv(file,sep=',',header=None,dtype=np.float32, usecols=[i for i in cols if i not in col_select.astype(int)])\n",
    "        testfolds_U.append(testdata_U)\n",
    "        #testdata_U = pd.read_csv(file,sep=',',header=None,dtype=np.float32)\n",
    "        #testfolds_U.append(testdata_U)\n",
    "        print('Subject ' + str(subject_predict) + ' loaded: ' + str(len(testdata_U)) + ' windows.')\n",
    "\n",
    "        nmuscles=int((len(testdata_U.columns)-1)/spw)\n",
    "        print('\\nEach window is composed by ' + str(len(testdata_U.columns)) + ' samples.')\n",
    "        print('The number of muscles is ' + str(nmuscles))\n",
    "        \n",
    "        for i in range(1, nfold + 1):\n",
    "            setSeeds(0)\n",
    "    \n",
    "                # Classe per il Testset UNLEARNED\n",
    "            class Testdataset_U(Dataset):\n",
    "                def __init__(self):\n",
    "                    self.data=testfolds_U[i-1]\n",
    "                    self.x_data=torch.from_numpy(np.asarray(self.data.iloc[:, 0:-1]))\n",
    "                    self.len=self.data.shape[0]\n",
    "                    self.y_data = torch.from_numpy(np.asarray(self.data.iloc[:, [-1]]))\n",
    "                    if (use_cuda):\n",
    "                        self.x_data = self.x_data.cuda()\n",
    "                        self.y_data = self.y_data.cuda()\n",
    "                def __getitem__(self, index):\n",
    "                    return self.x_data[index], self.y_data[index]\n",
    "                def __len__(self):\n",
    "                    return self.len\n",
    "                \n",
    "            testdataset_U = Testdataset_U()    # Test dataset UNLEARNED\n",
    "            test_loader = torch.utils.data.DataLoader(testdataset_U, batch_size = batch_size, \n",
    "                                                          drop_last = True)\n",
    "\n",
    "            current_fold_number = fold_offset + i - 1\n",
    "            \n",
    "            modelClass = \"Model\" + str(k)\n",
    "            model = eval(modelClass)()\n",
    "            if (use_cuda):\n",
    "                model = model.cuda()\n",
    "                \n",
    "            if use_cuda:                        \n",
    "                state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'))\n",
    "            else:\n",
    "                state = torch.load(os.path.join(folder,'F'+str(current_fold_number)+'best.pth.tar'), map_location=lambda storage, loc: storage)\n",
    "            stop_epoch = state['epoch']\n",
    "            model.load_state_dict(state['state_dict'])\n",
    "            if not use_cuda:\n",
    "                model.cpu()\n",
    "            accuracy_dev = state['best_acc_dev']\n",
    "            model.eval()\n",
    "\n",
    "                ## -- QUESTO DEVI CONTROLLARLO! SE ESCE CHE i È DIVERSO DALLA FOLD DEVI RIFARE TUTTO!\n",
    "            print('F'+str(current_fold_number)+'best.pth.tar')\n",
    "            predicted_baso = save_predicted_baso(test_loader, model, subject_predict, model_lst[k], current_fold_number)  \n",
    "            print('Subject Predicted!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am using CUDA: SUCCESS!\n",
      "+++++++++++++++++++++++++++++++++++++++++++++++++\n",
      "MODEL: FF6\n",
      "Fold: 23\n",
      "+++++++++++++++++++++++++++++++++++++++++++++++++\n",
      "\n",
      "\n",
      "Trainset fold190 shape: 458378x201\n",
      "Testset (L) fold190 shape: 50965x201\n",
      "Testset (U) fold190 shape: 13593x201\n",
      "\n",
      "Train Set dimension: 11459\n",
      "Dev Set dimension: 2864\n",
      "Test Set (L) dimension: 1592\n",
      "Test Set (U) dimension: 424\n",
      "Accuracy on test set before training: 40.588\n",
      "\n",
      "[1,  2000] loss: 0.842\n",
      "[1,  4000] loss: 0.329\n",
      "[1,  6000] loss: 0.307\n",
      "[1,  8000] loss: 0.290\n",
      "[1, 10000] loss: 0.290\n",
      "Accuracy on dev set:94.314\n",
      "=> Saving a new best. Epoch: 1\n",
      "[2,  2000] loss: 0.275\n",
      "[2,  4000] loss: 0.270\n",
      "[2,  6000] loss: 0.267\n",
      "[2,  8000] loss: 0.261\n",
      "[2, 10000] loss: 0.265\n",
      "Accuracy on dev set:94.439\n",
      "=> Saving a new best. Epoch: 2\n",
      "[3,  2000] loss: 0.254\n",
      "[3,  4000] loss: 0.251\n",
      "[3,  6000] loss: 0.255\n",
      "[3,  8000] loss: 0.245\n",
      "[3, 10000] loss: 0.245\n",
      "Accuracy on dev set:95.174\n",
      "=> Saving a new best. Epoch: 3\n",
      "[4,  2000] loss: 0.242\n",
      "[4,  4000] loss: 0.240\n",
      "[4,  6000] loss: 0.237\n",
      "[4,  8000] loss: 0.237\n",
      "[4, 10000] loss: 0.233\n",
      "Accuracy on dev set:95.329\n",
      "=> Saving a new best. Epoch: 4\n",
      "[5,  2000] loss: 0.232\n",
      "[5,  4000] loss: 0.230\n",
      "[5,  6000] loss: 0.226\n",
      "[5,  8000] loss: 0.224\n",
      "[5, 10000] loss: 0.229\n",
      "Accuracy on dev set:95.481\n",
      "=> Saving a new best. Epoch: 5\n",
      "[6,  2000] loss: 0.221\n",
      "[6,  4000] loss: 0.221\n",
      "[6,  6000] loss: 0.218\n",
      "[6,  8000] loss: 0.220\n",
      "[6, 10000] loss: 0.216\n",
      "Accuracy on dev set:95.637\n",
      "=> Saving a new best. Epoch: 6\n",
      "[7,  2000] loss: 0.218\n",
      "[7,  4000] loss: 0.213\n",
      "[7,  6000] loss: 0.213\n",
      "[7,  8000] loss: 0.213\n",
      "[7, 10000] loss: 0.214\n",
      "Accuracy on dev set:96.101\n",
      "=> Saving a new best. Epoch: 7\n",
      "[8,  2000] loss: 0.208\n",
      "[8,  4000] loss: 0.208\n",
      "[8,  6000] loss: 0.208\n",
      "[8,  8000] loss: 0.208\n",
      "[8, 10000] loss: 0.208\n",
      "Accuracy on dev set:95.601\n",
      "=> Validation accuracy did not improve. Epoch: 8\n",
      "[9,  2000] loss: 0.204\n",
      "[9,  4000] loss: 0.199\n",
      "[9,  6000] loss: 0.207\n",
      "[9,  8000] loss: 0.200\n",
      "[9, 10000] loss: 0.203\n",
      "Accuracy on dev set:95.966\n",
      "=> Validation accuracy did not improve. Epoch: 9\n",
      "[10,  2000] loss: 0.196\n",
      "[10,  4000] loss: 0.196\n",
      "[10,  6000] loss: 0.201\n",
      "[10,  8000] loss: 0.202\n",
      "[10, 10000] loss: 0.194\n",
      "Accuracy on dev set:95.907\n",
      "=> Validation accuracy did not improve. Epoch: 10\n",
      "[11,  2000] loss: 0.195\n",
      "[11,  4000] loss: 0.196\n",
      "[11,  6000] loss: 0.193\n",
      "[11,  8000] loss: 0.196\n",
      "[11, 10000] loss: 0.189\n",
      "Accuracy on dev set:96.343\n",
      "=> Saving a new best. Epoch: 11\n",
      "[12,  2000] loss: 0.192\n",
      "[12,  4000] loss: 0.186\n",
      "[12,  6000] loss: 0.185\n",
      "[12,  8000] loss: 0.193\n",
      "[12, 10000] loss: 0.188\n",
      "Accuracy on dev set:96.548\n",
      "=> Saving a new best. Epoch: 12\n",
      "[13,  2000] loss: 0.186\n",
      "[13,  4000] loss: 0.184\n",
      "[13,  6000] loss: 0.186\n",
      "[13,  8000] loss: 0.184\n",
      "[13, 10000] loss: 0.182\n",
      "Accuracy on dev set:96.467\n",
      "=> Validation accuracy did not improve. Epoch: 13\n",
      "[14,  2000] loss: 0.179\n",
      "[14,  4000] loss: 0.185\n",
      "[14,  6000] loss: 0.182\n",
      "[14,  8000] loss: 0.181\n",
      "[14, 10000] loss: 0.179\n",
      "Accuracy on dev set:96.12\n",
      "=> Validation accuracy did not improve. Epoch: 14\n",
      "[15,  2000] loss: 0.182\n",
      "[15,  4000] loss: 0.177\n",
      "[15,  6000] loss: 0.180\n",
      "[15,  8000] loss: 0.175\n",
      "[15, 10000] loss: 0.176\n",
      "Accuracy on dev set:96.465\n",
      "=> Validation accuracy did not improve. Epoch: 15\n",
      "[16,  2000] loss: 0.178\n",
      "[16,  4000] loss: 0.178\n",
      "[16,  6000] loss: 0.173\n",
      "[16,  8000] loss: 0.172\n",
      "[16, 10000] loss: 0.175\n",
      "Accuracy on dev set:95.378\n",
      "=> Validation accuracy did not improve. Epoch: 16\n",
      "[17,  2000] loss: 0.171\n",
      "[17,  4000] loss: 0.177\n",
      "[17,  6000] loss: 0.175\n",
      "[17,  8000] loss: 0.170\n",
      "[17, 10000] loss: 0.170\n",
      "Accuracy on dev set:96.401\n",
      "=> Validation accuracy did not improve. Epoch: 17\n",
      "[18,  2000] loss: 0.172\n",
      "[18,  4000] loss: 0.172\n",
      "[18,  6000] loss: 0.167\n",
      "[18,  8000] loss: 0.172\n",
      "[18, 10000] loss: 0.166\n",
      "Accuracy on dev set:96.714\n",
      "=> Saving a new best. Epoch: 18\n",
      "[19,  2000] loss: 0.171\n",
      "[19,  4000] loss: 0.167\n",
      "[19,  6000] loss: 0.167\n",
      "[19,  8000] loss: 0.162\n",
      "[19, 10000] loss: 0.166\n",
      "Accuracy on dev set:96.835\n",
      "=> Saving a new best. Epoch: 19\n",
      "[20,  2000] loss: 0.163\n",
      "[20,  4000] loss: 0.166\n",
      "[20,  6000] loss: 0.164\n",
      "[20,  8000] loss: 0.163\n",
      "[20, 10000] loss: 0.166\n",
      "Accuracy on dev set:96.71\n",
      "=> Validation accuracy did not improve. Epoch: 20\n",
      "[21,  2000] loss: 0.160\n",
      "[21,  4000] loss: 0.161\n",
      "[21,  6000] loss: 0.164\n",
      "[21,  8000] loss: 0.165\n",
      "[21, 10000] loss: 0.165\n",
      "Accuracy on dev set:96.326\n",
      "=> Validation accuracy did not improve. Epoch: 21\n",
      "[22,  2000] loss: 0.160\n",
      "[22,  4000] loss: 0.160\n",
      "[22,  6000] loss: 0.156\n",
      "[22,  8000] loss: 0.163\n",
      "[22, 10000] loss: 0.160\n",
      "Accuracy on dev set:96.738\n",
      "=> Validation accuracy did not improve. Epoch: 22\n",
      "[23,  2000] loss: 0.160\n",
      "[23,  4000] loss: 0.157\n",
      "[23,  6000] loss: 0.155\n",
      "[23,  8000] loss: 0.162\n",
      "[23, 10000] loss: 0.157\n",
      "Accuracy on dev set:96.679\n",
      "=> Validation accuracy did not improve. Epoch: 23\n",
      "[24,  2000] loss: 0.157\n",
      "[24,  4000] loss: 0.160\n",
      "[24,  6000] loss: 0.154\n",
      "[24,  8000] loss: 0.158\n",
      "[24, 10000] loss: 0.155\n",
      "Accuracy on dev set:96.432\n",
      "=> Validation accuracy did not improve. Epoch: 24\n",
      "[25,  2000] loss: 0.155\n",
      "[25,  4000] loss: 0.157\n",
      "[25,  6000] loss: 0.152\n",
      "[25,  8000] loss: 0.153\n",
      "[25, 10000] loss: 0.156\n",
      "Accuracy on dev set:96.819\n",
      "=> Validation accuracy did not improve. Epoch: 25\n",
      "[26,  2000] loss: 0.153\n",
      "[26,  4000] loss: 0.150\n",
      "[26,  6000] loss: 0.156\n",
      "[26,  8000] loss: 0.154\n",
      "[26, 10000] loss: 0.156\n",
      "Accuracy on dev set:96.214\n",
      "=> Validation accuracy did not improve. Epoch: 26\n",
      "[27,  2000] loss: 0.153\n",
      "[27,  4000] loss: 0.150\n",
      "[27,  6000] loss: 0.150\n",
      "[27,  8000] loss: 0.152\n",
      "[27, 10000] loss: 0.151\n",
      "Accuracy on dev set:96.847\n",
      "=> Saving a new best. Epoch: 27\n",
      "[28,  2000] loss: 0.147\n",
      "[28,  4000] loss: 0.150\n",
      "[28,  6000] loss: 0.152\n",
      "[28,  8000] loss: 0.148\n",
      "[28, 10000] loss: 0.150\n",
      "Accuracy on dev set:96.213\n",
      "=> Validation accuracy did not improve. Epoch: 28\n",
      "[29,  2000] loss: 0.149\n",
      "[29,  4000] loss: 0.145\n",
      "[29,  6000] loss: 0.150\n",
      "[29,  8000] loss: 0.146\n",
      "[29, 10000] loss: 0.146\n",
      "Accuracy on dev set:96.958\n",
      "=> Saving a new best. Epoch: 29\n",
      "[30,  2000] loss: 0.146\n",
      "[30,  4000] loss: 0.143\n",
      "[30,  6000] loss: 0.146\n",
      "[30,  8000] loss: 0.148\n",
      "[30, 10000] loss: 0.151\n",
      "Accuracy on dev set:96.702\n",
      "=> Validation accuracy did not improve. Epoch: 30\n",
      "[31,  2000] loss: 0.141\n",
      "[31,  4000] loss: 0.148\n",
      "[31,  6000] loss: 0.146\n",
      "[31,  8000] loss: 0.145\n",
      "[31, 10000] loss: 0.147\n",
      "Accuracy on dev set:96.942\n",
      "=> Validation accuracy did not improve. Epoch: 31\n",
      "[32,  2000] loss: 0.141\n",
      "[32,  4000] loss: 0.147\n",
      "[32,  6000] loss: 0.144\n",
      "[32,  8000] loss: 0.141\n",
      "[32, 10000] loss: 0.145\n",
      "Accuracy on dev set:96.807\n",
      "=> Validation accuracy did not improve. Epoch: 32\n",
      "[33,  2000] loss: 0.142\n",
      "[33,  4000] loss: 0.142\n",
      "[33,  6000] loss: 0.142\n",
      "[33,  8000] loss: 0.141\n",
      "[33, 10000] loss: 0.142\n",
      "Accuracy on dev set:96.711\n",
      "=> Validation accuracy did not improve. Epoch: 33\n",
      "[34,  2000] loss: 0.141\n",
      "[34,  4000] loss: 0.143\n",
      "[34,  6000] loss: 0.140\n",
      "[34,  8000] loss: 0.142\n",
      "[34, 10000] loss: 0.142\n",
      "Accuracy on dev set:96.456\n",
      "=> Validation accuracy did not improve. Epoch: 34\n",
      "[35,  2000] loss: 0.142\n",
      "[35,  4000] loss: 0.139\n",
      "[35,  6000] loss: 0.143\n",
      "[35,  8000] loss: 0.139\n",
      "[35, 10000] loss: 0.140\n",
      "Accuracy on dev set:96.839\n",
      "=> Validation accuracy did not improve. Epoch: 35\n",
      "[36,  2000] loss: 0.143\n",
      "[36,  4000] loss: 0.138\n",
      "[36,  6000] loss: 0.138\n",
      "[36,  8000] loss: 0.137\n",
      "[36, 10000] loss: 0.136\n",
      "Accuracy on dev set:97.008\n",
      "=> Saving a new best. Epoch: 36\n",
      "[37,  2000] loss: 0.139\n",
      "[37,  4000] loss: 0.139\n",
      "[37,  6000] loss: 0.138\n",
      "[37,  8000] loss: 0.133\n",
      "[37, 10000] loss: 0.138\n",
      "Accuracy on dev set:96.951\n",
      "=> Validation accuracy did not improve. Epoch: 37\n",
      "[38,  2000] loss: 0.133\n",
      "[38,  4000] loss: 0.138\n",
      "[38,  6000] loss: 0.142\n",
      "[38,  8000] loss: 0.139\n",
      "[38, 10000] loss: 0.137\n",
      "Accuracy on dev set:96.979\n",
      "=> Validation accuracy did not improve. Epoch: 38\n",
      "[39,  2000] loss: 0.131\n",
      "[39,  4000] loss: 0.133\n",
      "[39,  6000] loss: 0.139\n",
      "[39,  8000] loss: 0.140\n",
      "[39, 10000] loss: 0.136\n",
      "Accuracy on dev set:95.522\n",
      "=> Validation accuracy did not improve. Epoch: 39\n",
      "[40,  2000] loss: 0.131\n",
      "[40,  4000] loss: 0.138\n",
      "[40,  6000] loss: 0.136\n",
      "[40,  8000] loss: 0.133\n",
      "[40, 10000] loss: 0.135\n",
      "Accuracy on dev set:96.233\n",
      "=> Validation accuracy did not improve. Epoch: 40\n",
      "[41,  2000] loss: 0.133\n",
      "[41,  4000] loss: 0.132\n",
      "[41,  6000] loss: 0.134\n",
      "[41,  8000] loss: 0.135\n",
      "[41, 10000] loss: 0.138\n",
      "Accuracy on dev set:96.884\n",
      "=> Validation accuracy did not improve. Epoch: 41\n",
      "[42,  2000] loss: 0.129\n",
      "[42,  4000] loss: 0.130\n",
      "[42,  6000] loss: 0.133\n",
      "[42,  8000] loss: 0.134\n",
      "[42, 10000] loss: 0.134\n",
      "Accuracy on dev set:96.976\n",
      "=> Validation accuracy did not improve. Epoch: 42\n",
      "[43,  2000] loss: 0.132\n",
      "[43,  4000] loss: 0.129\n",
      "[43,  6000] loss: 0.133\n",
      "[43,  8000] loss: 0.134\n",
      "[43, 10000] loss: 0.132\n",
      "Accuracy on dev set:96.694\n",
      "=> Validation accuracy did not improve. Epoch: 43\n",
      "[44,  2000] loss: 0.129\n",
      "[44,  4000] loss: 0.131\n",
      "[44,  6000] loss: 0.130\n",
      "[44,  8000] loss: 0.134\n",
      "[44, 10000] loss: 0.131\n",
      "Accuracy on dev set:97.063\n",
      "=> Saving a new best. Epoch: 44\n",
      "[45,  2000] loss: 0.131\n",
      "[45,  4000] loss: 0.129\n",
      "[45,  6000] loss: 0.129\n",
      "[45,  8000] loss: 0.130\n",
      "[45, 10000] loss: 0.130\n",
      "Accuracy on dev set:95.902\n",
      "=> Validation accuracy did not improve. Epoch: 45\n",
      "[46,  2000] loss: 0.127\n",
      "[46,  4000] loss: 0.127\n",
      "[46,  6000] loss: 0.130\n",
      "[46,  8000] loss: 0.128\n",
      "[46, 10000] loss: 0.130\n",
      "Accuracy on dev set:97.113\n",
      "=> Saving a new best. Epoch: 46\n",
      "[47,  2000] loss: 0.129\n",
      "[47,  4000] loss: 0.129\n",
      "[47,  6000] loss: 0.131\n",
      "[47,  8000] loss: 0.122\n",
      "[47, 10000] loss: 0.128\n",
      "Accuracy on dev set:97.131\n",
      "=> Saving a new best. Epoch: 47\n",
      "[48,  2000] loss: 0.124\n",
      "[48,  4000] loss: 0.123\n",
      "[48,  6000] loss: 0.133\n",
      "[48,  8000] loss: 0.128\n",
      "[48, 10000] loss: 0.127\n",
      "Accuracy on dev set:96.813\n",
      "=> Validation accuracy did not improve. Epoch: 48\n",
      "[49,  2000] loss: 0.124\n",
      "[49,  4000] loss: 0.128\n",
      "[49,  6000] loss: 0.125\n",
      "[49,  8000] loss: 0.126\n",
      "[49, 10000] loss: 0.129\n",
      "Accuracy on dev set:96.709\n",
      "=> Validation accuracy did not improve. Epoch: 49\n",
      "[50,  2000] loss: 0.124\n",
      "[50,  4000] loss: 0.124\n",
      "[50,  6000] loss: 0.126\n",
      "[50,  8000] loss: 0.126\n",
      "[50, 10000] loss: 0.128\n",
      "Accuracy on dev set:96.932\n",
      "=> Validation accuracy did not improve. Epoch: 50\n",
      "[51,  2000] loss: 0.124\n",
      "[51,  4000] loss: 0.121\n",
      "[51,  6000] loss: 0.124\n",
      "[51,  8000] loss: 0.123\n",
      "[51, 10000] loss: 0.127\n",
      "Accuracy on dev set:97.019\n",
      "=> Validation accuracy did not improve. Epoch: 51\n",
      "[52,  2000] loss: 0.119\n",
      "[52,  4000] loss: 0.127\n",
      "[52,  6000] loss: 0.125\n",
      "[52,  8000] loss: 0.124\n",
      "[52, 10000] loss: 0.122\n",
      "Accuracy on dev set:96.763\n",
      "=> Validation accuracy did not improve. Epoch: 52\n",
      "[53,  2000] loss: 0.122\n",
      "[53,  4000] loss: 0.118\n",
      "[53,  6000] loss: 0.126\n",
      "[53,  8000] loss: 0.121\n",
      "[53, 10000] loss: 0.128\n",
      "Accuracy on dev set:96.675\n",
      "=> Validation accuracy did not improve. Epoch: 53\n",
      "[54,  2000] loss: 0.122\n",
      "[54,  4000] loss: 0.121\n",
      "[54,  6000] loss: 0.122\n",
      "[54,  8000] loss: 0.119\n",
      "[54, 10000] loss: 0.125\n",
      "Accuracy on dev set:96.4\n",
      "=> Validation accuracy did not improve. Epoch: 54\n",
      "[55,  2000] loss: 0.118\n",
      "[55,  4000] loss: 0.125\n",
      "[55,  6000] loss: 0.121\n",
      "[55,  8000] loss: 0.123\n",
      "[55, 10000] loss: 0.119\n",
      "Accuracy on dev set:96.972\n",
      "=> Validation accuracy did not improve. Epoch: 55\n",
      "[56,  2000] loss: 0.117\n",
      "[56,  4000] loss: 0.123\n",
      "[56,  6000] loss: 0.120\n",
      "[56,  8000] loss: 0.118\n",
      "[56, 10000] loss: 0.121\n",
      "Accuracy on dev set:96.569\n",
      "=> Validation accuracy did not improve. Epoch: 56\n",
      "[57,  2000] loss: 0.119\n",
      "[57,  4000] loss: 0.123\n",
      "[57,  6000] loss: 0.118\n",
      "[57,  8000] loss: 0.119\n",
      "[57, 10000] loss: 0.121\n",
      "Accuracy on dev set:97.057\n",
      "=> Validation accuracy did not improve. Epoch: 57\n",
      "+------+-------+-------+-------+-------+-------+-------+----------+------------+\n",
      "| Fold | Acc_L | Acc_U | R_0_U | R_1_U | R_0_L | R_1_L | Stop_epo | Accuracy_d |\n",
      "|      |       |       |       |       |       |       |    ch    |     ev     |\n",
      "+------+-------+-------+-------+-------+-------+-------+----------+------------+\n",
      "|  1   | 95.98 | 98.01 | 97.74 | 98.18 | 93.87 | 97.65 |    47    |   97.131   |\n",
      "|      |   6   |       |   8   |   9   |   7   |   7   |          |            |\n",
      "+------+-------+-------+-------+-------+-------+-------+----------+------------+\n",
      "----------------------------------------------------------------------\n",
      "+--------------+--------+---------+\n",
      "|     FF6      |  Avg   | Std_dev |\n",
      "+--------------+--------+---------+\n",
      "|    Acc_U     | 98.01  |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    Acc_L     | 95.986 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    P_0_U     | 97.359 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    R_0_U     | 97.748 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    F1_0_U    | 97.553 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    P_1_U     | 98.458 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    R_1_U     | 98.189 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    F1_1_U    | 98.323 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    P_0_L     | 96.946 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    R_0_L     | 93.877 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    F1_0_L    | 95.387 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    P_1_L     | 95.267 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    R_1_L     | 97.657 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "|    F1_1_L    | 96.447 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "| Accuracy_dev | 97.131 |   0.0   |\n",
      "+--------------+--------+---------+\n",
      "Elapsed time: 1:31:38\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nmuscles=int((len(traindata.columns)-1)/spw)\n",
    "if use_cuda and not use_gputil and cuda_device!=None and torch.cuda.is_available():\n",
    "    with torch.cuda.device(cuda_device):\n",
    "        print('I am using CUDA: SUCCESS!')\n",
    "        train_test()\n",
    "else:\n",
    "    print('I am NOT using CUDA: SUCCESS!')\n",
    "    train_test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}