titanic-pytorch-nn-tutorial.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nimport torchvision\nfrom torchvision import datasets, models, transforms\nfrom torchvision.transforms import functional as F\nimport random\nimport tensorflow as tf\n\nimport time\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.set_random_seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nkaeru_seed = 1337\nseed_everything(seed=kaeru_seed)\n\nbatch_size = 32\ntrain_epochs = 6","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntarget = train[\"Survived\"]\n\ntrain = pd.concat([train, test], sort=True)\n\nprint(train.shape)\n#train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"feature engineering"},{"metadata":{"trusted":true,"_uuid":"4864de0e92b42922e80e3a2e1e3b033d306e42cf"},"cell_type":"code","source":"def get_text_features(train):\n    train['Length_Name'] = train['Name'].astype(str).map(len)\n    return train\n\ntrain = get_text_features(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e4f6158f920e84f5f0a604eb465299cc997d6d7"},"cell_type":"code","source":"cat_cols = [\n     'Cabin','Embarked','Name','Sex','Ticket',\n]\n\nnum_cols = list(set(train.columns) - set(cat_cols) - set([\"Survived\"]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d465534be8d97dcacbc82ecca1c1912228016bb"},"cell_type":"markdown","source":"**handling categorical feats**"},{"metadata":{"trusted":true,"_uuid":"2bbce141eedd550a7cf9e6b72d930624ec3fa6aa"},"cell_type":"code","source":"def encode(encoder, x):\n    len_encoder = len(encoder)\n    try:\n        id = encoder[x]\n    except KeyError:\n        id = len_encoder\n    return id\n\nencoders = [{} for cat in cat_cols]\n\n\nfor i, cat in enumerate(cat_cols):\n    print('encoding %s ...' % cat, end=' ')\n    encoders[i] = {l: id for id, l in enumerate(train.loc[:, cat].astype(str).unique())}\n    train[cat] = train[cat].astype(str).apply(lambda x: encode(encoders[i], x))\n    print('Done')\n\nembed_sizes = [len(encoder) for encoder in encoders]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8982b7f9e96cefd1207a9a27d1db426f3499f4d9"},"cell_type":"markdown","source":"**handling numerical feats**"},{"metadata":{"trusted":true,"_uuid":"109ae46a6fab5e86eb13b2bbf15c083ebbf9daa5"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n \ntrain[num_cols] = train[num_cols].fillna(0)\nprint('scaling numerical columns')\n\nscaler = StandardScaler()\ntrain[num_cols] = scaler.fit_transform(train[num_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define PyTorch NN Architecture**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomLinear(nn.Module):\n    def __init__(self, in_features,\n                 out_features,\n                 bias=True, p=0.5):\n        super().__init__()\n        self.linear = nn.Linear(in_features,\n                               out_features,\n                               bias)\n        self.relu = nn.ReLU()\n        self.drop = nn.Dropout(p)\n        \n    def forward(self, x):\n        x = self.linear(x)\n        x = self.relu(x)\n        x = self.drop(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = nn.Sequential(CustomLinear(12, 32),\n                    nn.Linear(32, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split train and test"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train.loc[np.isfinite(train.Survived), :]\nX_train = X_train.drop([\"Survived\"], axis=1).values\ny_train = target.values\n\nX_test = train.loc[~np.isfinite(train.Survived), :]\n\nX_test = X_test.drop([\"Survived\"], axis=1).values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"StratifiedKfold"},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=kaeru_seed).split(X_train, y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preds = np.zeros((len(X_train)))\ntest_preds = np.zeros((len(X_test)))\n\nseed_everything(kaeru_seed)\n\nx_test_cuda = torch.tensor(X_test, dtype=torch.float32).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):\n    x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.float32).cuda()\n    y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.float32).cuda()\n    y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    model = net\n    model.cuda()\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n    optimizer = torch.optim.Adam(model.parameters())\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    for epoch in range(train_epochs):\n        start_time = time.time()\n        \n        model.train()\n        avg_loss = 0.\n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_pred = model(x_batch)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n        \n        model.eval()\n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros(len(X_test))\n        avg_val_loss = 0.\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_pred = model(x_batch).detach()\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))\n        \n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n\n        test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n    train_preds[valid_idx] = valid_preds_fold\n    test_preds += test_preds_fold / len(splits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_preds_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"help(x_batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = accuracy_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'accuracy_score': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"search_result = threshold_search(y_train, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/gender_submission.csv')\nsub.Survived = (test_preds > search_result['threshold']).astype(np.int8)\nsub.to_csv('simple_nn_submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}