Theano_Lasagne_CIFAR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level Theano + Lasagne Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cuDNN version 6021 on context None\n",
      "Mapped name None to device cuda: Tesla K80 (BF4E:00:00.0)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import theano.tensor as T\n",
    "import theano\n",
    "import lasagne\n",
    "import lasagne.layers as L\n",
    "import lasagne.nonlinearities as nl\n",
    "import lasagne.objectives as obj\n",
    "import lasagne.updates as upd\n",
    "from common.params import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Numpy:  1.13.1\n",
      "Theano:  0.10.0beta1.dev-RELEASE\n",
      "Lasagne:  0.2.dev1\n",
      "GPU:  ['Tesla K80', 'Tesla K80']\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"Theano: \", theano.__version__)\n",
    "print(\"Lasagne: \", lasagne.__version__)\n",
    "print(\"GPU: \", get_gpu_name())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#CuDNN auto-tune\n",
    "theano.config.dnn.conv.algo_fwd = \"time_once\"\n",
    "theano.config.dnn.conv.algo_bwd_filter = \"time_once\"\n",
    "theano.config.dnn.conv.algo_bwd_data = \"time_once\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_symbol():\n",
    "    conv1 = L.Conv2DLayer(X, num_filters=50, filter_size=(3, 3), pad='same')\n",
    "    conv2 = L.Conv2DLayer(conv1, num_filters=50, filter_size=(3, 3), pad='same')\n",
    "    pool1 = L.MaxPool2DLayer(conv2, pool_size=(2, 2), stride=(2, 2))\n",
    "    drop1 = L.DropoutLayer(pool1, 0.25)\n",
    "    \n",
    "    conv3 = L.Conv2DLayer(drop1, num_filters=100, filter_size=(3, 3), pad='same')\n",
    "    conv4 = L.Conv2DLayer(conv3, num_filters=100, filter_size=(3, 3), pad='same')\n",
    "    pool2 = L.MaxPool2DLayer(conv4, pool_size=(2, 2), stride=(2, 2))\n",
    "    drop2 = L.DropoutLayer(pool2, 0.25)\n",
    "    \n",
    "    flatten = L.FlattenLayer(drop2)\n",
    "    fc1 = L.DenseLayer(flatten, 512)\n",
    "    drop4 = L.DropoutLayer(fc1, 0.5)\n",
    "    pred = L.DenseLayer(drop4, N_CLASSES, name=\"output\", nonlinearity=nl.softmax)\n",
    "    \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_model(net):\n",
    "    pred = L.get_output(net)\n",
    "    params = L.get_all_params(net)\n",
    "    xentropy = obj.categorical_crossentropy(pred, y)\n",
    "    loss = T.mean(xentropy)\n",
    "    # The tensorflow LR, MOMENTUM are slightly different\n",
    "    updates = upd.momentum(loss, params, learning_rate=LR, momentum=MOMENTUM)\n",
    "    return pred, loss, updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
      "Done.\n",
      "Extracting files...\n",
      "Done.\n",
      "Preparing train set...\n",
      "Preparing test set...\n",
      "Done.\n",
      "(50000, 3, 32, 32) (10000, 3, 32, 32) (50000,) (10000,)\n",
      "float32 float32 int32 int32\n",
      "CPU times: user 2.97 s, sys: 1.49 s, total: 4.46 s\n",
      "Wall time: 35.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = cifar_for_library(channel_first=True)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 48 ms, sys: 4 ms, total: 52 ms\n",
      "Wall time: 55.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Place-holders\n",
    "X = L.InputLayer(shape=(None, 3, 32, 32))\n",
    "y = T.ivector(\"y\")\n",
    "# Initialise model\n",
    "net = create_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 344 ms, sys: 12 ms, total: 356 ms\n",
      "Wall time: 874 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pred, loss, updates = init_model(net)\n",
    "# Accuracy for logging\n",
    "accuracy = obj.categorical_accuracy(pred, y)\n",
    "accuracy = T.mean(T.cast(accuracy, theano.config.floatX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.96 s, sys: 288 ms, total: 3.25 s\n",
      "Wall time: 13.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Compile functions\n",
    "train_func = theano.function([X.input_var, y], [loss, accuracy], updates=updates)\n",
    "pred = L.get_output(net, deterministic=True)\n",
    "pred_func = theano.function([X.input_var], T.argmax(pred, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.40625\n",
      "1 Train accuracy: 0.484375\n",
      "2 Train accuracy: 0.5625\n",
      "3 Train accuracy: 0.734375\n",
      "4 Train accuracy: 0.625\n",
      "5 Train accuracy: 0.703125\n",
      "6 Train accuracy: 0.75\n",
      "7 Train accuracy: 0.828125\n",
      "8 Train accuracy: 0.765625\n",
      "9 Train accuracy: 0.8125\n",
      "CPU times: user 2min 29s, sys: 54.7 s, total: 3min 23s\n",
      "Wall time: 4min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for j in range(EPOCHS):\n",
    "    for data, label in yield_mb(x_train, y_train, BATCHSIZE, shuffle=True):\n",
    "        loss, acc_train = train_func(data, label)\n",
    "    # Log\n",
    "    print(j, \"Train accuracy:\", acc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 640 ms, sys: 308 ms, total: 948 ms\n",
      "Wall time: 1.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
    "y_guess = np.zeros(n_samples, dtype=np.int)\n",
    "y_truth = y_test[:n_samples]\n",
    "c = 0\n",
    "for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
    "    output = pred_func(data)\n",
    "    y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = output\n",
    "    c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.768930288462\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}