cnn_emotion_0.3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages loaded\n",
      "['trainlabel', 'trainimg', 'imgsize', 'testimg', 'testlabel']\n",
      "2614 train images loaded\n",
      "1743 test images loaded\n",
      "2304 dimensional input\n",
      "Image size is [48 48]\n",
      "8 classes\n"
     ]
    }
   ],
   "source": [
    "#Ryan Shin: sungjin7127@gmail.com\n",
    "#Date: 160927\n",
    "#Obj: find the best model for emotion recogntion\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "#%matplotlib inline\n",
    "print (\"Packages loaded\")\n",
    "\n",
    "#Load data\n",
    "cwd = os.getcwd()\n",
    "loadpath = cwd + \"/dataset/data_gray.npz\"\n",
    "l = np.load(loadpath)\n",
    "\n",
    "#Check what's included\n",
    "print (l.files)\n",
    "\n",
    "# Parse data\n",
    "trainimg = l['trainimg']\n",
    "trainlabel = l['trainlabel']\n",
    "testimg = l['testimg']\n",
    "testlabel = l['testlabel']\n",
    "imgsize = l['imgsize']\n",
    "#use_gray = l['use_gray']\n",
    "ntrain = trainimg.shape[0]\n",
    "nclass = trainlabel.shape[1]\n",
    "dim    = trainimg.shape[1]\n",
    "ntest  = testimg.shape[0]\n",
    "print (\"%d train images loaded\" % (ntrain))\n",
    "print (\"%d test images loaded\" % (ntest))\n",
    "print (\"%d dimensional input\" % (dim))\n",
    "print (\"Image size is %s\" % (imgsize))\n",
    "print (\"%d classes\" % (nclass))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "total size of new array must be unchanged",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-22e57d8bc401>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainimg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0ma2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0ma2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: total size of new array must be unchanged"
     ]
    }
   ],
   "source": [
    "a = trainimg[5]\n",
    "a2 = a.reshape(28,28)\n",
    "a2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "plt.imshow(a2)\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "device_type = \"/cpu:0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN ready\n"
     ]
    }
   ],
   "source": [
    "with tf.device(device_type): # <= This is optional\n",
    "    n_input  = 48 * 48\n",
    "    n_output = 8\n",
    "    weights  = {\n",
    "        'wc1': tf.Variable(tf.random_normal([3, 3, 1, 96], stddev=0.1)),\n",
    "        'wd1': tf.Variable(tf.random_normal([24*24*96, n_output], stddev=0.1))\n",
    "    }\n",
    "    biases   = {\n",
    "        'bc1': tf.Variable(tf.random_normal([96], stddev=0.1)),\n",
    "        'bd1': tf.Variable(tf.random_normal([n_output], stddev=0.1))\n",
    "    }\n",
    "    def conv_simple(_input, _w, _b):\n",
    "        # Reshape input\n",
    "        _input_r = tf.reshape(_input, shape=[-1, 48, 48, 1])\n",
    "        # Convolution\n",
    "        _conv1 = tf.nn.conv2d(_input_r, _w['wc1'], strides=[1, 1, 1, 1], padding='SAME')\n",
    "        # Add-bias\n",
    "        _conv2 = tf.nn.bias_add(_conv1, _b['bc1'])\n",
    "        # Pass ReLu\n",
    "        _conv3 = tf.nn.relu(_conv2)\n",
    "        # Max-pooling\n",
    "        _pool  = tf.nn.max_pool(_conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        # Vectorize\n",
    "        _dense = tf.reshape(_pool, [-1, _w['wd1'].get_shape().as_list()[0]])\n",
    "        # Fully-connected layer\n",
    "        _out = tf.add(tf.matmul(_dense, _w['wd1']), _b['bd1'])\n",
    "        #output = tf.nn.softmax_cross_entropy_with_logits(_dense, weights['wd1'], biases['bd1'])\n",
    "        # Return everything\n",
    "        out = {\n",
    "            'input_r': _input_r, 'conv1': _conv1, 'conv2': _conv2, 'conv3': _conv3\n",
    "            , 'pool': _pool, 'dense': _dense, 'out': _out\n",
    "        }\n",
    "        return out\n",
    "print (\"CNN ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready to Go!\n"
     ]
    }
   ],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "#keepratio = tf.placeholder(tf.float32)\n",
    "\n",
    "# Parameters\n",
    "learning_rate   = 0.001\n",
    "training_epochs = 5\n",
    "batch_size      = 100\n",
    "display_step    = 1\n",
    "# Functions! \n",
    "with tf.device(device_type): # <= This is optional\n",
    "    _pred = conv_simple(x, weights, biases)['out']\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_pred, y))\n",
    "    optm = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "    _corr = tf.equal(tf.argmax(_pred,1), tf.argmax(y,1)) # Count corrects\n",
    "    accr = tf.reduce_mean(tf.cast(_corr, tf.float32)) # Accuracy\n",
    "    init = tf.initialize_all_variables()\n",
    "# Saver \n",
    "save_step = 1;\n",
    "savedir = \"./nets\"\n",
    "if not os.path.exists(savedir):\n",
    "    os.makedirs(savedir)\n",
    "\n",
    "saver = tf.train.Saver(max_to_keep=3) \n",
    "print (\"Network Ready to Go!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "do_train = 1\n",
    "sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/005 cost: 5.257679833\n",
      " Training accuracy: 0.150\n",
      " Test accuracy: 0.195\n",
      "Epoch: 001/005 cost: 2.119039235\n",
      " Training accuracy: 0.230\n",
      " Test accuracy: 0.264\n",
      "Epoch: 002/005 cost: 1.753020560\n",
      " Training accuracy: 0.410\n",
      " Test accuracy: 0.373\n",
      "Epoch: 003/005 cost: 1.564771069\n",
      " Training accuracy: 0.550\n",
      " Test accuracy: 0.452\n",
      "Epoch: 004/005 cost: 1.356885323\n",
      " Training accuracy: 0.630\n",
      " Test accuracy: 0.523\n",
      "Optimization Finished.\n"
     ]
    }
   ],
   "source": [
    "if do_train == 1:\n",
    "    for epoch in range(training_epochs):\n",
    "        avg_cost = 0.\n",
    "        total_batch = int(ntrain/batch_size)+1\n",
    "        #total_batch = int(mnist.train.num_examples/batch_size)\n",
    "        # Loop over all batches\n",
    "        for i in range(total_batch):\n",
    "            randidx = np.random.randint(ntrain, size=batch_size)\n",
    "            batch_xs = trainimg[randidx, :]\n",
    "            batch_ys = trainlabel[randidx, :]\n",
    "            #batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            # Fit training using batch data\n",
    "            sess.run(optm, feed_dict={x: batch_xs, y: batch_ys})\n",
    "            # Compute average loss\n",
    "            avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/total_batch\n",
    "\n",
    "        # Display logs per epoch step\n",
    "        if epoch % display_step == 0: \n",
    "            print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch, training_epochs, avg_cost))\n",
    "            train_acc = sess.run(accr, feed_dict={x: batch_xs, y: batch_ys})\n",
    "            print (\" Training accuracy: %.3f\" % (train_acc))\n",
    "            test_acc = sess.run(accr, feed_dict={x: testimg, y: testlabel})\n",
    "            print (\" Test accuracy: %.3f\" % (test_acc))\n",
    "\n",
    "        # Save Net\n",
    "        if epoch % save_step == 0:\n",
    "            saver.save(sess, \"nets/cnn_mnist_simple.ckpt-\" + str(epoch))\n",
    "    print (\"Optimization Finished.\")   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "do_train == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if do_train == 0:\n",
    "    epoch = training_epochs-1\n",
    "    saver.restore(sess, \"nets/cnn_mnist_simple.ckpt-\" + str(epoch))\n",
    "    print (\"NETWORK RESTORED\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False) \n",
    "    \n",
    "with tf.Session(graph = graph) as session:\n",
    "    ckpt = tf.train.get_checkpoint_state(savedir)\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "    start = global_step.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def save(checkpoint_file='hello.chk'):\n",
    "    with tf.Session() as session:\n",
    "        x = tf.Variable([42.0, 42.1, 42.3], name='x')\n",
    "        y = tf.Variable([[1.0, 2.0], [3.0, 4.0]], name='y')\n",
    "        not_saved = tf.Variable([-1, -2], name='not_saved')\n",
    "        session.run(tf.initialize_all_variables())\n",
    "\n",
    "        print(session.run(tf.all_variables()))\n",
    "        saver = tf.train.Saver([x, y])\n",
    "        saver.save(session, checkpoint_file)\n",
    "\n",
    "def restore(checkpoint_file='hello.chk'):\n",
    "    x = tf.Variable(-1.0, validate_shape=False, name='x')\n",
    "    y = tf.Variable(-1.0, validate_shape=False, name='y')\n",
    "    with tf.Session() as session:\n",
    "        saver = tf.train.Saver()\n",
    "        saver.restore(session, checkpoint_file)\n",
    "        print(session.run(tf.all_variables()))\n",
    "        \n",
    "def reset():\n",
    "    tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(session, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if do_train == 0:\n",
    "    epoch = training_epochs-1\n",
    "    saver.restore(sess, \"nets/cnn_mnist_simple.ckpt-\" + str(epoch))\n",
    "    print (\"NETWORK RESTORED\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False) \n",
    "    \n",
    "with tf.Session(graph = graph) as session:\n",
    "    ckpt = tf.train.get_checkpoint_state(savedir)\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "    start = global_step.eval()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "REAL_DATA_PATH = './dataset/anger/0.jpg'\n",
    "if (os.path.exists(REAL_DATA_PATH)):\n",
    "    print('Full Filename : ' + REAL_DATA_PATH)\n",
    "    img = cv2.imread(REAL_DATA_PATH)\n",
    "    print(img.shape)\n",
    "    if len(img.shape) == 2:\n",
    "        print(img.shape)\n",
    "        image = img.reshape([-1, IMAGE_SIZE, IMAGE_SIZE, 1])\n",
    "        print(image.shape)\n",
    "        print(image)\n",
    "        image = (image.astype(float) - PIXEL_DEPTH / 2) / PIXEL_DEPTH\n",
    "        print(image)\n",
    "        image = np.float32(image)\n",
    "        print(type(image))\n",
    "        ckpt = tf.train.get_checkpoint_state(savedir)\n",
    "        if ckpt and ckpt.model_checkpoint_path:\n",
    "            saver.restore(session, ckpt.model_checkpoint_path) # restore all variables\n",
    "            start = global_step.eval() # get last global_step\n",
    "            global_step.assign(start).eval()\n",
    "            logits = model(image, layer1_weights, layer1_biases, layer2_weights, \n",
    "            layer2_biases, layer3_weights, layer3_biases, layer4_weights, layer4_biases, 1.0, 1.0)\n",
    "            #train_prediction = tf.argmax(logits, 1)\n",
    "            train_prediction = session.run(tf.argmax(logits, 1))\n",
    "            print('=====================================================')\n",
    "            print(train_prediction)\n",
    "            print('=====================================================')            \n",
    "        else:\n",
    "            print('saver data load Fail\\t')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "savedir2 = tf.train.get_checkpoint_state(savedir)\n",
    "savedir2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if(os.path.exists(REAL_DATA_PATH)):\n",
    "                print('Full Filename : ' + REAL_DATA_PATH)\n",
    "                img = cv2.imread(REAL_DATA_PATH)\n",
    "                img = facecrop(img)\n",
    "                print(img.shape)\n",
    "                if len(img.shape) == 2:\n",
    "                    print(img.shape)\n",
    "                    image = img.reshape([-1, IMAGE_SIZE, IMAGE_SIZE, 1])\n",
    "                    print(image.shape)\n",
    "                    print(image)\n",
    "                    image = (image.astype(float) - PIXEL_DEPTH / 2) / PIXEL_DEPTH\n",
    "                    print(image)\n",
    "                    image = np.float32(image)\n",
    "                    print(type(image))\n",
    "                    ckpt = tf.train.get_checkpoint_state(ckpt_dir)\n",
    "                    if ckpt and ckpt.model_checkpoint_path:\n",
    "                        saver.restore(session, ckpt.model_checkpoint_path) # restore all variables\n",
    "                        start = global_step.eval() # get last global_step\n",
    "                        global_step.assign(start).eval()\n",
    "                        logits = model(image, layer1_weights, layer1_biases, layer2_weights, \n",
    "                        layer2_biases, layer3_weights, layer3_biases, layer4_weights, layer4_biases, 1.0, 1.0)\n",
    "                        #train_prediction = tf.argmax(logits, 1)\n",
    "                        train_prediction = session.run(tf.argmax(logits, 1))\n",
    "                        print('=====================================================')\n",
    "                        print(train_prediction)\n",
    "                        print('=====================================================')            \n",
    "                    else:\n",
    "                        print('saver data load Fail\\t')\n",
    "                else:\n",
    "                    print('Prediction \\t Fail')               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.device(device_type):\n",
    "    conv_out = conv_simple(x, weights, biases)\n",
    "\n",
    "input_r = sess.run(conv_out['input_r'], feed_dict={x: trainimg[0:1, :]})\n",
    "conv1   = sess.run(conv_out['conv1'], feed_dict={x: trainimg[0:1, :]})\n",
    "conv2   = sess.run(conv_out['conv2'], feed_dict={x: trainimg[0:1, :]})\n",
    "conv3   = sess.run(conv_out['conv3'], feed_dict={x: trainimg[0:1, :]})\n",
    "pool    = sess.run(conv_out['pool'], feed_dict={x: trainimg[0:1, :]})\n",
    "dense   = sess.run(conv_out['dense'], feed_dict={x: trainimg[0:1, :]})\n",
    "out     = sess.run(conv_out['out'], feed_dict={x: trainimg[0:1, :]})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0~7\n",
    "\n",
    "paths = {\"dataset/anger/\", \"dataset/contempt/\", \"dataset/disgust/\", \"dataset/fear\",\n",
    "        \"dataset/happy/\", \"dataset/neutral/\", \"dataset/sadness/\", \"dataset/surprise/\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Let's see 'input_r'\n",
    "print (\"Size of 'input_r' is %s\" % (input_r.shape,))\n",
    "label = np.argmax(trainlabel[0, :])\n",
    "print (\"Label is %d\" % (label))\n",
    "\n",
    "# Plot ! \n",
    "plt.matshow(input_r[0, :, :, 0], cmap=plt.get_cmap('gray'))\n",
    "plt.title(\"Label of this image is \" + str(label) + \"\")\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "#Size of 'input_r' is (1, 28, 28"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}