Sparse_Coding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "## referenced from \n",
    "## https://blog.metaflow.fr/sparse-coding-a-simple-exploration-152a3c900a7c#.uo3kxjenw\n",
    "import time, os\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "dir = os.getcwd()\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "\n",
    "# Fully connected model\n",
    "# Number of parameters: (784 * 784 + 784) + (784 * 10 + 10) = 615440 + 7850 = 623290\n",
    "# Dimensionality: R^784 -> R^784 -> R^10\n",
    "\n",
    "# Placeholder\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "y_true = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "\n",
    "sparsity_constraint = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('NeuralLayer'):\n",
    "    W = tf.get_variable('W', shape=[784, 784], initializer=tf.random_normal_initializer(stddev=1e-1))\n",
    "    b = tf.get_variable('b', shape=[784], initializer=tf.constant_initializer(0.1))\n",
    "\n",
    "    z = tf.matmul(x, W) + b\n",
    "    a = tf.nn.relu(z)\n",
    "\n",
    "    # We graph the average density of neurons activation\n",
    "    average_density = tf.reduce_mean(tf.reduce_sum(tf.cast((a > 0), tf.float32), axis=[1]))\n",
    "    tf.summary.scalar('AverageDensity', average_density)\n",
    "\n",
    "with tf.variable_scope('SoftmaxLayer'):\n",
    "    W_s = tf.get_variable('W_s', shape=[784, 10], initializer=tf.random_normal_initializer(stddev=1e-1))\n",
    "    b_s = tf.get_variable('b_s', shape=[10], initializer=tf.constant_initializer(0.1))\n",
    "\n",
    "    out = tf.matmul(a, W_s) + b_s\n",
    "    y = tf.nn.relu(out)\n",
    "\n",
    "with tf.variable_scope('Loss'):\n",
    "    epsilon = 1e-7 # After some training, y can be 0 on some classes which lead to NaN \n",
    "    diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y)\n",
    "    # We add our sparsity constraint on the activations\n",
    "    cross_entropy = tf.reduce_mean(diff)\n",
    "    loss = cross_entropy + sparsity_constraint * tf.reduce_sum(a)\n",
    "\n",
    "    tf.summary.scalar('loss', loss) # Graph the loss\n",
    "\n",
    "summaries = tf.summary.merge_all() # This is convenient\n",
    "\n",
    "with tf.variable_scope('Accuracy'):\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_true, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    acc_summary = tf.summary.scalar('accuracy', accuracy) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch: 100, loss: 0.422984, accuracy: 0.830300\n",
      "batch: 200, loss: 0.475206, accuracy: 0.845500\n",
      "batch: 300, loss: 0.555954, accuracy: 0.857900\n",
      "batch: 400, loss: 0.439589, accuracy: 0.858900\n",
      "batch: 500, loss: 0.450527, accuracy: 0.864400\n",
      "batch: 600, loss: 0.381175, accuracy: 0.869300\n",
      "batch: 700, loss: 0.451939, accuracy: 0.874800\n",
      "batch: 800, loss: 0.393449, accuracy: 0.876000\n",
      "batch: 900, loss: 0.361880, accuracy: 0.877400\n",
      "batch: 1000, loss: 0.421486, accuracy: 0.877400\n",
      "batch: 1100, loss: 0.312802, accuracy: 0.878200\n",
      "batch: 1200, loss: 0.394193, accuracy: 0.881100\n",
      "batch: 1300, loss: 0.073932, accuracy: 0.968900\n",
      "batch: 1400, loss: 0.116551, accuracy: 0.969100\n",
      "batch: 1500, loss: 0.129034, accuracy: 0.970600\n",
      "batch: 1600, loss: 0.145322, accuracy: 0.973300\n",
      "batch: 1700, loss: 0.027795, accuracy: 0.976600\n",
      "batch: 1800, loss: 0.058118, accuracy: 0.975300\n",
      "batch: 1900, loss: 0.096179, accuracy: 0.973400\n",
      "batch: 2000, loss: 0.094231, accuracy: 0.977700\n",
      "batch: 2100, loss: 0.016156, accuracy: 0.976100\n",
      "batch: 2200, loss: 0.076875, accuracy: 0.977600\n",
      "batch: 2300, loss: 0.026827, accuracy: 0.976600\n",
      "batch: 2400, loss: 0.019858, accuracy: 0.978000\n",
      "batch: 2500, loss: 0.036339, accuracy: 0.979000\n",
      "batch: 2600, loss: 0.007717, accuracy: 0.976700\n",
      "batch: 2700, loss: 0.022863, accuracy: 0.974400\n",
      "batch: 2800, loss: 0.019340, accuracy: 0.977600\n",
      "batch: 2900, loss: 0.010259, accuracy: 0.979800\n",
      "batch: 3000, loss: 0.025043, accuracy: 0.976700\n",
      "batch: 3100, loss: 0.021000, accuracy: 0.978900\n",
      "batch: 3200, loss: 0.013523, accuracy: 0.978800\n",
      "batch: 3300, loss: 0.058691, accuracy: 0.975800\n",
      "batch: 3400, loss: 0.004304, accuracy: 0.980700\n",
      "batch: 3500, loss: 0.020460, accuracy: 0.978600\n",
      "batch: 3600, loss: 0.003356, accuracy: 0.979400\n",
      "batch: 3700, loss: 0.016526, accuracy: 0.978000\n",
      "batch: 3800, loss: 0.015064, accuracy: 0.979500\n",
      "batch: 3900, loss: 0.002650, accuracy: 0.980800\n",
      "batch: 4000, loss: 0.006054, accuracy: 0.980100\n",
      "batch: 4100, loss: 0.010178, accuracy: 0.981400\n",
      "batch: 4200, loss: 0.013791, accuracy: 0.980600\n",
      "batch: 4300, loss: 0.018243, accuracy: 0.980400\n",
      "batch: 4400, loss: 0.004923, accuracy: 0.976300\n",
      "batch: 4500, loss: 0.004527, accuracy: 0.980000\n",
      "batch: 4600, loss: 0.002858, accuracy: 0.981500\n",
      "batch: 4700, loss: 0.029303, accuracy: 0.981700\n",
      "batch: 4800, loss: 0.002325, accuracy: 0.980400\n",
      "batch: 4900, loss: 0.001662, accuracy: 0.980000\n",
      "batch: 5000, loss: 0.006373, accuracy: 0.980600\n",
      "batch: 5100, loss: 0.000907, accuracy: 0.978600\n",
      "batch: 5200, loss: 0.005701, accuracy: 0.976900\n",
      "batch: 5300, loss: 0.010166, accuracy: 0.978600\n",
      "batch: 5400, loss: 0.001430, accuracy: 0.977000\n",
      "batch: 5500, loss: 0.005279, accuracy: 0.978300\n",
      "batch: 5600, loss: 0.008969, accuracy: 0.978600\n",
      "batch: 5700, loss: 0.002080, accuracy: 0.978000\n",
      "batch: 5800, loss: 0.011670, accuracy: 0.981000\n",
      "batch: 5900, loss: 0.002080, accuracy: 0.980800\n",
      "batch: 6000, loss: 0.022755, accuracy: 0.974500\n",
      "batch: 6100, loss: 0.003905, accuracy: 0.978000\n",
      "batch: 6200, loss: 0.001175, accuracy: 0.980000\n",
      "batch: 6300, loss: 0.007899, accuracy: 0.979200\n",
      "batch: 6400, loss: 0.023489, accuracy: 0.979500\n",
      "batch: 6500, loss: 0.004425, accuracy: 0.978600\n",
      "batch: 6600, loss: 0.001368, accuracy: 0.979300\n",
      "batch: 6700, loss: 0.001891, accuracy: 0.981600\n",
      "batch: 6800, loss: 0.004551, accuracy: 0.980700\n",
      "batch: 6900, loss: 0.001913, accuracy: 0.981500\n",
      "batch: 7000, loss: 0.015556, accuracy: 0.981500\n",
      "batch: 7100, loss: 0.000908, accuracy: 0.978700\n",
      "batch: 7200, loss: 0.005783, accuracy: 0.978100\n",
      "batch: 7300, loss: 0.000739, accuracy: 0.980900\n",
      "batch: 7400, loss: 0.000962, accuracy: 0.982100\n",
      "batch: 7500, loss: 0.027615, accuracy: 0.981000\n",
      "batch: 7600, loss: 0.001458, accuracy: 0.978400\n",
      "batch: 7700, loss: 0.042200, accuracy: 0.978300\n",
      "batch: 7800, loss: 0.000454, accuracy: 0.981300\n",
      "batch: 7900, loss: 0.004001, accuracy: 0.980900\n",
      "batch: 8000, loss: 0.001648, accuracy: 0.976800\n",
      "batch: 8100, loss: 0.005987, accuracy: 0.980300\n",
      "batch: 8200, loss: 0.002910, accuracy: 0.979400\n",
      "batch: 8300, loss: 0.000645, accuracy: 0.980600\n",
      "batch: 8400, loss: 0.006329, accuracy: 0.982000\n",
      "batch: 8500, loss: 0.000333, accuracy: 0.983300\n",
      "batch: 8600, loss: 0.007980, accuracy: 0.982800\n",
      "batch: 8700, loss: 0.000445, accuracy: 0.983400\n",
      "batch: 8800, loss: 0.001106, accuracy: 0.983200\n",
      "batch: 8900, loss: 0.024071, accuracy: 0.980600\n",
      "batch: 9000, loss: 0.000185, accuracy: 0.984000\n",
      "batch: 9100, loss: 0.000210, accuracy: 0.982200\n",
      "batch: 9200, loss: 0.000095, accuracy: 0.982700\n",
      "batch: 9300, loss: 0.023929, accuracy: 0.981400\n",
      "batch: 9400, loss: 0.001869, accuracy: 0.978300\n",
      "batch: 9500, loss: 0.001444, accuracy: 0.974400\n",
      "batch: 9600, loss: 0.007540, accuracy: 0.977700\n",
      "batch: 9700, loss: 0.001108, accuracy: 0.979200\n",
      "batch: 9800, loss: 0.002910, accuracy: 0.979900\n",
      "batch: 9900, loss: 0.018534, accuracy: 0.981400\n",
      "batch: 10000, loss: 0.000311, accuracy: 0.982300\n",
      "batch: 10100, loss: 0.001148, accuracy: 0.981600\n",
      "batch: 10200, loss: 0.004687, accuracy: 0.979500\n",
      "batch: 10300, loss: 0.000660, accuracy: 0.980000\n",
      "batch: 10400, loss: 0.006889, accuracy: 0.981000\n",
      "batch: 10500, loss: 0.001124, accuracy: 0.979300\n",
      "batch: 10600, loss: 0.025029, accuracy: 0.981600\n",
      "batch: 10700, loss: 0.008028, accuracy: 0.979200\n",
      "batch: 10800, loss: 0.000461, accuracy: 0.981300\n",
      "batch: 10900, loss: 0.000576, accuracy: 0.982200\n",
      "batch: 11000, loss: 0.002702, accuracy: 0.982900\n",
      "batch: 11100, loss: 0.000299, accuracy: 0.983000\n",
      "batch: 11200, loss: 0.002920, accuracy: 0.982600\n",
      "batch: 11300, loss: 0.001642, accuracy: 0.982100\n",
      "batch: 11400, loss: 0.000689, accuracy: 0.982300\n",
      "batch: 11500, loss: 0.000198, accuracy: 0.980700\n",
      "batch: 11600, loss: 0.000486, accuracy: 0.980100\n",
      "batch: 11700, loss: 0.000256, accuracy: 0.981200\n",
      "batch: 11800, loss: 0.005031, accuracy: 0.982600\n",
      "batch: 11900, loss: 0.015067, accuracy: 0.978600\n",
      "batch: 12000, loss: 0.005177, accuracy: 0.976100\n",
      "batch: 12100, loss: 0.021568, accuracy: 0.976100\n",
      "batch: 12200, loss: 0.000565, accuracy: 0.982700\n",
      "batch: 12300, loss: 0.000489, accuracy: 0.980300\n",
      "batch: 12400, loss: 0.010257, accuracy: 0.980900\n",
      "batch: 12500, loss: 0.000029, accuracy: 0.983000\n",
      "batch: 12600, loss: 0.000652, accuracy: 0.982900\n",
      "batch: 12700, loss: 0.000277, accuracy: 0.983600\n",
      "batch: 12800, loss: 0.001639, accuracy: 0.983800\n",
      "batch: 12900, loss: 0.000115, accuracy: 0.984600\n",
      "batch: 13000, loss: 0.000028, accuracy: 0.983800\n",
      "batch: 13100, loss: 0.000029, accuracy: 0.985000\n",
      "batch: 13200, loss: 0.000110, accuracy: 0.984200\n",
      "batch: 13300, loss: 0.000088, accuracy: 0.984100\n",
      "batch: 13400, loss: 0.000123, accuracy: 0.984100\n",
      "batch: 13500, loss: 0.000234, accuracy: 0.985000\n",
      "batch: 13600, loss: 0.000008, accuracy: 0.985400\n",
      "batch: 13700, loss: 0.000075, accuracy: 0.984700\n",
      "batch: 13800, loss: 0.000020, accuracy: 0.984800\n",
      "batch: 13900, loss: 0.000109, accuracy: 0.984700\n",
      "batch: 14000, loss: 0.000052, accuracy: 0.984800\n",
      "batch: 14100, loss: 0.000050, accuracy: 0.984800\n",
      "batch: 14200, loss: 0.000034, accuracy: 0.984700\n",
      "batch: 14300, loss: 0.000007, accuracy: 0.984800\n",
      "batch: 14400, loss: 0.000012, accuracy: 0.984900\n",
      "batch: 14500, loss: 0.000075, accuracy: 0.984700\n",
      "batch: 14600, loss: 0.000048, accuracy: 0.984700\n",
      "batch: 14700, loss: 0.000024, accuracy: 0.984800\n",
      "batch: 14800, loss: 0.000029, accuracy: 0.984900\n",
      "batch: 14900, loss: 0.000043, accuracy: 0.984700\n",
      "batch: 15000, loss: 0.000029, accuracy: 0.984500\n",
      "batch: 15100, loss: 0.000042, accuracy: 0.984300\n",
      "batch: 15200, loss: 0.000069, accuracy: 0.984600\n",
      "batch: 15300, loss: 0.000012, accuracy: 0.984800\n",
      "batch: 15400, loss: 0.000016, accuracy: 0.984900\n",
      "batch: 15500, loss: 0.000015, accuracy: 0.985100\n",
      "batch: 15600, loss: 0.000037, accuracy: 0.985000\n",
      "batch: 15700, loss: 0.000023, accuracy: 0.984500\n",
      "batch: 15800, loss: 0.000035, accuracy: 0.984500\n",
      "batch: 15900, loss: 0.000040, accuracy: 0.984500\n",
      "batch: 16000, loss: 0.023055, accuracy: 0.984700\n",
      "batch: 16100, loss: 0.000013, accuracy: 0.984400\n",
      "batch: 16200, loss: 0.000029, accuracy: 0.984500\n",
      "batch: 16300, loss: 0.000024, accuracy: 0.984600\n",
      "batch: 16400, loss: 0.000008, accuracy: 0.984500\n",
      "batch: 16500, loss: 0.000016, accuracy: 0.984800\n",
      "batch: 16600, loss: 0.000035, accuracy: 0.984500\n",
      "batch: 16700, loss: 0.000018, accuracy: 0.984600\n",
      "batch: 16800, loss: 0.000003, accuracy: 0.984400\n",
      "batch: 16900, loss: 0.000010, accuracy: 0.984400\n",
      "batch: 17000, loss: 0.000016, accuracy: 0.984500\n",
      "batch: 17100, loss: 0.000014, accuracy: 0.984500\n",
      "batch: 17200, loss: 0.000022, accuracy: 0.984500\n",
      "batch: 17300, loss: 0.000009, accuracy: 0.984500\n",
      "batch: 17400, loss: 0.023073, accuracy: 0.984800\n",
      "batch: 17500, loss: 0.000019, accuracy: 0.984700\n",
      "batch: 17600, loss: 0.000028, accuracy: 0.984500\n",
      "batch: 17700, loss: 0.000028, accuracy: 0.984500\n",
      "batch: 17800, loss: 0.000032, accuracy: 0.984600\n",
      "batch: 17900, loss: 0.000005, accuracy: 0.984800\n",
      "batch: 18000, loss: 0.000009, accuracy: 0.984500\n",
      "batch: 18100, loss: 0.000013, accuracy: 0.984900\n",
      "batch: 18200, loss: 0.000006, accuracy: 0.984500\n",
      "batch: 18300, loss: 0.000006, accuracy: 0.984200\n",
      "batch: 18400, loss: 0.000018, accuracy: 0.984600\n",
      "batch: 18500, loss: 0.000008, accuracy: 0.984200\n",
      "batch: 18600, loss: 0.000004, accuracy: 0.984300\n",
      "batch: 18700, loss: 0.000012, accuracy: 0.982200\n",
      "batch: 18800, loss: 0.113530, accuracy: 0.971700\n",
      "batch: 18900, loss: 0.005545, accuracy: 0.975200\n",
      "batch: 19000, loss: 0.008829, accuracy: 0.981600\n",
      "batch: 19100, loss: 0.000204, accuracy: 0.980000\n",
      "batch: 19200, loss: 0.001275, accuracy: 0.980300\n",
      "batch: 19300, loss: 0.006147, accuracy: 0.981100\n",
      "batch: 19400, loss: 0.000132, accuracy: 0.981800\n",
      "batch: 19500, loss: 0.000137, accuracy: 0.980400\n",
      "batch: 19600, loss: 0.000262, accuracy: 0.982000\n",
      "batch: 19700, loss: 0.003406, accuracy: 0.982700\n",
      "batch: 19800, loss: 0.000334, accuracy: 0.982700\n",
      "batch: 19900, loss: 0.024198, accuracy: 0.982300\n",
      "batch: 20000, loss: 0.000357, accuracy: 0.981900\n",
      "batch: 100, loss: 0.799340, accuracy: 0.911100\n",
      "batch: 200, loss: 0.595641, accuracy: 0.933100\n",
      "batch: 300, loss: 0.529316, accuracy: 0.943100\n",
      "batch: 400, loss: 0.476803, accuracy: 0.947800\n",
      "batch: 500, loss: 0.368166, accuracy: 0.953800\n",
      "batch: 600, loss: 0.349278, accuracy: 0.960200\n",
      "batch: 700, loss: 0.358639, accuracy: 0.959900\n",
      "batch: 800, loss: 0.312309, accuracy: 0.962800\n",
      "batch: 900, loss: 0.318567, accuracy: 0.965400\n",
      "batch: 1000, loss: 0.288142, accuracy: 0.968500\n",
      "batch: 1100, loss: 0.309541, accuracy: 0.970100\n",
      "batch: 1200, loss: 0.213278, accuracy: 0.970200\n",
      "batch: 1300, loss: 0.187620, accuracy: 0.970700\n",
      "batch: 1400, loss: 0.227435, accuracy: 0.971700\n",
      "batch: 1500, loss: 0.231115, accuracy: 0.973200\n",
      "batch: 1600, loss: 0.182227, accuracy: 0.972200\n",
      "batch: 1700, loss: 0.242122, accuracy: 0.972700\n",
      "batch: 1800, loss: 0.199319, accuracy: 0.973100\n",
      "batch: 1900, loss: 0.169500, accuracy: 0.972500\n",
      "batch: 2000, loss: 0.214864, accuracy: 0.972700\n",
      "batch: 2100, loss: 0.166399, accuracy: 0.975100\n",
      "batch: 2200, loss: 0.155111, accuracy: 0.972100\n",
      "batch: 2300, loss: 0.167019, accuracy: 0.975000\n",
      "batch: 2400, loss: 0.139423, accuracy: 0.975200\n",
      "batch: 2500, loss: 0.127805, accuracy: 0.974700\n",
      "batch: 2600, loss: 0.153125, accuracy: 0.975500\n",
      "batch: 2700, loss: 0.165393, accuracy: 0.974500\n",
      "batch: 2800, loss: 0.178675, accuracy: 0.972200\n",
      "batch: 2900, loss: 0.139425, accuracy: 0.973900\n",
      "batch: 3000, loss: 0.138589, accuracy: 0.974000\n",
      "batch: 3100, loss: 0.222720, accuracy: 0.974300\n",
      "batch: 3200, loss: 0.129425, accuracy: 0.976500\n",
      "batch: 3300, loss: 0.120829, accuracy: 0.974700\n",
      "batch: 3400, loss: 0.170888, accuracy: 0.974400\n",
      "batch: 3500, loss: 0.118562, accuracy: 0.975000\n",
      "batch: 3600, loss: 0.098176, accuracy: 0.976400\n",
      "batch: 3700, loss: 0.097191, accuracy: 0.976500\n",
      "batch: 3800, loss: 0.099361, accuracy: 0.975900\n",
      "batch: 3900, loss: 0.098241, accuracy: 0.975100\n",
      "batch: 4000, loss: 0.107478, accuracy: 0.974400\n",
      "batch: 4100, loss: 0.100569, accuracy: 0.976000\n",
      "batch: 4200, loss: 0.124511, accuracy: 0.974700\n",
      "batch: 4300, loss: 0.108320, accuracy: 0.976000\n",
      "batch: 4400, loss: 0.103891, accuracy: 0.974700\n",
      "batch: 4500, loss: 0.088168, accuracy: 0.974600\n",
      "batch: 4600, loss: 0.084789, accuracy: 0.975800\n",
      "batch: 4700, loss: 0.107942, accuracy: 0.973800\n",
      "batch: 4800, loss: 0.092075, accuracy: 0.973700\n",
      "batch: 4900, loss: 0.082554, accuracy: 0.974700\n",
      "batch: 5000, loss: 0.117734, accuracy: 0.976900\n",
      "batch: 5100, loss: 0.104632, accuracy: 0.976600\n",
      "batch: 5200, loss: 0.082895, accuracy: 0.976300\n",
      "batch: 5300, loss: 0.088871, accuracy: 0.975200\n",
      "batch: 5400, loss: 0.084952, accuracy: 0.975300\n",
      "batch: 5500, loss: 0.094927, accuracy: 0.976900\n",
      "batch: 5600, loss: 0.103687, accuracy: 0.976200\n",
      "batch: 5700, loss: 0.103049, accuracy: 0.975500\n",
      "batch: 5800, loss: 0.104532, accuracy: 0.975900\n",
      "batch: 5900, loss: 0.074222, accuracy: 0.975300\n",
      "batch: 6000, loss: 0.093915, accuracy: 0.975000\n",
      "batch: 6100, loss: 0.072217, accuracy: 0.975400\n",
      "batch: 6200, loss: 0.087227, accuracy: 0.975000\n",
      "batch: 6300, loss: 0.074172, accuracy: 0.973800\n",
      "batch: 6400, loss: 0.101785, accuracy: 0.974900\n",
      "batch: 6500, loss: 0.073143, accuracy: 0.973600\n",
      "batch: 6600, loss: 0.072178, accuracy: 0.974600\n",
      "batch: 6700, loss: 0.064636, accuracy: 0.975400\n",
      "batch: 6800, loss: 0.083098, accuracy: 0.974800\n",
      "batch: 6900, loss: 0.064779, accuracy: 0.974700\n",
      "batch: 7000, loss: 0.058673, accuracy: 0.975200\n",
      "batch: 7100, loss: 0.066167, accuracy: 0.975100\n",
      "batch: 7200, loss: 0.067173, accuracy: 0.973500\n",
      "batch: 7300, loss: 0.067565, accuracy: 0.972800\n",
      "batch: 7400, loss: 0.067639, accuracy: 0.973700\n",
      "batch: 7500, loss: 0.063838, accuracy: 0.973800\n",
      "batch: 7600, loss: 0.062042, accuracy: 0.974800\n",
      "batch: 7700, loss: 0.066948, accuracy: 0.974300\n",
      "batch: 7800, loss: 0.057654, accuracy: 0.974200\n",
      "batch: 7900, loss: 0.075228, accuracy: 0.973100\n",
      "batch: 8000, loss: 0.056135, accuracy: 0.972700\n",
      "batch: 8100, loss: 0.054581, accuracy: 0.973200\n",
      "batch: 8200, loss: 0.060296, accuracy: 0.972300\n",
      "batch: 8300, loss: 0.060089, accuracy: 0.973700\n",
      "batch: 8400, loss: 0.076239, accuracy: 0.972100\n",
      "batch: 8500, loss: 0.054865, accuracy: 0.973100\n",
      "batch: 8600, loss: 0.053774, accuracy: 0.974000\n",
      "batch: 8700, loss: 0.051379, accuracy: 0.971800\n",
      "batch: 8800, loss: 0.055406, accuracy: 0.972300\n",
      "batch: 8900, loss: 0.058912, accuracy: 0.973000\n",
      "batch: 9000, loss: 0.055131, accuracy: 0.972400\n",
      "batch: 9100, loss: 0.052640, accuracy: 0.972300\n",
      "batch: 9200, loss: 0.076340, accuracy: 0.971800\n",
      "batch: 9300, loss: 0.049929, accuracy: 0.970700\n",
      "batch: 9400, loss: 0.051146, accuracy: 0.972400\n",
      "batch: 9500, loss: 0.053192, accuracy: 0.972500\n",
      "batch: 9600, loss: 0.045904, accuracy: 0.972400\n",
      "batch: 9700, loss: 0.052332, accuracy: 0.970900\n",
      "batch: 9800, loss: 0.045231, accuracy: 0.970700\n",
      "batch: 9900, loss: 0.046571, accuracy: 0.971300\n",
      "batch: 10000, loss: 0.055630, accuracy: 0.973800\n",
      "batch: 10100, loss: 0.046109, accuracy: 0.970600\n",
      "batch: 10200, loss: 0.070322, accuracy: 0.971100\n",
      "batch: 10300, loss: 0.048589, accuracy: 0.971100\n",
      "batch: 10400, loss: 0.044010, accuracy: 0.972300\n",
      "batch: 10500, loss: 0.043318, accuracy: 0.971500\n",
      "batch: 10600, loss: 0.047952, accuracy: 0.969800\n",
      "batch: 10700, loss: 0.043690, accuracy: 0.970200\n",
      "batch: 10800, loss: 0.046496, accuracy: 0.970100\n",
      "batch: 10900, loss: 0.042604, accuracy: 0.971100\n",
      "batch: 11000, loss: 0.048055, accuracy: 0.972800\n",
      "batch: 11100, loss: 0.050795, accuracy: 0.973700\n",
      "batch: 11200, loss: 0.049396, accuracy: 0.971700\n",
      "batch: 11300, loss: 0.046902, accuracy: 0.970400\n",
      "batch: 11400, loss: 0.041012, accuracy: 0.971200\n",
      "batch: 11500, loss: 0.046784, accuracy: 0.972000\n",
      "batch: 11600, loss: 0.044466, accuracy: 0.971100\n",
      "batch: 11700, loss: 0.042182, accuracy: 0.969900\n",
      "batch: 11800, loss: 0.043913, accuracy: 0.971900\n",
      "batch: 11900, loss: 0.038281, accuracy: 0.969900\n",
      "batch: 12000, loss: 0.043897, accuracy: 0.971100\n",
      "batch: 12100, loss: 0.042468, accuracy: 0.969800\n",
      "batch: 12200, loss: 0.040614, accuracy: 0.969100\n",
      "batch: 12300, loss: 0.037483, accuracy: 0.969500\n",
      "batch: 12400, loss: 0.044213, accuracy: 0.970300\n",
      "batch: 12500, loss: 0.042226, accuracy: 0.970100\n",
      "batch: 12600, loss: 0.038174, accuracy: 0.971600\n",
      "batch: 12700, loss: 0.039200, accuracy: 0.971600\n",
      "batch: 12800, loss: 0.047719, accuracy: 0.968600\n",
      "batch: 12900, loss: 0.045574, accuracy: 0.967900\n",
      "batch: 13000, loss: 0.038729, accuracy: 0.967600\n",
      "batch: 13100, loss: 0.034608, accuracy: 0.968900\n",
      "batch: 13200, loss: 0.041573, accuracy: 0.968300\n",
      "batch: 13300, loss: 0.045100, accuracy: 0.968000\n",
      "batch: 13400, loss: 0.035621, accuracy: 0.969500\n",
      "batch: 13500, loss: 0.039514, accuracy: 0.967500\n",
      "batch: 13600, loss: 0.046050, accuracy: 0.966500\n",
      "batch: 13700, loss: 0.036811, accuracy: 0.968800\n",
      "batch: 13800, loss: 0.034760, accuracy: 0.968500\n",
      "batch: 13900, loss: 0.037043, accuracy: 0.969000\n",
      "batch: 14000, loss: 0.039823, accuracy: 0.966300\n",
      "batch: 14100, loss: 0.037517, accuracy: 0.969600\n",
      "batch: 14200, loss: 0.033498, accuracy: 0.967800\n",
      "batch: 14300, loss: 0.032811, accuracy: 0.968000\n",
      "batch: 14400, loss: 0.032274, accuracy: 0.968200\n",
      "batch: 14500, loss: 0.053809, accuracy: 0.967600\n",
      "batch: 14600, loss: 0.038072, accuracy: 0.970100\n",
      "batch: 14700, loss: 0.033076, accuracy: 0.968700\n",
      "batch: 14800, loss: 0.035837, accuracy: 0.967200\n",
      "batch: 14900, loss: 0.035984, accuracy: 0.967800\n",
      "batch: 15000, loss: 0.049752, accuracy: 0.966900\n",
      "batch: 15100, loss: 0.035395, accuracy: 0.964800\n",
      "batch: 15200, loss: 0.039396, accuracy: 0.967300\n",
      "batch: 15300, loss: 0.035644, accuracy: 0.968900\n",
      "batch: 15400, loss: 0.033971, accuracy: 0.968100\n",
      "batch: 15500, loss: 0.033623, accuracy: 0.967600\n",
      "batch: 15600, loss: 0.034878, accuracy: 0.968000\n",
      "batch: 15700, loss: 0.038570, accuracy: 0.968200\n",
      "batch: 15800, loss: 0.029717, accuracy: 0.967500\n",
      "batch: 15900, loss: 0.032570, accuracy: 0.965800\n",
      "batch: 16000, loss: 0.036498, accuracy: 0.966800\n",
      "batch: 16100, loss: 0.034551, accuracy: 0.967100\n",
      "batch: 16200, loss: 0.031493, accuracy: 0.968200\n",
      "batch: 16300, loss: 0.032755, accuracy: 0.966500\n",
      "batch: 16400, loss: 0.033057, accuracy: 0.965800\n",
      "batch: 16500, loss: 0.030561, accuracy: 0.967100\n",
      "batch: 16600, loss: 0.030152, accuracy: 0.967800\n",
      "batch: 16700, loss: 0.034550, accuracy: 0.966500\n",
      "batch: 16800, loss: 0.052083, accuracy: 0.965500\n",
      "batch: 16900, loss: 0.036443, accuracy: 0.968200\n",
      "batch: 17000, loss: 0.031928, accuracy: 0.965800\n",
      "batch: 17100, loss: 0.039695, accuracy: 0.965400\n",
      "batch: 17200, loss: 0.031663, accuracy: 0.965400\n",
      "batch: 17300, loss: 0.032342, accuracy: 0.967500\n",
      "batch: 17400, loss: 0.071920, accuracy: 0.965600\n",
      "batch: 17500, loss: 0.044392, accuracy: 0.966700\n",
      "batch: 17600, loss: 0.033567, accuracy: 0.964300\n",
      "batch: 17700, loss: 0.045677, accuracy: 0.964600\n",
      "batch: 17800, loss: 0.033438, accuracy: 0.962200\n",
      "batch: 17900, loss: 0.048711, accuracy: 0.966400\n",
      "batch: 18000, loss: 0.037808, accuracy: 0.967700\n",
      "batch: 18100, loss: 0.030098, accuracy: 0.967700\n",
      "batch: 18200, loss: 0.031503, accuracy: 0.967000\n",
      "batch: 18300, loss: 0.026859, accuracy: 0.966700\n",
      "batch: 18400, loss: 0.028264, accuracy: 0.965300\n",
      "batch: 18500, loss: 0.030389, accuracy: 0.965800\n",
      "batch: 18600, loss: 0.061008, accuracy: 0.966200\n",
      "batch: 18700, loss: 0.028496, accuracy: 0.965700\n",
      "batch: 18800, loss: 0.033282, accuracy: 0.966800\n",
      "batch: 18900, loss: 0.025250, accuracy: 0.964700\n",
      "batch: 19000, loss: 0.028719, accuracy: 0.965500\n",
      "batch: 19100, loss: 0.028723, accuracy: 0.965000\n",
      "batch: 19200, loss: 0.027678, accuracy: 0.965900\n",
      "batch: 19300, loss: 0.025475, accuracy: 0.965400\n",
      "batch: 19400, loss: 0.036396, accuracy: 0.961500\n",
      "batch: 19500, loss: 0.035242, accuracy: 0.963300\n",
      "batch: 19600, loss: 0.028590, accuracy: 0.966800\n",
      "batch: 19700, loss: 0.028868, accuracy: 0.965000\n",
      "batch: 19800, loss: 0.030595, accuracy: 0.965000\n",
      "batch: 19900, loss: 0.029084, accuracy: 0.965900\n",
      "batch: 20000, loss: 0.026087, accuracy: 0.964900\n",
      "batch: 100, loss: 1.598626, accuracy: 0.809600\n",
      "batch: 200, loss: 1.176148, accuracy: 0.897300\n",
      "batch: 300, loss: 0.976207, accuracy: 0.921500\n",
      "batch: 400, loss: 0.784159, accuracy: 0.933700\n",
      "batch: 500, loss: 0.852232, accuracy: 0.936700\n",
      "batch: 600, loss: 0.695387, accuracy: 0.939500\n",
      "batch: 700, loss: 0.661438, accuracy: 0.943000\n",
      "batch: 800, loss: 0.608597, accuracy: 0.944500\n",
      "batch: 900, loss: 0.523651, accuracy: 0.946600\n",
      "batch: 1000, loss: 0.508024, accuracy: 0.947600\n",
      "batch: 1100, loss: 0.552541, accuracy: 0.948200\n",
      "batch: 1200, loss: 0.527390, accuracy: 0.952300\n",
      "batch: 1300, loss: 0.399479, accuracy: 0.951300\n",
      "batch: 1400, loss: 0.424543, accuracy: 0.952500\n",
      "batch: 1500, loss: 0.565762, accuracy: 0.952700\n",
      "batch: 1600, loss: 0.581953, accuracy: 0.954600\n",
      "batch: 1700, loss: 0.502441, accuracy: 0.955500\n",
      "batch: 1800, loss: 0.373509, accuracy: 0.955200\n",
      "batch: 1900, loss: 0.355643, accuracy: 0.956100\n",
      "batch: 2000, loss: 0.515131, accuracy: 0.956500\n",
      "batch: 2100, loss: 0.391472, accuracy: 0.956800\n",
      "batch: 2200, loss: 0.414913, accuracy: 0.956900\n",
      "batch: 2300, loss: 0.411258, accuracy: 0.958500\n",
      "batch: 2400, loss: 0.399418, accuracy: 0.958700\n",
      "batch: 2500, loss: 0.315854, accuracy: 0.959200\n",
      "batch: 2600, loss: 0.367848, accuracy: 0.959500\n",
      "batch: 2700, loss: 0.349193, accuracy: 0.959800\n",
      "batch: 2800, loss: 0.406162, accuracy: 0.959000\n",
      "batch: 2900, loss: 0.260636, accuracy: 0.960700\n",
      "batch: 3000, loss: 0.318246, accuracy: 0.960000\n",
      "batch: 3100, loss: 0.319930, accuracy: 0.960300\n",
      "batch: 3200, loss: 0.350624, accuracy: 0.960400\n",
      "batch: 3300, loss: 0.341392, accuracy: 0.961600\n",
      "batch: 3400, loss: 0.395383, accuracy: 0.961000\n",
      "batch: 3500, loss: 0.239906, accuracy: 0.961800\n",
      "batch: 3600, loss: 0.344195, accuracy: 0.962100\n",
      "batch: 3700, loss: 0.277841, accuracy: 0.963200\n",
      "batch: 3800, loss: 0.287502, accuracy: 0.960800\n",
      "batch: 3900, loss: 0.303167, accuracy: 0.960800\n",
      "batch: 4000, loss: 0.282615, accuracy: 0.962100\n",
      "batch: 4100, loss: 0.275749, accuracy: 0.962600\n",
      "batch: 4200, loss: 0.263474, accuracy: 0.962600\n",
      "batch: 4300, loss: 0.262328, accuracy: 0.961300\n",
      "batch: 4400, loss: 0.275131, accuracy: 0.962400\n",
      "batch: 4500, loss: 0.251968, accuracy: 0.963600\n",
      "batch: 4600, loss: 0.314973, accuracy: 0.962200\n",
      "batch: 4700, loss: 0.295825, accuracy: 0.963100\n",
      "batch: 4800, loss: 0.243790, accuracy: 0.961900\n",
      "batch: 4900, loss: 0.275580, accuracy: 0.963000\n",
      "batch: 5000, loss: 0.252239, accuracy: 0.962600\n",
      "batch: 5100, loss: 0.206629, accuracy: 0.963100\n",
      "batch: 5200, loss: 0.276758, accuracy: 0.962600\n",
      "batch: 5300, loss: 0.200882, accuracy: 0.962900\n",
      "batch: 5400, loss: 0.292414, accuracy: 0.962900\n",
      "batch: 5500, loss: 0.209267, accuracy: 0.961000\n",
      "batch: 5600, loss: 0.226860, accuracy: 0.962800\n",
      "batch: 5700, loss: 0.240989, accuracy: 0.962900\n",
      "batch: 5800, loss: 0.280763, accuracy: 0.961600\n",
      "batch: 5900, loss: 0.215833, accuracy: 0.962100\n",
      "batch: 6000, loss: 0.264998, accuracy: 0.961700\n",
      "batch: 6100, loss: 0.205731, accuracy: 0.961800\n",
      "batch: 6200, loss: 0.209155, accuracy: 0.962700\n",
      "batch: 6300, loss: 0.228747, accuracy: 0.962700\n",
      "batch: 6400, loss: 0.261059, accuracy: 0.963100\n",
      "batch: 6500, loss: 0.249220, accuracy: 0.961600\n",
      "batch: 6600, loss: 0.186797, accuracy: 0.962100\n",
      "batch: 6700, loss: 0.197404, accuracy: 0.960700\n",
      "batch: 6800, loss: 0.203143, accuracy: 0.962700\n",
      "batch: 6900, loss: 0.186063, accuracy: 0.962700\n",
      "batch: 7000, loss: 0.167855, accuracy: 0.965100\n",
      "batch: 7100, loss: 0.194702, accuracy: 0.962300\n",
      "batch: 7200, loss: 0.238506, accuracy: 0.963300\n",
      "batch: 7300, loss: 0.179761, accuracy: 0.960800\n",
      "batch: 7400, loss: 0.204239, accuracy: 0.960300\n",
      "batch: 7500, loss: 0.193109, accuracy: 0.963500\n",
      "batch: 7600, loss: 0.215740, accuracy: 0.960900\n",
      "batch: 7700, loss: 0.186054, accuracy: 0.959700\n",
      "batch: 7800, loss: 0.182661, accuracy: 0.961700\n",
      "batch: 7900, loss: 0.229264, accuracy: 0.961200\n",
      "batch: 8000, loss: 0.189895, accuracy: 0.963200\n",
      "batch: 8100, loss: 0.169061, accuracy: 0.961500\n",
      "batch: 8200, loss: 0.239267, accuracy: 0.960200\n",
      "batch: 8300, loss: 0.182213, accuracy: 0.960900\n",
      "batch: 8400, loss: 0.164942, accuracy: 0.963100\n",
      "batch: 8500, loss: 0.265847, accuracy: 0.962400\n",
      "batch: 8600, loss: 0.181469, accuracy: 0.960600\n",
      "batch: 8700, loss: 0.224032, accuracy: 0.961900\n",
      "batch: 8800, loss: 0.203321, accuracy: 0.962000\n",
      "batch: 8900, loss: 0.243300, accuracy: 0.960500\n",
      "batch: 9000, loss: 0.139444, accuracy: 0.960800\n",
      "batch: 9100, loss: 0.199656, accuracy: 0.958000\n",
      "batch: 9200, loss: 0.158150, accuracy: 0.960500\n",
      "batch: 9300, loss: 0.138270, accuracy: 0.960800\n",
      "batch: 9400, loss: 0.193110, accuracy: 0.958800\n",
      "batch: 9500, loss: 0.235481, accuracy: 0.960700\n",
      "batch: 9600, loss: 0.159184, accuracy: 0.958500\n",
      "batch: 9700, loss: 0.205321, accuracy: 0.960400\n",
      "batch: 9800, loss: 0.169734, accuracy: 0.958200\n",
      "batch: 9900, loss: 0.264720, accuracy: 0.962300\n",
      "batch: 10000, loss: 0.230559, accuracy: 0.958100\n",
      "batch: 10100, loss: 0.165419, accuracy: 0.958400\n",
      "batch: 10200, loss: 0.139989, accuracy: 0.959200\n",
      "batch: 10300, loss: 0.158586, accuracy: 0.958200\n",
      "batch: 10400, loss: 0.147285, accuracy: 0.959700\n",
      "batch: 10500, loss: 0.225517, accuracy: 0.959100\n",
      "batch: 10600, loss: 0.165267, accuracy: 0.958000\n",
      "batch: 10700, loss: 0.174860, accuracy: 0.959700\n",
      "batch: 10800, loss: 0.119446, accuracy: 0.960000\n",
      "batch: 10900, loss: 0.152536, accuracy: 0.957800\n",
      "batch: 11000, loss: 0.138052, accuracy: 0.958900\n",
      "batch: 11100, loss: 0.150864, accuracy: 0.959200\n",
      "batch: 11200, loss: 0.133165, accuracy: 0.959100\n",
      "batch: 11300, loss: 0.133785, accuracy: 0.960400\n",
      "batch: 11400, loss: 0.122843, accuracy: 0.958800\n",
      "batch: 11500, loss: 0.174624, accuracy: 0.957500\n",
      "batch: 11600, loss: 0.126654, accuracy: 0.958200\n",
      "batch: 11700, loss: 0.127831, accuracy: 0.960600\n",
      "batch: 11800, loss: 0.140850, accuracy: 0.958700\n",
      "batch: 11900, loss: 0.170695, accuracy: 0.960000\n",
      "batch: 12000, loss: 0.186481, accuracy: 0.956800\n",
      "batch: 12100, loss: 0.129841, accuracy: 0.959600\n",
      "batch: 12200, loss: 0.132074, accuracy: 0.958900\n",
      "batch: 12300, loss: 0.115783, accuracy: 0.959000\n",
      "batch: 12400, loss: 0.105881, accuracy: 0.957900\n",
      "batch: 12500, loss: 0.168758, accuracy: 0.959600\n",
      "batch: 12600, loss: 0.111002, accuracy: 0.958400\n",
      "batch: 12700, loss: 0.140523, accuracy: 0.958700\n",
      "batch: 12800, loss: 0.119881, accuracy: 0.958500\n",
      "batch: 12900, loss: 0.170783, accuracy: 0.955900\n",
      "batch: 13000, loss: 0.169498, accuracy: 0.957200\n",
      "batch: 13100, loss: 0.116349, accuracy: 0.958800\n",
      "batch: 13200, loss: 0.149985, accuracy: 0.957400\n",
      "batch: 13300, loss: 0.113934, accuracy: 0.955700\n",
      "batch: 13400, loss: 0.110180, accuracy: 0.956300\n",
      "batch: 13500, loss: 0.153579, accuracy: 0.957900\n",
      "batch: 13600, loss: 0.123067, accuracy: 0.957900\n",
      "batch: 13700, loss: 0.114596, accuracy: 0.955200\n",
      "batch: 13800, loss: 0.112013, accuracy: 0.956000\n",
      "batch: 13900, loss: 0.111679, accuracy: 0.957400\n",
      "batch: 14000, loss: 0.107662, accuracy: 0.958100\n",
      "batch: 14100, loss: 0.166005, accuracy: 0.956100\n",
      "batch: 14200, loss: 0.088773, accuracy: 0.956400\n",
      "batch: 14300, loss: 0.110076, accuracy: 0.956800\n",
      "batch: 14400, loss: 0.146417, accuracy: 0.954100\n",
      "batch: 14500, loss: 0.158407, accuracy: 0.957500\n",
      "batch: 14600, loss: 0.103466, accuracy: 0.955300\n",
      "batch: 14700, loss: 0.128492, accuracy: 0.956500\n",
      "batch: 14800, loss: 0.104607, accuracy: 0.955200\n",
      "batch: 14900, loss: 0.118394, accuracy: 0.958100\n",
      "batch: 15000, loss: 0.144942, accuracy: 0.956300\n",
      "batch: 15100, loss: 0.103451, accuracy: 0.957300\n",
      "batch: 15200, loss: 0.128361, accuracy: 0.957900\n",
      "batch: 15300, loss: 0.116135, accuracy: 0.956700\n",
      "batch: 15400, loss: 0.129118, accuracy: 0.956400\n",
      "batch: 15500, loss: 0.116008, accuracy: 0.954800\n",
      "batch: 15600, loss: 0.149153, accuracy: 0.954600\n",
      "batch: 15700, loss: 0.100677, accuracy: 0.958300\n",
      "batch: 15800, loss: 0.100070, accuracy: 0.956000\n",
      "batch: 15900, loss: 0.106050, accuracy: 0.956000\n",
      "batch: 16000, loss: 0.182776, accuracy: 0.951500\n",
      "batch: 16100, loss: 0.107099, accuracy: 0.957600\n",
      "batch: 16200, loss: 0.118849, accuracy: 0.954300\n",
      "batch: 16300, loss: 0.141735, accuracy: 0.954100\n",
      "batch: 16400, loss: 0.097759, accuracy: 0.954800\n",
      "batch: 16500, loss: 0.121652, accuracy: 0.956600\n",
      "batch: 16600, loss: 0.156183, accuracy: 0.955600\n",
      "batch: 16700, loss: 0.154068, accuracy: 0.951200\n",
      "batch: 16800, loss: 0.099755, accuracy: 0.954000\n",
      "batch: 16900, loss: 0.106753, accuracy: 0.952500\n",
      "batch: 17000, loss: 0.116806, accuracy: 0.954200\n",
      "batch: 17100, loss: 0.124067, accuracy: 0.956600\n",
      "batch: 17200, loss: 0.102815, accuracy: 0.952900\n",
      "batch: 17300, loss: 0.114903, accuracy: 0.952700\n",
      "batch: 17400, loss: 0.096046, accuracy: 0.955200\n",
      "batch: 17500, loss: 0.162890, accuracy: 0.953700\n",
      "batch: 17600, loss: 0.097867, accuracy: 0.954400\n",
      "batch: 17700, loss: 0.101302, accuracy: 0.955000\n",
      "batch: 17800, loss: 0.098062, accuracy: 0.956800\n",
      "batch: 17900, loss: 0.123589, accuracy: 0.955400\n",
      "batch: 18000, loss: 0.087029, accuracy: 0.954900\n",
      "batch: 18100, loss: 0.127444, accuracy: 0.950900\n",
      "batch: 18200, loss: 0.121410, accuracy: 0.954800\n",
      "batch: 18300, loss: 0.112348, accuracy: 0.951800\n",
      "batch: 18400, loss: 0.106891, accuracy: 0.954700\n",
      "batch: 18500, loss: 0.110302, accuracy: 0.954900\n",
      "batch: 18600, loss: 0.172551, accuracy: 0.953800\n",
      "batch: 18700, loss: 0.116077, accuracy: 0.953300\n",
      "batch: 18800, loss: 0.137878, accuracy: 0.953300\n",
      "batch: 18900, loss: 0.143477, accuracy: 0.952600\n",
      "batch: 19000, loss: 0.172294, accuracy: 0.953900\n",
      "batch: 19100, loss: 0.149220, accuracy: 0.953900\n",
      "batch: 19200, loss: 0.129542, accuracy: 0.953700\n",
      "batch: 19300, loss: 0.093003, accuracy: 0.954600\n",
      "batch: 19400, loss: 0.092015, accuracy: 0.954000\n",
      "batch: 19500, loss: 0.136537, accuracy: 0.955100\n",
      "batch: 19600, loss: 0.135304, accuracy: 0.951400\n",
      "batch: 19700, loss: 0.102869, accuracy: 0.952000\n",
      "batch: 19800, loss: 0.090460, accuracy: 0.953000\n",
      "batch: 19900, loss: 0.095337, accuracy: 0.956100\n",
      "batch: 20000, loss: 0.096361, accuracy: 0.953600\n",
      "batch: 100, loss: 2.158742, accuracy: 0.398300\n",
      "batch: 200, loss: 1.920126, accuracy: 0.552800\n",
      "batch: 300, loss: 1.533340, accuracy: 0.822300\n",
      "batch: 400, loss: 1.209904, accuracy: 0.891500\n",
      "batch: 500, loss: 1.082192, accuracy: 0.906100\n",
      "batch: 600, loss: 1.060161, accuracy: 0.912500\n",
      "batch: 700, loss: 1.043883, accuracy: 0.918200\n",
      "batch: 800, loss: 0.815459, accuracy: 0.921900\n",
      "batch: 900, loss: 0.778214, accuracy: 0.925700\n",
      "batch: 1000, loss: 0.743723, accuracy: 0.927500\n",
      "batch: 1100, loss: 0.711183, accuracy: 0.929800\n",
      "batch: 1200, loss: 0.755103, accuracy: 0.929100\n",
      "batch: 1300, loss: 0.733315, accuracy: 0.932800\n",
      "batch: 1400, loss: 0.679906, accuracy: 0.932400\n",
      "batch: 1500, loss: 0.631887, accuracy: 0.932100\n",
      "batch: 1600, loss: 0.656811, accuracy: 0.936000\n",
      "batch: 1700, loss: 0.693908, accuracy: 0.935400\n",
      "batch: 1800, loss: 0.679329, accuracy: 0.937100\n",
      "batch: 1900, loss: 0.629560, accuracy: 0.937300\n",
      "batch: 2000, loss: 0.601061, accuracy: 0.940400\n",
      "batch: 2100, loss: 0.575219, accuracy: 0.939300\n",
      "batch: 2200, loss: 0.630810, accuracy: 0.940500\n",
      "batch: 2300, loss: 0.649436, accuracy: 0.939300\n",
      "batch: 2400, loss: 0.521005, accuracy: 0.941000\n",
      "batch: 2500, loss: 0.544774, accuracy: 0.938600\n",
      "batch: 2600, loss: 0.637994, accuracy: 0.940700\n",
      "batch: 2700, loss: 0.588794, accuracy: 0.944200\n",
      "batch: 2800, loss: 0.547130, accuracy: 0.942800\n",
      "batch: 2900, loss: 0.489382, accuracy: 0.944500\n",
      "batch: 3000, loss: 0.607933, accuracy: 0.945400\n",
      "batch: 3100, loss: 0.398844, accuracy: 0.944100\n",
      "batch: 3200, loss: 0.473504, accuracy: 0.943500\n",
      "batch: 3300, loss: 0.462993, accuracy: 0.942700\n",
      "batch: 3400, loss: 0.493002, accuracy: 0.945600\n",
      "batch: 3500, loss: 0.530583, accuracy: 0.945500\n",
      "batch: 3600, loss: 0.443975, accuracy: 0.944300\n",
      "batch: 3700, loss: 0.425563, accuracy: 0.944100\n",
      "batch: 3800, loss: 0.451833, accuracy: 0.944800\n",
      "batch: 3900, loss: 0.438919, accuracy: 0.943300\n",
      "batch: 4000, loss: 0.408693, accuracy: 0.943100\n",
      "batch: 4100, loss: 0.497433, accuracy: 0.943900\n",
      "batch: 4200, loss: 0.397412, accuracy: 0.946600\n",
      "batch: 4300, loss: 0.586465, accuracy: 0.947100\n",
      "batch: 4400, loss: 0.481562, accuracy: 0.946900\n",
      "batch: 4500, loss: 0.379243, accuracy: 0.943900\n",
      "batch: 4600, loss: 0.480777, accuracy: 0.947300\n",
      "batch: 4700, loss: 0.438183, accuracy: 0.946800\n",
      "batch: 4800, loss: 0.401034, accuracy: 0.944400\n",
      "batch: 4900, loss: 0.415130, accuracy: 0.946900\n",
      "batch: 5000, loss: 0.405726, accuracy: 0.945900\n",
      "batch: 5100, loss: 0.509783, accuracy: 0.944600\n",
      "batch: 5200, loss: 0.358126, accuracy: 0.945400\n",
      "batch: 5300, loss: 0.498641, accuracy: 0.944600\n",
      "batch: 5400, loss: 0.356154, accuracy: 0.946500\n",
      "batch: 5500, loss: 0.325649, accuracy: 0.948400\n",
      "batch: 5600, loss: 0.366596, accuracy: 0.948300\n",
      "batch: 5700, loss: 0.339529, accuracy: 0.947400\n",
      "batch: 5800, loss: 0.363603, accuracy: 0.946800\n",
      "batch: 5900, loss: 0.420796, accuracy: 0.946900\n",
      "batch: 6000, loss: 0.387401, accuracy: 0.948100\n",
      "batch: 6100, loss: 0.366698, accuracy: 0.946800\n",
      "batch: 6200, loss: 0.369349, accuracy: 0.948000\n",
      "batch: 6300, loss: 0.355248, accuracy: 0.945200\n",
      "batch: 6400, loss: 0.390758, accuracy: 0.948500\n",
      "batch: 6500, loss: 0.419040, accuracy: 0.945900\n",
      "batch: 6600, loss: 0.338499, accuracy: 0.947100\n",
      "batch: 6700, loss: 0.339303, accuracy: 0.945100\n",
      "batch: 6800, loss: 0.363573, accuracy: 0.947200\n",
      "batch: 6900, loss: 0.382638, accuracy: 0.945300\n",
      "batch: 7000, loss: 0.331695, accuracy: 0.947000\n",
      "batch: 7100, loss: 0.310070, accuracy: 0.943100\n",
      "batch: 7200, loss: 0.302475, accuracy: 0.948000\n",
      "batch: 7300, loss: 0.442316, accuracy: 0.947300\n",
      "batch: 7400, loss: 0.396769, accuracy: 0.947000\n",
      "batch: 7500, loss: 0.234061, accuracy: 0.947300\n",
      "batch: 7600, loss: 0.335504, accuracy: 0.947100\n",
      "batch: 7700, loss: 0.243322, accuracy: 0.948800\n",
      "batch: 7800, loss: 0.299276, accuracy: 0.948500\n",
      "batch: 7900, loss: 0.229625, accuracy: 0.945900\n",
      "batch: 8000, loss: 0.319731, accuracy: 0.946900\n",
      "batch: 8100, loss: 0.315908, accuracy: 0.947500\n",
      "batch: 8200, loss: 0.308340, accuracy: 0.946200\n",
      "batch: 8300, loss: 0.338236, accuracy: 0.945100\n",
      "batch: 8400, loss: 0.288121, accuracy: 0.946400\n",
      "batch: 8500, loss: 0.361144, accuracy: 0.943900\n",
      "batch: 8600, loss: 0.312917, accuracy: 0.946900\n",
      "batch: 8700, loss: 0.392845, accuracy: 0.946100\n",
      "batch: 8800, loss: 0.234695, accuracy: 0.948000\n",
      "batch: 8900, loss: 0.361960, accuracy: 0.948300\n",
      "batch: 9000, loss: 0.313095, accuracy: 0.947600\n",
      "batch: 9100, loss: 0.222909, accuracy: 0.945100\n",
      "batch: 9200, loss: 0.282955, accuracy: 0.945100\n",
      "batch: 9300, loss: 0.272546, accuracy: 0.946800\n",
      "batch: 9400, loss: 0.248695, accuracy: 0.946000\n",
      "batch: 9500, loss: 0.293581, accuracy: 0.945900\n",
      "batch: 9600, loss: 0.409071, accuracy: 0.945700\n",
      "batch: 9700, loss: 0.332575, accuracy: 0.945600\n",
      "batch: 9800, loss: 0.312571, accuracy: 0.947300\n",
      "batch: 9900, loss: 0.366365, accuracy: 0.946800\n",
      "batch: 10000, loss: 0.329709, accuracy: 0.946300\n",
      "batch: 10100, loss: 0.239260, accuracy: 0.947200\n",
      "batch: 10200, loss: 0.257840, accuracy: 0.943600\n",
      "batch: 10300, loss: 0.285804, accuracy: 0.946700\n",
      "batch: 10400, loss: 0.382897, accuracy: 0.947300\n",
      "batch: 10500, loss: 0.375903, accuracy: 0.946300\n",
      "batch: 10600, loss: 0.205066, accuracy: 0.944200\n",
      "batch: 10700, loss: 0.204498, accuracy: 0.947100\n",
      "batch: 10800, loss: 0.281294, accuracy: 0.945700\n",
      "batch: 10900, loss: 0.285990, accuracy: 0.946500\n",
      "batch: 11000, loss: 0.220389, accuracy: 0.947100\n",
      "batch: 11100, loss: 0.236955, accuracy: 0.943800\n",
      "batch: 11200, loss: 0.225060, accuracy: 0.944900\n",
      "batch: 11300, loss: 0.270178, accuracy: 0.945600\n",
      "batch: 11400, loss: 0.306668, accuracy: 0.946300\n",
      "batch: 11500, loss: 0.254310, accuracy: 0.945400\n",
      "batch: 11600, loss: 0.242944, accuracy: 0.947700\n",
      "batch: 11700, loss: 0.209035, accuracy: 0.948400\n",
      "batch: 11800, loss: 0.290087, accuracy: 0.946100\n",
      "batch: 11900, loss: 0.229725, accuracy: 0.946700\n",
      "batch: 12000, loss: 0.221281, accuracy: 0.947600\n",
      "batch: 12100, loss: 0.251411, accuracy: 0.948400\n",
      "batch: 12200, loss: 0.309012, accuracy: 0.945500\n",
      "batch: 12300, loss: 0.235659, accuracy: 0.948300\n",
      "batch: 12400, loss: 0.186754, accuracy: 0.946900\n",
      "batch: 12500, loss: 0.203741, accuracy: 0.941700\n",
      "batch: 12600, loss: 0.268564, accuracy: 0.947200\n",
      "batch: 12700, loss: 0.293141, accuracy: 0.948400\n",
      "batch: 12800, loss: 0.256203, accuracy: 0.945700\n",
      "batch: 12900, loss: 0.211165, accuracy: 0.944100\n",
      "batch: 13000, loss: 0.282855, accuracy: 0.946000\n",
      "batch: 13100, loss: 0.355198, accuracy: 0.945600\n",
      "batch: 13200, loss: 0.198773, accuracy: 0.944500\n",
      "batch: 13300, loss: 0.293652, accuracy: 0.948000\n",
      "batch: 13400, loss: 0.306334, accuracy: 0.945700\n",
      "batch: 13500, loss: 0.212386, accuracy: 0.948000\n",
      "batch: 13600, loss: 0.212882, accuracy: 0.950600\n",
      "batch: 13700, loss: 0.227915, accuracy: 0.947400\n",
      "batch: 13800, loss: 0.173064, accuracy: 0.945800\n",
      "batch: 13900, loss: 0.175708, accuracy: 0.947400\n",
      "batch: 14000, loss: 0.279574, accuracy: 0.944500\n",
      "batch: 14100, loss: 0.226631, accuracy: 0.942100\n",
      "batch: 14200, loss: 0.279131, accuracy: 0.944500\n",
      "batch: 14300, loss: 0.316610, accuracy: 0.945000\n",
      "batch: 14400, loss: 0.371960, accuracy: 0.946700\n",
      "batch: 14500, loss: 0.217185, accuracy: 0.944700\n",
      "batch: 14600, loss: 0.261837, accuracy: 0.948300\n",
      "batch: 14700, loss: 0.228997, accuracy: 0.944000\n",
      "batch: 14800, loss: 0.244926, accuracy: 0.943900\n",
      "batch: 14900, loss: 0.287479, accuracy: 0.944800\n",
      "batch: 15000, loss: 0.246530, accuracy: 0.943400\n",
      "batch: 15100, loss: 0.249999, accuracy: 0.943000\n",
      "batch: 15200, loss: 0.198657, accuracy: 0.945600\n",
      "batch: 15300, loss: 0.251786, accuracy: 0.945300\n",
      "batch: 15400, loss: 0.265535, accuracy: 0.944300\n",
      "batch: 15500, loss: 0.360547, accuracy: 0.943500\n",
      "batch: 15600, loss: 0.178035, accuracy: 0.945100\n",
      "batch: 15700, loss: 0.235202, accuracy: 0.945100\n",
      "batch: 15800, loss: 0.208942, accuracy: 0.944600\n",
      "batch: 15900, loss: 0.282951, accuracy: 0.946000\n",
      "batch: 16000, loss: 0.257186, accuracy: 0.943600\n",
      "batch: 16100, loss: 0.251277, accuracy: 0.945200\n",
      "batch: 16200, loss: 0.217138, accuracy: 0.944000\n",
      "batch: 16300, loss: 0.166533, accuracy: 0.944300\n",
      "batch: 16400, loss: 0.280458, accuracy: 0.940800\n",
      "batch: 16500, loss: 0.198719, accuracy: 0.944300\n",
      "batch: 16600, loss: 0.266070, accuracy: 0.943900\n",
      "batch: 16700, loss: 0.226411, accuracy: 0.945700\n",
      "batch: 16800, loss: 0.173997, accuracy: 0.942600\n",
      "batch: 16900, loss: 0.206691, accuracy: 0.943600\n",
      "batch: 17000, loss: 0.241908, accuracy: 0.947400\n",
      "batch: 17100, loss: 0.217000, accuracy: 0.943500\n",
      "batch: 17200, loss: 0.178420, accuracy: 0.942300\n",
      "batch: 17300, loss: 0.279271, accuracy: 0.944300\n",
      "batch: 17400, loss: 0.274703, accuracy: 0.940700\n",
      "batch: 17500, loss: 0.167837, accuracy: 0.943500\n",
      "batch: 17600, loss: 0.237785, accuracy: 0.942900\n",
      "batch: 17700, loss: 0.230881, accuracy: 0.943800\n",
      "batch: 17800, loss: 0.208314, accuracy: 0.943000\n",
      "batch: 17900, loss: 0.246805, accuracy: 0.938400\n",
      "batch: 18000, loss: 0.168713, accuracy: 0.942100\n",
      "batch: 18100, loss: 0.241911, accuracy: 0.940100\n",
      "batch: 18200, loss: 0.194608, accuracy: 0.946100\n",
      "batch: 18300, loss: 0.246890, accuracy: 0.944500\n",
      "batch: 18400, loss: 0.187969, accuracy: 0.947200\n",
      "batch: 18500, loss: 0.277679, accuracy: 0.946000\n",
      "batch: 18600, loss: 0.299592, accuracy: 0.945800\n",
      "batch: 18700, loss: 0.306786, accuracy: 0.943800\n",
      "batch: 18800, loss: 0.223328, accuracy: 0.944100\n",
      "batch: 18900, loss: 0.154720, accuracy: 0.944800\n",
      "batch: 19000, loss: 0.218889, accuracy: 0.943500\n",
      "batch: 19100, loss: 0.175370, accuracy: 0.940300\n",
      "batch: 19200, loss: 0.195766, accuracy: 0.945400\n",
      "batch: 19300, loss: 0.298207, accuracy: 0.943000\n",
      "batch: 19400, loss: 0.290409, accuracy: 0.946000\n",
      "batch: 19500, loss: 0.250325, accuracy: 0.946000\n",
      "batch: 19600, loss: 0.179282, accuracy: 0.944200\n",
      "batch: 19700, loss: 0.269909, accuracy: 0.942800\n",
      "batch: 19800, loss: 0.185761, accuracy: 0.944200\n",
      "batch: 19900, loss: 0.294517, accuracy: 0.946500\n",
      "batch: 20000, loss: 0.171258, accuracy: 0.945800\n",
      "batch: 100, loss: 2.411296, accuracy: 0.150700\n",
      "batch: 200, loss: 2.350595, accuracy: 0.159700\n",
      "batch: 300, loss: 2.317443, accuracy: 0.134700\n",
      "batch: 400, loss: 2.312226, accuracy: 0.124800\n",
      "batch: 500, loss: 2.301046, accuracy: 0.120000\n",
      "batch: 600, loss: 2.233116, accuracy: 0.121200\n",
      "batch: 700, loss: 2.216657, accuracy: 0.210600\n",
      "batch: 800, loss: 2.211239, accuracy: 0.211900\n",
      "batch: 900, loss: 2.171452, accuracy: 0.211700\n",
      "batch: 1000, loss: 2.191990, accuracy: 0.212300\n",
      "batch: 1100, loss: 2.185893, accuracy: 0.212700\n",
      "batch: 1200, loss: 2.114900, accuracy: 0.212500\n",
      "batch: 1300, loss: 2.216179, accuracy: 0.212700\n",
      "batch: 1400, loss: 2.203327, accuracy: 0.212500\n",
      "batch: 1500, loss: 2.098653, accuracy: 0.212500\n",
      "batch: 1600, loss: 2.115290, accuracy: 0.212500\n",
      "batch: 1700, loss: 2.157744, accuracy: 0.213000\n",
      "batch: 1800, loss: 2.035249, accuracy: 0.213100\n",
      "batch: 1900, loss: 2.139236, accuracy: 0.213400\n",
      "batch: 2000, loss: 2.119855, accuracy: 0.215300\n",
      "batch: 2100, loss: 2.060729, accuracy: 0.214200\n",
      "batch: 2200, loss: 2.139871, accuracy: 0.220700\n",
      "batch: 2300, loss: 2.042457, accuracy: 0.299100\n",
      "batch: 2400, loss: 2.141603, accuracy: 0.318200\n",
      "batch: 2500, loss: 1.868330, accuracy: 0.440700\n",
      "batch: 2600, loss: 1.717018, accuracy: 0.547400\n",
      "batch: 2700, loss: 1.702477, accuracy: 0.557500\n",
      "batch: 2800, loss: 1.458082, accuracy: 0.566900\n",
      "batch: 2900, loss: 1.509936, accuracy: 0.586000\n",
      "batch: 3000, loss: 1.478194, accuracy: 0.576200\n",
      "batch: 3100, loss: 1.490776, accuracy: 0.599000\n",
      "batch: 3200, loss: 1.433245, accuracy: 0.574500\n",
      "batch: 3300, loss: 1.418068, accuracy: 0.580600\n",
      "batch: 3400, loss: 1.347851, accuracy: 0.598900\n",
      "batch: 3500, loss: 1.363400, accuracy: 0.603900\n",
      "batch: 3600, loss: 1.369457, accuracy: 0.609400\n",
      "batch: 3700, loss: 1.316319, accuracy: 0.616100\n",
      "batch: 3800, loss: 1.297120, accuracy: 0.610700\n",
      "batch: 3900, loss: 1.304538, accuracy: 0.627100\n",
      "batch: 4000, loss: 1.237063, accuracy: 0.695800\n",
      "batch: 4100, loss: 1.256887, accuracy: 0.714400\n",
      "batch: 4200, loss: 1.325883, accuracy: 0.715900\n",
      "batch: 4300, loss: 1.169132, accuracy: 0.714300\n",
      "batch: 4400, loss: 1.196955, accuracy: 0.733100\n",
      "batch: 4500, loss: 1.248577, accuracy: 0.731500\n",
      "batch: 4600, loss: 1.252205, accuracy: 0.732600\n",
      "batch: 4700, loss: 1.166627, accuracy: 0.747600\n",
      "batch: 4800, loss: 1.074284, accuracy: 0.753300\n",
      "batch: 4900, loss: 1.186921, accuracy: 0.753900\n",
      "batch: 5000, loss: 1.054811, accuracy: 0.755500\n",
      "batch: 5100, loss: 1.108908, accuracy: 0.767000\n",
      "batch: 5200, loss: 1.018541, accuracy: 0.768500\n",
      "batch: 5300, loss: 1.064614, accuracy: 0.781700\n",
      "batch: 5400, loss: 0.955482, accuracy: 0.781300\n",
      "batch: 5500, loss: 1.027627, accuracy: 0.780000\n",
      "batch: 5600, loss: 0.988222, accuracy: 0.787000\n",
      "batch: 5700, loss: 1.105976, accuracy: 0.789000\n",
      "batch: 5800, loss: 0.869178, accuracy: 0.795300\n",
      "batch: 5900, loss: 1.047105, accuracy: 0.797700\n",
      "batch: 6000, loss: 0.850407, accuracy: 0.795200\n",
      "batch: 6100, loss: 0.820721, accuracy: 0.791500\n",
      "batch: 6200, loss: 0.958446, accuracy: 0.793600\n",
      "batch: 6300, loss: 0.901438, accuracy: 0.793500\n",
      "batch: 6400, loss: 0.854161, accuracy: 0.792500\n",
      "batch: 6500, loss: 1.013904, accuracy: 0.790600\n",
      "batch: 6600, loss: 0.854836, accuracy: 0.790300\n",
      "batch: 6700, loss: 1.041383, accuracy: 0.799400\n",
      "batch: 6800, loss: 0.831534, accuracy: 0.799600\n",
      "batch: 6900, loss: 0.967393, accuracy: 0.799700\n",
      "batch: 7000, loss: 0.866282, accuracy: 0.806400\n",
      "batch: 7100, loss: 0.950604, accuracy: 0.803400\n",
      "batch: 7200, loss: 0.856411, accuracy: 0.795700\n",
      "batch: 7300, loss: 0.954649, accuracy: 0.809000\n",
      "batch: 7400, loss: 0.987203, accuracy: 0.802200\n",
      "batch: 7500, loss: 0.939193, accuracy: 0.799800\n",
      "batch: 7600, loss: 0.937222, accuracy: 0.802500\n",
      "batch: 7700, loss: 0.948701, accuracy: 0.804800\n",
      "batch: 7800, loss: 0.941994, accuracy: 0.804300\n",
      "batch: 7900, loss: 0.864209, accuracy: 0.807400\n",
      "batch: 8000, loss: 0.812955, accuracy: 0.805500\n",
      "batch: 8100, loss: 0.921735, accuracy: 0.806000\n",
      "batch: 8200, loss: 0.741668, accuracy: 0.803900\n",
      "batch: 8300, loss: 0.724565, accuracy: 0.809200\n",
      "batch: 8400, loss: 0.844863, accuracy: 0.810600\n",
      "batch: 8500, loss: 0.874536, accuracy: 0.810900\n",
      "batch: 8600, loss: 0.670937, accuracy: 0.809900\n",
      "batch: 8700, loss: 0.893099, accuracy: 0.809400\n",
      "batch: 8800, loss: 0.804866, accuracy: 0.807100\n",
      "batch: 8900, loss: 0.721768, accuracy: 0.805800\n",
      "batch: 9000, loss: 0.836150, accuracy: 0.813100\n",
      "batch: 9100, loss: 0.784307, accuracy: 0.806800\n",
      "batch: 9200, loss: 0.923043, accuracy: 0.814900\n",
      "batch: 9300, loss: 0.818607, accuracy: 0.813000\n",
      "batch: 9400, loss: 0.631095, accuracy: 0.810600\n",
      "batch: 9500, loss: 0.865538, accuracy: 0.833400\n",
      "batch: 9600, loss: 0.835404, accuracy: 0.860800\n",
      "batch: 9700, loss: 0.700251, accuracy: 0.866200\n",
      "batch: 9800, loss: 0.830094, accuracy: 0.870600\n",
      "batch: 9900, loss: 0.725377, accuracy: 0.878800\n",
      "batch: 10000, loss: 0.667150, accuracy: 0.865800\n",
      "batch: 10100, loss: 0.873979, accuracy: 0.872900\n",
      "batch: 10200, loss: 0.718247, accuracy: 0.874200\n",
      "batch: 10300, loss: 0.743699, accuracy: 0.878200\n",
      "batch: 10400, loss: 0.617049, accuracy: 0.878800\n",
      "batch: 10500, loss: 0.678958, accuracy: 0.878300\n",
      "batch: 10600, loss: 0.718788, accuracy: 0.878600\n",
      "batch: 10700, loss: 0.840176, accuracy: 0.880700\n",
      "batch: 10800, loss: 0.790217, accuracy: 0.881300\n",
      "batch: 10900, loss: 0.618621, accuracy: 0.880400\n",
      "batch: 11000, loss: 0.656704, accuracy: 0.885500\n",
      "batch: 11100, loss: 0.822553, accuracy: 0.883100\n",
      "batch: 11200, loss: 0.616551, accuracy: 0.881100\n",
      "batch: 11300, loss: 0.643115, accuracy: 0.881900\n",
      "batch: 11400, loss: 0.706217, accuracy: 0.885700\n",
      "batch: 11500, loss: 0.709952, accuracy: 0.886400\n",
      "batch: 11600, loss: 0.772231, accuracy: 0.875300\n",
      "batch: 11700, loss: 0.673858, accuracy: 0.879400\n",
      "batch: 11800, loss: 0.631502, accuracy: 0.878300\n",
      "batch: 11900, loss: 0.675524, accuracy: 0.884000\n",
      "batch: 12000, loss: 0.752943, accuracy: 0.880400\n",
      "batch: 12100, loss: 0.595716, accuracy: 0.886800\n",
      "batch: 12200, loss: 0.845407, accuracy: 0.883700\n",
      "batch: 12300, loss: 0.720722, accuracy: 0.880700\n",
      "batch: 12400, loss: 0.662958, accuracy: 0.880700\n",
      "batch: 12500, loss: 0.579511, accuracy: 0.884000\n",
      "batch: 12600, loss: 0.615262, accuracy: 0.881900\n",
      "batch: 12700, loss: 0.597738, accuracy: 0.886600\n",
      "batch: 12800, loss: 0.570774, accuracy: 0.883700\n",
      "batch: 12900, loss: 0.565228, accuracy: 0.888400\n",
      "batch: 13000, loss: 0.669345, accuracy: 0.886700\n",
      "batch: 13100, loss: 0.637433, accuracy: 0.884500\n",
      "batch: 13200, loss: 0.725062, accuracy: 0.880700\n",
      "batch: 13300, loss: 0.667876, accuracy: 0.888600\n",
      "batch: 13400, loss: 0.686438, accuracy: 0.884900\n",
      "batch: 13500, loss: 0.661194, accuracy: 0.881600\n",
      "batch: 13600, loss: 0.805686, accuracy: 0.885900\n",
      "batch: 13700, loss: 0.563369, accuracy: 0.878400\n",
      "batch: 13800, loss: 0.683016, accuracy: 0.886300\n",
      "batch: 13900, loss: 0.529349, accuracy: 0.883100\n",
      "batch: 14000, loss: 0.627770, accuracy: 0.887100\n",
      "batch: 14100, loss: 0.585023, accuracy: 0.886900\n",
      "batch: 14200, loss: 0.593997, accuracy: 0.884800\n",
      "batch: 14300, loss: 0.593502, accuracy: 0.884100\n",
      "batch: 14400, loss: 0.540672, accuracy: 0.879600\n",
      "batch: 14500, loss: 0.550146, accuracy: 0.880800\n",
      "batch: 14600, loss: 0.558753, accuracy: 0.888200\n",
      "batch: 14700, loss: 0.781071, accuracy: 0.879100\n",
      "batch: 14800, loss: 0.639559, accuracy: 0.887600\n",
      "batch: 14900, loss: 0.456541, accuracy: 0.886800\n",
      "batch: 15000, loss: 0.556688, accuracy: 0.884000\n",
      "batch: 15100, loss: 0.612444, accuracy: 0.880200\n",
      "batch: 15200, loss: 0.672431, accuracy: 0.888700\n",
      "batch: 15300, loss: 0.595775, accuracy: 0.884900\n",
      "batch: 15400, loss: 0.547272, accuracy: 0.889400\n",
      "batch: 15500, loss: 0.615566, accuracy: 0.887300\n",
      "batch: 15600, loss: 0.599542, accuracy: 0.882600\n",
      "batch: 15700, loss: 0.545993, accuracy: 0.887200\n",
      "batch: 15800, loss: 0.623329, accuracy: 0.891100\n",
      "batch: 15900, loss: 0.738292, accuracy: 0.885700\n",
      "batch: 16000, loss: 0.583700, accuracy: 0.883400\n",
      "batch: 16100, loss: 0.614708, accuracy: 0.888200\n",
      "batch: 16200, loss: 0.643762, accuracy: 0.889900\n",
      "batch: 16300, loss: 0.596647, accuracy: 0.882900\n",
      "batch: 16400, loss: 0.512153, accuracy: 0.887500\n",
      "batch: 16500, loss: 0.477547, accuracy: 0.884900\n",
      "batch: 16600, loss: 0.730187, accuracy: 0.881800\n",
      "batch: 16700, loss: 0.533657, accuracy: 0.885500\n",
      "batch: 16800, loss: 0.471786, accuracy: 0.887600\n",
      "batch: 16900, loss: 0.679805, accuracy: 0.890900\n",
      "batch: 17000, loss: 0.560967, accuracy: 0.885600\n",
      "batch: 17100, loss: 0.629321, accuracy: 0.888400\n",
      "batch: 17200, loss: 0.625432, accuracy: 0.891700\n",
      "batch: 17300, loss: 0.620647, accuracy: 0.884300\n",
      "batch: 17400, loss: 0.617275, accuracy: 0.882000\n",
      "batch: 17500, loss: 0.585064, accuracy: 0.888600\n",
      "batch: 17600, loss: 0.493174, accuracy: 0.882100\n",
      "batch: 17700, loss: 0.599709, accuracy: 0.883100\n",
      "batch: 17800, loss: 0.479017, accuracy: 0.888900\n",
      "batch: 17900, loss: 0.556445, accuracy: 0.891500\n",
      "batch: 18000, loss: 0.646759, accuracy: 0.890400\n",
      "batch: 18100, loss: 0.685434, accuracy: 0.877000\n",
      "batch: 18200, loss: 0.593324, accuracy: 0.889400\n",
      "batch: 18300, loss: 0.741116, accuracy: 0.888700\n",
      "batch: 18400, loss: 0.531608, accuracy: 0.879500\n",
      "batch: 18500, loss: 0.567390, accuracy: 0.886100\n",
      "batch: 18600, loss: 0.629272, accuracy: 0.886400\n",
      "batch: 18700, loss: 0.642936, accuracy: 0.893200\n",
      "batch: 18800, loss: 0.692222, accuracy: 0.885800\n",
      "batch: 18900, loss: 0.553751, accuracy: 0.885800\n",
      "batch: 19000, loss: 0.586882, accuracy: 0.889700\n",
      "batch: 19100, loss: 0.659456, accuracy: 0.881600\n",
      "batch: 19200, loss: 0.604095, accuracy: 0.895500\n",
      "batch: 19300, loss: 0.560300, accuracy: 0.890900\n",
      "batch: 19400, loss: 0.614791, accuracy: 0.888100\n",
      "batch: 19500, loss: 0.661798, accuracy: 0.889600\n",
      "batch: 19600, loss: 0.549081, accuracy: 0.889900\n",
      "batch: 19700, loss: 0.429393, accuracy: 0.893500\n",
      "batch: 19800, loss: 0.518973, accuracy: 0.892100\n",
      "batch: 19900, loss: 0.627685, accuracy: 0.891400\n",
      "batch: 20000, loss: 0.642310, accuracy: 0.892200\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "adam = tf.train.AdamOptimizer(learning_rate=1e-3)\n",
    "train_op = adam.minimize(loss)\n",
    "sess = None\n",
    "# We iterate over different sparsity constraint\n",
    "for sc in [0, 1e-4, 5e-4, 1e-3, 2.7e-3]:\n",
    "    result_folder = dir + '/results/' + str(int(time.time())) + '-fc-sc' + str(sc)\n",
    "    config = tf.ConfigProto(allow_soft_placement=True)\n",
    "    config.gpu_options.allow_growth=True\n",
    "    with tf.Session(config=config) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        sw = tf.summary.FileWriter(result_folder, sess.graph)\n",
    "        \n",
    "        for i in range(20000):\n",
    "            batch = mnist.train.next_batch(100)\n",
    "            current_loss, summary, _ = sess.run([loss, summaries, train_op], feed_dict={\n",
    "                x: batch[0],\n",
    "                y_true: batch[1],\n",
    "                sparsity_constraint: sc\n",
    "            })\n",
    "            sw.add_summary(summary, i + 1)\n",
    "\n",
    "            if (i + 1) % 100 == 0:\n",
    "                acc, acc_sum = sess.run([accuracy, acc_summary], feed_dict={\n",
    "                    x: mnist.test.images, \n",
    "                    y_true: mnist.test.labels\n",
    "                })\n",
    "                sw.add_summary(acc_sum, i + 1)\n",
    "                print('batch: %d, loss: %f, accuracy: %f' % (i + 1, current_loss, acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}