W06: Beatiful code and figures for problem 4.

w06/w06.py

@@ -0,0 +1,50 @@
+"""
+Code to plot decision boundary of ANN.
+
+Stephen Brennan, EECS 440 Written 6, 10/06/2015.
+"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+plt.style.use('ggplot')
+
+
+def sigmoid(X):
+    return (1 + np.exp(-X)) ** -1
+
+
+def output(X, weights, out_weights):
+    hidden_outs = sigmoid(np.dot(weights, X.T))
+    return sigmoid(np.dot(hidden_outs.T, out_weights))
+
+
+def plot_decision_boundary(wmin, wmax):
+    hidden_weights = np.random.uniform(wmin, wmax, (2, 2))
+    output_weights = np.random.uniform(wmin, wmax, 2)
+
+    x1 = np.arange(-5.0, 5.1, step=0.1)
+    x2 = np.arange(-5.0, 5.1, step=0.1)
+    X = np.transpose([np.tile(x1, len(x2)), np.repeat(x2, len(x1))])
+    y = output(X, hidden_weights, output_weights)
+
+    fig, ax = plt.subplots()
+    pos_idx = y > 0.5
+    pos = X[pos_idx]
+    neg = X[~pos_idx]
+    print(y)
+    print('%r positives, %r negatives' % (len(pos), len(neg)))
+    print(pos)
+    print(neg)
+    ax.scatter(*pos.T, color='b')
+    ax.scatter(*neg.T, color='r')
+    return ax
+
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) < 2:
+        print('Need weight bounds.')
+    else:
+        bound = float(sys.argv[1])
+        ax = plot_decision_boundary(-bound, bound)
+        ax.figure.savefig(sys.argv[1] + '.pdf')

w06/w06.tex

@@ -5,9 +5,52 @@
 \assignment{Written 6}
 \duedate{October 6, 2015}
 
+% Solarized colors
+%\usepackage[x11names, rgb, html]{xcolor}
+\definecolor{sbase03}{HTML}{002B36}
+\definecolor{sbase02}{HTML}{073642}
+\definecolor{sbase01}{HTML}{586E75}
+\definecolor{sbase00}{HTML}{657B83}
+\definecolor{sbase0}{HTML}{839496}
+\definecolor{sbase1}{HTML}{93A1A1}
+\definecolor{sbase2}{HTML}{EEE8D5}
+\definecolor{sbase3}{HTML}{FDF6E3}
+\definecolor{syellow}{HTML}{B58900}
+\definecolor{sorange}{HTML}{CB4B16}
+\definecolor{sred}{HTML}{DC322F}
+\definecolor{smagenta}{HTML}{D33682}
+\definecolor{sviolet}{HTML}{6C71C4}
+\definecolor{sblue}{HTML}{268BD2}
+\definecolor{scyan}{HTML}{2AA198}
+\definecolor{sgreen}{HTML}{859900}
+
+\usepackage{listings}
+\lstset{
+    % How/what to match
+    sensitive=true,
+    % Border (above and below)
+    frame=lines,
+    % Extra margin on line (align with paragraph)
+    xleftmargin=\parindent,
+    % Put extra space under caption
+    belowcaptionskip=1\baselineskip,
+    % Colors
+    backgroundcolor=\color{sbase3},
+    basicstyle=\color{sbase00}\ttfamily,
+    keywordstyle=\color{scyan},
+    commentstyle=\color{sbase1},
+    stringstyle=\color{sblue},
+    numberstyle=\color{sviolet},
+    identifierstyle=\color{sbase00},
+    % Break long lines into multiple lines?
+    breaklines=true,
+    % Show a character for spaces?
+    showstringspaces=false,
+    tabsize=2
+}
+
 \usepackage{mathtools}
 \newcommand{\sigmoid}{\:\text{sigmoid}}
-%\usepackage{graphicx}
 
 \begin{document}
   \maketitle
@@ -126,14 +169,48 @@
       Using R/Matlab/Mathematica/your favorite math software, plot the decision
       boundary for an ANN with two inputs, two hidden units and one output. All
       activation functions are sigmoids.  Each layer is fully connected to the
-      next. Assume the inputs range between −5 to 5 and fix all activation
+      next. Assume the inputs range between -5 to 5 and fix all activation
       thresholds to 0. Plot the decision boundaries for the weights except the
       thresholds randomly chosen between \textbf{(i)} $(−10,10)$, \textbf{(ii)}
       $(−3,3)$, \textbf{(iii)} $(−0.1,0.1)$ (one random set for each case is
       enough). Use your plots to show that weight decay can be used to control
       overfitting for ANNs. (If you use Matlab, the following commands might be
       useful: meshgrid and surf). (10 points)
     \end{question}
+
+    \begin{figure}
+      \centering
+      \caption{Decision boundary for random ANN with weights in (-10, 10).}
+      \includegraphics[width=0.7\textwidth]{10.pdf}
+      \label{fig:10}
+    \end{figure}
+
+    \begin{figure}
+      \centering
+      \caption{Decision boundary for random ANN with weights in (-3, 3).}
+      \includegraphics[width=0.7\textwidth]{3.pdf}
+      \label{fig:3}
+    \end{figure}
+
+    \begin{figure}
+      \centering
+      \caption{Decision boundary for random ANN with weights in (-0.1, 0.1).}
+      \includegraphics[width=0.7\textwidth]{0_1.pdf}
+      \label{fig:0.1}
+    \end{figure}
+
+    Plots for \textbf{(i)}, \textbf{(ii)}, and \textbf{(iii)} are provided as
+    Figures~\ref{fig:10},~\ref{fig:3}, and~\ref{fig:0.1}, respectively.  In the
+    figures, the examples predicted positive are blue, and the examples
+    predicted negative are red.  The decision boundary is the boundary between
+    the blue and red regions.  You can see that as the weights become closer to
+    zero, the decision boundaries become less complex (that is, have fewer
+    ``bends'' in them).  Therefore, it stands to reason that creating an
+    artificial incentive to have smaller weights would reduce the ``bendiness'',
+    and therefore reduce overfitting.
+
+    The Python source code that produced the images is provided in
+    Appendix~\ref{p4src}.
   \end{problem}
 
   \begin{problem}{5}
@@ -145,4 +222,11 @@
     \end{question}
   \end{problem}
 
+  \newpage
+  \appendix
+  \section{Source Code for Problem 4}
+  \label{p4src}
+  \lstinputlisting[language=Python]{w06.py}
+
+  Example: \texttt{\$ python w06.py 10}
 \end{document}