diff --git a/w06/0_1.pdf b/w06/0_1.pdf new file mode 100644 index 0000000..2a9919e Binary files /dev/null and b/w06/0_1.pdf differ diff --git a/w06/10.pdf b/w06/10.pdf new file mode 100644 index 0000000..f46ddba Binary files /dev/null and b/w06/10.pdf differ diff --git a/w06/3.pdf b/w06/3.pdf new file mode 100644 index 0000000..826af6c Binary files /dev/null and b/w06/3.pdf differ diff --git a/w06/w06.py b/w06/w06.py new file mode 100644 index 0000000..9bb1308 --- /dev/null +++ b/w06/w06.py @@ -0,0 +1,50 @@ +""" +Code to plot decision boundary of ANN. + +Stephen Brennan, EECS 440 Written 6, 10/06/2015. +""" + +import numpy as np +import matplotlib.pyplot as plt +plt.style.use('ggplot') + + +def sigmoid(X): + return (1 + np.exp(-X)) ** -1 + + +def output(X, weights, out_weights): + hidden_outs = sigmoid(np.dot(weights, X.T)) + return sigmoid(np.dot(hidden_outs.T, out_weights)) + + +def plot_decision_boundary(wmin, wmax): + hidden_weights = np.random.uniform(wmin, wmax, (2, 2)) + output_weights = np.random.uniform(wmin, wmax, 2) + + x1 = np.arange(-5.0, 5.1, step=0.1) + x2 = np.arange(-5.0, 5.1, step=0.1) + X = np.transpose([np.tile(x1, len(x2)), np.repeat(x2, len(x1))]) + y = output(X, hidden_weights, output_weights) + + fig, ax = plt.subplots() + pos_idx = y > 0.5 + pos = X[pos_idx] + neg = X[~pos_idx] + print(y) + print('%r positives, %r negatives' % (len(pos), len(neg))) + print(pos) + print(neg) + ax.scatter(*pos.T, color='b') + ax.scatter(*neg.T, color='r') + return ax + + +if __name__ == '__main__': + import sys + if len(sys.argv) < 2: + print('Need weight bounds.') + else: + bound = float(sys.argv[1]) + ax = plot_decision_boundary(-bound, bound) + ax.figure.savefig(sys.argv[1] + '.pdf') diff --git a/w06/w06.tex b/w06/w06.tex index 781583a..bd75c30 100644 --- a/w06/w06.tex +++ b/w06/w06.tex @@ -5,9 +5,52 @@ \assignment{Written 6} \duedate{October 6, 2015} +% Solarized colors +%\usepackage[x11names, rgb, html]{xcolor} +\definecolor{sbase03}{HTML}{002B36} +\definecolor{sbase02}{HTML}{073642} +\definecolor{sbase01}{HTML}{586E75} +\definecolor{sbase00}{HTML}{657B83} +\definecolor{sbase0}{HTML}{839496} +\definecolor{sbase1}{HTML}{93A1A1} +\definecolor{sbase2}{HTML}{EEE8D5} +\definecolor{sbase3}{HTML}{FDF6E3} +\definecolor{syellow}{HTML}{B58900} +\definecolor{sorange}{HTML}{CB4B16} +\definecolor{sred}{HTML}{DC322F} +\definecolor{smagenta}{HTML}{D33682} +\definecolor{sviolet}{HTML}{6C71C4} +\definecolor{sblue}{HTML}{268BD2} +\definecolor{scyan}{HTML}{2AA198} +\definecolor{sgreen}{HTML}{859900} + +\usepackage{listings} +\lstset{ + % How/what to match + sensitive=true, + % Border (above and below) + frame=lines, + % Extra margin on line (align with paragraph) + xleftmargin=\parindent, + % Put extra space under caption + belowcaptionskip=1\baselineskip, + % Colors + backgroundcolor=\color{sbase3}, + basicstyle=\color{sbase00}\ttfamily, + keywordstyle=\color{scyan}, + commentstyle=\color{sbase1}, + stringstyle=\color{sblue}, + numberstyle=\color{sviolet}, + identifierstyle=\color{sbase00}, + % Break long lines into multiple lines? + breaklines=true, + % Show a character for spaces? + showstringspaces=false, + tabsize=2 +} + \usepackage{mathtools} \newcommand{\sigmoid}{\:\text{sigmoid}} -%\usepackage{graphicx} \begin{document} \maketitle @@ -126,7 +169,7 @@ Using R/Matlab/Mathematica/your favorite math software, plot the decision boundary for an ANN with two inputs, two hidden units and one output. All activation functions are sigmoids. Each layer is fully connected to the - next. Assume the inputs range between −5 to 5 and fix all activation + next. Assume the inputs range between -5 to 5 and fix all activation thresholds to 0. Plot the decision boundaries for the weights except the thresholds randomly chosen between \textbf{(i)} $(−10,10)$, \textbf{(ii)} $(−3,3)$, \textbf{(iii)} $(−0.1,0.1)$ (one random set for each case is @@ -134,6 +177,40 @@ overfitting for ANNs. (If you use Matlab, the following commands might be useful: meshgrid and surf). (10 points) \end{question} + + \begin{figure} + \centering + \caption{Decision boundary for random ANN with weights in (-10, 10).} + \includegraphics[width=0.7\textwidth]{10.pdf} + \label{fig:10} + \end{figure} + + \begin{figure} + \centering + \caption{Decision boundary for random ANN with weights in (-3, 3).} + \includegraphics[width=0.7\textwidth]{3.pdf} + \label{fig:3} + \end{figure} + + \begin{figure} + \centering + \caption{Decision boundary for random ANN with weights in (-0.1, 0.1).} + \includegraphics[width=0.7\textwidth]{0_1.pdf} + \label{fig:0.1} + \end{figure} + + Plots for \textbf{(i)}, \textbf{(ii)}, and \textbf{(iii)} are provided as + Figures~\ref{fig:10},~\ref{fig:3}, and~\ref{fig:0.1}, respectively. In the + figures, the examples predicted positive are blue, and the examples + predicted negative are red. The decision boundary is the boundary between + the blue and red regions. You can see that as the weights become closer to + zero, the decision boundaries become less complex (that is, have fewer + ``bends'' in them). Therefore, it stands to reason that creating an + artificial incentive to have smaller weights would reduce the ``bendiness'', + and therefore reduce overfitting. + + The Python source code that produced the images is provided in + Appendix~\ref{p4src}. \end{problem} \begin{problem}{5} @@ -145,4 +222,11 @@ \end{question} \end{problem} + \newpage + \appendix + \section{Source Code for Problem 4} + \label{p4src} + \lstinputlisting[language=Python]{w06.py} + + Example: \texttt{\$ python w06.py 10} \end{document} \ No newline at end of file