Skip to content

Commit 28d1ea5

Browse files
arvoelkesamir-alazzam
arvoelke
and
samir-alazzam
committed
Converted paper branch into minimal LMU repository
 - Updated to TensorFlow2 and added get_config method - Removed all non-essential code, apart from psMNIST example - Removed nru and .gitmodules - Used nengo-bones to template NengoLMU project - Set version to 0.1.0dev - Added CHANGES - Redid the README - Support Python>=3.5 - Simplified package requirements - Added .gitlint - Passed all static checks and black reformatted - Added docstrings - Simplified and renamed psMNIST notebook Co-authored-by: Samir Alazzam <[email protected]>
1 parent 0237f61 commit 28d1ea5

17 files changed

+1439
-0
lines changed

Diff for: ‎.codecov.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Automatically generated by nengo-bones, do not edit this file directly
2+
3+
codecov:
4+
ci:
5+
- "!ci.appveyor.com"
6+
notify:
7+
require_ci_to_pass: no
8+
9+
coverage:
10+
status:
11+
project:
12+
default:
13+
enabled: yes
14+
target: auto
15+
patch:
16+
default:
17+
enabled: yes
18+
target: 100%
19+
changes: no

Diff for: ‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
lmu.egg-info
2+
*.swo
3+
*.swp
4+
*.ipynb_checkpoints
5+
__pycache__

Diff for: ‎.gitlint

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[general]
2+
ignore=body-is-missing
3+
4+
[title-max-length]
5+
line-length=50
6+
7+
[B1]
8+
# body line length
9+
line-length=72
10+
11+
[title-match-regex]
12+
regex=^[A-Z]

Diff for: ‎.nengobones.yml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
project_name: NengoLMU
2+
pkg_name: lmu
3+
repo_name: abr/lmu
4+
description: Legendre Memory Units
5+
6+
copyright_start: 2019
7+
8+
license_rst: {}
9+
10+
contributing_rst: {}
11+
12+
contributors_rst: {}
13+
14+
setup_py:
15+
install_req:
16+
- nengolib>=0.5.1
17+
- tensorflow>=2.0.0
18+
docs_req:
19+
- matplotlib>=3.0.2
20+
- IPython>=7.2.0
21+
- notebook>=5.7.4
22+
- seaborn>=0.9.0
23+
optional_req:
24+
- scipy
25+
classifiers:
26+
- "Development Status :: 3 - Alpha"
27+
- "Framework :: Nengo"
28+
- "Intended Audience :: Science/Research"
29+
- "License :: Free for non-commercial use"
30+
- "Operating System :: OS Independent"
31+
- "Programming Language :: Python "
32+
- "Programming Language :: Python :: 3.5"
33+
- "Programming Language :: Python :: 3.6"
34+
- "Programming Language :: Python :: 3.7"
35+
- "Topic :: Scientific/Engineering "
36+
- "Topic :: Scientific/Engineering :: Artificial Intelligence"
37+
38+
setup_cfg: {}
39+
40+
travis_yml:
41+
python: 3.6
42+
jobs:
43+
- script: static
44+
pypi_user: arvoelke
45+
slack_notifications: "vv0lGEj/xNMFSZDbFsdoMJyEcr9BO8p43KUefmgHfjIQtodZqXdesLl+XJcXW0jxCJlNyy3H8LHYP/mEGJpIsK+tQ7dxlWscLjSGWfcoNiZGy4a1Jp4fF+MZyYvIGlTRJqxrazIrj73tGFptVr2XDE74eO0Z9YaVSJVQw4twEDrWFEAq4foWxV30SkcXfdCkhBwX+43CJyuGE3YFDD/+03me/mdccjNRqCfJ0lURRk7H5tcztryrZy2gpwHV+W73raGTybxlP1xEa1hyLYJO40eH/JfeqBqIDxa5m61Aw+BH/HJ5ZLNlTEUyUB6p7kcIYO9lyko5TY3QSqlX9pK+tK+2DojDlzI97QwgQVbx4WvTJ1JEidfgRqNcTlJOG16RvlyxQjW1u3/QV67bmINus470qQqzIBbdLfM70v+E5Ga/bk+Gk1Z29btB7DxXt4z9dH9z3NXTOLhDpH5WZzpcatrbfSrgMzKtxC+z6oLfDzzio9Fx20RiuHv3P8GtXyyR9WkelMH9GVi7xUBHVCveRVVhNKL555u7NbP5TI6Jc9NZqf7OtrNsRKIY4MfGc9KKjYa+Ks+3PT+yQZ8u/ZMMddMTv73nzLH0pU715/CBl1hQGkKkopukGtKbCpdc666PnRrFy9l21hBqSNqLo/FGPF/Yqr+yTXhuhBhvNZnvFQU="
46+
deploy_dists:
47+
- sdist
48+
- bdist_wheel
49+
50+
ci_scripts:
51+
- template: static
52+
53+
codecov_yml: {}

Diff for: ‎.travis.yml

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Automatically generated by nengo-bones, do not edit this file directly
2+
3+
language: python
4+
python: 3.6
5+
notifications:
6+
email:
7+
on_success: change
8+
on_failure: change
9+
slack:
10+
if: branch = master
11+
on_pull_requests: false
12+
on_success: change
13+
on_failure: always
14+
rooms:
15+
- secure: "vv0lGEj/xNMFSZDbFsdoMJyEcr9BO8p43KUefmgHfjIQtodZqXdesLl+XJcXW0jxCJlNyy3H8LHYP/mEGJpIsK+tQ7dxlWscLjSGWfcoNiZGy4a1Jp4fF+MZyYvIGlTRJqxrazIrj73tGFptVr2XDE74eO0Z9YaVSJVQw4twEDrWFEAq4foWxV30SkcXfdCkhBwX+43CJyuGE3YFDD/+03me/mdccjNRqCfJ0lURRk7H5tcztryrZy2gpwHV+W73raGTybxlP1xEa1hyLYJO40eH/JfeqBqIDxa5m61Aw+BH/HJ5ZLNlTEUyUB6p7kcIYO9lyko5TY3QSqlX9pK+tK+2DojDlzI97QwgQVbx4WvTJ1JEidfgRqNcTlJOG16RvlyxQjW1u3/QV67bmINus470qQqzIBbdLfM70v+E5Ga/bk+Gk1Z29btB7DxXt4z9dH9z3NXTOLhDpH5WZzpcatrbfSrgMzKtxC+z6oLfDzzio9Fx20RiuHv3P8GtXyyR9WkelMH9GVi7xUBHVCveRVVhNKL555u7NbP5TI6Jc9NZqf7OtrNsRKIY4MfGc9KKjYa+Ks+3PT+yQZ8u/ZMMddMTv73nzLH0pU715/CBl1hQGkKkopukGtKbCpdc666PnRrFy9l21hBqSNqLo/FGPF/Yqr+yTXhuhBhvNZnvFQU="
16+
cache: pip
17+
18+
dist: xenial
19+
20+
env:
21+
global:
22+
- SCRIPT="test"
23+
- TEST_ARGS=""
24+
- BRANCH_NAME="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}"
25+
26+
jobs:
27+
include:
28+
-
29+
env:
30+
SCRIPT="static"
31+
- stage: deploy
32+
if: branch =~ ^release-candidate-* OR tag =~ ^v[0-9]*
33+
env: SCRIPT="deploy"
34+
cache: false
35+
deploy:
36+
- provider: pypi
37+
server: https://test.pypi.org/legacy/
38+
user: arvoelke
39+
password: $PYPI_TOKEN
40+
distributions: "sdist bdist_wheel "
41+
on:
42+
all_branches: true
43+
tags: false
44+
condition: $TRAVIS_BRANCH =~ ^release-candidate-*
45+
- provider: pypi
46+
user: arvoelke
47+
password: $PYPI_TOKEN
48+
distributions: "sdist bdist_wheel "
49+
on:
50+
all_branches: true
51+
tags: true
52+
condition: $TRAVIS_TAG =~ ^v[0-9]*
53+
54+
before_install:
55+
# export travis_terminate for use in scripts, from here:
56+
# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_terminate.bash
57+
- export -f travis_terminate
58+
_travis_terminate_agent
59+
_travis_terminate_freebsd
60+
_travis_terminate_linux
61+
_travis_terminate_osx
62+
_travis_terminate_unix
63+
_travis_terminate_windows
64+
# upgrade pip
65+
- pip install pip --upgrade
66+
# install/run nengo-bones
67+
- pip install git+https://github.com/nengo/nengo-bones#egg=nengo-bones
68+
- bones-generate --output-dir .ci ci-scripts
69+
- if [[ "$TRAVIS_PYTHON_VERSION" < "3.6" ]]; then
70+
echo "Skipping bones-check because Python $TRAVIS_PYTHON_VERSION < 3.6";
71+
else
72+
bones-check --verbose;
73+
fi
74+
# display environment info
75+
- pip freeze
76+
77+
install:
78+
- .ci/$SCRIPT.sh install
79+
- pip freeze
80+
81+
before_script:
82+
- .ci/$SCRIPT.sh before_script
83+
84+
script:
85+
- .ci/$SCRIPT.sh script
86+
87+
before_cache:
88+
- .ci/$SCRIPT.sh before_cache
89+
90+
after_success:
91+
- .ci/$SCRIPT.sh after_success
92+
93+
after_failure:
94+
- .ci/$SCRIPT.sh after_failure
95+
96+
before_deploy:
97+
- .ci/$SCRIPT.sh before_deploy
98+
99+
after_deploy:
100+
- .ci/$SCRIPT.sh after_deploy
101+
102+
after_script:
103+
- .ci/$SCRIPT.sh after_script

Diff for: ‎CHANGES.rst

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
***************
2+
Release history
3+
***************
4+
5+
.. Changelog entries should follow this format:
6+
7+
version (release date)
8+
======================
9+
10+
**section**
11+
12+
- One-line description of change (link to Github issue/PR)
13+
14+
.. Changes should be organized in one of several sections:
15+
16+
- Added
17+
- Changed
18+
- Deprecated
19+
- Removed
20+
- Fixed
21+
22+
0.1.0 (unreleased)
23+
==================
24+
25+
Initial release of LMU 0.1.0! Supports Python 3.5+.
26+
27+
The API is considered unstable; parts are likely to change in the future.
28+
29+
Thanks to all of the contributors for making this possible!

Diff for: ‎CONTRIBUTING.rst

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
.. Automatically generated by nengo-bones, do not edit this file directly
2+
3+
************************
4+
Contributing to NengoLMU
5+
************************
6+
7+
Issues and pull requests are always welcome!
8+
We appreciate help from the community to make NengoLMU better.
9+
10+
Filing issues
11+
=============
12+
13+
If you find a bug in NengoLMU,
14+
or think that a certain feature is missing,
15+
please consider
16+
`filing an issue <https://github.com/abr/lmu/issues>`_!
17+
Please search the currently open issues first
18+
to see if your bug or feature request already exists.
19+
If so, feel free to add a comment to the issue
20+
so that we know that multiple people are affected.
21+
22+
Making pull requests
23+
====================
24+
25+
If you want to fix a bug or add a feature to NengoLMU,
26+
we welcome pull requests.
27+
Ensure that you fill out all sections of the pull request template,
28+
deleting the comments as you go.
29+
We check most aspects of code style automatically.
30+
Please refer to our
31+
`code style guide <https://www.nengo.ai/nengo-bones/style.html>`_
32+
for things that we check manually.
33+
34+
Contributor agreement
35+
=====================
36+
37+
We require that all contributions be covered under
38+
our contributor assignment agreement. Please see
39+
`the agreement <https://www.nengo.ai/caa/>`_
40+
for instructions on how to sign.
41+
42+
More details
43+
============
44+
45+
For more details on how to contribute to Nengo,
46+
please see the `developer guide <https://www.nengo.ai/contributing/>`_.

Diff for: ‎CONTRIBUTORS.rst

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
.. Automatically generated by nengo-bones, do not edit this file directly
2+
3+
*********************
4+
NengoLMU contributors
5+
*********************
6+
7+
See https://github.com/abr/lmu/graphs/contributors
8+
for a list of the people who have committed to NengoLMU.
9+
Thank you for your contributions!
10+
11+
For the full list of the many contributors to the Nengo ecosystem,
12+
see https://www.nengo.ai/people/.

Diff for: ‎LICENSE.rst

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
.. Automatically generated by nengo-bones, do not edit this file directly
2+
3+
****************
4+
NengoLMU license
5+
****************
6+
7+
Copyright (c) 2019-2020 Applied Brain Research
8+
9+
NengoLMU is made available under a proprietary license
10+
that permits using, copying, sharing, and making derivative works from
11+
NengoLMU and its source code for any non-commercial purpose,
12+
as long as the above copyright notice and this permission notice
13+
are included in all copies or substantial portions of the software.
14+
15+
If you would like to use NengoLMU commercially,
16+
licenses can be purchased from Applied Brain Research.
17+
Please contact [email protected] for more information.
18+
19+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25+
SOFTWARE.

Diff for: ‎README.md

Whitespace-only changes.

Diff for: ‎README.rst

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks
2+
----------------------------------------------------------------------------------
3+
4+
`Paper <https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.pdf>`_
5+
6+
We propose a novel memory cell for recurrent neural networks that dynamically maintains information across long windows of time using relatively few resources. The Legendre Memory Unit (LMU) is mathematically derived to orthogonalize its continuous-time history – doing so by solving d coupled ordinary differential equations (ODEs), whose phase space linearly maps onto sliding windows of time via the Legendre polynomials up to degree d − 1 (example d=12, shown below).
7+
8+
.. image:: https://i.imgur.com/Uvl6tj5.png
9+
:target: https://i.imgur.com/Uvl6tj5.png
10+
:alt: Legendre polynomials
11+
12+
A single ``LMUCell`` expresses the following computational graph in Keras as an RNN layer, which couples the optimal linear memory (``m``) with a nonlinear hidden state (``h``):
13+
14+
.. image:: https://i.imgur.com/IJGUVg6.png
15+
:target: https://i.imgur.com/IJGUVg6.png
16+
:alt: Computational graph
17+
18+
The discretized ``(A, B)`` matrices are initialized according to the LMU's mathematical derivation with respect to some chosen window length, θ. Backpropagation can be used to learn this time-scale, or fine-tune ``(A, B)``, if necessary. By default the coupling between the hidden state (``h``) and the memory vector (``m``) is trained via backpropagation, while the dynamics of the memory remain fixed (`see paper for details <https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.pdf>`_).
19+
20+
The ``docs`` includes an example for how to use the ``LMUCell``.
21+
22+
The ``paper`` branch in the ``lmu`` GitHub repository includes a pre-trained Keras/TensorFlow model, located at ``models/psMNIST-standard.hdf5``, which obtains the current best-known psMNIST result (using an RNN) of **97.15%**. Note, the network is using fewer internal state-variables and neurons than there are pixels in the input sequence. To reproduce the results from the paper, run the notebooks in the ``experiments`` directory within the ``paper`` branch.
23+
24+
Nengo Examples
25+
--------------
26+
27+
* `Spiking LMUs in Nengo (with online learning) <https://www.nengo.ai/nengo/examples/learning/lmu.html>`_
28+
* `Spiking LMUs in Nengo Loihi (with online learning) <https://www.nengo.ai/nengo-loihi/examples/lmu.html>`_
29+
* `LMUs in NengoDL (reproducing SotA on psMNIST) <https://www.nengo.ai/nengo-dl/examples/lmu.html>`_
30+
31+
Citation
32+
--------
33+
34+
.. code-block::
35+
36+
@inproceedings{voelker2019lmu,
37+
title={Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks},
38+
author={Aaron R. Voelker and Ivana Kaji\'c and Chris Eliasmith},
39+
booktitle={Advances in Neural Information Processing Systems},
40+
pages={15544--15553},
41+
year={2019}
42+
}

0 commit comments

Comments
 (0)