[Enhancement] SIMD Extension for CPUTensor

GNUmakefile

@@ -6,6 +6,8 @@ MKTEMP              := mktemp
 RLWRAP              := rlwrap
 LOGFILE             := $(shell mktemp)
 
+PYTHON              := "python"
+GCC                 := "gcc"
 .DEFAULT_GOAL := help
 
 # This code taken from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -16,19 +18,19 @@ help:
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: compile
-compile: ## Compile whole project
+compile: ## Compiles the whole project
 	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:compile-system :cl-waffe2)' \
 		--quit
 
 .PHONY: test
-test: ## Run test harness
+test: ## Running a test harness
 	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:test-system :cl-waffe2)' \
 		--quit
 
 .PHONY: recordtest
-recordtest: ## Run test harness with logging
+recordtest: ## Running a test harness with recording logs
 	$(warning This session will be recorded in $(LOGFILE))
 	$(RLWRAP) --logfile $(LOGFILE) \
 	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
@@ -37,7 +39,7 @@ recordtest: ## Run test harness with logging
 	@printf 'This session has been recorded in %s\n' $(LOGFILE)
 
 .PHONY: repl
-repl: ## Launch REPL
+repl: ## Launch REPL with loading cl-waffe2
 	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
 
 .PHONY: rlrepl
@@ -70,6 +72,10 @@ docs: ## Generate documents
 		--eval '(cl-waffe2.docs:generate)' \
 		--quit
 
+.PHONY: mkdocs-serve
+mkdocs-serve: ## Launchs the documentation server.
+	cd ./docs/cl-waffe2-docs && mkdocs serve
+
 .PHONY: rt
 rt: recordtest ## Alias for recordtest
 
@@ -107,3 +113,17 @@ add_to_init_file: ## Enable Quicklisp autoloading
 		--non-interactive \
 		--load ~/quicklisp/setup.lisp \
 		--eval '(ql-util:without-prompting (ql:add-to-init-file))'
+
+.PHONY: build_simd_extension
+build_simd_extension: ## Installs SIMD Extension shared library for the CPUTensor backend.
+	$(GCC) -O3 -march=native -shared -o \
+		./source/backends/cpu/cl-waffe2-simd/kernels/cl-waffe2-simd.so \
+		-fpic ./source/backends/cpu/cl-waffe2-simd/kernels/cl-waffe2-simd.c -lm
+
+.PHONY: delete_simd_extension
+delete_simd_extension: ## Deletes Compiled SIMD Extension shared library so that CPUTensor works under OpenBLAS
+	rm -rf ./source/backends/cpu/cl-waffe2-simd/kernels/cl-waffe2-simd.so
+
+.PHONY: download_assets
+download_assets: ## Downloads training data sample codes use.
+	cd ./examples/mnist && $(PYTHON) train_data.py

cl-waffe2.asd

@@ -1,15 +1,33 @@
 
 (in-package :cl-user)
 
+
+(defpackage :cl-waffe2-simd-asd
+  (:use :cl :asdf :uiop))
+
+(in-package :cl-waffe2-simd-asd)
+
+(defsystem :cl-waffe2/simd-extension
+  :author "hikettei"
+  :licence "MIT"
+  :description "Utils for SIMD-Enabled Extension, CPUTensor."
+  :pathname "source/backends/cpu/cl-waffe2-simd"
+  :serial t
+  :depends-on (:cffi)
+  :components ((:file "package")
+	       (:file "shared-object")
+	       (:file "api")))
+
+
 (defpackage :cl-waffe2-asd
   (:use :cl :asdf :uiop))
 
 (in-package :cl-waffe2-asd)
 
 (defsystem :cl-waffe2
-  :author "hikettei"
+  :author "hikettei <[email protected]>"
   :licence "MIT"
-  :description "Deep Learning Framework"
+  :description "Programmable Deep Learning Framework for Common Lisp"
   :pathname "source"
   :serial t
   :depends-on (:cl-ppcre
@@ -21,7 +39,8 @@
 	       :bordeaux-threads
 	       :closer-mop
 	       :optima
-	       :trivial-garbage)
+	       :trivial-garbage
+	       :cl-waffe2/simd-extension)
   ;; TODO: Use components and split dependencies.
   :components ((:file "threads")
 	       (:file "vm/generic-tensor/package")
@@ -76,6 +95,7 @@
 	       (:file "base-impl/transform")
 	       (:file "base-impl/ir")
 	       (:file "base-impl/reshapers")
+	       (:file "base-impl/unfold")
 
 
 	       (:file "vm/ir")
@@ -99,6 +119,8 @@
 	       (:file "backends/cpu/blas-functions")
 	       (:file "backends/cpu/arithmetic")
 	       (:file "backends/cpu/matrix-ops")
+	       (:file "backends/cpu/logical")
+
 
 	       (:file "distributions/package")
 	       (:file "distributions/generic")
@@ -160,6 +182,8 @@
 	       (:file "viz/package")
 	       (:file "viz/ast")
 	       (:file "cl-waffe2-repl")
+
+
 
 	       )
   :in-order-to ((test-op (test-op cl-waffe2/test))))
@@ -268,4 +292,3 @@
 	       (:file "model")
 	       (:file "reverse")
 	       (:file "profile")))
-

docs/apis/nn.lisp

@@ -33,5 +33,6 @@
 	"(Conv2D 3 5 '(3 3))"))
 
     (with-nn-doc (find-class 'MaxPool2D) 't)
-    (with-nn-doc (find-class 'AvgPool2D) 't)))
+    (with-nn-doc (find-class 'AvgPool2D) 't)
+    (with-nn-doc 'unfold 'function)))
 

docs/apis/optimizers.lisp

@@ -10,5 +10,6 @@
     (with-op-doc (find-class 'AbstractOptimizer) 't)
     (with-op-doc (macro-function 'defoptimizer) 'function)
     (with-op-doc (find-class 'SGD) 't)
+    (with-op-doc (find-class 'Adam) 't)
     ))
 

docs/apis/reference.lisp

@@ -106,6 +106,9 @@
 
     (nodedoc Where-Operation-Node)
     (nodedoc Compare-Operation-Node)
+
+    (nodedoc Im2ColNode)
+    (nodedoc Col2ImNode)
     ))
 
 
@@ -256,7 +259,12 @@ set-input:
 
 predict:
   describe ..
-```"))))
+```"))
+
+    (with-op-doc #'show-backends 'function)
+    (with-op-doc #'set-devices-toplevel 'function)
+
+    ))
 
 (with-page *lisp-tensor-backend* "[package] :cl-waffe2/backends.lisp"
   (insert
@@ -274,6 +282,12 @@ It is recommended that `LispTensor` are installed in the lowest priority of `*us
 (with-page *cpu-tensor-backend* "[package] :cl-waffe2/backends.cpu"
   (insert
    "The package `:cl-waffe2/backends.cpu` provides an AbstractTensor `CPUTensor` where most of its implementation relies on foreign libraries (e.g.: OpenBLAS, oneDNN in the coming future).")
+
+  (insert "
+## Enabling the SIMD Extension
+
+For some instructions (e.g.: `!max` `!min`, sparse matrix supports, `SLEEF`, etc...), packages that provide SIMD-enabled CPUTensor implementations are not enabled by default as a design. To enable it, run `make build_simd_extension` in the same directory as cl-waffe2.asd. You can check that it is loaded properly with the `(show-backends)` function.
+")
 
   (macrolet ((with-op-doc (name type &body body)
 	       `(progn

docs/cl-waffe2-docs/docs/base-impl-nodes.md

@@ -1616,8 +1616,7 @@ Where-Operation-Node is a node which set `true-then`, if the result of calling `
 ✅ Already defined. 
 
 ```lisp
-((self dout da do) (declare (ignore dout da do)) ;; todo: :no-grad t
- (values nil nil))
+((self dout da do) (declare (ignore dout da do)) (values nil nil))
 ```
 
 No need to implement backwards at `define-impl`. (they'd be ignored.)
@@ -1647,8 +1646,57 @@ Compare-Operation-Node is a node which set `true-then`, if the result of calling
 ✅ Already defined. 
 
 ```lisp
-((self dout da db do) (declare (ignore dout da db do)) ;; todo: :no-grad t
- (values nil nil nil))
+((self dout da db do) (declare (ignore dout da db do)) (values nil nil nil))
+```
+
+No need to implement backwards at `define-impl`. (they'd be ignored.)
+## [node] IM2COLNODE
+
+```
+(X[N C H W] COL[N C K-H K-W H-OUT W-OUT] -> COL[N C K-H K-W H-OUT W-OUT])
 ```
 
-No need to implement backwards at `define-impl`. (they'd be ignored.)
+### Description
+
+Im2ColNode is `AbstractNode` which implements forward propagation of [nn.Unfold](https://pytorch.org/docs/stable/generated/torch.nn.Unfold.html).
+
+The node is only executed through the `cl-waffe2/nn:unfold` function, so arguments for constructors are dispatched automatically. In addition, the tensor `X` it receive will be the one after padding has been performed.
+
+`N` indicates the number of batch-size, `C` is a channel-size. `k-h`, `k-w` represents the size of kernel, height and width respectively. `h-out` `w-out` is the size of output. `stride-x` `stride-y` is the number of stride, for the most case, specified by the stride argument in `Pooling2D` or `Conv2D`. `img-out` is AbstractTensor with the shape of `(N C H-in W-in)`, can be read by `img-out-of`. All symbols are exported from `cl-waffe2/base-impl` package.
+
+In order to implement device-specific implementation of `Unfold`, define-impl `Im2ColNode` and `Col2ImNode`.
+
+
+### Backward
+
+✅ Already defined. 
+
+```lisp
+((self dout x col) (declare (ignore x col))
+ (with-slots ((n n) (c c) (k-h k-h) (k-w k-w) (h-out h-out) (w-out w-out)
+              (stride-x stride-x) (stride-y stride-y))
+     self
+   (values
+    (call
+     (col2imnode n c (h-of self) (w-of self) k-h k-w h-out w-out stride-x
+      stride-y (img-out-of self))
+     dout)
+    nil)))
+```
+
+No need to implement backwards at `define-impl`. (they'd be ignored.)
+## [node] COL2IMNODE
+
+```
+(COL[N C K-H K-W H-OUT W-OUT] -> X[N C H W])
+```
+
+### Description
+
+Col2ImNode is `AbstractNode` which implements backward propagation of [nn.Unfold](https://pytorch.org/docs/stable/generated/torch.nn.Unfold.html).
+
+See also: `Im2ColNode` documentation for argument descriptions.
+
+### Backward
+
+❌ Undefined. (To make it differentiable, must be defined with `define-impl` macro.)

docs/cl-waffe2-docs/docs/base-impl.md

@@ -1693,7 +1693,7 @@ The function a>scal sets `true-then` if the equation: `element > scal` is t, oth
 `A` AbstractTensor
 `scal` number (not a ScalarTensor)
 
-(TODO: ScalarTensor as scal)
+(TODO: ScalarTensor as a `scal` argument)
 ## [function] a<scal
 
 ```
@@ -1707,7 +1707,7 @@ The function a<scal sets `true-then` if the equation: `element < scal` is t, oth
 `A` AbstractTensor
 `scal` number (not a ScalarTensor)
 
-(TODO: ScalarTensor as scal)
+(TODO: ScalarTensor as a `scal` argument)
 ## [function] a>=scal
 
 ```
@@ -1721,7 +1721,7 @@ The function a>=scal sets `true-then` if the equation: `element >= scal` is t, o
 `A` AbstractTensor
 `scal` number (not a ScalarTensor)
 
-(TODO: ScalarTensor as scal)
+(TODO: ScalarTensor as a `scal` argument)
 ## [function] a<=scal
 
 ```
@@ -1735,7 +1735,7 @@ The function a<=scal sets `true-then` if the equation: `element <= scal` is t, o
 `A` AbstractTensor
 `scal` number (not a ScalarTensor)
 
-(TODO: ScalarTensor as scal)
+(TODO: ScalarTensor as a `scal` argument)
 ## [function] a=scal
 
 ```
@@ -1749,7 +1749,7 @@ The function a=scal sets `true-then` if the equation: `element = scal` is t, oth
 `A` AbstractTensor
 `scal` number (not a ScalarTensor)
 
-(TODO: ScalarTensor as scal)
+(TODO: ScalarTensor as a `scal` argument)
 ## [function] a>b
 
 ```

docs/cl-waffe2-docs/docs/cpu-tensor-backend.md

@@ -1,5 +1,8 @@
 
 # [package] :cl-waffe2/backends.cpu
 The package `:cl-waffe2/backends.cpu` provides an AbstractTensor `CPUTensor` where most of its implementation relies on foreign libraries (e.g.: OpenBLAS, oneDNN in the coming future).
-## [AbstractTensor] CPUTensor
+## Enabling the SIMD Extension
+
+For some instructions (e.g.: `!max` `!min`, sparse matrix supports, `SLEEF`, etc...), packages that provide SIMD-enabled CPUTensor implementations are not enabled by default as a design. To enable it, run `make build_simd_extension` in the same directory as cl-waffe2.asd. You can check that it is loaded properly with the `(show-backends)` function.
 
+## [AbstractTensor] CPUTensor