[WIP] From ONNX Mode

.github/workflows/test_on_push.yml

@@ -9,8 +9,10 @@ on:
         - develop
 
 jobs:
-  test:
-    name: ${{ matrix.lisp }} on ${{ matrix.os }}
+  # TODO: Metal/CUDA Runtime
+  UnitTest-CPUTensor:
+    # TODO: Metal/CUDA Runtime
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Unittest-CPUTensor
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -36,10 +38,97 @@ jobs:
       - name: Check $PATH
         run: echo $PATH
 
-      - name: Downloading OpenBLAS
+      - name: Installing OpenBLAS
         run: |
           sudo apt install libblas-dev
-      - name: Operating tests
+
+      #- name: Installing Extensions
+      #  run: |
+      #    git submodule update --init --recursive
+      #    sudo make build_sleef
+      #    sudo make build_simd_extension
+
+      - name: Unittest (CPU+Lisp)
         run: |
           ros config set dynamic-space-size 4gb
-          ros --eval '(progn (defparameter cl-user::*cl-waffe-config* `((:libblas "libblas.so"))))' ./roswell/cl-waffe2-test.ros
+          ./roswell/waffe2.ros test -b CPUTensor -b LispTensor
+  UnitTest-LispTensor:
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Unittest-LispTensor
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        lisp: [sbcl-bin/2.3.9]
+        os: [ubuntu-latest]
+        target:
+        - normal
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setting up environments
+        env:
+          LISP: ${{ matrix.lisp }}
+        run: |
+          curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh
+      - name: Update $PATH
+        run: |
+          echo $PATH
+          echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+      - name: Check $PATH
+        run: echo $PATH
+      - name: Unittest (Lisp+CPU)
+        run: |
+          ros config set dynamic-space-size 4gb
+          ./roswell/waffe2.ros test -b LispTensor -b CPUTensor
+  Test-Classification-CPU:
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Test-Classification-CPU
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        lisp: [sbcl-bin/2.3.9]
+        os: [ubuntu-latest]
+        target:
+        - normal
+        archcpu:
+        - AVX2
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setting up environments
+        env:
+          LISP: ${{ matrix.lisp }}
+        run: |
+          curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh
+      - name: Update $PATH
+        run: |
+          echo $PATH
+          echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+      - name: Check $PATH
+        run: echo $PATH
+      - name: Installing OpenBLAS
+        run: |
+          sudo apt install libblas-dev
+
+      - name: Set up Python
+        id: set_up_python_python_ci
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Downloading assets
+        run: |
+          pip install torch torchvision numpy
+          cd ./examples/mnist 
+          python train_data.py
+          cd ../../
+
+      #- name: Installing Extensions
+      #  run: |
+      #    git submodule update --init --recursive
+      #    sudo make build_sleef
+      #    sudo make build_simd_extension
+
+      - name: Training w/ LispTensor
+        run: |
+          ros config set dynamic-space-size 4gb
+          ./roswell/waffe2.ros demo --example mnist -b LispTensor -b CPUTensor --config epoch-num=2
+
+

GNUmakefile

@@ -1,7 +1,9 @@
+ROSWELL             := ros
 SBCL                := sbcl
 # Avoid loading system-wide libraries (--no-sysinit)
 SBCL_OPTIONS        := --noinform --no-sysinit
-QUICKLOAD_WAFFE2    := --load cl-waffe2.asd --eval '(ql:quickload :cl-waffe2)'
+ROSWELL_OPTIONS     := dynamic-space-size=4096
+QUICKLOAD_WAFFE2    := --eval '(load "cl-waffe2.asd")' --eval '(ql:quickload :cl-waffe2)'
 MKTEMP              := mktemp
 RLWRAP              := rlwrap
 LOGFILE             := $(shell mktemp)
@@ -26,32 +28,32 @@ compile: ## Compiles the whole project
 
 .PHONY: test
 test: ## Running a test harness
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:test-system :cl-waffe2)' \
 		--quit
 
 .PHONY: recordtest
 recordtest: ## Running a test harness with recording logs
 	$(warning This session will be recorded in $(LOGFILE))
 	$(RLWRAP) --logfile $(LOGFILE) \
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:test-system :cl-waffe2)' \
 		--quit
 	@printf 'This session has been recorded in %s\n' $(LOGFILE)
 
 .PHONY: repl
 repl: ## Launch REPL with loading cl-waffe2
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 
 .PHONY: rlrepl
 rlrepl: ## Launch REPL with rlwrap
-	$(RLWRAP) $(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+	$(RLWRAP) $(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 
 .PHONY: record
 record: ## Launch REPL with logging
 	$(warning This session will be recorded in $(LOGFILE))
 	$(RLWRAP) --logfile $(LOGFILE) \
-		$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+		$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 	@printf 'This session has been recorded in %s\n' $(LOGFILE)
 
 .PHONY: slynk
@@ -68,7 +70,7 @@ swank: ## Launch Swank server
 
 .PHONY: docs
 docs: ## Generate documents
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(ql:quickload :cl-waffe2/docs)' \
 		--eval '(cl-waffe2.docs:generate)' \
 		--quit
@@ -139,14 +141,14 @@ download_assets: ## Downloads training data sample codes use.
 
 .PHONY: example_mnist
 example_mnist: ## Start MNIST Example and Benchmarking. (download_assets must be called in advance)
-	$(SBCL) $(SBCL_OPTIONS) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) \
 		--load ./cl-waffe2.asd --load ./examples/mnist/mnist.asd \
 		--eval '(ql:quickload :mnist-sample)' \
 		--eval '(mnist-sample::train-and-valid-mlp :epoch-num 10)'
 
 .PHONY: example_gpt
 example_gpt: ## Launch GPT2 Inference (download_assets must be called in advance)
-	$(SBCL) --dynamic-space-size 4096 $(SBCL_OPTIONS) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) \
 		--load ./cl-waffe2.asd --load ./examples/gpt-2/gpt-2.asd \
 		--eval '(ql:quickload :gpt-2-example)' \
 		--eval '(gpt-2-example:launch-repl)'

cl-waffe2.asd

@@ -42,7 +42,6 @@
 	       :closer-mop
 	       :trivial-garbage
 	       :cl-waffe2/simd-extension
-
 	       :cl-environments
 	       :numpy-file-format
 	       :jonathan)
@@ -158,20 +157,7 @@
 	       (:file "package")
 
 
-	       (:file "backends/lisp/wf2model")
-	       (:file "backends/JITCPUTensor/package")
-	       (:file "backends/JITCPUTensor/tensor")
-	       (:file "backends/JITCPUTensor/compiler")
-	       (:file "backends/JITCPUTensor/blueprint")	       
-	       (:file "backends/JITCPUTensor/ir")
-	       (:file "backends/JITCPUTensor/on-finalizing")
-	       (:file "backends/JITCPUTensor/dtype")
-	       (:file "backends/JITCPUTensor/foreign-function")
-
-
-	       (:file "backends/JITCPUTensor/impls/arithmetic")
-	       (:file "backends/JITCPUTensor/impls/math")
-
+	       (:file "backends/lisp/wf2model")	       
 	       (:file "optimizers/defoptimizer")
 
 	       (:file "array-converter")
@@ -223,7 +209,7 @@
   :description "Tests for cl-waffe2"
   :serial t
   :pathname "source"
-  :depends-on (:cl-waffe2 :fiveam)
+  :depends-on (:cl-waffe2 :fiveam :rove)
   :components ((:file "vm/iterator/test-suites")
 	       (:file "vm/t/package")
 	       (:file "vm/t/lazy-axis")
@@ -251,30 +237,16 @@
 	       (:file "backends/cpu/t/arithmetic")
 
 	       (:file "backends/lisp/t/package")
-
-	       (:file "backends/JITCPUTensor/t/package")
-	       (:file "backends/JITCPUTensor/t/jit")
 
 	       (:file "nn/t/package")
 	       (:file "nn/t/conv")
 	       (:file "nn/t/activation")
 	       (:file "nn/t/criterion")
-	       (:file "nn/t/regression")	       
+	       (:file "nn/t/regression")
+
+	       (:file "test-suites")
 	       )
-  :perform (test-op (o s)
-		    (symbol-call :fiveam :run! :iterator-test)
-		    (symbol-call :fiveam :run! :test-nodes)
-		    (symbol-call :fiveam :run! :test-tensor)
-
-		    (symbol-call :fiveam :run! :base-impl-test)
-		    (symbol-call :fiveam :run! :jit-lisp-test)
-
-		    (symbol-call :fiveam :run! :lisp-backend-test)
-		    (symbol-call :fiveam :run! :test-backends-cpu)
-		    (symbol-call :fiveam :run! :jit-cpu-test)
-
-		    (symbol-call :fiveam :run! :nn-test)
-		    (symbol-call :fiveam :run! :vm-test)))
+  :perform (test-op (o s) (error "moved to source/test-suites.lisp")))
 
 
 (defpackage :cl-waffe2-docs-asdf

cl-waffe2.frontends.asd

@@ -0,0 +1,30 @@
+
+;; [WIP] Feature roadmap on frontends.
+;;
+;;  frontend/onnx (From ONNX To cl-waffe2 translator)
+;;  frontend/qnn  (Quantized Neural Network Op Supports (basically relies on JITCompiler))
+;;  frontend/backends
+;;  frontend/backends/cuda (Additional backends which requires more dependency, should be placed at frontend)
+;;
+
+(asdf:defsystem :cl-waffe2.frontends
+  :description "A set of frontend toolkits for cl-waffe2"
+  :author "hikettei <[email protected]>"
+  :Licence "MIT")
+
+(asdf:defsystem :cl-waffe2.frontends/onnx
+  :description "ONNX Graph Translator"
+  :author      "hikettei <[email protected]>"
+  :licence     "MIT"
+  :pathname   "frontends/onnx"
+  :depends-on ("cl-onnx" "alexandria")
+  :components ((:file "package")
+	       (:file "utils")
+	       (:file "opset")
+	       (:file "from-model-proto")))
+
+;; Other frontend system follows...
+;;(asdf:defsystem :cl-waffe2.frontends/qnn
+;;  :description "[WIP] Quantization Tools"
+;;  :author      "hikettei <[email protected]>"
+;;  :licence     "MIT")

docs/apis/nn.lisp

@@ -20,13 +20,13 @@
       (with-example
 	"(proceed (!sigmoid (randn `(10 10))))"))
 
-    (with-nn-doc '!leakey-relu 'function
+    (with-nn-doc '!leaky-relu 'function
       (with-example
-	"(proceed (!leakey-relu (randn `(10 10))))"))
+	"(proceed (!leaky-relu (randn `(10 10))))"))
 
     (with-nn-doc '!elu 'function
       (with-example
-	"(proceed (!leakey-relu (randn `(10 10))))"))
+	"(proceed (!elu (randn `(10 10))))"))
 
     (with-nn-doc '!softmax 'function
       (with-example

docs/apis/reference.lisp

@@ -51,13 +51,6 @@ The package `cl-waffe2/vm` is the central of system, and features are focused on
     (nodedoc SubNode)
     (nodedoc MulNode)
     (nodedoc DivNode)
-
-    (nodedoc InverseTensorNode)
-
-    (nodedoc ScalarAdd)
-    (nodedoc ScalarSub)
-    (nodedoc ScalarMul)
-    (nodedoc ScalarDiv)
 
     (nodedoc MoveTensorNode)
 
@@ -158,7 +151,7 @@ The package `cl-waffe2/vm` is the central of system, and features are focused on
     (caller-doc !matrix-mul)
     (caller-doc !matrix-div)
 
-    (caller-doc !inverse)
+    (caller-doc !reciprocal)
     (caller-doc !scalar-add)
     (caller-doc !scalar-sub)
     (caller-doc !scalar-mul)
@@ -337,35 +330,3 @@ To get further performance on CPU, SIMD Extension must be installed on your devi
 		  ,@body)))
     (with-op-doc (find-class 'CPUTensor) 't)))
 
-(with-page *cpu-jit-tensor-backend* "[backend] :cl-waffe2/backends.jit.cpu"
-  (insert "
-Those backends without JIT, relies on `do-compiled-loop` or `call-with-view` to calculate only part of matrices: complicated offsets and permution (i.e: `!view` and `!permute`). However, under certain circumstances this can be difficult to parallelise by simply calling a CFFI function; and this backend is intended to solve the problem.
-
-This package provides a `JITCPUTensor` backend which works by jit-compiling whole code to vectorized C, and is only for the purpose of optimising memory layout, so currently only four arithmetic operations and copying are implemented. (OpFusion is remained to be implemented; but it is definitely possible to fuse several ops)
-
-Optimise the memory layout by enclosing the code you want to optimise the layout in:
-
-```lisp
-(with-cpu-jit (CPUTensor LispTensor)
-    ;; body
-    )
-```
-
-Tips: Use the `proceed-bench` function to know the bottleneck; if `MoveTensorNode` combined with `PermuteNode` is slow compared to other nodes, the memory layout is remained to be optimized for example.
-")
-
-  (macrolet ((with-op-doc (name type &body body)
-	       `(progn
-		  (placedoc ,name ,type)
-		  ,@body)))
-    (with-op-doc '*default-c-compiler* 'variable)
-    (with-op-doc '*compiler-flags* 'variable)
-    (with-op-doc '*viz-compiled-code* 'variable)
-
-    (with-op-doc (find-class 'JITCPUTensor) 't)
-
-    (with-op-doc #'cpujit-set-config 'function)
-    (with-op-doc (macro-function 'with-cpu-jit) 'function)
-
-    ))
-