Merge pull request #154 from hikettei/develop

[WIP] Various new features: Enhanced JIT Compiler targeted to multiple devices, ONNX Support, et al.

.github/workflows/test_on_push.yml

@@ -9,8 +9,10 @@ on:
         - develop
 
 jobs:
-  test:
-    name: ${{ matrix.lisp }} on ${{ matrix.os }}
+  # TODO: Metal/CUDA Runtime
+  UnitTest-CPUTensor:
+    # TODO: Metal/CUDA Runtime
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Unittest-CPUTensor
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -36,10 +38,146 @@ jobs:
       - name: Check $PATH
         run: echo $PATH
 
-      - name: Downloading OpenBLAS
+      - name: Installing OpenBLAS
         run: |
           sudo apt install libblas-dev
-      - name: Operating tests
+          
+      #- name: Installing Extensions
+      #  run: |
+      #    git submodule update --init --recursive
+      #    sudo make build_sleef
+      #    sudo make build_simd_extension
+
+      - name: Unittest (CPU+Lisp)
         run: |
           ros config set dynamic-space-size 4gb
-          ros --eval '(progn (defparameter cl-user::*cl-waffe-config* `((:libblas "libblas.so"))))' ./roswell/cl-waffe2-test.ros
+          ./roswell/waffe2.ros test -b CPUTensor -b LispTensor
+  UnitTest-LispTensor:
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Unittest-LispTensor
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        lisp: [sbcl-bin/2.3.9]
+        os: [ubuntu-latest]
+        target:
+        - normal
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setting up environments
+        env:
+          LISP: ${{ matrix.lisp }}
+        run: |
+          curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh
+      - name: Update $PATH
+        run: |
+          echo $PATH
+          echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+      - name: Check $PATH
+        run: echo $PATH
+      - name: Unittest (Lisp+CPU)
+        run: |
+          ros config set dynamic-space-size 4gb
+          ./roswell/waffe2.ros test -b LispTensor -b CPUTensor
+  Test-Classification-CPU:
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Test-Classification-CPU
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        lisp: [sbcl-bin/2.3.9]
+        os: [ubuntu-latest]
+        target:
+        - normal
+        archcpu:
+        - AVX2
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setting up environments
+        env:
+          LISP: ${{ matrix.lisp }}
+        run: |
+          curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh
+      - name: Update $PATH
+        run: |
+          echo $PATH
+          echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+      - name: Check $PATH
+        run: echo $PATH
+      - name: Installing OpenBLAS
+        run: |
+          sudo apt install libblas-dev
+  
+      - name: Set up Python
+        id: set_up_python_python_ci
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Downloading assets
+        run: |
+          pip install torch torchvision numpy
+          cd ./examples/mnist 
+          python train_data.py
+          cd ../../
+      
+      #- name: Installing Extensions
+      #  run: |
+      #    git submodule update --init --recursive
+      #    sudo make build_sleef
+      #    sudo make build_simd_extension
+
+      - name: Training w/ LispTensor
+        run: |
+          ros config set dynamic-space-size 4gb
+          ./roswell/waffe2.ros demo --example mnist -b LispTensor -b CPUTensor --config epoch-num=2
+  Test-Classification-JIT:
+    name: (${{ matrix.os }} on ${{ matrix.lisp }}) | Test-Classification-JIT
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        lisp: [sbcl-bin/2.3.9]
+        os: [ubuntu-latest]
+        target:
+        - normal
+        archcpu:
+        - AVX2
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setting up environments
+        env:
+          LISP: ${{ matrix.lisp }}
+        run: |
+          curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh
+      - name: Update $PATH
+        run: |
+          echo $PATH
+          echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+      - name: Check $PATH
+        run: echo $PATH
+      - name: Installing OpenBLAS
+        run: |
+          sudo apt install libblas-dev
+  
+      - name: Set up Python
+        id: set_up_python_python_ci
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Downloading assets
+        run: |
+          pip install torch torchvision numpy
+          cd ./examples/mnist 
+          python train_data.py
+          cd ../../
+      - name: Installing Qlot
+        run: |
+          curl -L https://qlot.tech/installer | sh
+          export PATH="/home/runner/.qlot/bin:$PATH"
+          qlot install
+      
+      - name: Training w/ Aten[Clang]
+        run: |
+          ros config set dynamic-space-size 4gb
+          ./roswell/waffe2.ros demo --example mnist -b "(Aten[Clang] :debug 1)" -b LispTensor -b CPUTensor --config epoch-num=2
+  

GNUmakefile

@@ -1,7 +1,9 @@
+ROSWELL             := ros
 SBCL                := sbcl
 # Avoid loading system-wide libraries (--no-sysinit)
 SBCL_OPTIONS        := --noinform --no-sysinit
-QUICKLOAD_WAFFE2    := --load cl-waffe2.asd --eval '(ql:quickload :cl-waffe2)'
+ROSWELL_OPTIONS     := dynamic-space-size=4096
+QUICKLOAD_WAFFE2    := --eval '(load "cl-waffe2.asd")' --eval '(ql:quickload :cl-waffe2)'
 MKTEMP              := mktemp
 RLWRAP              := rlwrap
 LOGFILE             := $(shell mktemp)
@@ -26,32 +28,32 @@ compile: ## Compiles the whole project
 
 .PHONY: test
 test: ## Running a test harness
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:test-system :cl-waffe2)' \
 		--quit
 
 .PHONY: recordtest
 recordtest: ## Running a test harness with recording logs
 	$(warning This session will be recorded in $(LOGFILE))
 	$(RLWRAP) --logfile $(LOGFILE) \
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(asdf:test-system :cl-waffe2)' \
 		--quit
 	@printf 'This session has been recorded in %s\n' $(LOGFILE)
 
 .PHONY: repl
 repl: ## Launch REPL with loading cl-waffe2
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 
 .PHONY: rlrepl
 rlrepl: ## Launch REPL with rlwrap
-	$(RLWRAP) $(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+	$(RLWRAP) $(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 
 .PHONY: record
 record: ## Launch REPL with logging
 	$(warning This session will be recorded in $(LOGFILE))
 	$(RLWRAP) --logfile $(LOGFILE) \
-		$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2)
+		$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2)
 	@printf 'This session has been recorded in %s\n' $(LOGFILE)
 
 .PHONY: slynk
@@ -68,7 +70,7 @@ swank: ## Launch Swank server
 
 .PHONY: docs
 docs: ## Generate documents
-	$(SBCL) $(SBCL_OPTIONS) $(QUICKLOAD_WAFFE2) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) $(QUICKLOAD_WAFFE2) \
 		--eval '(ql:quickload :cl-waffe2/docs)' \
 		--eval '(cl-waffe2.docs:generate)' \
 		--quit
@@ -139,14 +141,14 @@ download_assets: ## Downloads training data sample codes use.
 
 .PHONY: example_mnist
 example_mnist: ## Start MNIST Example and Benchmarking. (download_assets must be called in advance)
-	$(SBCL) $(SBCL_OPTIONS) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) \
 		--load ./cl-waffe2.asd --load ./examples/mnist/mnist.asd \
 		--eval '(ql:quickload :mnist-sample)' \
 		--eval '(mnist-sample::train-and-valid-mlp :epoch-num 10)'
 
 .PHONY: example_gpt
 example_gpt: ## Launch GPT2 Inference (download_assets must be called in advance)
-	$(SBCL) --dynamic-space-size 4096 $(SBCL_OPTIONS) \
+	$(ROSWELL) run $(ROSWELL_OPTIONS) \
 		--load ./cl-waffe2.asd --load ./examples/gpt-2/gpt-2.asd \
 		--eval '(ql:quickload :gpt-2-example)' \
 		--eval '(gpt-2-example:launch-repl)'

cl-waffe2.asd

@@ -42,7 +42,6 @@
 	       :closer-mop
 	       :trivial-garbage
 	       :cl-waffe2/simd-extension
-
 	       :cl-environments
 	       :numpy-file-format
 	       :jonathan)
@@ -158,20 +157,7 @@
 	       (:file "package")
 
 
-	       (:file "backends/lisp/wf2model")
-	       (:file "backends/JITCPUTensor/package")
-	       (:file "backends/JITCPUTensor/tensor")
-	       (:file "backends/JITCPUTensor/compiler")
-	       (:file "backends/JITCPUTensor/blueprint")	       
-	       (:file "backends/JITCPUTensor/ir")
-	       (:file "backends/JITCPUTensor/on-finalizing")
-	       (:file "backends/JITCPUTensor/dtype")
-	       (:file "backends/JITCPUTensor/foreign-function")
-
-
-	       (:file "backends/JITCPUTensor/impls/arithmetic")
-	       (:file "backends/JITCPUTensor/impls/math")
-
+	       (:file "backends/lisp/wf2model")	       
 	       (:file "optimizers/defoptimizer")
 
 	       (:file "array-converter")
@@ -223,7 +209,7 @@
   :description "Tests for cl-waffe2"
   :serial t
   :pathname "source"
-  :depends-on (:cl-waffe2 :fiveam)
+  :depends-on (:cl-waffe2 :fiveam :rove)
   :components ((:file "vm/iterator/test-suites")
 	       (:file "vm/t/package")
 	       (:file "vm/t/lazy-axis")
@@ -251,30 +237,16 @@
 	       (:file "backends/cpu/t/arithmetic")
 
 	       (:file "backends/lisp/t/package")
-
-	       (:file "backends/JITCPUTensor/t/package")
-	       (:file "backends/JITCPUTensor/t/jit")
 
 	       (:file "nn/t/package")
 	       (:file "nn/t/conv")
 	       (:file "nn/t/activation")
 	       (:file "nn/t/criterion")
-	       (:file "nn/t/regression")	       
+	       (:file "nn/t/regression")
+
+	       (:file "test-suites")
 	       )
-  :perform (test-op (o s)
-		    (symbol-call :fiveam :run! :iterator-test)
-		    (symbol-call :fiveam :run! :test-nodes)
-		    (symbol-call :fiveam :run! :test-tensor)
-
-		    (symbol-call :fiveam :run! :base-impl-test)
-		    (symbol-call :fiveam :run! :jit-lisp-test)
-
-		    (symbol-call :fiveam :run! :lisp-backend-test)
-		    (symbol-call :fiveam :run! :test-backends-cpu)
-		    (symbol-call :fiveam :run! :jit-cpu-test)
-
-		    (symbol-call :fiveam :run! :nn-test)
-		    (symbol-call :fiveam :run! :vm-test)))
+  :perform (test-op (o s) (error "moved to source/test-suites.lisp")))
 
 
 (defpackage :cl-waffe2-docs-asdf

cl-waffe2.frontends.asd

@@ -0,0 +1,30 @@
+
+;; [WIP] Feature roadmap on frontends.
+;;
+;;  frontend/onnx (From ONNX To cl-waffe2 translator)
+;;  frontend/qnn  (Quantized Neural Network Op Supports (basically relies on JITCompiler))
+;;  frontend/backends
+;;  frontend/backends/cuda (Additional backends which requires more dependency, should be placed at frontend)
+;;
+
+(asdf:defsystem :cl-waffe2.frontends
+  :description "A set of frontend toolkits for cl-waffe2"
+  :author "hikettei <[email protected]>"
+  :Licence "MIT")
+
+(asdf:defsystem :cl-waffe2.frontends/onnx
+  :description "ONNX Graph Translator"
+  :author      "hikettei <[email protected]>"
+  :licence     "MIT"
+  :pathname   "frontends/onnx"
+  :depends-on ("cl-onnx" "alexandria")
+  :components ((:file "package")
+	       (:file "utils")
+	       (:file "opset")
+	       (:file "from-model-proto")))
+
+;; Other frontend system follows...
+;;(asdf:defsystem :cl-waffe2.frontends/qnn
+;;  :description "[WIP] Quantization Tools"
+;;  :author      "hikettei <[email protected]>"
+;;  :licence     "MIT")

docs/apis/nn.lisp

@@ -20,13 +20,13 @@
       (with-example
 	"(proceed (!sigmoid (randn `(10 10))))"))
 
-    (with-nn-doc '!leakey-relu 'function
+    (with-nn-doc '!leaky-relu 'function
       (with-example
-	"(proceed (!leakey-relu (randn `(10 10))))"))
+	"(proceed (!leaky-relu (randn `(10 10))))"))
 
     (with-nn-doc '!elu 'function
       (with-example
-	"(proceed (!leakey-relu (randn `(10 10))))"))
+	"(proceed (!elu (randn `(10 10))))"))
 
     (with-nn-doc '!softmax 'function
       (with-example