feat: support building AO wasm module with Rust code

dev-cli/container/Dockerfile

@@ -3,7 +3,7 @@ LABEL maintainer="tom wilson <tom@hyper.io>"
 
 # The working directory used by the base image is /src, so we can mount volumes to there
 # to expose files on the host to the ao container
-# 
+#
 # https://github.com/emscripten-core/emsdk/blob/9b0db91883452051aca8deddc932363aab29060b/docker/Dockerfile#L120
 
 RUN apt-get update --fix-missing -qq -y
@@ -53,12 +53,12 @@ RUN cp -r /lua-${LUA_VERSION} /lua-${LUA_VERSION}-32
 # And, re-compile lua with "generic WASM"
 RUN cd /lua-${LUA_VERSION} && \
   make clean && \
-  make generic CC='emcc -s WASM=1 -s MEMORY64=1 -s SUPPORT_LONGJMP=1' 
+  make generic CC='emcc -s WASM=1 -s MEMORY64=1 -s SUPPORT_LONGJMP=1'
 
 # And, re-compile lua with "generic WASM 32-bit"
 RUN cd /lua-${LUA_VERSION}-32 && \
   make clean && \
-  make generic CC='emcc -s WASM=1 -s SUPPORT_LONGJMP=1' 
+  make generic CC='emcc -s WASM=1 -s SUPPORT_LONGJMP=1'
 
 
 #############################
@@ -115,7 +115,7 @@ ENV NM='/emsdk/upstream/bin/llvm-nm'
 ###########################################
 # We first create a directory for the node impls to be placed
 # and dependencies installed
-# 
+#
 # By running npm link, we allow any commands exposed by
 # the node module to be ran globally within the container
 RUN mkdir -p /opt/node
@@ -124,3 +124,21 @@ RUN cd /opt/node && \
   npm install --omit="dev" && \
   npm link
 
+
+###########################################
+### Rust and WASM toolchains ###
+###########################################
+
+# Set environment variables
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:/usr/local/rustup/bin:$PATH
+
+# Install Rust, components, and cbindgen in a single step
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly && \
+    rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu && \
+    cargo install cbindgen && \
+    rustc --version && cargo --version && cbindgen --version && rustup show
+
+# Make build directory
+RUN mkdir /opt/aorustlib

dev-cli/container/src/ao-build-module

@@ -37,7 +37,7 @@ def determine_language():
     #     raise Exception('Multiple languages detected in the module')
     if len(langs) == 0:
         raise Exception('No language detected in the module')
-    
+
     if langs.count('lua') == 0 and langs.count('c') >= 1:
         lang = 'c'
     elif langs.count('lua') == 0 and langs.count('rust') >= 1:
@@ -51,7 +51,7 @@ def determine_language():
 def main():
     # Load the definition file
     definition = Definition('/opt/definition.yml')
-    
+
     # Load the config file
     config = Config('/src/config.yml')
 
@@ -67,26 +67,26 @@ def main():
     c_source_files = []
     injected_lua_files = []
     dependency_libraries = []
-    
+
     # Inject c files into c_program if language is c
     if(language == 'c'):
-        c_program = inject_c_files(definition, c_program, c_source_files)
-    
+        c_program = inject_c_files(definition, c_program, c_source_files, link_libraries)
+
     # Inject rust files into c_program if language is rust
     if(language == 'rust'):
-        c_program = inject_rust_files(definition, c_program)
+        c_program = inject_rust_files(definition, c_program, config)
 
     # Inject lua files into c_program always to load lua files and replace placeholders
     if(language == 'lua'):
         c_program = inject_lua_files(definition, c_program, injected_lua_files)
-    
+
     # Load all libraries
     c_program = load_libraries(config, definition, c_program, link_libraries, c_source_files, injected_lua_files,dependency_libraries)
-                    
+
     # Inject the lua files into the c program
     c_program = c_program.replace(
         '__INJECT_LUA_FILES__', '\n'.join(injected_lua_files))
-    
+
     #Generate compile target file
     debug_print('Start to generate complie.c')
 
@@ -97,25 +97,25 @@ def main():
     debug_print('Start to compile as WASM')
 
     # Setup the compile command
-    cmd = ['emcc', '-O3', 
+    cmd = ['emcc', '-O3',
       '-g2',
     #   '-s', 'MAIN_MODULE', # This is required to load dynamic libraries at runtime
       '-s', 'ASYNCIFY=1',
       '-s', 'STACK_SIZE=' + str(config.stack_size),
       '-s', 'ASYNCIFY_STACK_SIZE=' + str(config.stack_size),
-      '-s', 'ALLOW_MEMORY_GROWTH=1', 
-      '-s', 'INITIAL_MEMORY=' + str(config.initial_memory), 
-      '-s', 'MAXIMUM_MEMORY=' + str(config.maximum_memory), 
-      '-s', 'WASM=1', 
+      '-s', 'ALLOW_MEMORY_GROWTH=1',
+      '-s', 'INITIAL_MEMORY=' + str(config.initial_memory),
+      '-s', 'MAXIMUM_MEMORY=' + str(config.maximum_memory),
+      '-s', 'WASM=1',
       '-s', 'MODULARIZE',
-      # '-s', 'FILESYSTEM=0', 
-      '-s', 'DETERMINISTIC=1', 
-      '-s', 'NODERAWFS=0', 
+      # '-s', 'FILESYSTEM=0',
+      '-s', 'DETERMINISTIC=1',
+      '-s', 'NODERAWFS=0',
       '-s', 'FORCE_FILESYSTEM=1',
       '-msimd128',
       '--pre-js', '/opt/pre.js'
     ]
-    
+
     # If the target is 64 bit, add the MEMORY64 flag and link against 64 bit libraries
     if config.target == 64:
         cmd.extend(['-sMEMORY64=1'])
@@ -128,7 +128,7 @@ def main():
 
     # Link Rust library
     if(language == 'rust'):
-        cmd.extend(['-L/opt/aorustlib', '-l:libaorust.a'])
+        cmd.extend(['-I/opt/aorustlib', '-L/opt/aorustlib', '-laorust'])
 
     cmd.extend(['-s','ASSERTIONS=1'])
     cmd.extend(definition.get_extra_args())
@@ -138,7 +138,7 @@ def main():
     # Add all c source files if there are any
     for f in c_source_files:
         cmd.append(quote(f))
-    
+
     # Add the compile file and link libraries
     cmd.extend(['/tmp/compile.c'])
     cmd.extend([quote(v.filepath) for v in link_libraries])
@@ -157,6 +157,7 @@ def main():
     # add metering library
     # meter_cmd = ['node', '/opt/node/apply-metering.cjs']
     # shell_exec(*meter_cmd)
+
     if config.keep_js is False:
         shell_exec(*['rm', os.path.join(os.getcwd(), 'process.js')])
 

dev-cli/container/src/ao_module_lib/languages/rust.py

@@ -4,61 +4,67 @@
 
 from ao_module_lib.helper import shell_exec
 from ao_module_lib.definition import Definition
+from ao_module_lib.config import Config
 
 RUST_DIR = '/opt/aorustlib'
-def inject_rust_files(definition: Definition, c_program: str):
+def inject_rust_files(definition: Definition, c_program: str, config: Config):
 
     c_header_files = []
-    rust_source_files = []
+    # rust_source_files = []
 
-    c_header_files += glob.glob('/src/**/*.h', recursive=True)    
+    c_header_files += glob.glob('/src/**/*.h', recursive=True)
     c_header_files += glob.glob('/src/**/*.hpp', recursive=True)
 
     inc = []
     for header in c_header_files:
         c_program = '#include "{}"\n'.format(header) + c_program
 
-    # c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program
+#    c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program
 
 
     c_program = c_program.replace('__FUNCTION_DECLARATIONS__', definition.make_c_function_delarations())
+    c_program = c_program.replace('__LUA_BASE__', "")
+    c_program = c_program.replace('__LUA_MAIN__', "")
 
 
-    rust_source_files += glob.glob('/src/**/*.rs', recursive=True)
-    rust_src_dir = os.path.join(RUST_DIR, "src")
+    # rust_source_files += glob.glob('/src/**/*.rs', recursive=True)
+    # rust_src_dir = os.path.join(RUST_DIR, "src")
 
-    for file in rust_source_files:
-        if os.path.isfile(file):
-            shutil.copy2(file, os.path.join(rust_src_dir))
+    # for file in rust_source_files:
+    #     if os.path.isfile(file):
+    #         shutil.copy2(file, os.path.join(rust_src_dir))
 
-    prev_dir = os.getcwd()
-    os.chdir(RUST_DIR)
+#    prev_dir = os.getcwd()
+#    os.chdir(RUST_DIR)
 
+    # build rust code at '/src'
+    target = config.target == 64 and 'wasm64-unknown-unknown' or 'wasm32-unknown-unknown'
     os.environ["RUSTFLAGS"] = "--cfg=web_sys_unstable_apis --Z wasm_c_abi=spec"
     cmd = [
         'cargo',
         '+nightly',
         'build',
         '-Zbuild-std=std,panic_unwind,panic_abort',
         # '-Zbuild-std-features=panic_immediate_abort',
-        '--target=wasm64-unknown-unknown',
+        f'--target={target}',
         '--release'
         ]
 
     shell_exec(*cmd)
-    rust_lib = glob.glob('/opt/aorustlib/**/release/*.a', recursive=True)[0]
+    rust_lib = glob.glob(f'/src/target/{target}/release/*.a', recursive=True)[0]
     rust_lib = shutil.copy2(rust_lib, RUST_DIR)
 
     cmd = [
-        'cbindgen', 
-        '--config', 
+        'cbindgen',
+        '--config',
         'cbindgen.toml',
-            '--crate', 
-            'aorust', 
-            '--output', 
+            '--crate',
+            'aorust',
+            '--output',
             'aorust.h'
         ]
     shell_exec(*cmd)
-    c_program = '#include "{}"\n'.format('/opt/aorustlib/aorust.h') + c_program
-    os.chdir(prev_dir)
+    rust_header = shutil.copy2('/src/aorust.h', RUST_DIR)
+    c_program = '#include "{}"\n'.format('aorust.h') + c_program
+#    os.chdir(prev_dir)
     return c_program

dev-cli/src/commands/build.js

@@ -3,8 +3,10 @@
 import { Command } from '../deps.js'
 import { VERSION } from '../versions.js'
 
-export async function build () {
+export async function build(customImage) {
   const pwd = Deno.cwd()
+  const image = customImage || `p3rmaw3b/ao:${VERSION.IMAGE}`
+
   const p = Deno.run({
     cmd: [
       'docker',
@@ -13,13 +15,16 @@ export async function build () {
       'linux/amd64',
       '-v',
       `${pwd}:/src`,
-      `p3rmaw3b/ao:${VERSION.IMAGE}`,
+      image,
       'ao-build-module'
     ]
   })
   await p.status()
 }
 
 export const command = new Command()
-  .description('Build the Lua Project into WASM')
-  .action(build)
+  .description('Build the Project into WASM')
+  .option('-i, --image <image:string>', 'Specify a custom Docker image to use')
+  .action(async (options) => {
+    await build(options.image)
+  })

dev-cli/src/commands/init.js

@@ -19,10 +19,10 @@ export async function init ({ lang = 'lua' }, name) {
 
 export const command = new Command()
   .description('Create an ao Process Source Project')
-  .usage('-l cpp <my-project-name>')
+  .usage('-l c <my-project-name>')
   .option(
     '-l, --lang <language:string>',
-    'The starter to use. Defaults to Lua. Options are "lua" and "cpp"'
+    'The starter to use. Defaults to Lua. Options are "lua" and "c"'
   )
   .arguments('<name:string>')
   .action(init)

dev-cli/src/deps.js

@@ -1,4 +1,4 @@
-export { Command } from 'https://deno.land/x/cliffy@v1.0.0-rc.3/command/mod.ts'
+export { Command } from 'https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts'
 export { basename, resolve } from 'https://deno.land/std@0.224.0/path/mod.ts'
 export { copy } from 'https://deno.land/std@0.224.0/fs/copy.ts'
 export { parse } from 'jsr:@std/yaml'

dev-cli/src/starters/rust/.gitignore

@@ -0,0 +1,11 @@
+/target/
+Cargo.lock
+**/*.log
+**/*.tmp
+*.swp
+*.bak
+*.bk
+*.tmp
+*.old
+aorust.h
+process.wasm

dev-cli/src/starters/rust/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "aorust"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+serde = { version = "1.0", default-features = false, features = ["derive"] }
+serde_json = { version="1.0", default-features = false, features = ["alloc"] }
+
+[profile.dev]
+panic = "abort"
+
+[profile.release]
+panic = "abort"

dev-cli/src/starters/rust/Makefile

@@ -0,0 +1,18 @@
+all:build bindgen
+
+build:
+	RUSTFLAGS="--Z wasm_c_abi=spec " \
+	cargo +nightly build \
+	    -Zbuild-std=std,panic_unwind,panic_abort \
+	    --target=wasm64-unknown-unknown \
+	    --release
+
+bindgen:
+	cbindgen --config cbindgen.toml --crate aorust --output aorust.h
+
+clean:
+	cargo clean
+	rm aorust.h
+
+opt:
+	wasm-opt --post-emscripten -O3 --low-memory-unused process.wasm --enable-bulk-memory --enable-memory64 --no-validation -o process.wasm

dev-cli/src/starters/rust/cbindgen.toml

@@ -0,0 +1,7 @@
+language = "C"
+
+[defines]
+MY_DEFINE = "1"
+
+[export]
+include = ["process_handler"]

dev-cli/src/starters/rust/config.yml

@@ -0,0 +1,2 @@
+target: 32
+preset: 'sm'