ESP32: External modules

.gitignore

@@ -8,9 +8,10 @@ build/
 app/
 components/*/.output/
 tools/toolchains
+extmods.ini
 
 #ignore Eclipse project files
 .cproject
 .project
 .settings/
-.vscode/**
+.vscode/**

Makefile

@@ -12,6 +12,9 @@ TOOLCHAIN_RELEASES_BASEURL:=https://github.com/jmattsson/esp-toolchains/releases
 TOOLCHAIN_VERSION:=20181106.1
 PLATFORM:=linux-$(shell uname --machine)
 
+## Directory to place external modules:
+export EXTRA_COMPONENT_DIRS:=$(THIS_DIR)/components/modules/external
+
 ESP32_BIN:=$(THIS_DIR)/tools/toolchains/esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION)/bin
 ESP32_GCC:=$(ESP32_BIN)/xtensa-esp32-elf-gcc
 ESP32_TOOLCHAIN_DL:=$(THIS_DIR)/cache/toolchain-esp32-$(PLATFORM)-$(TOOLCHAIN_VERSION).tar.xz
@@ -61,3 +64,9 @@ include $(IDF_PATH)/make/project.mk
 CC:=$(CC) $(BASIC_TYPES) -D__ESP32__ $(MORE_CFLAGS)
 
 endif
+
+extmod-update:
+	@tools/extmod/extmod.sh update
+
+extmod-clean:
+	@tools/extmod/extmod.sh clean

components/base_nodemcu/include/module.h

@@ -70,3 +70,25 @@
     luaR_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_LUA_MODULE_,cfgname))) \
     = {luaname, LRO_ROVAL(map ## _map)}
 #endif
+
+
+// helper stringing macros
+#define xstr(s) str(s)
+#define str(s) #s
+
+// EXTMODNAME is injected by the generated component.mk
+#ifdef EXTMODNAME
+#define MODNAME xstr(EXTMODNAME)
+#else
+#define MODNAME "module"
+#endif
+
+// use NODEMCU_MODULE_METATABLE() to generate a unique metatable name for your objects:
+#define NODEMCU_MODULE_METATABLE() MODULE_EXPAND_(MODNAME xstr(__COUNTER__))
+
+// NODEMCU_MODULE_STD() defines the entry points for an external module:
+#define NODEMCU_MODULE_STD()                                             \
+    static const LOCK_IN_SECTION(libs)                                   \
+        luaR_entry lua_lib_module = {MODNAME, LRO_FUNCVAL(module_init)}; \
+    const LOCK_IN_SECTION(rotable)                                       \
+        luaR_entry MODULE_EXPAND_PASTE_(EXTMODNAME, _entry) = {MODNAME, LRO_ROVAL(module_map)};

components/modules/.gitignore

@@ -0,0 +1 @@
+external/

docs/modules/extmods.md

@@ -0,0 +1,134 @@
+
+# External modules. Plugging your own C modules. **BETA**
+
+Note: this feature is still in beta. Configuration files / API may change.
+
+In order to make the most of NodeMCU, you will undoubtedly have to connect your ESP device to many different hardware modules, which will require you to write driver code in C and expose a Lua interface.
+
+To make this easy, we have come up with the concept of "external modules". External modules allow you to refer to an external git repository containing the module, which will be downloaded/updated and built along the firmware. It is similar to git submodules, but without the complexity and having to alter the parent repository, while also adapted to the compilation requirements of NodeMCU and Lua.
+
+## How to use external modules:
+
+To use external modules, simply create an `extmods.ini` file in the repository root, with the following syntax:
+
+```ini
+[lua_mount_point]
+url=<git repo URL>
+ref=<branch name, tag or commit id>
+...
+```
+
+Where:
+* **lua_mount_point**: Name you want the referenced module to have in Lua.
+* **url**: Repository URL where to fetch the code from
+* **ref**: branch, tag or commit hash of the version of the module you want to pull in.
+* **disabled**: (optional) Whether or not to actually compile the module (see below)
+
+For example:
+
+```ini
+[helloworld]
+[email protected]:espore-ide/nodemcu-module-helloworld.git
+ref=master
+```
+
+You can add further sections to `extmods.ini`, one for each module you want to add. Once this file is ready, run the update command:
+
+```shell
+make extmod-update
+```
+
+This will download or update the modules to the external modules directory, `components/modules/external`.
+
+You can now compile the firmware with `make` as usual. The build system will find your external modules and compile them along the core modules.
+
+After this is flashed to the device, the module in the example will be available in lua as `helloworld`.
+
+### Updating to the latest code
+
+If your external module entry in `extmods.ini` points to a branch, e.g., `master`, you can update your local version to the latest code anytime by simply running `make extmod-update`.
+
+### Temporarily disabling an external module
+
+If you want to stop compiling and including a module in the build for any reason, without having to remove the entry from `extmods.ini`, simply add a `disabled=true` entry in the module section and run `make extmod-update`.
+
+Example:
+```ini
+[helloworld]
+url=https://github.com/espore-ide/nodemcu-module-helloworld.git
+ref=master
+disabled=true
+```
+
+### Mounting different module versions
+
+Provided the module is well written (no global variables, etc), it is even possible to easily mount different versions of the same module simultaneously for testing:
+
+```ini
+[helloworld]
+url=https://github.com/espore-ide/nodemcu-module-helloworld.git
+ref=master
+
+[helloworld_dev]
+url=https://github.com/espore-ide/nodemcu-module-helloworld.git
+ref=dev
+```
+
+Note that the second one points to a different branch, `dev`. Both modules will be visible in Lua under `helloworld` and `helloworld_dev` respectively.
+
+## How to write external modules:
+
+To write your own external module do the following:
+
+1. Create an empty repository in your favorite remote, GitHub, BitBucket, GitLab, etc, or fork the helloworld example.
+2. Create an entry in `extmods.ini` as explained above, with the `url=` key pointing to your repository. For modules that you author, it is recommended to use an updateable git URL in SSH format, such as `[email protected]:espore-ide/nodemcu-module-helloworld.git`.
+3. Run `make extmod-update`
+
+You can now change to `components/modules/external/your_module` and begin work. Since that is your own repository, you can work normally, commit, create branches, etc.
+
+### External module scaffolding
+
+External modules must follow a specific structure to declare the module in C. Please refer to the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) example, or use it as a template. In particular:
+
+1. Include `module.h` 
+2. Define a Module Function Map with name `module`
+3. Define a `module_init` function
+4. Include the module lua entries by adding a call to the `NODEMCU_MODULE_STD` macro
+
+Here is a bare minimum module:
+```c
+#include "module.h"
+
+// Module function map
+LROT_BEGIN(module)
+/* module-level functions go here*/
+LROT_END(module, NULL, 0)
+
+// module_init is invoked on device startup
+static int module_init(lua_State* L) {
+    // initialize your module, register metatables, etc
+    return 0;
+}
+
+NODEMCU_MODULE_STD();  // define Lua entries
+```
+For a full example module boilerplate, check the [helloworld.c](https://github.com/nodemcu/nodemcu-firmware/blob/dev-esp32/tools/example/helloworld.c) file.
+
+
+### Special makefile or compilation requirements
+
+If your module has special makefile requirements, you can create a `module.mk` file at the root of your module repository. This will be executed during compilation.
+
+### What is this "component.mk" file that appeared in my repo when running `make extmod-update` ?
+
+This file is ignored by your repository. Do not edit or check it in!. This file contains the necessary information to compile your module along with the others.
+
+Note that you don't even need to add this file to your `.gitignore`, since the `make extmod-update` operation configures your local copy to ignore it (via `.git/info/exclude`). 
+
+## Further work:
+
+* Support for per-module menuconfig (`Kconfig`). This is actually possible already, but need to work around potential config variable collisions in case two module authors happen to choose the same variable names.
+* Module registry: Create an official repository of known external modules.
+* Move all non-essential and specific hardware-related C modules currently in the NodeMCU repository to external modules, each in their own repository.
+* Create the necessary scaffolding to facilitate writing modules that will work both in ESP8266 and ESP32.
+* Port this work to the ESP8266 branch. Mostly, the scripts that make this possible could work in both branches directly.

mkdocs.yml

@@ -31,6 +31,7 @@ pages:
     - Extension Developer FAQ: 'extn-developer-faq.md'
 - Whitepapers:
     - Filesystem on SD card: 'sdcard.md'
+    - Writing external C modules: 'modules/extmods.md'
 - C Modules:
     - 'adc':          'modules/adc.md'
     - 'bit':          'modules/bit.md'

tools/extmod/component.mk.template

@@ -0,0 +1,16 @@
+# This file is autogenerated! DO NOT EDIT!!
+# To add special makefile directives, create a module.mk file in your repo
+# Make sure you have this file in your .gitignore to avoid checking it in your module repo
+
+MODNAME="%%MODNAME%%"
+MODULE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+
+COMPONENT_ADD_LDFLAGS=-u $(MODNAME)_entry -l$(MODNAME)
+
+CFLAGS += \
+	-DEXTMODNAME=$(MODNAME) \
+	-Werror=unused-function \
+	-Werror=unused-but-set-variable \
+	-Werror=unused-variable 
+
+-include $(MODULE_DIR)module.mk

tools/extmod/example/helloworld.c

@@ -0,0 +1,61 @@
+// Helloworld sample module
+
+#include "esp_log.h"
+#include "lauxlib.h"
+#include "lnodeaux.h"
+#include "module.h"
+
+static const char* HELLOWORLD_METATABLE = NODEMCU_MODULE_METATABLE();
+
+// hello_context_t struct contains information to wrap a "hello world object"
+typedef struct {
+    char* my_name;  // pointer to the greeter's name
+} hello_context_t;
+
+// Lua: helloworldobj:hello(text)
+static int helloworld_hello(lua_State* L) {
+    hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
+    printf("Hello, %s: %s\n", context->my_name, luaL_optstring(L, 2, "How are you?"));
+    return 0;
+}
+
+// helloworld_delete is called on garbage collection
+static int helloworld_delete(lua_State* L) {
+    hello_context_t* context = (hello_context_t*)luaL_checkudata(L, 1, HELLOWORLD_METATABLE);
+    printf("Helloworld object with name '%s' garbage collected\n", context->my_name);
+    luaX_free_string(L, context->my_name);
+    return 0;
+}
+
+// Lua: modulename.new(string)
+static int helloworld_new(lua_State* L) {
+    //create a new lua userdata object and initialize to 0.
+    hello_context_t* context = (hello_context_t*)lua_newuserdata(L, sizeof(hello_context_t));
+
+    context->my_name = luaX_alloc_string(L, 1, 100);
+
+    luaL_getmetatable(L, HELLOWORLD_METATABLE);
+    lua_setmetatable(L, -2);
+
+    return 1;  //one object returned, the helloworld context wrapped in a lua userdata object
+}
+
+// object function map:
+LROT_BEGIN(helloworld_metatable)
+LROT_FUNCENTRY(hello, helloworld_hello)
+LROT_FUNCENTRY(__gc, helloworld_delete)
+LROT_TABENTRY(__index, helloworld_metatable)
+LROT_END(helloworld_metatable, NULL, 0)
+
+// Module function map
+LROT_BEGIN(module)
+LROT_FUNCENTRY(new, helloworld_new)
+LROT_END(module, NULL, 0)
+
+// module_init is invoked on device startup
+static int module_init(lua_State* L) {
+    luaL_rometatable(L, HELLOWORLD_METATABLE, (void*)helloworld_metatable_map);  // create metatable for helloworld
+    return 0;
+}
+
+NODEMCU_MODULE_STD();  // define Lua entries