From cef0a1a09cb5a617f7054624c1837f48c890d5ff Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 07:34:12 +0100 Subject: [PATCH 1/8] add module loader builder via require() indexer --- main.cc | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/main.cc b/main.cc index 7c9caad..1446976 100644 --- a/main.cc +++ b/main.cc @@ -165,23 +165,29 @@ static xoptOption options[] = { #ifdef NDEBUG # define topcheck(...) ((void)0) +# define disarmtopcheck() ((void)0) #else struct _topcheck { _topcheck(lua_State *_L, int ret = 0) : top(lua_gettop(_L)) { + this->armed = true; this->L = _L; this->ret = ret; } ~_topcheck() { - assert(lua_gettop(this->L) == (this->top + this->ret)); + if (armed) { + assert(lua_gettop(this->L) == (this->top + this->ret)); + } } + bool armed; const int top; int ret; lua_State *L; }; # define topcheck(...) _topcheck __tc(__VA_ARGS__) +# define disarmtopcheck() __tc.armed = false #endif static int errfn_ref = -1; @@ -408,6 +414,77 @@ static bool inject_link_flags(lua_State *L, config &conf) { return true; } +static int try_module_load(lua_State *L) { + topcheck(L, 1); + lua_getglobal(L, "terralib"); + lua_getfield(L, -1, "loadmodule"); + lua_remove(L, -2); + if (!lua_isfunction(L, -1)) { + lua_pop(L, 1); + disarmtopcheck(); + luaL_error(L, "terralib.loadmodule is not a function"); + return 0; + } + lua_pushvalue(L, 1); // the module being require()'d + lua_pushvalue(L, lua_upvalueindex(1)); // the origin filename + lua_pushvalue(L, lua_upvalueindex(2)); // the original (lua-provided) require() + lua_call(L, 3, 1); + return 1; +} + +static int make_module_loader(lua_State *L) { + topcheck(L, 1); + + if (string(lua_tostring(L, 2)) != "require") { + // fallback to regular index + lua_pushvalue(L, lua_upvalueindex(1)); + lua_pushvalue(L, 1); + lua_pushvalue(L, 2); + lua_call(L, 2, 1); + return 1; + } + + lua_Debug dbg; + assert(lua_getstack(L, 1, &dbg) == 1); + assert(lua_getinfo(L, "S", &dbg) != 0); + + const char *source = dbg.source; + if (*source == '@') { + ++source; + } else { + // @ signifies a module-loadable chunk. + // this chunk doesn't start with it, + // so give it the old require. + lua_pushvalue(L, lua_upvalueindex(2)); + return 1; + } + + lua_pushstring(L, source); + lua_pushvalue(L, lua_upvalueindex(2)); + lua_pushcclosure(L, &try_module_load, 2); + + return 1; +} + +static void inject_require(lua_State *L) { + topcheck(L); + + // let's tango. + // + // - push original _G.__index and _G.require as upvalues to make_module_loader + // - set make_module_loader as _G.__index + // - nil-out _G.require in order to start triggering new module loader + lua_getglobal(L, "_G"); + lua_getmetatable(L, -1); + lua_getfield(L, -1, "__index"); + lua_getfield(L, -2, "require"); + lua_pushcclosure(L, &make_module_loader, 2); + lua_setfield(L, -2, "__index"); + lua_pushnil(L); + lua_setfield(L, -3, "require"); + lua_pop(L, 2); +} + /* static void print_table(lua_State *L, int t) { topcheck(L); @@ -620,6 +697,12 @@ int pmain(config &conf) { lua_pop(L, 2); } + // inject module loader + { + topcheck(L); + inject_require(L); + } + // create terrac object { topcheck(L); From a2218cc0bb8150da529d1384d7753b7465a58040 Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 09:32:04 +0100 Subject: [PATCH 2/8] working module loading --- main.cc | 177 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 2 deletions(-) diff --git a/main.cc b/main.cc index 1446976..10eee1b 100644 --- a/main.cc +++ b/main.cc @@ -15,6 +15,12 @@ #include "xopt.h" #include "filesystem.hpp" +#ifdef _WIN32 +# define TERRA_PATHSEP ';' +#else +# define TERRA_PATHSEP ':' +#endif + using namespace std; // https://github.com/stevedonovan/Penlight/blob/d90956418cdf9e315719a7e4ce314db6b512ae00/lua/pl/compat.lua#L145-L155 @@ -414,6 +420,159 @@ static bool inject_link_flags(lua_State *L, config &conf) { return true; } +static filesystem::path mod_to_path(string mod) { + istringstream iss(mod); + filesystem::path result; + string leaf; + while (getline(iss, leaf, '.')) { + result.push_back(leaf); + } + return result; +} + +static filesystem::path mod_to_relpath(string mod) { + assert(mod[0] == '.'); + + filesystem::path result; + size_t i = 0; + while (mod[i] == '.') { + result.push_back(".."); + ++i; + } + + return (result / mod_to_path(mod.substr(i))); +} + +static void pathenv_to_paths(string pathenv, vector &paths) { + istringstream iss(pathenv); + string path; + while (getline(iss, path, TERRA_PATHSEP)) { + paths.emplace_back(path); + } +} + +static int terra_loadmodule(lua_State *L) { + /* + The only parameter that is required is + the first one - the module name. It must + also not be empty. + + The second parameter - the origin file - is + only required if the module is a relative + path. + + The third parameter - the original require() + function - is only used if passed and provides + a fallback mechanism for requiring files. + */ + string mod = luaL_checkstring(L, 1); + string origin = lua_isstring(L, 2) ? lua_tostring(L, 2) : ""; + stringstream reason; + char *terrapathenv; + vector terrapaths; + filesystem::path modpath; + filesystem::path trypath; + + if (mod.empty()) { + luaL_error(L, "module path cannot be empty"); + return 0; /* not hit */ + } + + // Fail fast: valid module paths are only [.a-z0-9_-]i + for (int i = 0, len = mod.length(); i < len; i++) { + char c = mod[i]; + if (!( + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '.' || c == '_' || c == '-')) + { + reason << "\n\tnot a valid module string (contains invalid characters)"; + goto fallback_loader; + } + } + + if (mod[0] == '.') { + if (origin.empty()) { + reason << "\n\tmodule looked like a relative path but no origin was provided"; + goto fallback_loader; + } + + modpath = mod_to_relpath(mod); + + trypath = (filesystem::path(origin) / modpath.with_extension(".t")).resolve(false); + if (trypath.exists()) { + modpath = trypath; + goto resolved; + } + reason << "\n\tno terra module '" << trypath << "'"; + + trypath = (filesystem::path(origin) / modpath / "init.t").resolve(false); + if (trypath.exists()) { + modpath = trypath; + goto resolved; + } + reason << "\n\tno terra module '" << trypath << "'"; + } else { + // get the path + terrapathenv = getenv("TERRA_MODPATH"); + + if (terrapathenv == NULL) { + reason << "\n\tTERRA_MODPATH is empty"; + goto fallback_loader; + } + + modpath = mod_to_path(mod); + pathenv_to_paths(terrapathenv, terrapaths); + + for (const auto &path : terrapaths) { + trypath = (path / modpath).with_extension(".t"); + if (trypath.exists()) { + modpath = trypath; + goto resolved; + } + reason << "\n\tno terra module '" << trypath << "'"; + + trypath = path / modpath / "init.t"; + if (trypath.exists()) { + modpath = trypath; + goto resolved; + } + reason << "\n\tno terra module '" << trypath << "'"; + } + } + +fallback_loader: + if (lua_isfunction(L, 3)) { + // try to load it + lua_pushvalue(L, 3); + lua_pushvalue(L, 1); + if (lua_pcall(L, 1, 1, 0) != 0) { + // better error message opportunity here + luaL_error(L, "terra module '%s' not found:%s\ndefault loader also failed: %s", + lua_tostring(L, 1), + reason.str().c_str(), + lua_tostring(L, -1)); + return 0; /* not hit */ + } + + return 1; + } + + luaL_error(L, "could not find module '%s'"); + return 0; /* not hit */ + +resolved: + if (terra_loadfile(L, modpath.str().c_str())) { + luaL_error(L, "terra module '%s' could not be opened for reading", + modpath.str().c_str()); + return 0; /* not hit */ + } + + lua_call(L, 0, 1); + return 1; +} + static int try_module_load(lua_State *L) { topcheck(L, 1); lua_getglobal(L, "terralib"); @@ -423,7 +582,7 @@ static int try_module_load(lua_State *L) { lua_pop(L, 1); disarmtopcheck(); luaL_error(L, "terralib.loadmodule is not a function"); - return 0; + return 0; /* not hit */ } lua_pushvalue(L, 1); // the module being require()'d lua_pushvalue(L, lua_upvalueindex(1)); // the origin filename @@ -477,12 +636,19 @@ static void inject_require(lua_State *L) { lua_getglobal(L, "_G"); lua_getmetatable(L, -1); lua_getfield(L, -1, "__index"); - lua_getfield(L, -2, "require"); + lua_getfield(L, -3, "require"); + assert(!lua_isnil(L, -1)); lua_pushcclosure(L, &make_module_loader, 2); lua_setfield(L, -2, "__index"); lua_pushnil(L); lua_setfield(L, -3, "require"); lua_pop(L, 2); + + // install the default module loader + lua_getglobal(L, "terralib"); + lua_pushcfunction(L, &terra_loadmodule); + lua_setfield(L, -2, "loadmodule"); + lua_pop(L, 1); } /* @@ -839,6 +1005,7 @@ int main(int argc, const char **argv) { const char *err = nullptr; int extrac = 0; const char **extrav = nullptr; + string absfilename; config conf; conf.help = false; @@ -882,6 +1049,12 @@ int main(int argc, const char **argv) { conf.filename = extrav[0]; + // get absolute filename of input + // this is important since the module loader has to + // be able to navigate the filesystem efficiently. + absfilename = filesystem::path(conf.filename).resolve().str(); + conf.filename = absfilename.c_str(); + error: if (err) { cerr << "terrac: error: " << err << endl; From 78013f858f77a2a1a81c5ebd355699bf5730451b Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 09:45:30 +0100 Subject: [PATCH 3/8] use terralib.modpath for module path storage --- main.cc | 63 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/main.cc b/main.cc index 10eee1b..2cb7f63 100644 --- a/main.cc +++ b/main.cc @@ -23,6 +23,10 @@ using namespace std; +const char * const DEFAULT_MODPATH = + "/usr/share/terra/modules" + ":/usr/local/share/terra/modules"; + // https://github.com/stevedonovan/Penlight/blob/d90956418cdf9e315719a7e4ce314db6b512ae00/lua/pl/compat.lua#L145-L155 // MIT License // TODO convert to C @@ -51,6 +55,8 @@ struct config { unique_ptr> include_dirs; unique_ptr> lib_dirs; unique_ptr> libs; + unique_ptr> modulepaths; + bool nosysmods; }; static void on_verbose(const char *v, void *data, const struct xoptOption *option, bool longArg, const char **err) { @@ -121,6 +127,24 @@ static xoptOption options[] = { "name", "Specifies a library to be linked against the resulting binary" }, + { + "mod-dir", + 'm', + offsetof(config, modulepaths), + &on_multi_path, + XOPT_TYPE_STRING, + "dir", + "Adds a module search path" + }, + { + "nosysmods", + 0, + offsetof(config, nosysmods), + 0, + XOPT_TYPE_BOOL, + 0, + "If specified, default system module paths are omitted from the modpath" + }, { "depfile", 'D', @@ -468,7 +492,7 @@ static int terra_loadmodule(lua_State *L) { string mod = luaL_checkstring(L, 1); string origin = lua_isstring(L, 2) ? lua_tostring(L, 2) : ""; stringstream reason; - char *terrapathenv; + string terramodpath; vector terrapaths; filesystem::path modpath; filesystem::path trypath; @@ -515,15 +539,18 @@ static int terra_loadmodule(lua_State *L) { reason << "\n\tno terra module '" << trypath << "'"; } else { // get the path - terrapathenv = getenv("TERRA_MODPATH"); + lua_getglobal(L, "terralib"); + lua_getfield(L, -1, "modpath"); + terramodpath = lua_isnil(L, -1) ? "" : lua_tostring(L, -1); + lua_pop(L, 2); - if (terrapathenv == NULL) { + if (terramodpath.empty()) { reason << "\n\tTERRA_MODPATH is empty"; goto fallback_loader; } modpath = mod_to_path(mod); - pathenv_to_paths(terrapathenv, terrapaths); + pathenv_to_paths(terramodpath, terrapaths); for (const auto &path : terrapaths) { trypath = (path / modpath).with_extension(".t"); @@ -863,6 +890,24 @@ int pmain(config &conf) { lua_pop(L, 2); } + // inject module path + { + topcheck(L); + string modpath = conf.nosysmods ? "" : DEFAULT_MODPATH; + char *terramodpath = getenv("TERRA_MODPATH"); + if (terramodpath != NULL) { + if (conf.nosysmods) { + modpath = string(terramodpath); + } else { + modpath += TERRA_PATHSEP + string(terramodpath); + } + } + lua_getglobal(L, "terralib"); + lua_pushstring(L, modpath.c_str()); + lua_setfield(L, -2, "modpath"); + lua_pop(L, 1); + } + // inject module loader { topcheck(L); @@ -936,11 +981,11 @@ int pmain(config &conf) { lua_getfield(L, -1, "saveobj"); assert(!lua_isnil(L, -1)); - lua_pushstring(L, conf.output); // 1 - lua_pushvalue(L, -4); // 2 - if (!get_link_flags(L)) return 1; // 3 - lua_pushnil(L); // 4 - lua_pushboolean(L, (int) !conf.debug); // 5 + lua_pushstring(L, conf.output); + lua_pushvalue(L, -4); + if (!get_link_flags(L)) return 1; + lua_pushnil(L); + lua_pushboolean(L, (int) !conf.debug); if (conf.verbosity > 0) { cerr << "terrac: exporting public symbols to " << conf.output << endl; From 5c89bf236de905407321579a79c8ffa1c36c38e8 Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 20:16:58 +0100 Subject: [PATCH 4/8] correct error message regarding empty terralib.modpath --- main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.cc b/main.cc index 2cb7f63..6ad2bd3 100644 --- a/main.cc +++ b/main.cc @@ -545,7 +545,7 @@ static int terra_loadmodule(lua_State *L) { lua_pop(L, 2); if (terramodpath.empty()) { - reason << "\n\tTERRA_MODPATH is empty"; + reason << "\n\tterralib.modpath is empty"; goto fallback_loader; } From 51ee03fdd6ca87d1321dbfda42cb71fece6e0e3f Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 20:23:07 +0100 Subject: [PATCH 5/8] initialize module option fields --- main.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.cc b/main.cc index 6ad2bd3..4a8a53c 100644 --- a/main.cc +++ b/main.cc @@ -1064,6 +1064,8 @@ int main(int argc, const char **argv) { conf.lib_dirs.reset(new list()); conf.libs.reset(new list()); conf.depfiles.reset(new list()); + conf.modulepaths.reset(new list()); + conf.nosysmods = false; XOPT_SIMPLE_PARSE( argv[0], From 857101a5b1ec66f91d7a2451c89613aa5b267169 Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 20:32:13 +0100 Subject: [PATCH 6/8] fix modpath initializers --- main.cc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/main.cc b/main.cc index 4a8a53c..5b50365 100644 --- a/main.cc +++ b/main.cc @@ -137,7 +137,7 @@ static xoptOption options[] = { "Adds a module search path" }, { - "nosysmods", + "nostdmod", 0, offsetof(config, nosysmods), 0, @@ -893,17 +893,25 @@ int pmain(config &conf) { // inject module path { topcheck(L); - string modpath = conf.nosysmods ? "" : DEFAULT_MODPATH; + stringstream modpath; + char *terramodpath = getenv("TERRA_MODPATH"); if (terramodpath != NULL) { - if (conf.nosysmods) { - modpath = string(terramodpath); - } else { - modpath += TERRA_PATHSEP + string(terramodpath); - } + modpath << TERRA_PATHSEP << terramodpath; + } + + for (const auto &mpath : *conf.modulepaths) { + modpath << TERRA_PATHSEP << mpath; } + + if (!conf.nosysmods) { + modpath << TERRA_PATHSEP << DEFAULT_MODPATH; + } + + string modpaths = modpath.str().substr(1); + lua_getglobal(L, "terralib"); - lua_pushstring(L, modpath.c_str()); + lua_pushstring(L, modpaths.c_str()); lua_setfield(L, -2, "modpath"); lua_pop(L, 1); } From fcf4d29d05990f280c2521f3ce3632e5db698726 Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Fri, 15 Feb 2019 21:23:43 +0100 Subject: [PATCH 7/8] emit terra module paths as depfiles --- main.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/main.cc b/main.cc index 5b50365..d748dd2 100644 --- a/main.cc +++ b/main.cc @@ -489,6 +489,8 @@ static int terra_loadmodule(lua_State *L) { function - is only used if passed and provides a fallback mechanism for requiring files. */ + config &conf = *((config *)lua_touserdata(L, lua_upvalueindex(1))); + string mod = luaL_checkstring(L, 1); string origin = lua_isstring(L, 2) ? lua_tostring(L, 2) : ""; stringstream reason; @@ -583,6 +585,8 @@ static int terra_loadmodule(lua_State *L) { return 0; /* not hit */ } + // success! + conf.depfiles->push_back(lua_tostring(L, 1)); return 1; } @@ -597,6 +601,9 @@ static int terra_loadmodule(lua_State *L) { } lua_call(L, 0, 1); + + // success! + conf.depfiles->push_back(modpath.str()); return 1; } @@ -652,7 +659,7 @@ static int make_module_loader(lua_State *L) { return 1; } -static void inject_require(lua_State *L) { +static void inject_mod_loader(lua_State *L, config &conf) { topcheck(L); // let's tango. @@ -673,7 +680,8 @@ static void inject_require(lua_State *L) { // install the default module loader lua_getglobal(L, "terralib"); - lua_pushcfunction(L, &terra_loadmodule); + lua_pushlightuserdata(L, &conf); + lua_pushcclosure(L, &terra_loadmodule, 1); lua_setfield(L, -2, "loadmodule"); lua_pop(L, 1); } @@ -919,7 +927,7 @@ int pmain(config &conf) { // inject module loader { topcheck(L); - inject_require(L); + inject_mod_loader(L, conf); } // create terrac object From e48f92de2f90bc677845e699eba56a6886300a31 Mon Sep 17 00:00:00 2001 From: Josh Junon Date: Sat, 16 Feb 2019 05:59:29 +0100 Subject: [PATCH 8/8] add error for failed terra module loads --- main.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.cc b/main.cc index d748dd2..e14f694 100644 --- a/main.cc +++ b/main.cc @@ -595,8 +595,7 @@ static int terra_loadmodule(lua_State *L) { resolved: if (terra_loadfile(L, modpath.str().c_str())) { - luaL_error(L, "terra module '%s' could not be opened for reading", - modpath.str().c_str()); + lua_error(L); return 0; /* not hit */ }