From da363fcb88c117c89e3efbc04ed92762602b8b71 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 19 Sep 2023 19:09:58 +0200 Subject: [PATCH] try to fix freetype init for full dynamic wheel --- packages.d/pygame/pygame.sh | 24 +++++ pygbag/__main__.py | 2 +- pygbag/support/cross/aio/pep0723.py | 50 ++++++---- pygbag/support/pythonrc.py | 150 +++++++++++++++------------- static/pythons.js | 115 ++++++++++----------- 5 files changed, 186 insertions(+), 155 deletions(-) diff --git a/packages.d/pygame/pygame.sh b/packages.d/pygame/pygame.sh index e63533f..ace2520 100755 --- a/packages.d/pygame/pygame.sh +++ b/packages.d/pygame/pygame.sh @@ -73,6 +73,30 @@ then #unsure wget -O- https://patch-diff.githubusercontent.com/raw/pmp-p/pygame-ce-wasm/pull/3.diff | patch -p1 + patch -p1 << END +diff --git a/src_c/static.c b/src_c/static.c +index 03cc7c61..a00a51a7 100644 +--- a/src_c/static.c ++++ b/src_c/static.c +@@ -255,9 +255,17 @@ static struct PyModuleDef mod_pygame_static = {PyModuleDef_HEAD_INIT, + "pygame_static", NULL, -1, + mod_pygame_static_methods}; + ++#include ++ + PyMODINIT_FUNC + PyInit_pygame_static() + { ++ { ++ if (TTF_Init()) ++ fprintf(stderr, "ERROR: TTF_Init error"); ++ SDL_SetHint("SDL_EMSCRIPTEN_KEYBOARD_ELEMENT", "1"); ++ } ++ + load_submodule("pygame", PyInit_base(), "base"); + load_submodule("pygame", PyInit_constants(), "constants"); + load_submodule("pygame", PyInit_surflock(), "surflock"); +END # cython3 / merged # wget -O- https://patch-diff.githubusercontent.com/raw/pygame-community/pygame-ce/pull/2395.diff | patch -p1 diff --git a/pygbag/__main__.py b/pygbag/__main__.py index 1d8a4a1..c5564bb 100644 --- a/pygbag/__main__.py +++ b/pygbag/__main__.py @@ -186,7 +186,7 @@ async def async_imports(cls, callback, *wanted, **kw): ... @classmethod - def list_imports(cls, code=None, file=None): + def list_imports(cls, code=None, file=None, hint=""): return [] def eval(self, source): diff --git a/pygbag/support/cross/aio/pep0723.py b/pygbag/support/cross/aio/pep0723.py index b2cab77..ab0415c 100644 --- a/pygbag/support/cross/aio/pep0723.py +++ b/pygbag/support/cross/aio/pep0723.py @@ -204,30 +204,9 @@ async def pip_install(pkg, sconf={}): print("ERROR", wheel_url) - -async def check_list(code=None, filename=None): - print() - print("-" * 11, "computing required packages", "-" * 10) - - # store installed wheel somewhere - env = Path(os.getcwd()) / "build" / "env" - env.mkdir(parents=True, exist_ok=True) - - # we want host to load wasm packages too - # so make pure/bin folder first for imports - sys.path.insert(0, env.as_posix()) - - sconf = __import__("sysconfig").get_paths() - sconf["purelib"] = sconf["platlib"] = env.as_posix() - - # mandatory - importlib.invalidate_caches() - +async def parse_code(code, env): maybe_missing = [] - if code is None: - code = open(filename, "r").read() - if Config.READ_722: for req in read_dependency_block_722(code): pkg = str(req) @@ -256,6 +235,33 @@ async def check_list(code=None, filename=None): else: print("found in path :", dep) + return still_missing + + +async def check_list(code=None, filename=None): + print() + print("-" * 11, "computing required packages", "-" * 10) + + # store installed wheel somewhere + env = Path(os.getcwd()) / "build" / "env" + env.mkdir(parents=True, exist_ok=True) + + # we want host to load wasm packages too + # so make pure/bin folder first for imports + sys.path.insert(0, env.as_posix()) + + sconf = __import__("sysconfig").get_paths() + sconf["purelib"] = sconf["platlib"] = env.as_posix() + + # mandatory + importlib.invalidate_caches() + + + if code is None: + code = open(filename, "r").read() + + still_missing = await parse_code(code, env) + # nothing to do if not len(still_missing): return diff --git a/pygbag/support/pythonrc.py b/pygbag/support/pythonrc.py index d576809..42d9078 100644 --- a/pygbag/support/pythonrc.py +++ b/pygbag/support/pythonrc.py @@ -611,42 +611,31 @@ async def perf_index(): else: print(f"last frame : {aio.spent / 0.016666666666666666:.4f}") - # cannot deal with HTML trail properly - # @classmethod - # async def preload_file(cls, main, callback=None): - # # get a relevant list of modules likely to be imported - # # and prefetch them if found in repo trees - # imports = TopLevel_async_handler.list_imports(code=None, file=main) - # print(f"579: missing imports: {list(imports)}") - # await TopLevel_async_handler.async_imports(callback, *imports) - # PyConfig.imports_ready = True - # return True - @classmethod - async def preload_code(cls, code, callback=None): + async def preload_code(cls, code, callback=None, hint=""): # get a relevant list of modules likely to be imported + DBG(f"617: preload_code({len(code)=} {hint=}") + maybe_wanted = list(TopLevel_async_handler.list_imports(code, file=None, hint=hint)) - maybe_wanted = list(TopLevel_async_handler.list_imports(code, file=None)) - if 'numpy' in maybe_wanted: - print(f""" + import aio + import aio.pep0723 -630 : ================== NUMPY ====================== + if not aio.cross.simulator: + # don't use an env path, but site-packages instead + # we can only do purelib for now until pypi host wasm wheels + sconf = __import__("sysconfig").get_paths() + env = Path( sconf["purelib"] ) -{maybe_wanted} + DBG(f"628: aio.pep0723.check_list {env=}") + deps = await aio.pep0723.parse_code(code, env) + DBG(f"629: aio.pep0723.pip_install {deps=}") + else: # sim use a local folder venv model -""") - if 0: - import aio + await aio.pep0723.check_list(code=code, filename=None) - if not aio.cross.simulator: - print("636:","@" * 40) - import aio.pep0723 - - await aio.pep0723.check_list(code=code, filename=None) - print("@" * 40) await TopLevel_async_handler.async_imports(callback, *maybe_wanted) # await TopLevel_async_handler.async_imports(callback, *TopLevel_async_handler.list_imports(code, file=None)) @@ -659,7 +648,7 @@ def interactive(cls, prompt=False): return # if you don't reach that step # your main.py has an infinite sync loop somewhere ! - DBG("642: starting EventTarget in a few seconds") + DBG("651: starting EventTarget in a few seconds") print() TopLevel_async_handler.instance.banner() @@ -676,8 +665,6 @@ def interactive(cls, prompt=False): @classmethod async def runpy(cls, main, *args, **kw): - code = "" - shell.pgzrunning = None def check_code(file_name): nonlocal code @@ -696,16 +683,20 @@ def check_code(file_name): shell.pgzrunning = True if code.find("asyncio.run") < 0: - DBG("622: possibly synchronous code found") + DBG("606: possibly synchronous code found") maybe_sync = True has_pygame = code.find("display.flip(") > 0 or code.find("display.update(") > 0 if maybe_sync and has_pygame: - DBG("628: possibly synchronous+pygame code found") + DBG("694: possibly synchronous+pygame code found") return False return True + code = "" + shell.pgzrunning = None + DBG(f"690: : runpy({main=})") +# REMOVE THAT IT SHOULD BE DONE IN SIM ANALYSER AND HANDLED PROPERLY if not check_code(main): for base in ("pygame", "pg"): for func in ("flip", "update"): @@ -720,18 +711,23 @@ def check_code(file_name): cls.HOME = Path(realpath).parent os.chdir(cls.HOME) +# TODO: should be $0 / sys.argv[0] from there and while running + kw.setdefault('hint', main) + # get requirements await cls.preload_code(code, **kw) + # get an async executor to catch import errors if TopLevel_async_handler.instance: - DBG("646: starting shell") + DBG("715: starting shell") TopLevel_async_handler.instance.start_console(shell) else: - pdb("651: no async handler loader, starting a default async console") + pdb("718: no async handler loader, starting a default async console") shell.debug() await TopLevel_async_handler.start_toplevel(platform.shell, console=True) +# TODO: check if that thing really works if shell.pgzrunning: - print("600 : pygame zero detected") + DBG("728 : pygame zero detected") __main__ = __import__("__main__") sys._pgzrun = True sys.modules["pgzrun"] = type(__main__)("pgzrun") @@ -744,18 +740,19 @@ def check_code(file_name): import pgzero.runner pgzero.runner.prepare_mod(__main__) - + # finally eval async TopLevel_async_handler.instance.eval(code) - + # go back to prompt if not TopLevel_async_handler.muted: print("going interactive") - DBG("643: TODO detect input/print to select repl debug") + DBG("746: TODO detect input/print to select repl debug") cls.interactive() return code @classmethod async def source(cls, main, *args, **kw): + # this is not interactive turn off prompting TopLevel_async_handler.muted = True try: return await cls.runpy(main, *args, **kw) @@ -1052,7 +1049,7 @@ def eval(self, source): DBG(f"1039: {count} lines queued for async eval") @classmethod - def scan_imports(cls, code, filename, load_try=False): + def scan_imports(cls, code, filename, load_try=False, hint=""): required = [] try: root = ast.parse(code, filename) @@ -1100,11 +1097,12 @@ def scan_imports(cls, code, filename, load_try=False): if not mod in required: required.append(mod) - DBG(f"1095: import scan {filename=} {len(code)=} {required}") + DBG(f"1095: scan_imports {hint=} {filename=} {len(code)=} {required}") return required @classmethod - def list_imports(cls, code=None, file=None): + def list_imports(cls, code=None, file=None, hint=""): + DBG(f"1103: list_imports {len(code)=} {file=} {hint=}") if code is None: if file: with open(file) as fcode: @@ -1114,29 +1112,28 @@ def list_imports(cls, code=None, file=None): file = file or "" - for want in cls.scan_imports(code, file): - # DBG(f"1171: requesting module {want=} for {file=} ") + for want in cls.scan_imports(code, file, hint=hint): + # DBG(f"1114: requesting module {want=} for {file=} ") repo = None for repo in PyConfig.pkg_repolist: if want in cls.may_need: - DBG(f"1113: skip module {want=} reason: already requested") + DBG(f"1118: skip module {want=} reason: already requested") break if want in sys.modules: - DBG(f"1117: skip module {want=} reason: sys.modules") + DBG(f"1122: skip module {want=} reason: sys.modules") break if want in repo: cls.may_need.append(want) - # DBG(f"1184: module {want=} requested") + # DBG(f"1127: module {want=} requested") yield want break else: if repo: - DBG(f"1063: {repo['-CDN-']=} does not provide {want=}") + DBG(f"1132: {repo['-CDN-']=} does not provide {want=}") else: - pdb(f"1081: no pkg repository available") - + pdb("1134: no pkg repository available") # TODO: re order repo on failures @@ -1322,7 +1319,6 @@ async def import_now(mod): __import__(mod) - # always put numpy first await import_now('numpy') @@ -1726,49 +1722,60 @@ def log(*argv, **kw): async def import_site(__file__, run=True): global LOCK if LOCK: - print("1742: import_site IS NOT RE ENTRANT") + print("1728: import_site IS NOT RE ENTRANT") return - LOCK = True - from pathlib import Path - - embed = False - hint = "main.py" + try: + LOCK = True + from pathlib import Path - is_py = sys.argv[0].endswith(".py") + embed = False + hint = "main.py" - # if not imported by simulator then aio is handled externally - if "pygbag.aio" not in sys.modules: - import aio - sys.modules["pygbag.aio"] = aio + is_py = sys.argv[0].endswith(".py") + # if not imported by simulator then aio is handled externally + if "pygbag.aio" not in sys.modules: + import aio + sys.modules["pygbag.aio"] = aio - TopLevel_async_handler.mute_state = '.py' in ''.join(sys.argv) + # if running a script be silent for prompt + TopLevel_async_handler.mute_state = '.py' in ''.join(sys.argv) - try: - # always start async handler or we could not do imports. + # always start async handler or we could not do imports on import errors. await TopLevel_async_handler.start_toplevel(platform.shell, console=True) + + # RUNNING GIVEN DISK FILE with no prompt + # this is usually the import site given by javascript loader or a template loader (pygbag apk mode) + # or the user script (script mode). + if Path(__file__).is_file(): - DBG(f"1762: running {__file__=}") - TopLevel_async_handler.muted = True + DBG(f"1755: shell.source({__file__=})") await shell.source(__file__) - # allow to set user site customization + # allow to set user site customization network, or embedded js to be processed await asyncio.sleep(0) + if PyConfig.user_site_directory: DBG(f"1768: {__file__=} done, giving hand to user_site") return __file__ else: - DBG(f"1772: {__file__=} done : now trying remote sources") + DBG(f"1764: {__file__=} done : now trying user sources") else: - DBG(f"1774: {__file__=} NOT FOUND : now trying remote sources") + DBG(f"1767: {__file__=} NOT FOUND : now trying user sources") + + + # NOW CHECK OTHER SOURCES # where to retrieve import tempfile tmpdir = Path(tempfile.gettempdir()) + + # maybe a script filename or content passed as frozen config. + source = getattr(PyConfig, "frozen", "") if source: if Path(source).is_file(): @@ -1803,8 +1810,9 @@ async def main(): file.write(handle.read()) embed = True else: - print(f"1639: invalid embed {source=}") + print(f"1814: invalid embed {source=}") return None + # file has been retrieved stored in local else: local = None @@ -1829,7 +1837,7 @@ async def main(): # TODO: test tar.bz2 lzma tar.xz elif ext in ("zip", "gz", "tar", "apk", "jar"): - DBG(f"1664: found archive source {source=}") + DBG(f"1841: found archive source {source=}") # download and unpack into tmpdir fname = tmpdir / source.rsplit("/")[-1] @@ -1850,7 +1858,7 @@ async def main(): if file.find(hint) > 0: local = tmpdir / file break - DBG("1725: import_site: found ", local) + DBG("1862: import_site: found ", local) else: # maybe base64 or frozen code in html. ... diff --git a/static/pythons.js b/static/pythons.js index b7a82d7..de4aa3e 100644 --- a/static/pythons.js +++ b/static/pythons.js @@ -1,5 +1,26 @@ "use strict"; +/* BF2 is still broken see https://github.com/jvilk/BrowserFS/issues/325 +import { configure, BFSRequire, EmscriptenFS } from './browserfs.mjs'; +//import { Buffer } from 'buffer'; + +window.BrowserFS = {} +window.BrowserFS.configure = configure +window.BrowserFS.BFSRequire = BFSRequire +window.BrowserFS.EmscriptenFS = EmscriptenFS +window.BrowserFS.Buffer = BFSRequire('buffer') +*/ +var bfs2 = false + +async function import_browserfs() { + console.warn("late import", config.cdn+"browserfs.min.js" ) + var script = document.createElement("script") + script.src = vm.config.cdn + "browserfs.min.js" + document.head.appendChild(script) + await _until(defined)("BrowserFS") +} + + /* Facilities implemented in js js.SVG : convert svg to png @@ -268,14 +289,14 @@ function prerun(VM) { VM.FS = FS - +/* if (window.BrowserFS) { VM.BFS = new BrowserFS.EmscriptenFS() VM.BFS.Buffer = BrowserFS.BFSRequire('buffer').Buffer } else { - console.error("VM.prefun","BrowserFS not found") + console.error("VM.prerun","BrowserFS not found") } - +*/ const sixel_prefix = String.fromCharCode(27)+"Pq" @@ -845,22 +866,15 @@ console.log(" @@@@@@@@@@@@@@@@@@@@@@ 3D CANVAS @@@@@@@@@@@@@@@@@@@@@@") - - // file transfer (upload) async function feat_fs(debug_hidden) { var uploaded_file_count = 0 if (!window.BrowserFS) { - console.warn("late import", config.cdn+"browserfs.min.js" ) - var script = document.createElement("script") - script.src = vm.config.cdn + "browserfs.min.js" - document.head.appendChild(script) - await _until(defined)("BrowserFS") + await import_browserfs() } - function readFileAsArrayBuffer(file, success, error) { var fr = new FileReader(); fr.addEventListener('error', error, false); @@ -1102,6 +1116,8 @@ async function media_prepare(trackid) { if (track.type === "mount") { if (!vm.BFS) { + await import_browserfs() + // how is passed the FS object ??? vm.BFS = new BrowserFS.EmscriptenFS() // {FS:vm.FS} @@ -1115,66 +1131,21 @@ async function media_prepare(trackid) { const hint = `${track.mount.path}@${track.mount.point}:${trackid}` - var track_media - - if (!vm) { - vm={} - + if (!vm.BFS) { // how is passed the FS object ??? vm.BFS = new BrowserFS.EmscriptenFS() // {FS:vm.FS} - vm.BFS.Buffer = BrowserFS.BFSRequire('buffer').Buffer - const data = await (await fetch('test.apk')).arrayBuffer() - track_media = await vm.BFS.Buffer.from(data) - } else { - track_media = MM[trackid].media } - var bfs2 = false - if (!BrowserFS.InMemory) { + const track_media = MM[trackid].media + + if (!bfs2) { console.warn(" ==================== BFS1 ===============") BrowserFS.InMemory = BrowserFS.FileSystem.InMemory BrowserFS.OverlayFS = BrowserFS.FileSystem.OverlayFS BrowserFS.MountableFileSystem = BrowserFS.FileSystem.MountableFileSystem BrowserFS.ZipFS = BrowserFS.FileSystem.ZipFS - } else { - console.warn(" ==================== BFS2 ===============") - bfs2 = true - } - - if (bfs2) { - const zipfs = await BrowserFS.ZipFS.Create({ - zipData: track_media, - "name": hint - }) - await BrowserFS.initialize( zipfs ) - - const memfs = await BrowserFS.InMemory.Create(); - await BrowserFS.initialize( memfs) - - - const ovfs = await BrowserFS.OverlayFS.Create({ - writable : memfs, - readable: zipfs - }) - - await BrowserFS.initialize( ovfs) - - // this alone does not work ? why ?? - // const mfs = await BrowserFS.MountableFileSystem.Create({"/" : ovfs}) - // console.log( mfs ) - - const mfs = await BrowserFS.initialize( await BrowserFS.MountableFileSystem.Create({"/" : ovfs}) ) - - // where is the link beetween (BFSEmscriptenFS)vm.BFS and MountableFileSystem(mfs) ? - // vm.BFS.mount( ???????? ) - - const emfs = await vm.FS.mount(vm.BFS, {root: "/"}, "/data/data/test" ); - - setTimeout(()=>{track.ready=true}, 0) - - } else { function apk_cb(e, apkfs){ console.log(__FILE__, "930 mounting", hint, "onto", track.mount.point) @@ -1195,12 +1166,34 @@ async function media_prepare(trackid) { ); } - //await BrowserFS.FileSystem.ZipFS.Create( await BrowserFS.ZipFS.Create( {"zipData" : track_media, "name": hint}, apk_cb ) + + } else { + console.warn(" ==================== BFS2 ===============") + + // assuming FS is from Emscripten + await BrowserFS.configure({ + fs: 'MountableFileSystem', + options: { + '/': { + fs: 'OverlayFS', + options: { + readable: { fs: 'ZipFS', options: { zipData: track_media, name: 'hint'  } }, + writable: { fs: 'InMemory' } + } + } + } + }); + + vm.FS.mount(vm.BFS, { root: track.mount.path, }, track.mount.point); + } // bfs2 + + setTimeout(()=>{track.ready=true}, 0) + } // track type mount }