From 80db3a39ef2a964eaf36d58e291ae3814b0169c5 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 3 Oct 2023 20:16:56 +0200 Subject: [PATCH] 3.12 + cochonnet basic aio support --- .github/workflows/ci.yml | 23 +- pygbag/__init__.py | 2 +- pygbag/support/cross/__EMSCRIPTEN__.py | 20 +- pygbag/support/cross/aio/__init__.py | 39 +- pygbag/support/cross/aio/cross.py | 54 +-- pygbag/support/cross/aio/filelike.py | 16 +- pygbag/support/cross/aio/toplevel.py | 523 +++++++++++++------------ pygbag/support/pythonrc.py | 65 ++- pygbag/testserver.py | 18 +- scripts/build-loader.sh | 72 ++-- scripts/build-rootfs.sh | 2 +- static/pythons.js | 84 ++-- support/__EMSCRIPTEN__.c | 186 ++++++--- wasm-build-all.sh | 10 +- 14 files changed, 632 insertions(+), 482 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26c95ad..4557194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: build: runs-on: ubuntu-22.04 env: - SDK_VERSION: 3.1.46.2bi + SDK_VERSION: 3.1.47.0bi SYS_PYTHON: /usr/bin/python3 PACKAGES: emsdk hpy _ctypes pygame BUILD_STATIC: emsdk _ctypes hpy @@ -47,16 +47,17 @@ jobs: cd $GITHUB_WORKSPACE PYBUILD=3.11 bash ./scripts/build-loader.sh -# - name: build 3.12 packages -# run: | -# cd $GITHUB_WORKSPACE -# PYBUILD=3.12 bash ./scripts/build-pkg.sh -# -# - name: build 3.12 loader -# run: | -# cd $GITHUB_WORKSPACE -# PYBUILD=3.12 bash ./scripts/build-loader.sh -# + + - name: build 3.12 packages + run: | + cd $GITHUB_WORKSPACE + PYBUILD=3.12 BUILD_STATIC="emsdk hpy" bash ./scripts/build-pkg.sh + + - name: build 3.12 loader + run: | + cd $GITHUB_WORKSPACE + PYBUILD=3.12 BUILD_STATIC="emsdk hpy" bash ./scripts/build-loader.sh + - name: publishing run: | diff --git a/pygbag/__init__.py b/pygbag/__init__.py index 4c21161..53db400 100644 --- a/pygbag/__init__.py +++ b/pygbag/__init__.py @@ -7,7 +7,7 @@ from pathlib import Path -__version__ = "0.8.2" +__version__ = "0.8.3" # hack to test git cdn build without upgrading pygbag # beware can have side effects when file packager behaviour must change ! diff --git a/pygbag/support/cross/__EMSCRIPTEN__.py b/pygbag/support/cross/__EMSCRIPTEN__.py index 78698a0..7449658 100644 --- a/pygbag/support/cross/__EMSCRIPTEN__.py +++ b/pygbag/support/cross/__EMSCRIPTEN__.py @@ -22,8 +22,6 @@ def shed_yield(): return True -sys.breakpointhook = breakpointhook - this = __import__(__name__) # those __dunder__ are usually the same used in C conventions. @@ -31,7 +29,7 @@ def shed_yield(): try: __UPY__ except: - if hasattr(sys.implementation, "mpy"): + if hasattr(sys.implementation, "_mpy"): builtins.__UPY__ = this else: builtins.__UPY__ = None @@ -41,14 +39,17 @@ def shed_yield(): if __UPY__: sys.modules["sys"] = sys sys.modules["builtins"] = builtins - try: - from . import uasyncio as uasyncio - print("Warning : using WAPY uasyncio") - except Exception as e: - sys.print_exception(e) +# try: +# from . import uasyncio as uasyncio +# +# print("Warning : using WAPY uasyncio") +# except Exception as e: +# sys.print_exception(e) else: + sys.breakpointhook = breakpointhook + # fallback to asyncio based implementation try: from . import uasyncio_cpy as uasyncio @@ -56,7 +57,7 @@ def shed_yield(): pdb("INFO: no uasyncio implementation found") uasyncio = aio -sys.modules["uasyncio"] = uasyncio + sys.modules["uasyncio"] = uasyncio # detect if cpython is really running on a emscripten browser @@ -97,6 +98,7 @@ async def jsprom(prom): else: is_browser = False + pdb("101: no emscripten browser interface") builtins.__EMSCRIPTEN__ = None diff --git a/pygbag/support/cross/aio/__init__.py b/pygbag/support/cross/aio/__init__.py index 1275c32..4ff25ad 100644 --- a/pygbag/support/cross/aio/__init__.py +++ b/pygbag/support/cross/aio/__init__.py @@ -1,7 +1,7 @@ import sys import builtins import inspect -from time import time as time_time + DEBUG = True @@ -33,17 +33,25 @@ def pdb(*argv): # cascade debug by default from . import cross -cross.DEBUG = DEBUG +if not __UPY__: + from time import time as time_time + + # file+socket support fopen/sopen + from .filelike import * +else: + import utime + time_time = utime.ticks_ms + +cross.DEBUG = DEBUG -# file+socket support fopen/sopen -from .filelike import * # ========================================================================= # TODO: dbg stuff should be in the platform module in aio.cross # usefull https://pymotw.com/3/sys/tracing.html -if DEBUG: +# upy has no trace module +if DEBUG and not __UPY__: import trace _tracer = trace.Trace(count=False, trace=True) @@ -139,12 +147,17 @@ def overloaded(i, *attrs): spent = 0.00001 leave = enter + spent - from asyncio import * __run__ = run -import asyncio.events as events +if __UPY__: + def _set_running_loop(l):pass + sys.modules['asyncio.events'] = aio + aio.get_running_loop = aio.get_event_loop + events = aio +else: + from asyncio.events import _set_running_loop # Within a coroutine, simply use `asyncio.get_running_loop()`, @@ -157,9 +170,9 @@ def overloaded(i, *attrs): loop = get_event_loop() -import asyncio.events - -asyncio.events._set_running_loop(loop) +#import asyncio.events +#asyncio.events._set_running_loop(loop) +_set_running_loop(loop) sys.modules["asyncio"] = __import__(__name__) @@ -390,7 +403,7 @@ async def __main__(): # fallback to blocking asyncio else: - asyncio.events._set_running_loop(None) + _set_running_loop(None) # TODO: implement RaF from here try: loop.run_forever() @@ -461,7 +474,9 @@ async def atexit(coro): exit_now(maybecoro) -sys.exit = aio_exit +if not __UPY__: + sys.exit = aio_exit + # check if we have a Time handler. try: diff --git a/pygbag/support/cross/aio/cross.py b/pygbag/support/cross/aio/cross.py index 27ec931..23d10cc 100644 --- a/pygbag/support/cross/aio/cross.py +++ b/pygbag/support/cross/aio/cross.py @@ -9,18 +9,39 @@ # platform_impl = False -# that sym cannot be overloaded in a simulator +# ================== leverage known python implementations ==================== -if not defined("__WASM__"): - if __import__("os").uname().machine.startswith("wasm"): - import __WASM__ - else: - __WASM__ = False +# always come down to upy choices because cpython syntax can more easily be adapted - define("__WASM__", __WASM__) +if not defined("__UPY__"): + define("__UPY__", hasattr(sys, "print_exception") ) + + if not __UPY__: + # setup exception display with same syntax as upy + import traceback + + def print_exception(e, out=sys.stderr, **kw): + kw["file"] = out + traceback.print_exc(**kw) -# those can + sys.print_exception = print_exception + del print_exception + + + +if not defined("__WASM__"): + try: + # that sym cannot be overloaded in the simulator + if __import__("os").uname().machine.startswith("wasm"): + import __WASM__ + else: + __WASM__ = False + except AttributeError: + # upy does not have os.uname + __WASM__ = True + + define("__WASM__", __WASM__) if not defined("__wasi__"): @@ -29,19 +50,10 @@ else: __wasi__ = False - # setup exception display with same syntax as upy - import traceback - - def print_exception(e, out=sys.stderr, **kw): - kw["file"] = out - traceback.print_exc(**kw) - - sys.print_exception = print_exception - del print_exception - define("__wasi__", __wasi__) + # this *is* the cpython way if hasattr(sys, "getandroidapilevel"): platform_impl = defined("__ANDROID__") @@ -121,13 +133,7 @@ def print_exception(e, out=sys.stderr, **kw): sys.modules["platform"] = platform_impl -# ================== leverage known python implementations ==================== -# always come down to upy choices because cpython syntax can more easily be adapted - - -if not defined("__UPY__"): - define("__UPY__", hasattr(sys.implementation, "mpy")) if not __UPY__: diff --git a/pygbag/support/cross/aio/filelike.py b/pygbag/support/cross/aio/filelike.py index 76eba2a..2e1d320 100644 --- a/pygbag/support/cross/aio/filelike.py +++ b/pygbag/support/cross/aio/filelike.py @@ -1,10 +1,16 @@ import sys import io -import socket + +if not __UPY__: + import socket + socket.setdefaulttimeout(0.0) +else: + print("7: usocket implementation required") + import os # unlink -socket.setdefaulttimeout(0.0) + import aio import platform @@ -51,7 +57,8 @@ def mktemp(suffix=""): class fopen: - if __WASM__: + if __WASM__ and hasattr(platform, "ffi"): + flags = platform.ffi( { "redirect": "follow", @@ -59,13 +66,14 @@ class fopen: } ) else: + print("69: platform has no object serializer") flags = {} def __init__(self, maybe_url, mode="r", flags=None): self.url = fix_url(maybe_url) self.mode = mode flags = flags or self.__class__.flags - print(f'68: fopen: fetching "{self.url}"') + print(f'76: fopen: fetching "{self.url}"') self.tmpfile = None diff --git a/pygbag/support/cross/aio/toplevel.py b/pygbag/support/cross/aio/toplevel.py index 33dcfa1..c1a9340 100644 --- a/pygbag/support/cross/aio/toplevel.py +++ b/pygbag/support/cross/aio/toplevel.py @@ -6,10 +6,8 @@ import asyncio import ast -import code import types import inspect -import zipfile HISTORY = [] @@ -18,299 +16,306 @@ except: embed = False +if not __UPY__: -def install(pkg_file, sconf=None): - global HISTORY - from installer import install - from installer.destinations import SchemeDictionaryDestination - from installer.sources import WheelFile - - # Handler for installation directories and writing into them. - destination = SchemeDictionaryDestination( - sconf or __import_("sysconfig").get_paths(), - interpreter=sys.executable, - script_kind="posix", - ) - - try: - with WheelFile.open(pkg_file) as source: - install( - source=source, - destination=destination, - # Additional metadata that is generated by the installation tool. - additional_metadata={ - "INSTALLER": b"pygbag", - }, - ) - HISTORY.append(pkg_file) - except FileExistsError: - print(f"47: {pkg_file} already installed") - except Exception as ex: - pdb(f"49: cannot install {pkg_file}") - sys.print_exception(ex) - - -async def get_repo_pkg(pkg_file, pkg, resume, ex): - global HISTORY - - # print("-"*40) - import platform - import json - import sysconfig - import importlib - from pathlib import Path + import code - if not pkg_file in HISTORY: - sconf = sysconfig.get_paths() - # sconf["platlib"] = os.environ.get("HOME","/tmp") - platlib = sconf["platlib"] - Path(platlib).mkdir(exist_ok=True) + def install(pkg_file, sconf=None): + global HISTORY + from installer import install + from installer.destinations import SchemeDictionaryDestination + from installer.sources import WheelFile - if platlib not in sys.path: - sys.path.append(platlib) + # Handler for installation directories and writing into them. + destination = SchemeDictionaryDestination( + sconf or __import_("sysconfig").get_paths(), + interpreter=sys.executable, + script_kind="posix", + ) try: - aio.toplevel.install(pkg_file, sconf) - except Exception as rx: - pdb(f"75: failed to install {pkg_file}") - sys.print_exception(rx) + with WheelFile.open(pkg_file) as source: + install( + source=source, + destination=destination, + # Additional metadata that is generated by the installation tool. + additional_metadata={ + "INSTALLER": b"pygbag", + }, + ) + HISTORY.append(pkg_file) + except FileExistsError: + print(f"47: {pkg_file} already installed") + except Exception as ex: + pdb(f"49: cannot install {pkg_file}") + sys.print_exception(ex) + + + async def get_repo_pkg(pkg_file, pkg, resume, ex): + global HISTORY + + # print("-"*40) + import platform + import json + import sysconfig + import importlib + from pathlib import Path + + if not pkg_file in HISTORY: + sconf = sysconfig.get_paths() + # sconf["platlib"] = os.environ.get("HOME","/tmp") + platlib = sconf["platlib"] + Path(platlib).mkdir(exist_ok=True) + + if platlib not in sys.path: + sys.path.append(platlib) - # let wasm compilation happen - await asyncio.sleep(0) + try: + aio.toplevel.install(pkg_file, sconf) + except Exception as rx: + pdb(f"75: failed to install {pkg_file}") + sys.print_exception(rx) - try: - platform.explore(platlib) + # let wasm compilation happen await asyncio.sleep(0) - importlib.invalidate_caches() - # print(f"{pkg_file} installed, preloading", embed.preloading()) - except Exception as rx: - pdb(f"failed to preload {pkg_file}") - sys.print_exception(rx) - else: - print(f"90: {pkg_file} already installed") - - if pkg in platform.patches: - print("93:", pkg, "requires patching") - platform.patches.pop(pkg)() - - if resume and ex: - try: - if inspect.isawaitable(resume): - print(f"{resume=} is an awaitable") - return resume() - else: - print(f"{resume=} is not awaitable") - resume() - return asyncio.sleep(0) - except Exception as resume_ex: - sys.print_exception(ex, limit=-1) - return None - - -class AsyncInteractiveConsole(code.InteractiveConsole): - instance = None - console = None - # TODO: use PyConfig interactive flag - muted = True - - def __init__(self, locals, **kw): - super().__init__(locals) - self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT - self.line = "" - self.buffer = [] - self.one_liner = True - self.opts = kw - self.shell = self.opts.get("shell", None) - - if self.shell is None: - - class shell: - coro = [] - is_interactive = None - - @classmethod - def parse_sync(shell, line, **env): - print("NoOp shell", line) - - self.shell = shell - self.rv = None - - # need to subclass - # @staticmethod - # def get_pkg(want, ex=None, resume=None): - - def runsource(self, source, filename="", symbol="single"): - if len(self.buffer) > 1: - symbol = "exec" - try: - code = self.compile(source, filename, symbol) - except SyntaxError: - if self.one_liner: - if self.shell.parse_sync(self.line): - return - self.showsyntaxerror(filename) - return False + try: + platform.explore(platlib) + await asyncio.sleep(0) + importlib.invalidate_caches() + # print(f"{pkg_file} installed, preloading", embed.preloading()) + except Exception as rx: + pdb(f"failed to preload {pkg_file}") + sys.print_exception(rx) + else: + print(f"90: {pkg_file} already installed") + + if pkg in platform.patches: + print("93:", pkg, "requires patching") + platform.patches.pop(pkg)() + + if resume and ex: + try: + if inspect.isawaitable(resume): + print(f"{resume=} is an awaitable") + return resume() + else: + print(f"{resume=} is not awaitable") + resume() + return asyncio.sleep(0) + except Exception as resume_ex: + sys.print_exception(ex, limit=-1) + return None + + + class AsyncInteractiveConsole(code.InteractiveConsole): + instance = None + console = None + # TODO: use PyConfig interactive flag + muted = True + + def __init__(self, locals, **kw): + super().__init__(locals) + self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT + self.line = "" + self.buffer = [] + self.one_liner = True + self.opts = kw + self.shell = self.opts.get("shell", None) - except (OverflowError, ValueError): - # Case 1 - self.showsyntaxerror(filename) - return False + if self.shell is None: - if code is None: - # Case 2 - return True + class shell: + coro = [] + is_interactive = None - # Case 3 - self.runcode(code) - return False + @classmethod + def parse_sync(shell, line, **env): + print("NoOp shell", line) - def runcode(self, code): - if embed: - embed.set_ps1() - self.rv = undefined + self.shell = shell + self.rv = None - bc = types.FunctionType(code, self.locals) - try: - self.rv = bc() - except SystemExit: - aio.exit_now(0) + # need to subclass + # @staticmethod + # def get_pkg(want, ex=None, resume=None): + + def runsource(self, source, filename="", symbol="single"): + if len(self.buffer) > 1: + symbol = "exec" - except KeyboardInterrupt as ex: - print(ex, file=sys.__stderr__) - raise - - except ModuleNotFoundError as ex: - print("181: FIXME dependency table for manually built modules") - get_pkg = self.opts.get("get_pkg", self.async_get_pkg) - if get_pkg: - want = str(ex).split("'")[1] - self.shell.coro.append(get_pkg(want, ex, bc)) - - except BaseException as ex: - if self.one_liner: - shell = self.opts.get("shell", None) - if shell: - # coro maybe be filled by shell exec - if shell.parse_sync(self.line): + try: + code = self.compile(source, filename, symbol) + except SyntaxError: + if self.one_liner: + if self.shell.parse_sync(self.line): return - sys.print_exception(ex, limit=-1) + self.showsyntaxerror(filename) + return False - finally: - self.one_liner = True + except (OverflowError, ValueError): + # Case 1 + self.showsyntaxerror(filename) + return False - def banner(self): - if self.muted: - return - cprt = 'Type "help", "copyright", "credits" or "license" for more information.' + if code is None: + # Case 2 + return True - self.write("\nPython %s on %s\n%s\n" % (sys.version, sys.platform, cprt)) + # Case 3 + self.runcode(code) + return False - def prompt(self): - if not self.__class__.muted and self.shell.is_interactive: + def runcode(self, code): if embed: - embed.prompt() + embed.set_ps1() + self.rv = undefined - async def interact(self): - # multiline input clumsy sentinel - last_line = "" + bc = types.FunctionType(code, self.locals) + try: + self.rv = bc() + except SystemExit: + aio.exit_now(0) + + except KeyboardInterrupt as ex: + print(ex, file=sys.__stderr__) + raise + + except ModuleNotFoundError as ex: + print("181: FIXME dependency table for manually built modules") + get_pkg = self.opts.get("get_pkg", self.async_get_pkg) + if get_pkg: + want = str(ex).split("'")[1] + self.shell.coro.append(get_pkg(want, ex, bc)) + + except BaseException as ex: + if self.one_liner: + shell = self.opts.get("shell", None) + if shell: + # coro maybe be filled by shell exec + if shell.parse_sync(self.line): + return + sys.print_exception(ex, limit=-1) + + finally: + self.one_liner = True - try: - sys.ps1 - except AttributeError: - sys.ps1 = ">>> " + def banner(self): + if self.muted: + return + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' - try: - sys.ps2 - except AttributeError: - sys.ps2 = "--- " + self.write("\nPython %s on %s\n%s\n" % (sys.version, sys.platform, cprt)) - prompt = sys.ps1 + def prompt(self): + if not self.__class__.muted and self.shell.is_interactive: + if embed: + embed.prompt() - while not aio.exit: - await asyncio.sleep(0) + async def interact(self): + # multiline input clumsy sentinel + last_line = "" - if aio.exit: - return + try: + sys.ps1 + except AttributeError: + sys.ps1 = ">>> " try: - try: - self.line = await self.raw_input(prompt) - if self.line is None: - continue + sys.ps2 + except AttributeError: + sys.ps2 = "--- " - except EOFError: - self.write("\n") - break - else: - if self.push(self.line): - if self.one_liner: - prompt = sys.ps2 - if embed: - embed.set_ps2() - print("Sorry, multi line input editing is not supported", file=sys.stderr) - self.one_liner = False - self.resetbuffer() - else: + prompt = sys.ps1 + + while not aio.exit: + await asyncio.sleep(0) + + if aio.exit: + return + + try: + try: + self.line = await self.raw_input(prompt) + if self.line is None: continue + + except EOFError: + self.write("\n") + break else: - prompt = sys.ps1 + if self.push(self.line): + if self.one_liner: + prompt = sys.ps2 + if embed: + embed.set_ps2() + print("Sorry, multi line input editing is not supported", file=sys.stderr) + self.one_liner = False + self.resetbuffer() + else: + continue + else: + prompt = sys.ps1 - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - self.one_liner = True + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + self.one_liner = True - if aio.exit: - return + if aio.exit: + return - try: - # if async prepare is required - while len(self.shell.coro): - self.rv = await self.shell.coro.pop(0) - - # if self.rv not in [undefined, None, False, True]: - if inspect.isawaitable(self.rv): - await self.rv - except RuntimeError as re: - if str(re).endswith("awaited coroutine"): - ... - else: - sys.print_exception(ex) + try: + # if async prepare is required + while len(self.shell.coro): + self.rv = await self.shell.coro.pop(0) + + # if self.rv not in [undefined, None, False, True]: + if inspect.isawaitable(self.rv): + await self.rv + except RuntimeError as re: + if str(re).endswith("awaited coroutine"): + ... + else: + sys.print_exception(ex) - except Exception as ex: - print(type(self.rv), self.rv) - sys.print_exception(ex) + except Exception as ex: + print(type(self.rv), self.rv) + sys.print_exception(ex) - self.prompt() + self.prompt() - aio.exit_now(0) + aio.exit_now(0) - @classmethod - def make_instance(cls, shell, ns="__main__"): - cls.instance = cls( - vars(__import__(ns)), - shell=shell, - ) - shell.runner = cls.instance - del AsyncInteractiveConsole.make_instance - - @classmethod - def start_console(cls, shell, ns="__main__"): - """will only start a console, not async import system""" - if cls.instance is None: - cls.make_instance(shell, ns) - - if cls.console is None: - asyncio.create_task(cls.instance.interact()) - cls.console = cls.instance - - @classmethod - async def start_toplevel(cls, shell, console=True, ns="__main__"): - """start async import system with optionnal async console""" - if cls.instance is None: - cls.make_instance(shell, ns) - await cls.instance.async_repos() - - if console: - cls.start_console(shell, ns=ns) + @classmethod + def make_instance(cls, shell, ns="__main__"): + cls.instance = cls( + vars(__import__(ns)), + shell=shell, + ) + shell.runner = cls.instance + del AsyncInteractiveConsole.make_instance + + @classmethod + def start_console(cls, shell, ns="__main__"): + """will only start a console, not async import system""" + if cls.instance is None: + cls.make_instance(shell, ns) + + if cls.console is None: + asyncio.create_task(cls.instance.interact()) + cls.console = cls.instance + + @classmethod + async def start_toplevel(cls, shell, console=True, ns="__main__"): + """start async import system with optionnal async console""" + if cls.instance is None: + cls.make_instance(shell, ns) + await cls.instance.async_repos() + + if console: + cls.start_console(shell, ns=ns) + +else: + class AsyncInteractiveConsole: + ... diff --git a/pygbag/support/pythonrc.py b/pygbag/support/pythonrc.py index e40cfc1..22dd2ac 100644 --- a/pygbag/support/pythonrc.py +++ b/pygbag/support/pythonrc.py @@ -195,7 +195,9 @@ def dump_code(): PyConfig["pkg_repolist"] = [] aio.cross.simulator = False - sys.argv = PyConfig.pop("argv", []) + sys.argv.clear() + sys.argv.extend( PyConfig.pop("argv", []) ) + except Exception as e: sys.print_exception(e) @@ -203,7 +205,13 @@ def dump_code(): PyConfig = {} PyConfig["dev_mode"] = 1 PyConfig["run_filename"] = "main.py" - PyConfig["executable"] = sys.executable + +# TODO: use location of python js module. + if __UPY__: + PyConfig["executable"] = "upy" + else: + PyConfig["executable"] = sys.executable + PyConfig["interactive"] = 1 print(" - running in wasm simulator - ") aio.cross.simulator = True @@ -436,8 +444,9 @@ def spawn(cls, cmd, *argv, **env): print("a program is already running, using 'stop' cmd before retrying") cls.stop() cls.pgzrunning = None - aio.defer(cls.spawn, (cmd, *argv), env, delay=500) - + args = [cmd] + args.extend(argv) + aio.defer(cls.spawn, args, env, delay=500) else: execfile(cmd) return True @@ -826,12 +835,29 @@ async def exec(cls, sub, **env): await sub -PyConfig["shell"] = shell builtins.shell = shell # end shell -from types import SimpleNamespace + +if __UPY__: + import types + class SimpleNamespace: + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + + def __repr__(self): + keys = sorted(self.__dict__) + items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) + return "{}({})".format(type(self).__name__, ", ".join(items)) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + types.SimpleNamespace = SimpleNamespace +else: + from types import SimpleNamespace + import builtins builtins.PyConfig = SimpleNamespace(**PyConfig) @@ -866,7 +892,7 @@ def fix_url(maybe_url): url = "http:/" + url[5:] return url - __EMSCRIPTEN__.fix_url = fix_url + platform.fix_url = fix_url del fix_url @@ -978,11 +1004,16 @@ def File(path): import os # set correct umask ( emscripten default is 0 ) - os.umask(0o022) # already done in aio.toplevel + if hasattr(os, "umask"): + os.umask(0o022) # already done in aio.toplevel + import zipfile + else: + pdb("1010: missing os.umask") + pdb("1011: missing zipfile") + - import zipfile import aio.toplevel - import ast + #import ast from pathlib import Path class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): @@ -1049,6 +1080,7 @@ def eval(self, source): @classmethod def scan_imports(cls, code, filename, load_try=False, hint=""): + import ast required = [] try: root = ast.parse(code, filename) @@ -1425,13 +1457,14 @@ def patch(): global COLS, LINES, CONSOLE import platform - # DeprecationWarning: Using or importing the ABCs from 'collections' - # instead of from 'collections.abc' is deprecated since Python 3.3 - # and in 3.10 it will stop working - import collections - from collections.abc import MutableMapping + if not __UPY__: + # DeprecationWarning: Using or importing the ABCs from 'collections' + # instead of from 'collections.abc' is deprecated since Python 3.3 + # and in 3.10 it will stop working + import collections + from collections.abc import MutableMapping - collections.MutableMapping = MutableMapping + collections.MutableMapping = MutableMapping # could use that ? # import _sqlite3 diff --git a/pygbag/testserver.py b/pygbag/testserver.py index 1e7e3e0..4e93f67 100644 --- a/pygbag/testserver.py +++ b/pygbag/testserver.py @@ -58,11 +58,12 @@ def end_headers(self): self.send_header("cross-origin-resource-policy:", "cross-origin") self.send_header("cross-origin-opener-policy", "cross-origin") - # buggy for https://pygame-web.github.io/archives/repo/index.json + # not valid for Atomics # self.send_header("cross-origin-embedder-policy", "unsafe-none") - # at least raise - # net::ERR_BLOCKED_BY_RESPONSE.NotSameOriginAfterDefaultedToSameOriginByCoep 200 + # not -always- valid for Atomics (firefox) + # self.send_header("cross-origin-embedder-policy", "credentialless") + self.send_header("cross-origin-embedder-policy", "require-corp") super().end_headers() @@ -83,6 +84,10 @@ def do_HEAD(self): def send_head(self): global VERB, CDN, PROXY, BCDN, BPROXY, AUTO_REBUILD path = self.translate_path(self.path) + print(f""" + +{self.path=} {path=}""") + f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) @@ -155,14 +160,13 @@ def send_head(self): else: cached = False - if self.path.endswith(".apk"): + if path.endswith(".apk"): if AUTO_REBUILD: print() - print(self.path) AUTO_REBUILD() print() else: - print(f"{AUTO_REBUILD=} {self.path}") + print(f"{AUTO_REBUILD=} {path}") if f is None: try: @@ -245,8 +249,6 @@ def send_head(self): f = io.BytesIO(content) self.send_header("content-length", str(file_size)) - # self.send_header("Access-Control-Allow-Origin", "*") - # self.send_header("Cross-Origin-Embedder-Policy", "require-corp") if not cached: self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) diff --git a/scripts/build-loader.sh b/scripts/build-loader.sh index 7eea166..5b52a78 100755 --- a/scripts/build-loader.sh +++ b/scripts/build-loader.sh @@ -12,7 +12,7 @@ cp -rf ${SDKROOT}/prebuilt/emsdk/common/* ${SDKROOT}/prebuilt/emsdk/${PYBUILD}/ # pre populated site-packages -export REQUIREMENTS=$(realpath ${SDKROOT}/prebuilt/emsdk/${PYBUILD}/site-packages) +export REQUIREMENTS=${SDKROOT}/prebuilt/emsdk/${PYBUILD}/site-packages # and wasm libraries export DYNLOAD=${SDKROOT}/prebuilt/emsdk/${PYBUILD}/lib-dynload @@ -34,27 +34,8 @@ echo " " 1>&2 - -# SDL2_image turned off : -ltiff - -# CF_SDL="-sUSE_SDL=2 -sUSE_ZLIB=1 -sUSE_BZIP2=1" - - -# something triggers sdl2 *full* rebuild. -# also for SDL2_mixer, ogg and vorbis -# all pic - - -# / -# $EMPIC/libSDL2.a -# $EMPIC/libSDL2_gfx.a -# $EMPIC/libogg.a -# $EMPIC/libvorbis.a -# $EMPIC/libSDL2_mixer_ogg.a - EMPIC=/opt/python-wasm-sdk/emsdk/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/pic - SUPPORT_FS="" @@ -219,7 +200,7 @@ echo CPY_CFLAGS=$CPY_CFLAGS if emcc -fPIC -std=gnu99 -D__PYDK__=1 -DNDEBUG $CPY_CFLAGS $CF_SDL $CPOPTS \ -c -fwrapv -Wall -Werror=implicit-function-declaration -fvisibility=hidden\ - -I${PYDIR}/internal -I${PYDIR} -I./support -DPy_BUILD_CORE\ + -I${PYDIR}/internal -I${PYDIR} -I./support -I./src/hpy/hpy/devel/include -DPy_BUILD_CORE\ -o build/${MODE}.o support/__EMSCRIPTEN__-pymain.c then STDLIBFS="--preload-file build/stdlib-rootfs/python${PYBUILD}@/usr/lib/python${PYBUILD}" @@ -233,36 +214,9 @@ then # TODO: test -sWEBGL2_BACKWARDS_COMPATIBILITY_EMULATION + LDFLAGS="-sUSE_GLFW=3 -sUSE_WEBGL2 -sMIN_WEBGL_VERSION=2 -sOFFSCREENCANVAS_SUPPORT=1 -sFULL_ES2 -sFULL_ES3" - - -# /opt/python-wasm-sdk/emsdk/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/pic/libSDL2.a - - CF_SDL="-I${SDKROOT}/devices/emsdk/usr/include/SDL2" - #LD_SDL2="-lSDL2_gfx -lSDL2_mixer -lSDL2_ttf" - - LD_SDL2="$EMPIC/libSDL2.a" - LD_SDL2="$LD_SDL2 $EMPIC/libSDL2_gfx.a $EMPIC/libogg.a $EMPIC/libvorbis.a" - LD_SDL2="$LD_SDL2 $EMPIC/libSDL2_mixer_ogg.a $EMPIC/libSDL2_ttf.a" - LD_SDL2="$LD_SDL2 -lSDL2_image -lwebp -ljpeg -lpng -lharfbuzz -lfreetype" - - - #LDFLAGS="$LD_VENDOR -sUSE_GLFW=3 -sUSE_WEBGL2 -sMIN_WEBGL_VERSION=2 -sOFFSCREENCANVAS_SUPPORT=1 -sFULL_ES2 -sFULL_ES3" - LDFLAGS="$LD_SDL2" - -LDFLAGS="-sUSE_GLFW=3 -sUSE_WEBGL2 -sMIN_WEBGL_VERSION=2 -sOFFSCREENCANVAS_SUPPORT=1 -sFULL_ES2 -sFULL_ES3" - -# -sUSE_FREETYPE -sUSE_HARFBUZZ" - - - if echo ${PYBUILD}|grep -q 10$ - then - echo " - no sqlite3 for 3.10 -" - else - LDFLAGS="$LDFLAGS -lsqlite3" - fi - - + LDFLAGS="$LDFLAGS -lsqlite3" LDFLAGS="-L${SDKROOT}/devices/emsdk/usr/lib $LDFLAGS -lssl -lcrypto -lffi -lbz2 -lz -ldl -lm" @@ -373,3 +327,21 @@ else fi + + + + + + + + + + + + + + + + + + diff --git a/scripts/build-rootfs.sh b/scripts/build-rootfs.sh index af165a3..21a24a4 100755 --- a/scripts/build-rootfs.sh +++ b/scripts/build-rootfs.sh @@ -205,7 +205,7 @@ with open("build/stdlib.list","w") as tarlist: print(name, file=tarlist ) else: stdlp = stdlp.replace('$(arch)','emsdk') - #print(stdlp) + print(stdlp) tarcmd=f"tar --directory=/{stdlp}usr/lib --files-from=build/stdlib.list -cf build/stdl.tar" print(tarcmd) os.system(tarcmd) diff --git a/static/pythons.js b/static/pythons.js index fd94052..b56dcba 100644 --- a/static/pythons.js +++ b/static/pythons.js @@ -513,15 +513,14 @@ const vm = { } }, - + websocket : { "url" : "wss://" }, preRun : [ prerun ], postRun : [ function (VM) { - VM["websocket"]["url"] = "wss://" + window.VM = VM window.python = VM window.py = new bridge(VM) setTimeout(custom_postrun, 10) - - } ] + }] } @@ -549,22 +548,43 @@ async function run_pyrc(content) { } python.PyRun_SimpleString(`#!site -PyConfig = json.loads("""${JSON.stringify(python.PyConfig)}""") -verbose = PyConfig.get('quiet', False) import os, sys, json +PyConfig = json.loads("""${JSON.stringify(python.PyConfig)}""") pfx=PyConfig['prefix'] -if os.path.isdir(pfx): +def os_path_is_dir(path): + try: + os.listdir(str(path)) + return True + except: + return False +def os_path_is_file(path): + parent, file = str(path).rsplit('/',1) + try: + return file in os.listdir(parent) + except: + return False + +if os_path_is_dir(pfx): sys.path.append(pfx) os.chdir(pfx) + +print("581: Current Dir :", pfx) +del pfx __pythonrc__ = "${pyrc_file}" -if os.path.isfile(__pythonrc__): - exec(open(__pythonrc__).read(), globals(), globals()) +try: + if os_path_is_file(__pythonrc__): + exec(open(__pythonrc__).read(), globals(), globals()) + else: + raise Error("File not found") +except Exception as e: + print(f"579: invalid {__pythonrc__=}") + sys.print_exception(e) + +try: import asyncio asyncio.run(import_site("${main_file}")) -else: - print(f"510: invalid {__pythonrc__=}") -del pfx, verbose -# +except ImportError: + pass `) } @@ -2215,7 +2235,7 @@ console.warn("TODO: merge/replace location options over script options") if (url.endsWith(module_name)) { url = url + (window.location.search || "?") + ( window.location.hash || "#" ) - console.log("Location taking precedence over script", old_url ,'=>', url ) + console.log("Location of",module_name,"overrides script", old_url ,'=>', url ) } elems = url.rsplit('#',1) @@ -2243,24 +2263,32 @@ console.warn("TODO: merge/replace location options over script options") console.warn("1601: no inlined code found") } - // resolve python executable + // resolve python executable, cmdline first then script + var pystr = "cpython" - if (vm.cpy_argv.length) { - var orig_argv_py - if (vm.cpy_argv[0].search('cpython3')>=0) { - vm.script.interpreter = "cpython" - config.PYBUILD = vm.cpy_argv[0].substr(7) || "3.11" + if (vm.cpy_argv.length && vm.cpy_argv[0].search('py')>=0) { + pystr = vm.cpy_argv[0] + } else { + if (cfg.python.search('py')>=0) { + pystr = cfg.python + } + // fallback to cpython + } + + if (pystr.search('cpython3')>=0) { + vm.script.interpreter = "cpython" + config.PYBUILD = pystr.substr(7) || "3.11" + } else { + if (pystr.search('wapy')>=0) { + vm.script.interpreter = "wapy" + config.PYBUILD = pystr.substr(4) || "3.4" } else { - if (vm.cpy_argv[0].search('wapy')>=0) { -// TODO wapy is not versionned - vm.script.interpreter = "wapy" - } else { - vm.script.interpreter = config.python || "cpython" - config.PYBUILD = vm.cpy_argv[0].substr(7) || "3.11" - } + vm.script.interpreter = config.python || "cpython" + config.PYBUILD = pystr.substr(7) || "3.11" } } + // running pygbag proxy, lan testing or a module url ? if ( (location.hostname === "localhost") || cfg.module) { config.cdn = url.split("?",1)[0].replace(module_name, "") @@ -2389,7 +2417,7 @@ function auto_start(cfg) { cols : script.dataset.cols, lines : script.dataset.lines, url : script.src, - os : script.dataset.os, + os : script.dataset.os || "gui", text : code, id : script.id, autorun : "" diff --git a/support/__EMSCRIPTEN__.c b/support/__EMSCRIPTEN__.c index 817c072..4975a8d 100644 --- a/support/__EMSCRIPTEN__.c +++ b/support/__EMSCRIPTEN__.c @@ -4,7 +4,6 @@ static PyStatus pymain_init(const _PyArgv *args); static void pymain_free(void); - http://troubles.md/why-do-we-need-the-relooper-algorithm-again/ https://medium.com/leaningtech/solving-the-structured-control-flow-problem-once-and-for-all-5123117b1ee2 @@ -48,38 +47,85 @@ self hosting: #include +#if defined(WAPY) +//# include "mpconfigport.h" +# include "Python.h" + +#endif #include "../build/gen_static.h" #if PYDK_emsdk - #include - #include - #include "emscripten.h" - /* - #define SDL2 - #include - #include - */ -// #include // SDL_HINT_EMSCRIPTEN_KEYBOARD_ELEMENT - #define SDL_HINT_EMSCRIPTEN_KEYBOARD_ELEMENT "SDL_EMSCRIPTEN_KEYBOARD_ELEMENT" - +# include +# include +# include "emscripten.h" #define HOST_RETURN(value) return value PyObject *sysmod; PyObject *sysdict; -# include "__EMSCRIPTEN__.embed/sysmodule.c" +# include "sys/time.h" // gettimeofday +# include // umask +# if !defined(WAPY) +# include "__EMSCRIPTEN__.embed/sysmodule.c" +# endif #else - #error "wasi unsupported yet" +# error "wasi unsupported yet" #endif +#if !defined(WAPY) +# include "../build/gen_inittab.h" +#else +# pragma message " @@@@@@@@@@@ NOT YET ../build/gen_inittab.h @@@@@@@@@@@@" +#endif -#include "../build/gen_inittab.h" +// ============================================================================================== static int preloads = 0; static long long loops = 0; +// ============================================================================================== + + +#define HPY_ABI_UNIVERSAL +#include "hpy.h" + +HPyDef_METH(platform_run, "run", HPyFunc_VARARGS) + +static HPy +platform_run_impl(HPyContext *ctx, HPy self, const HPy *argv, size_t argc) { + puts("hpy runs"); + return HPyLong_FromLongLong(ctx, loops); +} + +static HPyDef *hpy_platform_Methods[] = { + &platform_run, + NULL, +}; + +static HPyModuleDef hpy_platform_def = { + .doc = "HPy _platform", + .defines = hpy_platform_Methods, +}; + +// The Python interpreter will create the module for us from the +// HPyModuleDef specification. Additional initialization can be +// done in the HPy_mod_exec slot +HPy_MODINIT(_platform, hpy_platform_def) + +extern PyModuleDef* _HPyModuleDef_AsPyInit(HPyModuleDef *hpydef); + +PyMODINIT_FUNC +PyInit__platform(void) +{ + return (PyObject *)_HPyModuleDef_AsPyInit(&hpy_platform_def); +} + + +// ============================================================================================== + + static PyObject * embed_run(PyObject *self, PyObject *argv, PyObject *kwds) { @@ -151,7 +197,6 @@ embed_test(PyObject *self, PyObject *args, PyObject *kwds) #include #include - static PyObject *embed_webgl(PyObject *self, PyObject *args, PyObject *kwds); @@ -360,10 +405,12 @@ static struct PyModuleDef mod_embed = { static PyObject *embed_dict; -PyMODINIT_FUNC init_embed(void) { +PyMODINIT_FUNC +PyInit_embed(void) { // activate javascript bindings that were moved from libpython to pymain. -#if defined(PYDK_emsdk) +// but not for wapy +#if defined(PYDK_emsdk) && !defined(WAPY) int res; sysmod = PyImport_ImportModule("sys"); // must call Py_DECREF when finished sysdict = PyModule_GetDict(sysmod); // borrowed ref, no need to delete @@ -378,13 +425,19 @@ PyMODINIT_FUNC init_embed(void) { Py_DECREF(sysmod); // release ref to sysMod err_occurred:; type_init_failed:; +#else + + puts("PyInit_embed"); #endif // helper module for pygbag api not well defined and need clean up. // callable as "platform" module. PyObject *embed_mod = PyModule_Create(&mod_embed); - embed_dict = PyModule_GetDict(embed_mod); - PyDict_SetItemString(embed_dict, "js2py", PyUnicode_FromString("{}")); + +// from old aiolink poc + //embed_dict = PyModule_GetDict(embed_mod); + //PyDict_SetItemString(embed_dict, "js2py", PyUnicode_FromString("{}")); + return embed_mod; } @@ -518,8 +571,6 @@ main_iteration(void) { fprintf( stderr, "%d: %s", lines, buf ); } - - rewind(file); if (lines>1) { @@ -534,11 +585,12 @@ main_iteration(void) { # undef file } - if ( (datalen = io_file_select(IO_RAW)) ) + if ( (datalen = io_file_select(IO_RAW)) ) { embed_os_read_bufsize += datalen; //printf("raw data %i\n", datalen); + } - if ( (datalen = io_file_select(0)) ) { + if ( (datalen = io_file_select(0)) ) { embed_readline_bufsize += datalen; //printf("stdin data q+%i / q=%i dq=%i\n", datalen, embed_readline_bufsize, embed_readline_cursor); } @@ -566,10 +618,6 @@ static void reprint(const char *fmt, PyObject *obj) { - - - - #define EGLTEST @@ -693,14 +741,14 @@ EMSCRIPTEN_KEEPALIVE void egl_test() { puts("EGL test failed"); } -#endif +#endif // EGLTEST static PyObject * embed_webgl(PyObject *self, PyObject *args, PyObject *kwds) { // setting up EmscriptenWebGLContextAttributes #if defined(EGLTEST) - egl_test(); + egl_test(); #endif EMSCRIPTEN_WEBGL_CONTEXT_HANDLE ctx = emscripten_webgl_get_current_context(); @@ -714,14 +762,16 @@ embed_webgl(PyObject *self, PyObject *args, PyObject *kwds) PyStatus status; -#if defined(FT) -#include -#include FT_FREETYPE_H -#endif - +#if defined(WAPY) +#define CPY 0 +MP_NOINLINE int +main_(int argc, char **argv) +#else +#define CPY 1 int main(int argc, char **argv) +#endif { gettimeofday(&time_last, NULL); @@ -732,7 +782,9 @@ main(int argc, char **argv) .wchar_argv = NULL }; - PyImport_AppendInittab("embed", init_embed); + PyImport_AppendInittab("embed", PyInit_embed); + + PyImport_AppendInittab("_platform", PyInit__platform); # include "../build/gen_inittab.c" @@ -759,7 +811,6 @@ main(int argc, char **argv) setenv("PYGLET_HEADLESS", "1", 1); - status = pymain_init(&args); if (_PyStatus_IS_EXIT(status)) { @@ -790,6 +841,7 @@ main(int argc, char **argv) LOG_V("no 'tmp' directory, creating one ..."); } + for (int i=0;i