diff --git a/pygbag/__init__.py b/pygbag/__init__.py index 9d8f720..f96f1c3 100644 --- a/pygbag/__init__.py +++ b/pygbag/__init__.py @@ -7,7 +7,7 @@ from pathlib import Path -__version__ = "0.8.4" +__version__ = "0.8.5" # hack to test git cdn build without upgrading pygbag # beware can have side effects when file packager behaviour must change ! @@ -21,7 +21,6 @@ ) __version__ = "0.0.0" - # make aio available sys.path.append(str(Path(__file__).parent / "support/cross")) diff --git a/pygbag/__main__.py b/pygbag/__main__.py index b8a6137..5341cd6 100644 --- a/pygbag/__main__.py +++ b/pygbag/__main__.py @@ -106,9 +106,8 @@ def truc(*argv, **kw): fakehost.fopen = aio.filelike.fopen - # cannot fake a cpu __WASM__ will be False + # cannot fake a wasm cpu but fake the platform AND the module - # but fake the platform AND the module sys.platform = "emscripten" class fake_EventTarget: @@ -187,10 +186,10 @@ def no_op(cls, *argv, **kw): class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): HTML_MARK = '"' * 3 + " # BEGIN -->" - @classmethod - async def async_repos(cls): - abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" - print(f"{abitag=}") +# @classmethod +# async def async_repos(cls): +# abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" +# print(f"{abitag=}") @classmethod async def async_imports(cls, callback, *wanted, **kw): diff --git a/pygbag/aio.py b/pygbag/aio.py index 60c56d3..c31293e 100644 --- a/pygbag/aio.py +++ b/pygbag/aio.py @@ -1,10 +1,13 @@ import sys import os + +# this aio is the support/cross one, not the current file. import aio # to allow "import pygbag.aio as asyncio" sys.modules["pygbag.aio"] = aio + # if not wasm cpu then run caller in simulator and block. if hasattr(os, "uname") and not os.uname().machine.startswith("wasm"): @@ -15,7 +18,6 @@ import aio.pep0723 - aio.pep0723.Config.dev_mode = ".-X.dev." in ".".join(sys.orig_argv) if aio.pep0723.Config.dev_mode: aio.pep0723.Config.PKG_INDEXES.extend(["http://localhost:8000/archives/repo/"]) else: @@ -52,9 +54,10 @@ async def custom_async_input(): dt = next - time.time() if dt < 0: past = int(-dt * 1000) - # do not spam for 2 ms late - if past > 2: + # do not spam for <4 ms late (50Hz vs 60Hz) + if past > 4: print(f"aio: violation frame is {past} ms late") + # too late do not sleep at all else: time.sleep(dt) print("sim loop exited") diff --git a/pygbag/app.py b/pygbag/app.py index 63a4d7c..3729d44 100644 --- a/pygbag/app.py +++ b/pygbag/app.py @@ -81,7 +81,8 @@ def set_args(program): app_folder = patharg.resolve() mainscript = DEFAULT_SCRIPT - sys.path.insert(0, str(app_folder)) + # print("84: prepending to sys.path", str(app_folder) ) + # sys.path.insert(0, str(app_folder)) if patharg.suffix == "pyw": required.append("79: Error, no .pyw allowed use .py for python script") diff --git a/pygbag/filtering.py b/pygbag/filtering.py index 47b3207..cc005bb 100644 --- a/pygbag/filtering.py +++ b/pygbag/filtering.py @@ -2,13 +2,36 @@ dbg = True +# q:what to do with the extreme case $HOME/main.py ? +# or folders > 512MiB total +# a: maybe break on too many files around the yield + + +IGNORE = """ +/.ssh +/.local +/.config +/.git +/.github +/.vscode +/.idea +/dist +/build +/venv +/ignore +/ATTIC +""".strip().split('\n') + +SKIP_EXT= ["pyc", "pyx", "pyd", "pyi", "exe", "log", "DS_Store"] def filter(walked): - global dbg + global dbg, IGNORE, SKIP_EXT for folder, filenames in walked: blocking = False - for block in ["/.git", "/.github", "/build", "/venv", "/ignore", "/.idea"]: + for block in IGNORE: + if not block: + continue if folder.match(block): if dbg: print("REJ 1", folder) @@ -33,7 +56,7 @@ def filter(walked): continue ext = filename.rsplit(".", 1)[-1].lower() - if ext in ["pyc", "pyx", "pyd", "pyi", "exe", "log", "DS_Store"]: + if ext in SKIP_EXT: if dbg: print("REJ 4", folder, filename) continue diff --git a/pygbag/optimizing.py b/pygbag/optimizing.py index 5fa0a8f..19fc764 100644 --- a/pygbag/optimizing.py +++ b/pygbag/optimizing.py @@ -1,6 +1,8 @@ import os import sys from pathlib import Path +import warnings + """ @@ -25,6 +27,14 @@ """ + +BAD = { + "wav" : "ogg", + "bmp" : "png", + "mp3" : "ogg", +} + + if sys.platform != "linux": def optimize(folder, filenames, **kw): @@ -34,11 +44,19 @@ def optimize(folder, filenames, **kw): else: def optimize(folder, filenames, **kw): + global BAD print("optimizing", folder) png_quality = 50 done_list = [] + try: + import black + print("Applying black format") + os.popen(f'black -t py311 -l 132 "{folder}"').read() + except ImportError: + warnings.warn(f"Black not found for processing {folder=}") + if os.popen("pngquant 2>&1").read().count("pngfile"): print(f" -> with pngquant --quality {png_quality}", folder) else: @@ -58,8 +76,15 @@ def translated(fn): if fp.stem.endswith("-pygbag"): continue +# TODO: still issue a warning if fp.suffix == ".mp3": - continue + ... + + if fp.suffix == ".wav": + ... + + if fp.suffix == ".bmp": + ... if fp not in done_list: done_list.append(fp) @@ -67,6 +92,30 @@ def translated(fn): return for fp in filenames: + fname = f"{folder}{fp}" + if fp.suffix == ".py": + tofix = [] + for bad in BAD.keys(): + with open(fname,"r") as source: + for l in source.readlines(): + if l.find(f'.{bad}"')>0: + tofix.append( [bad,BAD[bad]] ) + break + + if len(tofix): + fixname = Path(f"{fp.parent}/{fp.stem}-pygbag.py") + fixfull = Path(f"{folder}/{fixname}") + with open(fname,"r", encoding="utf-8") as source: + data = open(fname,"r").read() + with open(fixfull, "w", encoding="utf-8") as dest: + while len(tofix): + bad, good = tofix.pop(0) + warnings.warn(f"potential {bad.upper()} use in {fname}, prefer .{good} !") + data = data.replace(f'.{bad}"',f'.{good}"') + dest.write(data) + yield fixname + continue + if fp.suffix == ".png": if png_quality >= 0: if fp.stem.endswith("-pygbag"): @@ -80,7 +129,7 @@ def translated(fn): print("opt-skip(38)", fp) continue - osexec = f'pngquant -f --ext -pygbag.png --quality {png_quality} "{folder}{fp}"' + osexec = f'pngquant -f --ext -pygbag.png --quality {png_quality} "{fname}"' os.system(osexec) if opt.is_file(): yield translated(opt) @@ -98,7 +147,7 @@ def translated(fn): print("opt-skip(73)", fp) continue - osexec = f'ffmpeg -i "{folder}{fp}" -ac 1 -r 22000 "{opt}"' + osexec = f'ffmpeg -i "{fname}" -ac 1 -r 22000 "{opt}"' if has_ffmpeg: os.system(osexec) diff --git a/pygbag/support/cross/aio/filelike.py b/pygbag/support/cross/aio/filelike.py index 20d2ec2..8dc33b3 100644 --- a/pygbag/support/cross/aio/filelike.py +++ b/pygbag/support/cross/aio/filelike.py @@ -68,7 +68,7 @@ class fopen: print("69: platform has no object serializer") flags = {} - def __init__(self, maybe_url, mode="r", flags=None): + def __init__(self, maybe_url, mode="r", flags=None, encoding='UTF-8', errors=None, newline=None, closefd=True, opener=None): self.url = fix_url(maybe_url) self.mode = mode flags = flags or self.__class__.flags diff --git a/pygbag/support/cross/aio/pep0723.py b/pygbag/support/cross/aio/pep0723.py index 0e65ea4..6b11c0f 100644 --- a/pygbag/support/cross/aio/pep0723.py +++ b/pygbag/support/cross/aio/pep0723.py @@ -26,7 +26,7 @@ import platform_wasm.todo PATCHLIST = [] - +HISTORY = [] class Config: READ_722 = False @@ -38,7 +38,13 @@ class Config: REPO_DATA = "repodata.json" repos = [] pkg_repolist = [] - dev_mode = ".-X.dev." in ".".join(sys.orig_argv) + dev_mode = ".-X.dev." in ".".join(['']+sys.orig_argv+['']) + + mapping = { + "pygame": "pygame.base", + "pygame-ce": "pygame.base", + "python-i18n" : "i18n", + } def read_dependency_block_722(code): @@ -106,8 +112,6 @@ def read_dependency_block_723x(script): return None -HISTORY = [] - def install(pkg_file, sconf=None): global HISTORY @@ -140,24 +144,39 @@ def install(pkg_file, sconf=None): sys.print_exception(ex) + async def async_imports_init(): - for cdn in Config.PKG_INDEXES: - print("init cdn :", Config.PKG_INDEXES) - async with fopen(Path(cdn) / Config.REPO_DATA) as source: - Config.repos.append(json.loads(source.read())) + ... + +# see pythonrc +# if not len(Config.repos): +# for cdn in (Config.PKG_INDEXES or PyConfig.pkg_indexes): +# async with platform.fopen(Path(cdn) / Config.REPO_DATA) as source: +# Config.repos.append(json.loads(source.read())) +# +# DBG("1203: FIXME (this is pyodide maintened stuff, use PEP723 asap) referenced packages :", len(cls.repos[0]["packages"])) +# - pdb("referenced packages :", len(Config.repos[-1]["packages"])) async def async_repos(): abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" + + apitag = __import__("sysconfig").get_config_var("HOST_GNU_TYPE") + apitag = apitag.replace("-", "_") + print("163: async_repos", Config.PKG_INDEXES) for repo in Config.PKG_INDEXES: - async with fopen(f"{repo}index.json", "r") as index: + if apitag.find("mvp") > 0: + idx = f"{repo}index.json" + else: + idx = f"{repo}index-bi.json" + async with fopen(idx, "r", encoding='UTF-8') as index: try: data = index.read() if isinstance(data, bytes): data = data.decode() data = data.replace("", abitag) + data = data.replace("", apitag) repo = json.loads(data) except: pdb(f"110: {repo=}: malformed json index {data}") @@ -165,20 +184,32 @@ async def async_repos(): if repo not in Config.pkg_repolist: Config.pkg_repolist.append(repo) - if not aio.cross.simulator: - if window.location.href.startswith("https://pmp-p.ddns.net/pygbag"): - print(" =============== REDIRECTION TO DEV HOST ================ ") - for idx, repo in enumerate(PyConfig.pkg_repolist): - repo["-CDN-"] = "https://pmp-p.ddns.net/archives/repo/" - + repo = None if Config.dev_mode > 0: for idx, repo in enumerate(Config.pkg_repolist): try: - print("120:", repo["-CDN-"], idx, "REMAPPED TO", Config.PKG_INDEXES[idx]) repo["-CDN-"] = Config.PKG_INDEXES[idx] except Exception as e: sys.print_exception(e) + if not aio.cross.simulator: + import platform + print("193:", platform.window.location.href ) + if platform.window.location.href.startswith("https://pmp-p.ddns.net/pygbag"): + for idx, repo in enumerate(Config.pkg_repolist): + repo["-CDN-"] = "https://pmp-p.ddns.net/archives/repo/" + elif platform.window.location.href.startswith("http://192.168.1.66/pygbag"): + for idx, repo in enumerate(Config.pkg_repolist): + repo["-CDN-"] = "http://192.168.1.66/archives/repo/" + if repo: + print(f""" + +=============== REDIRECTION TO DEV HOST {repo['-CDN-']} ================ +{abitag=} +{apitag=} + +""") + async def install_pkg(sysconf, wheel_url, wheel_pkg): target_filename = f"/tmp/{wheel_pkg}" @@ -187,30 +218,61 @@ async def install_pkg(sysconf, wheel_url, wheel_pkg): target.write(pkg.read()) install(target_filename, sysconf) - +# FIXME: HISTORY and invalidate caches async def pip_install(pkg, sconf={}): - print("185: searching TODO: install from repo in found key", pkg) + if pkg in HISTORY: + return + print("225: searching", pkg) if not sconf: sconf = __import__("sysconfig").get_paths() wheel_url = "" - try: - async with fopen(f"https://pypi.org/simple/{pkg}/") as html: - if html: - for line in html.readlines(): - if line.find("href=") > 0: - if line.find("-py3-none-any.whl") > 0: - wheel_url = line.split('"', 2)[1] - else: - print("270: ERROR: cannot find package :", pkg) - except FileNotFoundError: - print("200: ERROR: cannot find package :", pkg) - return - except: - print("204: ERROR: cannot find package :", pkg) - return + # hack for WASM wheel repo + if pkg in Config.mapping: + pkg = Config.mapping[pkg] + if pkg in HISTORY: + return + print("228: package renamed to", pkg) + + for repo in Config.pkg_repolist: + if pkg in repo: + wheel_url = f"{repo['-CDN-']}{repo[pkg]}#" +# pkg_file = f"/tmp/{repo[want].rsplit('/',1)[-1]}" +# if pkg_file in aio.toplevel.HISTORY: +# break + +# cfg = {"io": "url", "type": "fs", "path": pkg_file} +# print(f"1205: async_get_pkg({pkg_url})") +# +# track = platform.window.MM.prepare(pkg_url, json.dumps(cfg)) +# +# try: +# await cls.pv(track) +# zipfile.ZipFile(pkg_file).close() +# break +# except (IOError, zipfile.BadZipFile): +# pdb(f"1294: network error on {repo['-CDN-']}, cannot install {pkg_file}") + + # try to get a pure python wheel from pypi + if not wheel_url: + try: + async with fopen(f"https://pypi.org/simple/{pkg}/") as html: + if html: + for line in html.readlines(): + if line.find("href=") > 0: + if line.find("-py3-none-any.whl") > 0: + wheel_url = line.split('"', 2)[1] + else: + print("270: ERROR: cannot find package :", pkg) + except FileNotFoundError: + print("200: ERROR: cannot find package :", pkg) + return + + except: + print("204: ERROR: cannot find package :", pkg) + return if wheel_url: try: @@ -221,11 +283,19 @@ async def pip_install(pkg, sconf={}): async def parse_code(code, env): - global PATCHLIST + global PATCHLIST, async_imports_init, async_repos # pythonrc is calling aio.pep0723.parse_code not check_list # so do patching here - platform_wasm.todo.patch() + patchlevel = platform_wasm.todo.patch() + if patchlevel: + print("264:parse_code() patches loaded :", list(patchlevel.keys()) ) + platform_wasm.todo.patch = lambda :None + # and only do that once and for all. + await async_imports_init() + await async_repos() + del async_imports_init, async_repos + maybe_missing = [] @@ -295,11 +365,6 @@ async def check_list(code=None, filename=None): if len(still_missing): importlib.invalidate_caches() - # only do that once and for all. - if not len(Config.repos): - await async_imports_init() - await async_repos() - # TODO: check for possible upgrade of env/* pkg maybe_missing = still_missing diff --git a/pygbag/support/cross/aio/toplevel.py b/pygbag/support/cross/aio/toplevel.py index 211983d..7299f19 100644 --- a/pygbag/support/cross/aio/toplevel.py +++ b/pygbag/support/cross/aio/toplevel.py @@ -308,7 +308,7 @@ async def start_toplevel(cls, shell, console=True, ns="__main__"): """start async import system with optionnal async console""" if cls.instance is None: cls.make_instance(shell, ns) - await cls.instance.async_repos() + #await cls.instance.async_repos() if console: cls.start_console(shell, ns=ns) diff --git a/pygbag/support/offline.png b/pygbag/support/offline.png new file mode 100644 index 0000000..c0b976c Binary files /dev/null and b/pygbag/support/offline.png differ diff --git a/pygbag/support/pygbag_tools.py b/pygbag/support/pygbag_tools.py new file mode 100644 index 0000000..472896a --- /dev/null +++ b/pygbag/support/pygbag_tools.py @@ -0,0 +1,81 @@ +import sys +import pygbag +from pathlib import Path + +# ====================== pygame + +def pg_load(fn, alpha=True): + import pygame + from pathlib import Path + + if Path(fn).is_file(): + media = pygame.image.load(fn) + else: + media = pygame.image.load( Path(__file__).parent / "offline.png" ) + + if alpha: + return media.convert_alpha() + else: + return media.convert() + + + +#======================= network + + + +import pygbag.aio as asyncio + +if aio.cross.simulator: + sys.path.append(str(Path(aio.__file__).parent.parent.parent)) + +print(" ================ PygBag utilities loaded ================== ") + + +def host(): + global proxy + print("serving sockv5 ...") + + from asyncio_socks_server.values import SocksAuthMethod + from asyncio_socks_server.config import Config + from asyncio_socks_server.proxyman import ProxyMan + + cfg = Config() + + cfg.update_config( + { + "LISTEN_HOST": "127.0.0.1", + "LISTEN_PORT": 8001, + "AUTH_METHOD": SocksAuthMethod.NO_AUTH, + "ACCESS_LOG": False, + "STRICT": False, + "DEBUG": False, + "USERS": {}, + } + ) + + proxy = ProxyMan(cfg) + aio.create_task(proxy.start_server()) + + +host() + + +async def connect(): + import aiohttp + from aiohttp_socks import ProxyType, ProxyConnector, ChainProxyConnector + + async def fetch(url): + connector = ProxyConnector.from_url("socks5://user:password@127.0.0.1:8001") + async with aiohttp.ClientSession(connector=connector) as session: + async with session.get(url) as response: + return await response.text() + + print(await fetch("https://example.com/")) + + +# await connect() + +if __name__ == "__main__": + async def main(): + asyncio.run(main()) diff --git a/pygbag/support/pygbag_ux.py b/pygbag/support/pygbag_ux.py new file mode 100644 index 0000000..69a5ce7 --- /dev/null +++ b/pygbag/support/pygbag_ux.py @@ -0,0 +1,47 @@ + +# screen pixels (real, hardware) +WIDTH = 1280 # {{cookiecutter.width}} +HEIGHT = 720 # {{cookiecutter.height}} + +def ux_dim(w,h): + global WIDTH, HEIGHT + WIDTH = int(w) + HEIGHT = int(h) + + +# reference/idealized screen pixels +REFX = 1980 +REFY = 1080 + +def u(real, ref, v): + if abs(v) < 0.9999999: + result = int((float(real) / 100.0) * (v * 1000)) + if v < 0: + return real - result + return result + return int((real / ref) * v) + +def ux(*argv): + global WIDTH, REFX + acc = 0 + for v in argv: + acc += u(WIDTH, REFX, v) + return acc + +def uy(*argv): + global HEIGHT, REFY + acc = 0 + for v in argv: + acc += u(HEIGHT, REFY, v) + return acc + +def ur(*argv): + x = ux(argv[0]) + y = uy(argv[1]) + ret = [x, y] + if len(argv) > 2: + w = ux(argv[2]) + h = uy(argv[3]) + ret.append(w) + ret.append(h) + return ret diff --git a/pygbag/support/pythonrc.py b/pygbag/support/pythonrc.py index 96c72b1..81ee492 100644 --- a/pygbag/support/pythonrc.py +++ b/pygbag/support/pythonrc.py @@ -192,15 +192,12 @@ def dump_code(): try: PyConfig - PyConfig["pkg_repolist"] = [] - - aio.cross.simulator = False - sys.argv.clear() - sys.argv.extend(PyConfig.pop("argv", [])) +except NameError: + PyConfig = None - -except Exception as e: - sys.print_exception(e) +# in simulator there's no PyConfig +# would need to get one from live cpython +if PyConfig is None: # TODO: build a pyconfig extracted from C here PyConfig = {} PyConfig["dev_mode"] = 1 @@ -215,6 +212,13 @@ def dump_code(): PyConfig["interactive"] = 1 print(" - running in wasm simulator - ") aio.cross.simulator = True +else: + PyConfig["pkg_repolist"] = [] + + aio.cross.simulator = False + sys.argv.clear() + sys.argv.extend(PyConfig.pop("argv", [])) + PyConfig["imports_ready"] = False PyConfig["pygbag"] = 0 @@ -625,11 +629,12 @@ async def perf_index(): @classmethod async def preload_code(cls, code, callback=None, hint=""): # get a relevant list of modules likely to be imported - DBG(f"617: preload_code({len(code)=} {hint=}") - maybe_wanted = list(TopLevel_async_handler.list_imports(code, file=None, hint=hint)) + PyConfig.dev_mode = 1 + DBG(f"632: preload_code({len(code)=} {hint=}") import aio import aio.pep0723 + from aio.pep0723 import Config if not aio.cross.simulator: # don't use an env path, but site-packages instead @@ -637,17 +642,37 @@ async def preload_code(cls, code, callback=None, hint=""): sconf = __import__("sysconfig").get_paths() env = Path(sconf["purelib"]) - DBG(f"628: aio.pep0723.check_list {env=}") + if not len(Config.repos): + if not len(Config.PKG_INDEXES): + Config.PKG_INDEXES = PyConfig.pkg_indexes + for cdn in Config.PKG_INDEXES: + async with platform.fopen(Path(cdn) / Config.REPO_DATA) as source: + Config.repos.append(json.loads(source.read())) + + DBG("650: FIXME (this is pyodide maintened stuff, use (auto)PEP723 asap)") + print("651: referenced packages :", len(Config.repos[0]["packages"])) + + DBG(f"644: aio.pep0723.check_list {env=}") deps = await aio.pep0723.parse_code(code, env) - DBG(f"629: aio.pep0723.pip_install {deps=}") + DBG(f"646: aio.pep0723.pip_install {deps=}") + + # auto import plumbing to avoid rely too much on import error + maybe_wanted = list(TopLevel_async_handler.list_imports(code, file=None, hint=hint)) + DBG(f"635: {maybe_wanted=}") + for dep in maybe_wanted: + if not dep in deps: + deps.append(dep) + for dep in deps: await aio.pep0723.pip_install(dep) + + await TopLevel_async_handler.async_imports(callback, *maybe_wanted) + # else: # sim use a local folder venv model # await aio.pep0723.check_list(code=code, filename=None) is done in pygbag.aio - await TopLevel_async_handler.async_imports(callback, *maybe_wanted) - # await TopLevel_async_handler.async_imports(callback, *TopLevel_async_handler.list_imports(code, file=None)) + PyConfig.imports_ready = True return True @@ -1030,9 +1055,7 @@ class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): HTML_MARK = '"' * 3 + " # BEGIN -->" repos = [] - mapping = { - "pygame": "pygame.base", - } + may_need = [] ignore = ["ctypes", "distutils", "installer", "sysconfig"] ignore += ["python-dateutil", "matplotlib-pyodide"] @@ -1046,7 +1069,7 @@ class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): "matplotlib": ["numpy", "six", "cycler", "PIL", "pygame-ce"], "bokeh": ["numpy", "yaml", "typing_extensions", "jinja2", "markupsafe"], "igraph": ["texttable"], - "pygame_gui": ["i18n"], + "pygame_gui": ["pygame.base", "i18n"], "ursina": ["numpy", "screeninfo", "gltf", "PIL", "pyperclip", "panda3d"], } @@ -1054,7 +1077,7 @@ class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): from pathlib import Path - repodata = "repodata.json" + #repodata = "repodata.json" async def raw_input(self, prompt=">>> "): if len(self.buffer): @@ -1085,6 +1108,7 @@ def eval(self, source): @classmethod def scan_imports(cls, code, filename, load_try=False, hint=""): + import aio.pep0723 import ast required = [] @@ -1092,7 +1116,7 @@ def scan_imports(cls, code, filename, load_try=False, hint=""): root = ast.parse(code, filename) except SyntaxError as e: print("_" * 40) - print("1004:", filename) + print("1111:", filename, hint) print("_" * 40) for count, line in enumerate(code.split("\n")): print(str(count).zfill(3), line) @@ -1113,7 +1137,7 @@ def scan_imports(cls, code, filename, load_try=False, hint=""): else: mod = n.name.split(".")[0] - mod = cls.mapping.get(mod, mod) + mod = aio.pep0723.Config.mapping.get(mod, mod) if mod in cls.ignore: continue @@ -1134,12 +1158,19 @@ def scan_imports(cls, code, filename, load_try=False, hint=""): if not mod in required: required.append(mod) - DBG(f"1095: scan_imports {hint=} {filename=} {len(code)=} {required}") + DBG(f"1153: scan_imports {hint=} {filename=} {len(code)=} {required}") return required @classmethod def list_imports(cls, code=None, file=None, hint=""): - DBG(f"1103: list_imports {len(code)=} {file=} {hint=}") + import aio.pep0723 + DBG(f""" + +1168: list_imports {len(code)=} {file=} {hint=}") +{aio.pep0723.Config.pkg_repolist[0]['-CDN-']=} + +""") + if code is None: if file: with open(file) as fcode: @@ -1152,7 +1183,7 @@ def list_imports(cls, code=None, file=None, hint=""): for want in cls.scan_imports(code, file, hint=hint): # DBG(f"1114: requesting module {want=} for {file=} ") repo = None - for repo in PyConfig.pkg_repolist: + for repo in aio.pep0723.Config.pkg_repolist: if want in cls.may_need: DBG(f"1118: skip module {want=} reason: already requested") break @@ -1168,9 +1199,9 @@ def list_imports(cls, code=None, file=None, hint=""): break else: if repo: - DBG(f"1132: {repo['-CDN-']=} does not provide {want=}") + DBG(f"1187: {repo['-CDN-']=} does not provide {want=}") else: - pdb("1134: no pkg repository available") + print("1189: no pkg repository available") # TODO: re order repo on failures # TODO: try to download from pypi with @@ -1190,8 +1221,8 @@ def import_one(cls, mod, lvl=0): if mod in cls.missing_fence: return [] - - for dep in cls.repos[0]["packages"].get(mod, {}).get("depends", []): + from aio.pep0723 import Config + for dep in Config.repos[0]["packages"].get(mod, {}).get("depends", []): if dep in cls.ignore: continue @@ -1262,6 +1293,7 @@ def imports(cls, *mods, lvl=0): @classmethod async def async_get_pkg(cls, want, ex, resume): + import aio.pep0723 pkg_file = "" miss_list = cls.imports(want) @@ -1273,7 +1305,7 @@ async def async_get_pkg(cls, want, ex, resume): DBG(f"1230: FIXME dependency table for manually built module '{want}' {miss_list=}") await cls.async_imports(None, *miss_list) - for repo in PyConfig.pkg_repolist: + for repo in aio.pep0723.Config.pkg_repolist: DBG(f"1234: {want=} found : {want in repo}") if want in repo: @@ -1305,27 +1337,6 @@ async def async_get_pkg(cls, want, ex, resume): def get_pkg(cls, want, ex=None, resume=None): return cls.async_get_pkg(want, ex, resume) - @classmethod - async def async_imports_init(cls): - for cdn in PyConfig.pkg_indexes: - async with platform.fopen(Path(cdn) / cls.repodata) as source: - cls.repos.append(json.loads(source.read())) - - DBG("referenced packages :", len(cls.repos[0]["packages"])) - - if not len(PyConfig.pkg_repolist): - await cls.async_repos() - - if window.location.href.startswith("https://pmp-p.ddns.net/pygbag/"): - print(" =============== REDIRECTION TO DEV HOST ================ ") - for idx, repo in enumerate(PyConfig.pkg_repolist): - repo["-CDN-"] = "https://pmp-p.ddns.net/archives/repo/" - elif PyConfig.pygbag > 0: - # if PyConfig.pygbag > 0: - for idx, repo in enumerate(PyConfig.pkg_repolist): - DBG("1264:", repo["-CDN-"], "REMAPPED TO", PyConfig.pkg_indexes[-1]) - repo["-CDN-"] = PyConfig.pkg_indexes[-1] - @classmethod async def async_imports(cls, callback, *wanted, **kw): def default_cb(pkg, error=None): @@ -1335,11 +1346,6 @@ def default_cb(pkg, error=None): callback = callback or default_cb - # init dep solver. - if not len(cls.repos): - await cls.async_imports_init() - del cls.async_imports_init - print("1302: ============= ", wanted) wants = cls.imports(*wanted) @@ -1409,39 +1415,39 @@ def print_pg_bar(total, iteration): # Print New Line on Complete print() - @classmethod - async def async_repos(cls): - abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" - apitag = __import__("sysconfig").get_config_var("HOST_GNU_TYPE") - apitag = apitag.replace("-", "_") - - for repo in PyConfig.pkg_indexes: - if apitag.find("mvp") > 0: - idx = f"{repo}index.json" - else: - idx = f"{repo}index-bi.json" - - async with platform.fopen(idx, "r") as index: - try: - data = index.read() - if isinstance(data, bytes): - data = data.decode() - data = data.replace("", abitag) - data = data.replace("", apitag) - repo = json.loads(data) - except: - pdb(f"1394: {repo=}: malformed json index {data}") - continue - if repo not in PyConfig.pkg_repolist: - PyConfig.pkg_repolist.append(repo) - - if PyConfig.dev_mode > 0: - for idx, repo in enumerate(PyConfig.pkg_repolist): - try: - print("1353:", repo["-CDN-"], idx, "REMAPPED TO", PyConfig.pkg_indexes[idx]) - repo["-CDN-"] = PyConfig.pkg_indexes[idx] - except Exception as e: - sys.print_exception(e) +# @classmethod +# async def async_repos(cls): +# abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" +# apitag = __import__("sysconfig").get_config_var("HOST_GNU_TYPE") +# apitag = apitag.replace("-", "_") +# +# for repo in PyConfig.pkg_indexes: +# if apitag.find("mvp") > 0: +# idx = f"{repo}index.json" +# else: +# idx = f"{repo}index-bi.json" +# +# async with platform.fopen(idx, "r") as index: +# try: +# data = index.read() +# if isinstance(data, bytes): +# data = data.decode() +# data = data.replace("", abitag) +# data = data.replace("", apitag) +# repo = json.loads(data) +# except: +# pdb(f"1394: {repo=}: malformed json index {data}") +# continue +# if repo not in PyConfig.pkg_repolist: +# PyConfig.pkg_repolist.append(repo) +# +# if PyConfig.dev_mode > 0: +# for idx, repo in enumerate(PyConfig.pkg_repolist): +# try: +# print("1353:", repo["-CDN-"], idx, "REMAPPED TO", PyConfig.pkg_indexes[idx]) +# repo["-CDN-"] = PyConfig.pkg_indexes[idx] +# except Exception as e: +# sys.print_exception(e) # end TopLevel_async_handler diff --git a/setup.cfg b/setup.cfg index bdece1f..d7a8198 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,24 +22,18 @@ classifiers = [options] packages = pygbag -# package_dir = pygbag include_package_data = True zip_safe = False python_requires = >=3.8 install_requires = - token-utils - aioconsole + token_utils pyparsing packaging installer + aioconsole aiohttp - -[options.packages.find] -where = pygbag - -[options.package_data] -pygbag = * + asyncio_socks_server [options.entry_points] console_scripts = diff --git a/static/pythons.js b/static/pythons.js index 380ac0d..5e3ce2c 100644 --- a/static/pythons.js +++ b/static/pythons.js @@ -13,6 +13,8 @@ window.BrowserFS.Buffer = BFSRequire('buffer') var bfs2 = false async function import_browserfs() { + if (window.BrowserFS) + return console.warn("late import", config.cdn+"browserfs.min.js" ) var script = document.createElement("script") script.src = vm.config.cdn + "browserfs.min.js" @@ -1222,7 +1224,7 @@ async function media_prepare(trackid) { BrowserFS.ZipFS = BrowserFS.FileSystem.ZipFS function apk_cb(e, apkfs){ - console.log(__FILE__, "930 mounting", hint, "onto", track.mount.point) + console.log(__FILE__, "1225: mounting", hint, "onto", track.mount.point) BrowserFS.InMemory.Create( function(e, memfs) { @@ -1231,8 +1233,9 @@ async function media_prepare(trackid) { BrowserFS.MountableFileSystem.Create({ '/' : ovfs }, async function(e, mfs) { - await BrowserFS.initialize(mfs); - await vm.FS.mount(vm.BFS, {root: track.mount.path}, track.mount.point); + await BrowserFS.initialize(mfs) + await vm.FS.mount(vm.BFS, {root: track.mount.path}, track.mount.point) + console.log("1236: mount complete") setTimeout(()=>{track.ready=true}, 0) }) } @@ -1264,11 +1267,9 @@ async function media_prepare(trackid) { }); vm.FS.mount(vm.BFS, { root: track.mount.path, }, track.mount.point); - + setTimeout(()=>{track.ready=true}, 0) } // bfs2 - setTimeout(()=>{track.ready=true}, 0) - } // track type mount } @@ -1287,7 +1288,7 @@ function MM_play(track, loops) { console.error(`** MEDIA USER ACTION REQUIRED [${track.test}] **`) if (track.test && track.test>0) { track.test += 1 - setTimeout(MM_play, 1000, track, loops) + setTimeout(MM_play, 2000, track, loops) } }); diff --git a/support/__EMSCRIPTEN__.c b/support/__EMSCRIPTEN__.c index 4975a8d..f9696aa 100644 --- a/support/__EMSCRIPTEN__.c +++ b/support/__EMSCRIPTEN__.c @@ -332,8 +332,6 @@ embed_prompt(PyObject *self, PyObject *_null) { Py_RETURN_NONE; } - - static PyObject * embed_isatty(PyObject *self, PyObject *argv) { int fd = 0; @@ -343,6 +341,159 @@ embed_isatty(PyObject *self, PyObject *argv) { return Py_BuildValue("i", isatty(fd) ); } +#include "pycore_ceval.h" +#include "pycore_function.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_frame.h" + +static void +_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) +{ + // Make sure that this is, indeed, the top frame. We can't check this in + // _PyThreadState_PopFrame, since f_code is already cleared at that point: + assert((PyObject **)frame + frame->f_code->co_nlocalsplus + + frame->f_code->co_stacksize + FRAME_SPECIALS_SIZE == tstate->datastack_top); + tstate->recursion_remaining--; + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + assert(frame->owner == FRAME_OWNED_BY_THREAD); + _PyFrame_Clear(frame); + tstate->recursion_remaining++; + _PyThreadState_PopFrame(tstate, frame); +} + +extern PyObject *WASM_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); + +static PyObject * +embed_bcrun(PyObject *self, PyObject *argv) { + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = NULL; + PyObject *locals = NULL; + +#if 0 + const char *bc = NULL; + if (!PyArg_ParseTuple(argv, "y", &bc)) { +#else + PyObject *co = NULL; + if (!PyArg_ParseTuple(argv, "OO", &co, &globals)) { + +#endif + return NULL; + } + + PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + if (builtins == NULL) { + return NULL; + } + + + locals = globals; + + puts("got bc"); + /* + PyObject* local_dict = PyDict_New(); + PyObject* obj = PyEval_EvalCode(co, globals , globals); +*/ + PyFrameConstructor desc = { + .fc_globals = globals, + .fc_builtins = builtins, + .fc_name = ((PyCodeObject *)co)->co_name, + .fc_qualname = ((PyCodeObject *)co)->co_name, + .fc_code = co, + .fc_defaults = NULL, + .fc_kwdefaults = NULL, + .fc_closure = NULL + }; + PyFunctionObject *func = _PyFunction_FromConstructor(&desc); + if (func == NULL) { + return NULL; + } + PyObject* const* args = NULL; + size_t argcount = 0; + PyObject *kwnames = NULL; + + //PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL); + /* _PyEvalFramePushAndInit consumes the references + * to func and all its arguments */ + Py_INCREF(func); + for (size_t i = 0; i < argcount; i++) { + Py_INCREF(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_INCREF(args[i+argcount]); + } + } +/* + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + tstate, func, locals, args, argcount, kwnames); +*/ + + PyCodeObject * code = (PyCodeObject *)func->func_code; + size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE; + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = _PyThreadState_BumpFramePointer(tstate, size); + if (frame == NULL) { + goto fail; + } + _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus); + PyObject **localsarray = &frame->localsplus[0]; + for (int i = 0; i < code->co_nlocalsplus; i++) { + localsarray[i] = NULL; + } + /* + if (initialize_locals(tstate, func, localsarray, args, argcount, kwnames)) { + assert(frame->owner != FRAME_OWNED_BY_GENERATOR); + _PyEvalFrameClearAndPop(tstate, frame); + } + */ + goto skip; + +fail: + /* Consume the references */ + for (size_t i = 0; i < argcount; i++) { + Py_DECREF(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_DECREF(args[i+argcount]); + } + } + PyErr_NoMemory(); + +skip: + +// _PyEvalFramePushAndInit + if (frame == NULL) { + return NULL; + } + + int throwflag = 0; +/* + PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0); + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +*/ + + //PyObject *retval = _PyEval_EvalFrameDefault(tstate, frame, throwflag); +puts("479"); + PyObject *retval = WASM_PyEval_EvalFrameDefault(tstate, frame, throwflag); +puts("481"); + + assert( + _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) || + _PyFrame_GetStackPointer(frame) == frame->localsplus + ); + + _PyEvalFrameClearAndPop(tstate, frame); + Py_DECREF(func); +//_PyEval_Vector + + puts("done"); + Py_RETURN_NONE; +} + + #if SDL2 static PyObject * embed_get_sdl_version(PyObject *self, PyObject *_null) @@ -357,6 +508,7 @@ embed_get_sdl_version(PyObject *self, PyObject *_null) static PyMethodDef mod_embed_methods[] = { {"run", (PyCFunction)embed_run, METH_VARARGS | METH_KEYWORDS, "start aio stepping"}, + {"bcrun", (PyCFunction)embed_bcrun, METH_VARARGS, ""}, {"preload", (PyCFunction)embed_preload, METH_VARARGS, "emscripten_run_preload_plugins"}, {"dlopen", (PyCFunction)embed_dlopen, METH_VARARGS | METH_KEYWORDS, ""}, {"dlcall", (PyCFunction)embed_dlcall, METH_VARARGS | METH_KEYWORDS, ""}, @@ -961,8 +1113,19 @@ EM_ASM({ SDL_SetHint(SDL_HINT_EMSCRIPTEN_KEYBOARD_ELEMENT, target); } #endif - +#if ASYNCIFIED + clock_t start = clock()+100; + while (1) { + main_iteration(); + clock_t current = clock(); + if (current > start) { + start = current+100; + emscripten_sleep(1); + } + } +#else emscripten_set_main_loop( (em_callback_func)main_iteration, 0, 1); +#endif puts("\nEnd"); return 0; }