From ae3d761ae14246c7ccdc4f043f383bda36cbe7ab Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 8 Nov 2023 15:05:49 +0100 Subject: [PATCH] more pep723, heuristic wav/mp3 code fixer --- .github/FUNDING.yml | 1 + .github/workflows/ci.yml | 2 +- pygbag/__init__.py | 4 +- pygbag/__main__.py | 27 +++++++----- pygbag/app.py | 18 ++++---- pygbag/filtering.py | 2 +- pygbag/optimizing.py | 7 +-- pygbag/pack.py | 3 +- pygbag/support/cross/aio/pep0723.py | 68 +++++++++++++++++++++-------- pygbag/support/pythonrc.py | 46 +++---------------- scripts/build-loader.sh | 3 +- setup.cfg | 9 ++-- static/vtx.js | 6 +-- support/__EMSCRIPTEN__.c | 10 +++++ 14 files changed, 111 insertions(+), 95 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 2634d36..04c9d79 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1,3 @@ open_collective: pythonseverywhere +liberapay: pmp-p diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 147809e..d023d2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: build: runs-on: ubuntu-22.04 env: - SDK_VERSION: 3.1.48.4bi + SDK_VERSION: 3.1.49.0bi SYS_PYTHON: /usr/bin/python3 PACKAGES: emsdk hpy _ctypes pygame BUILD_STATIC: emsdk _ctypes hpy diff --git a/pygbag/__init__.py b/pygbag/__init__.py index f96f1c3..1ed73cf 100644 --- a/pygbag/__init__.py +++ b/pygbag/__init__.py @@ -7,7 +7,7 @@ from pathlib import Path -__version__ = "0.8.5" +VERSION = "0.8.5" # hack to test git cdn build without upgrading pygbag # beware can have side effects when file packager behaviour must change ! @@ -19,7 +19,7 @@ """ ) - __version__ = "0.0.0" + VERSION = "0.0.0" # make aio available diff --git a/pygbag/__main__.py b/pygbag/__main__.py index 5341cd6..f307e49 100644 --- a/pygbag/__main__.py +++ b/pygbag/__main__.py @@ -1,18 +1,17 @@ +import sys import asyncio - asyncrun = asyncio.run -import sys - -from .__init__ import __version__ +from .__init__ import VERSION -print(f" *pygbag {__version__}*") +print(f" *pygbag {VERSION}*") from pathlib import Path async def import_site(sourcefile=None, simulator=False, async_input=None, async_pkg=None): import sys + import os from pathlib import Path if ("--sim" not in sys.argv) and ("--piny" not in sys.argv) and not simulator: @@ -27,7 +26,7 @@ async def import_site(sourcefile=None, simulator=False, async_input=None, async_ mod_dir = Path(__file__).parent support = mod_dir / "support" - + cur_dir = os.getcwd() print( f""" - single thread no-os wasm simulator - @@ -36,6 +35,7 @@ async def import_site(sourcefile=None, simulator=False, async_input=None, async_ platform support : {support} {sys.argv=} {sourcefile=} + {cur_dir=} """ ) @@ -117,6 +117,11 @@ class fake_EventTarget: async def process(self): ... + + def patch_platform_system(): + return 'Emscripten' + + # et = EventTarget() class __EMSCRIPTEN__(object): EventTarget = fake_EventTarget() @@ -124,7 +129,7 @@ class __EMSCRIPTEN__(object): def __init__(self): import platform - + platform.system = patch_platform_system self.platform = platform def __getattribute__(self, name): @@ -186,11 +191,6 @@ def no_op(cls, *argv, **kw): class TopLevel_async_handler(aio.toplevel.AsyncInteractiveConsole): HTML_MARK = '"' * 3 + " # BEGIN -->" -# @classmethod -# async def async_repos(cls): -# abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" -# print(f"{abitag=}") - @classmethod async def async_imports(cls, callback, *wanted, **kw): ... @@ -262,6 +262,9 @@ async def async_get_pkg(cls, want, ex, resume) print(__name__, "sim repl ready for", sourcefile) + if sys.path[0]!=cur_dir: + sys.path.insert(0,cur_dir) + await shell.runpy(sourcefile) shell.interactive() diff --git a/pygbag/app.py b/pygbag/app.py index 3729d44..02deebf 100644 --- a/pygbag/app.py +++ b/pygbag/app.py @@ -15,7 +15,7 @@ import shutil from datetime import datetime -from .__init__ import __version__ +from .__init__ import VERSION import pygbag @@ -31,7 +31,7 @@ CACHE_VERSION = CACHE_ROOT / "version.txt" CACHE_APP = CACHE_ROOT / "web" -cdn_dot = __version__.split(".") +cdn_dot = VERSION.split(".") cdn_dot.pop() cdn_version = ".".join(cdn_dot) del cdn_dot @@ -110,7 +110,7 @@ def set_args(program): def cache_check(app_folder, devmode=False): - global CACHE_PATH, CACHE_APP, __version__ + global CACHE_PATH, CACHE_APP, VERSION version_file = app_folder / CACHE_VERSION @@ -124,8 +124,8 @@ def cache_check(app_folder, devmode=False): try: with open(version_file, "r") as file: cache_ver = file.read() - if cache_ver != __version__: - print(f"115: cache {cache_ver} mismatch, want {__version__}, cleaning ...") + if cache_ver != VERSION: + print(f"115: cache {cache_ver} mismatch, want {VERSION}, cleaning ...") clear_cache = True except: # something's wrong in cache structure, try clean it up @@ -162,7 +162,7 @@ def make_cache_dirs(): make_cache_dirs() with open(version_file, "w") as file: - file.write(__version__) + file.write(VERSION) return build_dir, cache_dir @@ -235,8 +235,8 @@ async def main_run(app_folder, mainscript, cdn=DEFAULT_CDN): parser.add_argument( "--version", - default=__version__, - help="override prebuilt version path [default:%s]" % __version__, + default=VERSION, + help="override prebuilt version path [default:%s]" % VERSION, ) parser.add_argument("--build", action="store_true", help="build only, do not run test server") @@ -348,7 +348,7 @@ async def main_run(app_folder, mainscript, cdn=DEFAULT_CDN): "title": (args.title or app_name), "directory": app_name, "spdx": "cookiecutter.spdx", - "version": __version__, + "version": VERSION, "PYBUILD": args.PYBUILD, } diff --git a/pygbag/filtering.py b/pygbag/filtering.py index cc005bb..fc99d48 100644 --- a/pygbag/filtering.py +++ b/pygbag/filtering.py @@ -1,6 +1,6 @@ from pathlib import Path -dbg = True +dbg = False # q:what to do with the extreme case $HOME/main.py ? # or folders > 512MiB total diff --git a/pygbag/optimizing.py b/pygbag/optimizing.py index 19fc764..e293a16 100644 --- a/pygbag/optimizing.py +++ b/pygbag/optimizing.py @@ -104,16 +104,17 @@ def translated(fn): if len(tofix): fixname = Path(f"{fp.parent}/{fp.stem}-pygbag.py") - fixfull = Path(f"{folder}/{fixname}") + opt = Path(f"{folder}/{fixname}") with open(fname,"r", encoding="utf-8") as source: data = open(fname,"r").read() - with open(fixfull, "w", encoding="utf-8") as dest: + with open(opt, "w", encoding="utf-8") as dest: while len(tofix): bad, good = tofix.pop(0) warnings.warn(f"potential {bad.upper()} use in {fname}, prefer .{good} !") data = data.replace(f'.{bad}"',f'.{good}"') dest.write(data) - yield fixname + + yield translated(opt) continue if fp.suffix == ".png": diff --git a/pygbag/pack.py b/pygbag/pack.py index 9ef3821..92eb5f4 100644 --- a/pygbag/pack.py +++ b/pygbag/pack.py @@ -25,8 +25,9 @@ async def pack_files(zf, packlist, zfolders, target_folder): zpath.insert(0, str(target_folder)) zpath.append(str(asset)[1:]) - # print(f"\t{str(asset)[1:]}") zip_content = target_folder / str(asset)[1:] + print(f"\t{target_folder} : {str(asset)[1:]}") + zpath = list(zfolders) zpath.append(str(asset)[1:].replace("-pygbag.", ".")) diff --git a/pygbag/support/cross/aio/pep0723.py b/pygbag/support/cross/aio/pep0723.py index 6b11c0f..effc0cc 100644 --- a/pygbag/support/cross/aio/pep0723.py +++ b/pygbag/support/cross/aio/pep0723.py @@ -1,7 +1,9 @@ +# # https://peps.python.org/pep-0722/ – Dependency specification for single-file scripts +# https://peps.python.org/pep-0723 # https://peps.python.org/pep-0508/ – Dependency specification for Python Software Packages - # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html +# import sys import os @@ -20,9 +22,6 @@ from aio.filelike import fopen import platform - -print(platform) - import platform_wasm.todo PATCHLIST = [] @@ -136,7 +135,10 @@ def install(pkg_file, sconf=None): "INSTALLER": b"pygbag", }, ) + if pkg_file not in HISTORY: HISTORY.append(pkg_file) + importlib.invalidate_caches() + print("142: {pkg_file} installed") except FileExistsError as ex: print(f"38: {pkg_file} already installed (or partially)", ex) except Exception as ex: @@ -195,7 +197,10 @@ async def async_repos(): if not aio.cross.simulator: import platform print("193:", platform.window.location.href ) - if platform.window.location.href.startswith("https://pmp-p.ddns.net/pygbag"): + if platform.window.location.href.startswith("http://localhost:8"): + for idx, repo in enumerate(Config.pkg_repolist): + repo["-CDN-"] = "http://localhost:8000/archives/repo/" + elif platform.window.location.href.startswith("https://pmp-p.ddns.net/pygbag"): for idx, repo in enumerate(Config.pkg_repolist): repo["-CDN-"] = "https://pmp-p.ddns.net/archives/repo/" elif platform.window.location.href.startswith("http://192.168.1.66/pygbag"): @@ -283,19 +288,7 @@ async def pip_install(pkg, sconf={}): async def parse_code(code, env): - global PATCHLIST, async_imports_init, async_repos - - # pythonrc is calling aio.pep0723.parse_code not check_list - # so do patching here - patchlevel = platform_wasm.todo.patch() - if patchlevel: - print("264:parse_code() patches loaded :", list(patchlevel.keys()) ) - platform_wasm.todo.patch = lambda :None - # and only do that once and for all. - await async_imports_init() - await async_repos() - del async_imports_init, async_repos - + global PATCHLIST maybe_missing = [] @@ -338,10 +331,23 @@ async def parse_code(code, env): # parse_code does the patching # this is not called by pythonrc async def check_list(code=None, filename=None): - global PATCHLIST + global PATCHLIST, async_imports_init, async_repos print() print("-" * 11, "computing required packages", "-" * 10) + + # pythonrc is calling aio.pep0723.parse_code not check_list + # so do patching here + patchlevel = platform_wasm.todo.patch() + if patchlevel: + print("264:parse_code() patches loaded :", list(patchlevel.keys()) ) + platform_wasm.todo.patch = lambda :None + # and only do that once and for all. + await async_imports_init() + await async_repos() + del async_imports_init, async_repos + + # store installed wheel somewhere env = Path(os.getcwd()) / "build" / "env" env.mkdir(parents=True, exist_ok=True) @@ -353,6 +359,9 @@ async def check_list(code=None, filename=None): sconf = __import__("sysconfig").get_paths() sconf["purelib"] = sconf["platlib"] = env.as_posix() + if sconf["platlib"] not in sys.path: + sys.path.append(sconf["platlib"]) + # mandatory importlib.invalidate_caches() @@ -401,3 +410,24 @@ async def check_list(code=None, filename=None): print("-" * 40) print() + + return still_missing + + + + + + + + + + + + + + + + + + +# aio.pep0723 diff --git a/pygbag/support/pythonrc.py b/pygbag/support/pythonrc.py index 81ee492..e2bd1e5 100644 --- a/pygbag/support/pythonrc.py +++ b/pygbag/support/pythonrc.py @@ -414,10 +414,9 @@ async def pip(cls, *argv): for arg in argv: if arg == "install": continue - import aio.toplevel - + import aio.pep0723 # yield f"attempting to install {arg}" - await PyConfig.importer.async_imports(None, arg) + await aio.pep0723.pip_install(arg) @classmethod def cd(cls, *argv): @@ -486,11 +485,11 @@ def unzip(cls, *argv, **env): @classmethod def install(cls, *argv, **env): - import aio.toplevel + import aio.pep0723 for pkg_file in argv: try: - aio.toplevel.install(pkg_file) + aio.pep0723.install(pkg_file) yield f"{pkg_file} installed" except (IOError, zipfile.BadZipFile): pdb("397: invalid package", pkg_file) @@ -653,7 +652,8 @@ async def preload_code(cls, code, callback=None, hint=""): print("651: referenced packages :", len(Config.repos[0]["packages"])) DBG(f"644: aio.pep0723.check_list {env=}") - deps = await aio.pep0723.parse_code(code, env) + #deps = await aio.pep0723.parse_code(code, env) + deps = await aio.pep0723.check_list(code) #, env) DBG(f"646: aio.pep0723.pip_install {deps=}") # auto import plumbing to avoid rely too much on import error @@ -1415,40 +1415,6 @@ def print_pg_bar(total, iteration): # Print New Line on Complete print() -# @classmethod -# async def async_repos(cls): -# abitag = f"cp{sys.version_info.major}{sys.version_info.minor}" -# apitag = __import__("sysconfig").get_config_var("HOST_GNU_TYPE") -# apitag = apitag.replace("-", "_") -# -# for repo in PyConfig.pkg_indexes: -# if apitag.find("mvp") > 0: -# idx = f"{repo}index.json" -# else: -# idx = f"{repo}index-bi.json" -# -# async with platform.fopen(idx, "r") as index: -# try: -# data = index.read() -# if isinstance(data, bytes): -# data = data.decode() -# data = data.replace("", abitag) -# data = data.replace("", apitag) -# repo = json.loads(data) -# except: -# pdb(f"1394: {repo=}: malformed json index {data}") -# continue -# if repo not in PyConfig.pkg_repolist: -# PyConfig.pkg_repolist.append(repo) -# -# if PyConfig.dev_mode > 0: -# for idx, repo in enumerate(PyConfig.pkg_repolist): -# try: -# print("1353:", repo["-CDN-"], idx, "REMAPPED TO", PyConfig.pkg_indexes[idx]) -# repo["-CDN-"] = PyConfig.pkg_indexes[idx] -# except Exception as e: -# sys.print_exception(e) - # end TopLevel_async_handler diff --git a/scripts/build-loader.sh b/scripts/build-loader.sh index 5b52a78..347960e 100755 --- a/scripts/build-loader.sh +++ b/scripts/build-loader.sh @@ -263,7 +263,8 @@ emcc $FINAL_OPTS $LOPTS -std=gnu99 -D__PYDK__=1 -DNDEBUG \\ --preload-file ${DYNLOAD}@/usr/lib/python${PYBUILD}/lib-dynload \\ --preload-file ${REQUIREMENTS}@/data/data/org.python/assets/site-packages \\ -o ${DIST_DIR}/python${PYMAJOR}${PYMINOR}/${MODE}.js build/${MODE}.o \\ - $LDFLAGS + $LDFLAGS -sERROR_ON_UNDEFINED_SYMBOLS=0 + END chmod +x ./final_link.sh diff --git a/setup.cfg b/setup.cfg index d7a8198..41e7518 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,8 @@ [metadata] name = pygbag -version = attr: pygbag.__version__ +version = attr: pygbag.VERSION author = Paul Peny -description = pygame wasm, package and run python/pygame directly in modern web browsers. +description = package and run python/pygame-ce wasm (and more) directly in modern web browsers. long_description = file: README.md long_description_content_type = text/markdown url = https://pygame-web.github.io/ @@ -28,9 +28,12 @@ python_requires = >=3.8 install_requires = token_utils - pyparsing + pyparsing packaging installer + +[options.extras_require] +SIM = aioconsole aiohttp asyncio_socks_server diff --git a/static/vtx.js b/static/vtx.js index b203cfe..c906cf0 100644 --- a/static/vtx.js +++ b/static/vtx.js @@ -10,7 +10,7 @@ readline.complete = function (line) { } -function rawstdin(line) { +function rawstdin_send(line) { //console.log("RAW:", line ) python.rawstdin(line) } @@ -116,7 +116,7 @@ export class WasmTerminal { // TODO: check mouse pos if (window.RAW_MODE) - rawstdin(data) + rawstdin_send(data) // TODO: Handle ANSI escape sequences if (ord === 0x1b) { @@ -187,7 +187,7 @@ export class WasmTerminal { case 60: if (!window.RAW_MODE) - rawstdin(data) + rawstdin_send(data) break; default: diff --git a/support/__EMSCRIPTEN__.c b/support/__EMSCRIPTEN__.c index d91d45f..6d85883 100644 --- a/support/__EMSCRIPTEN__.c +++ b/support/__EMSCRIPTEN__.c @@ -301,6 +301,9 @@ embed_readline(PyObject *self, PyObject *_null); //forward static PyObject * embed_os_read(PyObject *self, PyObject *_null); //forward +static PyObject * +embed_stdin_select(PyObject *self, PyObject *_null); // forward + static PyObject * embed_flush(PyObject *self, PyObject *_null) { fprintf( stdout, "%c", 4); @@ -530,6 +533,8 @@ static PyMethodDef mod_embed_methods[] = { {"readline", (PyCFunction)embed_readline, METH_NOARGS, "get current line"}, {"os_read", (PyCFunction)embed_os_read, METH_NOARGS, "get current raw stdin"}, + {"stdin_select", (PyCFunction)embed_stdin_select, METH_NOARGS, "get current raw stdin bytes length"}, + {"flush", (PyCFunction)embed_flush, METH_NOARGS, "flush stdio+stderr"}, {"set_ps1", (PyCFunction)embed_set_ps1, METH_NOARGS, "set prompt output to >>> "}, @@ -682,6 +687,11 @@ embed_os_read(PyObject *self, PyObject *_null) { #undef file } +static PyObject * +embed_stdin_select(PyObject *self, PyObject *_null) { + return Py_BuildValue("i", embed_os_read_bufsize ); +} + int io_file_select(int fdnum) {