handle p8/lua files with invalid characters. also start requiring pyt…

…hon 3.8 or up, 3.7 is long-obsolete and hasn't eaten a walrus

.github/workflows/run-tests.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.7"
+        python-version: "3.8"
 
     - name: Install dependencies
       run: |

README.md

@@ -6,7 +6,7 @@ A set of Pico-8 & Picotron cart tools, with a focus on shrinking code size.
 
 [You can download a recent Windows Executable here.](https://github.com/thisismypassport/shrinko8/releases)
 
-Otherwise, requires [Python](https://www.python.org/) 3.7 or above to run.
+Otherwise, requires [Python](https://www.python.org/) 3.8 or above to run.
 
 Reading/Writing PNGs additionally requires the Pillow module (`python -m pip install pillow` to install)
 

pico_cart.py

@@ -415,7 +415,7 @@ def ext_nybbles(line):
             elif header == None and clean.startswith("version "):
                 cart.set_version(int(clean.split()[1]))
 
-        except Exception as e:
+        except ValueError:
             throw(f"Invalid {header} line in p8 file (line #{line_i + 1})")
 
     cart.code, cart.code_map = preprocess_code(path, "".join(code), code_line, preprocessor=preprocessor)

pico_defs.py

@@ -166,18 +166,20 @@ def mem_create_rom():
     k_charset.append(chr(i))
 k_charset += [
     '○',
-    '█','▒','🐱','⬇️','░','✽','●','♥','☉','웃','⌂','⬅️','😐','♪','🅾️','◆','…','➡️','★','⧗','⬆️','ˇ','∧','❎','▤','▥',
-    'あ','い','う','え','お','か','き','く','け','こ','さ','し','す','せ','そ','た','ち','つ','て','と','な','に','ぬ','ね','の','は','ひ','ふ','へ','ほ',
-    'ま','み','む','め','も','や','ゆ','よ','ら','り','る','れ','ろ','わ','を','ん','っ','ゃ','ゅ','ょ','ア','イ','ウ','エ','オ','カ','キ','ク','ケ','コ','サ',
-    'シ','ス','セ','ソ','タ','チ','ツ','テ','ト','ナ','ニ','ヌ','ネ','ノ','ハ','ヒ','フ','ヘ','ホ','マ','ミ','ム','メ','モ','ヤ','ユ','ヨ','ラ','リ','ル','レ',
-    'ロ','ワ','ヲ','ン','ッ','ャ','ュ','ョ','◜','◝'
+    # note: some of the below have a k_variant_char at the end
+    '█','▒','🐱','⬇️','░','✽','●','♥','☉','웃','⌂','⬅️','😐','♪','🅾️','◆',
+    '…','➡️','★','⧗','⬆️','ˇ','∧','❎','▤','▥','あ','い','う','え','お','か',
+    'き','く','け','こ','さ','し','す','せ','そ','た','ち','つ','て','と','な','に',
+    'ぬ','ね','の','は','ひ','ふ','へ','ほ','ま','み','む','め','も','や','ゆ','よ',
+    'ら','り','る','れ','ろ','わ','を','ん','っ','ゃ','ゅ','ょ','ア','イ','ウ','エ',
+    'オ','カ','キ','ク','ケ','コ','サ','シ','ス','セ','ソ','タ','チ','ツ','テ','ト',
+    'ナ','ニ','ヌ','ネ','ノ','ハ','ヒ','フ','ヘ','ホ','マ','ミ','ム','メ','モ','ヤ',
+    'ユ','ヨ','ラ','リ','ル','レ','ロ','ワ','ヲ','ン','ッ','ャ','ュ','ョ','◜','◝'
 ]
 assert len(k_charset) == 0x100
 
-# maps unicode character to pico8 char index
-k_charset_map = {ch[0]: i for i, ch in enumerate(k_charset) if ch != None}
-
-k_variant_char = '\uFE0F'
+# maps unicode characters or 2-char-strings to pico8 char index
+k_charset_map = {ch: i for i, ch in enumerate(k_charset) if ch != None}
 
 k_unicap_chars = ['𝘢','𝘣','𝘤','𝘥','𝘦','𝘧','𝘨','𝘩','𝘪','𝘫','𝘬','𝘭','𝘮','𝘯','𝘰','𝘱','𝘲','𝘳','𝘴','𝘵','𝘶','𝘷','𝘸','𝘹','𝘺','𝘻']
 
@@ -194,15 +196,22 @@ def mem_create_rom():
 def to_p8str(text):
     """Convert a unicode string to a pico8 string"""
     result = []
-    for ch in text:
+    ignore_next = False
+    for i, ch in enumerate(text):
         if ord(ch) < 0x80:
             result.append(ch)
         elif ch in k_charset_map:
             result.append(chr(k_charset_map[ch]))
-        elif ch == k_variant_char:
-            pass
+        elif ignore_next:
+            ignore_next = False
+            continue
+        elif (chpair := ch + str_get(text, i + 1, '')) in k_charset_map:
+            result.append(chr(k_charset_map[chpair]))
+            ignore_next = True
         else:
-            throw(f"invalid char: {ch} ({ord(ch)})")
+            # pico8 just reinterprets characters it doesn't understand as bytes
+            for byte in ch.encode():
+                result.append(chr(byte))
     return "".join(result)
 
 def from_p8str(text, unicaps=False):

run_tests.py

@@ -232,6 +232,7 @@ def run():
              "--local-builtin", "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z")
     run_test("tinyrom", "tiny.rom", "tiny.lua")
     run_test("title", "title.p8", "title.p8.png", update_version=False)
+    run_test("uni8", "uni8.lua", "uni8.p8")
 
     if run_test("repl", "repl.p8", "repl.p8", "--minify",
                 "--rename-map", "test_output/repl.map", extra_outputs=["repl.map"], pico8_output_val="finished"):

shrinko.py

@@ -78,7 +78,6 @@ def create_main(lang):
         fail("unknown language")
 
     def create_parser():
-        extend_arg = "extend" if sys.version_info >= (3,8) else None
         src_ext = CartFormatCls.default_src
 
         if is_pico8:
@@ -119,8 +118,8 @@ def create_parser():
         pgroup.add_argument("--no-minify-comments", action="store_true", help="disable comment removal in minification (requires --no-minify-spaces)")
         pgroup.add_argument("--no-minify-tokens", action="store_true", help="disable token removal/changes in minification")
         pgroup.add_argument("--no-minify-reorder", action="store_true", help="disable statement reordering in minification")
-        pgroup.add_argument("-p", "--preserve", type=SplitBySeps, action=extend_arg, help='preserve specific identifiers in minification, e.g. "global1, global2, *.member2, table3.*"')
-        pgroup.add_argument("--no-preserve", type=SplitBySeps, action=extend_arg, help='do not preserve specific built-in identifiers in minification, e.g. "circfill, rectfill"')
+        pgroup.add_argument("-p", "--preserve", type=SplitBySeps, action="extend", help='preserve specific identifiers in minification, e.g. "global1, global2, *.member2, table3.*"')
+        pgroup.add_argument("--no-preserve", type=SplitBySeps, action="extend", help='do not preserve specific built-in identifiers in minification, e.g. "circfill, rectfill"')
         pgroup.add_argument("--rename-members-as-globals", action="store_true", help='rename globals and members the same way (same as --preserve "*=*.*")')
         pgroup.add_argument("--rename-safe-only", action="store_true", help="only do renaming that's always safe to do (subset of --minify-safe-only)")
         pgroup.add_argument("--reorder-safe-only", action="store_true", help="only do statement reordering that's always safe to do (subset of --minify-safe-only)")
@@ -137,7 +136,7 @@ def create_parser():
         pgroup.add_argument("--no-lint-duplicate-global", action="store_true", help="don't print lint warnings on duplicate variables between a local and a global")
         pgroup.add_argument("--no-lint-undefined", action="store_true", help="don't print lint warnings on undefined variables")
         pgroup.add_argument("--no-lint-fail", action="store_true", help="create output cart even if there were lint warnings")
-        pgroup.add_argument("--lint-global", type=SplitBySeps, action=extend_arg, help="don't print lint warnings for these globals (same as '--lint:' comment)")
+        pgroup.add_argument("--lint-global", type=SplitBySeps, action="extend", help="don't print lint warnings for these globals (same as '--lint:' comment)")
         pgroup.add_argument("--error-format", type=EnumFromStr(ErrorFormat), help="how to format lint warnings & compilation errors {%s}" % EnumList(ErrorFormat))
 
         pgroup = parser.add_argument_group("count options")
@@ -163,10 +162,10 @@ def create_parser():
 
         if is_picotron:
             pgroup = parser.add_argument_group("picotron filesystem options")
-            pgroup.add_argument("--code-files", dest="code_sections", type=SplitBySeps, action=extend_arg,
+            pgroup.add_argument("--code-files", dest="code_sections", type=SplitBySeps, action="extend",
                                 help=f"specify a {sections_desc} that contain lua code to process (default: *.lua)")
             pgroup.add_argument("--list", action="store_true", help="list all files inside the cart")
-            pgroup.add_argument("--filter", type=SplitBySeps, action=extend_arg, help=f"specify a {sections_desc} to keep in the output")
+            pgroup.add_argument("--filter", type=SplitBySeps, action="extend", help=f"specify a {sections_desc} to keep in the output")
             pgroup.add_argument("--insert", nargs='+', action="append", metavar=(f"INPUT [FSPATH] [FILES_FILTER]", ""),
                                 help=f"insert the specified INPUT file or directory at FSPATH")
             pgroup.add_argument("--extract", nargs='+', action="append", metavar=(f"FSPATH [OUTPUT]", ""),
@@ -224,12 +223,12 @@ def create_parser():
 
         pgroup = parser.add_argument_group("other uninteresting options (semi-undocumented)")
         pgroup.add_argument("--ignore-hints", action="store_true", help="ignore shrinko8 hint comments")
-        pgroup.add_argument("--builtin", type=SplitBySeps, action=extend_arg, help="treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)")
-        pgroup.add_argument("--not-builtin", type=SplitBySeps, action=extend_arg, help="do not treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)")
+        pgroup.add_argument("--builtin", type=SplitBySeps, action="extend", help="treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)")
+        pgroup.add_argument("--not-builtin", type=SplitBySeps, action="extend", help="do not treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)")
         if is_pico8:
             pgroup.add_argument("--global-builtins-only", action="store_true", help="assume all builtins are global, corresponds to pico8's -global_api option")
-            pgroup.add_argument("--local-builtin", type=SplitBySeps, action=extend_arg, help="treat identifier(s) as a local builtin (probably no use outside of testing)")
-            pgroup.add_argument("--output-sections", type=SplitBySeps, action=extend_arg, help=f"specifies a {sections_desc} to write to the p8")
+            pgroup.add_argument("--local-builtin", type=SplitBySeps, action="extend", help="treat identifier(s) as a local builtin (probably no use outside of testing)")
+            pgroup.add_argument("--output-sections", type=SplitBySeps, action="extend", help=f"specifies a {sections_desc} to write to the p8")
             pgroup.add_argument("--export-name", help="name to use for the export (by default, taken from output name)")
             pgroup.add_argument("--custom-preprocessor", action="store_true", help=argparse.SUPPRESS) # might remove this one day
         else:

test_compare/uni8.p8

@@ -0,0 +1,8 @@
+pico-8 cartridge // http://www.pico-8.com
+version 42
+__lua__
+-- ウˇウめウめウほウやウみウむウな
+-- ユか▤█ユか▤⬇️
+-- ユか▤█ヤま◆ユか▤⬇️ヤま◆
+-- ⬇️⬆️➡️⬅️🐱ヤま◆❎ヤま◆🅾️
+-- ヌて♥ヌて●ヌおくヌて✽🐱❎ユか✽ゆ

test_input/uni8.lua

@@ -0,0 +1,5 @@
+-- Ελληνική
+-- 😀😃
+-- 😀️😃️
+-- ⬇️⬆️➡️⬅️🐱️❎️🅾️
+-- ⬇⬆➡⬅🐱❎🅾