diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ed68557..6288209 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.7" + python-version: "3.8" - name: Install dependencies run: | diff --git a/README.md b/README.md index 2677dd4..b09c1aa 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A set of Pico-8 & Picotron cart tools, with a focus on shrinking code size. [You can download a recent Windows Executable here.](https://github.com/thisismypassport/shrinko8/releases) -Otherwise, requires [Python](https://www.python.org/) 3.7 or above to run. +Otherwise, requires [Python](https://www.python.org/) 3.8 or above to run. Reading/Writing PNGs additionally requires the Pillow module (`python -m pip install pillow` to install) diff --git a/pico_cart.py b/pico_cart.py index 759fca5..19e8b6d 100644 --- a/pico_cart.py +++ b/pico_cart.py @@ -415,7 +415,7 @@ def ext_nybbles(line): elif header == None and clean.startswith("version "): cart.set_version(int(clean.split()[1])) - except Exception as e: + except ValueError: throw(f"Invalid {header} line in p8 file (line #{line_i + 1})") cart.code, cart.code_map = preprocess_code(path, "".join(code), code_line, preprocessor=preprocessor) diff --git a/pico_defs.py b/pico_defs.py index c490c42..984567e 100644 --- a/pico_defs.py +++ b/pico_defs.py @@ -166,18 +166,20 @@ def mem_create_rom(): k_charset.append(chr(i)) k_charset += [ '○', - '█','▒','đŸ±','âŹ‡ïž','░','✜','●','♄','☉','웃','⌂','âŹ…ïž','😐','â™Ș','đŸ…Ÿïž','◆','
','âžĄïž','★','⧗','âŹ†ïž','ˇ','∧','❎','â–€','â–„', - 'あ','い','う','え','お','か','き','く','け','こ','さ','し','す','せ','そ','た','づ','぀','お','ず','ăȘ','に','く','ね','た','は','ăČ','ご','ぞ','ほ', - 'ăŸ','み','む','め','も','や','ゆ','よ','ら','り','る','れ','ろ','わ','を','ん','っ','ゃ','ゅ','ょ','ケ','ă‚€','ォ','ス','ă‚Ș','ă‚«','キ','ク','ケ','コ','ă‚”', - 'ă‚·','ă‚č','ă‚»','ă‚œ','タ','チ','ツ','テ','ト','ナ','ニ','ヌ','ネ','ノ','ハ','ヒ','フ','ヘ','ホ','マ','ミ','ム','ュ','ヹ','ダ','ナ','ペ','ラ','ăƒȘ','ル','ハ', - 'ロ','ワ','ăƒČ','ン','ッ','ャ','ツ','ョ','◜','◝' + # note: some of the below have a k_variant_char at the end + '█','▒','đŸ±','âŹ‡ïž','░','✜','●','♄','☉','웃','⌂','âŹ…ïž','😐','â™Ș','đŸ…Ÿïž','◆', + '
','âžĄïž','★','⧗','âŹ†ïž','ˇ','∧','❎','â–€','â–„','あ','い','う','え','お','か', + 'き','く','け','こ','さ','し','す','せ','そ','た','づ','぀','お','ず','ăȘ','に', + 'く','ね','た','は','ăČ','ご','ぞ','ほ','ăŸ','み','む','め','も','や','ゆ','よ', + 'ら','り','る','れ','ろ','わ','を','ん','っ','ゃ','ゅ','ょ','ケ','ă‚€','ォ','ス', + 'ă‚Ș','ă‚«','キ','ク','ケ','コ','ă‚”','ă‚·','ă‚č','ă‚»','ă‚œ','タ','チ','ツ','テ','ト', + 'ナ','ニ','ヌ','ネ','ノ','ハ','ヒ','フ','ヘ','ホ','マ','ミ','ム','ュ','ヹ','ダ', + 'ナ','ペ','ラ','ăƒȘ','ル','ハ','ロ','ワ','ăƒČ','ン','ッ','ャ','ツ','ョ','◜','◝' ] assert len(k_charset) == 0x100 -# maps unicode character to pico8 char index -k_charset_map = {ch[0]: i for i, ch in enumerate(k_charset) if ch != None} - -k_variant_char = '\uFE0F' +# maps unicode characters or 2-char-strings to pico8 char index +k_charset_map = {ch: i for i, ch in enumerate(k_charset) if ch != None} k_unicap_chars = ['𝘱','𝘣','đ˜€','đ˜„','𝘩','𝘧','𝘹','đ˜©','đ˜Ș','đ˜«','𝘬','𝘭','𝘼','𝘯','𝘰','đ˜±','đ˜Č','𝘳','𝘮','đ˜”','đ˜¶','đ˜·','𝘾','đ˜č','đ˜ș','đ˜»'] @@ -194,15 +196,22 @@ def mem_create_rom(): def to_p8str(text): """Convert a unicode string to a pico8 string""" result = [] - for ch in text: + ignore_next = False + for i, ch in enumerate(text): if ord(ch) < 0x80: result.append(ch) elif ch in k_charset_map: result.append(chr(k_charset_map[ch])) - elif ch == k_variant_char: - pass + elif ignore_next: + ignore_next = False + continue + elif (chpair := ch + str_get(text, i + 1, '')) in k_charset_map: + result.append(chr(k_charset_map[chpair])) + ignore_next = True else: - throw(f"invalid char: {ch} ({ord(ch)})") + # pico8 just reinterprets characters it doesn't understand as bytes + for byte in ch.encode(): + result.append(chr(byte)) return "".join(result) def from_p8str(text, unicaps=False): diff --git a/run_tests.py b/run_tests.py index 5a75fbf..09dd8e4 100644 --- a/run_tests.py +++ b/run_tests.py @@ -232,6 +232,7 @@ def run(): "--local-builtin", "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z") run_test("tinyrom", "tiny.rom", "tiny.lua") run_test("title", "title.p8", "title.p8.png", update_version=False) + run_test("uni8", "uni8.lua", "uni8.p8") if run_test("repl", "repl.p8", "repl.p8", "--minify", "--rename-map", "test_output/repl.map", extra_outputs=["repl.map"], pico8_output_val="finished"): diff --git a/shrinko.py b/shrinko.py index 7ebf8b8..4d82d7f 100644 --- a/shrinko.py +++ b/shrinko.py @@ -78,7 +78,6 @@ def create_main(lang): fail("unknown language") def create_parser(): - extend_arg = "extend" if sys.version_info >= (3,8) else None src_ext = CartFormatCls.default_src if is_pico8: @@ -119,8 +118,8 @@ def create_parser(): pgroup.add_argument("--no-minify-comments", action="store_true", help="disable comment removal in minification (requires --no-minify-spaces)") pgroup.add_argument("--no-minify-tokens", action="store_true", help="disable token removal/changes in minification") pgroup.add_argument("--no-minify-reorder", action="store_true", help="disable statement reordering in minification") - pgroup.add_argument("-p", "--preserve", type=SplitBySeps, action=extend_arg, help='preserve specific identifiers in minification, e.g. "global1, global2, *.member2, table3.*"') - pgroup.add_argument("--no-preserve", type=SplitBySeps, action=extend_arg, help='do not preserve specific built-in identifiers in minification, e.g. "circfill, rectfill"') + pgroup.add_argument("-p", "--preserve", type=SplitBySeps, action="extend", help='preserve specific identifiers in minification, e.g. "global1, global2, *.member2, table3.*"') + pgroup.add_argument("--no-preserve", type=SplitBySeps, action="extend", help='do not preserve specific built-in identifiers in minification, e.g. "circfill, rectfill"') pgroup.add_argument("--rename-members-as-globals", action="store_true", help='rename globals and members the same way (same as --preserve "*=*.*")') pgroup.add_argument("--rename-safe-only", action="store_true", help="only do renaming that's always safe to do (subset of --minify-safe-only)") pgroup.add_argument("--reorder-safe-only", action="store_true", help="only do statement reordering that's always safe to do (subset of --minify-safe-only)") @@ -137,7 +136,7 @@ def create_parser(): pgroup.add_argument("--no-lint-duplicate-global", action="store_true", help="don't print lint warnings on duplicate variables between a local and a global") pgroup.add_argument("--no-lint-undefined", action="store_true", help="don't print lint warnings on undefined variables") pgroup.add_argument("--no-lint-fail", action="store_true", help="create output cart even if there were lint warnings") - pgroup.add_argument("--lint-global", type=SplitBySeps, action=extend_arg, help="don't print lint warnings for these globals (same as '--lint:' comment)") + pgroup.add_argument("--lint-global", type=SplitBySeps, action="extend", help="don't print lint warnings for these globals (same as '--lint:' comment)") pgroup.add_argument("--error-format", type=EnumFromStr(ErrorFormat), help="how to format lint warnings & compilation errors {%s}" % EnumList(ErrorFormat)) pgroup = parser.add_argument_group("count options") @@ -163,10 +162,10 @@ def create_parser(): if is_picotron: pgroup = parser.add_argument_group("picotron filesystem options") - pgroup.add_argument("--code-files", dest="code_sections", type=SplitBySeps, action=extend_arg, + pgroup.add_argument("--code-files", dest="code_sections", type=SplitBySeps, action="extend", help=f"specify a {sections_desc} that contain lua code to process (default: *.lua)") pgroup.add_argument("--list", action="store_true", help="list all files inside the cart") - pgroup.add_argument("--filter", type=SplitBySeps, action=extend_arg, help=f"specify a {sections_desc} to keep in the output") + pgroup.add_argument("--filter", type=SplitBySeps, action="extend", help=f"specify a {sections_desc} to keep in the output") pgroup.add_argument("--insert", nargs='+', action="append", metavar=(f"INPUT [FSPATH] [FILES_FILTER]", ""), help=f"insert the specified INPUT file or directory at FSPATH") pgroup.add_argument("--extract", nargs='+', action="append", metavar=(f"FSPATH [OUTPUT]", ""), @@ -224,12 +223,12 @@ def create_parser(): pgroup = parser.add_argument_group("other uninteresting options (semi-undocumented)") pgroup.add_argument("--ignore-hints", action="store_true", help="ignore shrinko8 hint comments") - pgroup.add_argument("--builtin", type=SplitBySeps, action=extend_arg, help="treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)") - pgroup.add_argument("--not-builtin", type=SplitBySeps, action=extend_arg, help="do not treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)") + pgroup.add_argument("--builtin", type=SplitBySeps, action="extend", help="treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)") + pgroup.add_argument("--not-builtin", type=SplitBySeps, action="extend", help="do not treat identifier(s) as a pico-8 builtin (for minify, lint, etc.)") if is_pico8: pgroup.add_argument("--global-builtins-only", action="store_true", help="assume all builtins are global, corresponds to pico8's -global_api option") - pgroup.add_argument("--local-builtin", type=SplitBySeps, action=extend_arg, help="treat identifier(s) as a local builtin (probably no use outside of testing)") - pgroup.add_argument("--output-sections", type=SplitBySeps, action=extend_arg, help=f"specifies a {sections_desc} to write to the p8") + pgroup.add_argument("--local-builtin", type=SplitBySeps, action="extend", help="treat identifier(s) as a local builtin (probably no use outside of testing)") + pgroup.add_argument("--output-sections", type=SplitBySeps, action="extend", help=f"specifies a {sections_desc} to write to the p8") pgroup.add_argument("--export-name", help="name to use for the export (by default, taken from output name)") pgroup.add_argument("--custom-preprocessor", action="store_true", help=argparse.SUPPRESS) # might remove this one day else: diff --git a/test_compare/uni8.p8 b/test_compare/uni8.p8 new file mode 100644 index 0000000..e7e8c82 --- /dev/null +++ b/test_compare/uni8.p8 @@ -0,0 +1,8 @@ +pico-8 cartridge // http://www.pico-8.com +version 42 +__lua__ +-- りˇりめりめりほりやりみりむりăȘ +-- ăƒŠă‹â–€â–ˆăƒŠă‹â–€âŹ‡ïž +-- ăƒŠă‹â–€â–ˆăƒ€ăŸâ—†ăƒŠă‹â–€âŹ‡ïžăƒ€ăŸâ—† +-- âŹ‡ïžâŹ†ïžâžĄïžâŹ…ïžđŸ±ăƒ€ăŸâ—†âŽăƒ€ăŸâ—†đŸ…Ÿïž +-- ăƒŒăŠâ™„ăƒŒăŠâ—ăƒŒăŠăăƒŒăŠâœœđŸ±âŽăƒŠă‹âœœă‚† diff --git a/test_input/uni8.lua b/test_input/uni8.lua new file mode 100644 index 0000000..5686209 --- /dev/null +++ b/test_input/uni8.lua @@ -0,0 +1,5 @@ +-- ΕλληΜÎčÎșÎź +-- 😀😃 +-- đŸ˜€ïžđŸ˜ƒïž +-- âŹ‡ïžâŹ†ïžâžĄïžâŹ…ïžđŸ±ïžâŽïžđŸ…Ÿïž +-- âŹ‡âŹ†âžĄâŹ…đŸ±âŽđŸ…Ÿ \ No newline at end of file