picotron pods - support b64 and unpod, added --base64-pods, added pod…

… format for easy cmdline manipulation, fail instead of warn if cannot parse

README.md

@@ -1012,7 +1012,22 @@ Cart manipulation features:
 * `--extract` - extract a file or directory from the cart. E.g. `--extract main.lua` or `--extract sfx/1.sfx MyDocuments/1.sfx`
 * `--merge` - merge another cart into the cart. E.g. `--merge other.p64` or `--merge other.p64 "sfx/*,gfx/*"`
 
+Cart formats:
+* p8 - Picotron cart source
+* dir - Picotron cart exported into a directory
+* png - Picotron cart exported into a png
+* rom - Picotron cart exported into a rom
+* tiny-rom - Picotron cart exported into a tiny rom (code only)
+* lua - raw lua code (as main.lua)
+* pod - raw POD file (for easier manipulation of single pods)
+* label - A 480x270 image of a cart's label (label only)
+
+POD files:
+By default, Shrinkotron repacks all POD files for better compression. There are options to change this:
+* `--uncompress-pods` (or `-u`) - uncompress all PODs
+* `--keep-pod-compression` - do not touch the PODs - keep them as-is
+* `--base64-pods` - output base64 version of PODs (probably not useful unless manipulating single pods via the 'pod' format)
+
 Notes:
 * Shrinkotron assumes calls to `include` are used to include other unmodified lua files. If this is not the case, minify may break even under `--minify-safe-only`
-* Shrinkotron repacks all POD files for better compression. (There are options to change this - `--uncompress-pods` and `--keep-pod-compression`)
 * As Picotron evolves, there might be new globals or table keys that Shrinkotron isn't aware of. You can report such cases and use [`--preserve`](#preserving-identifiers-across-the-entire-cart) meanwhile.

pico_cart.py

@@ -522,6 +522,12 @@ def iter_rect(width, height):
 k_base64_alt_chars = k_base64_chars[62:].encode()
 k_base64_char_map = {ch: i for i, ch in enumerate(k_base64_chars)}
 
+def pico_base64_encode(data):
+    return base64.b64encode(data, k_base64_alt_chars)
+
+def pico_base64_decode(data, validate=False):
+    return base64.b64decode(data, k_base64_alt_chars, validate=validate)
+
 def print_url_size(size, **kwargs):
     print_size("url", size - k_url_prefix_size, k_url_size, **kwargs)
 
@@ -550,7 +556,7 @@ def read_cart_from_url(url, size_handler=None, **opts):
     cart = Cart()
 
     if code:
-        codebuf = base64.b64decode(code, k_base64_alt_chars, validate=True).ljust(k_code_size, b'\0')
+        codebuf = pico_base64_decode(code, validate=True).ljust(k_code_size, b'\0')
         with BinaryReader(BytesIO(codebuf), big_end = True) as r:
             cart.code, cart.code_rom = read_code_from_rom(r, **opts)
 
@@ -577,7 +583,7 @@ def read_cart_from_url(url, size_handler=None, **opts):
 
 def write_cart_to_url(cart, url_prefix=k_url_prefix, force_compress=False, size_handler=None, **opts):
     raw_code = write_cart_to_tiny_rom(cart, **opts)        
-    code = base64.b64encode(raw_code, k_base64_alt_chars)
+    code = pico_base64_encode(raw_code)
 
     rect = iter_rect(128, 128)
     gfx = []

picotron_cart.py

@@ -1,16 +1,15 @@
 from utils import *
 from media_utils import Surface, Color
-from pico_cart import load_image_of_size, get_res_path, k_base64_alt_chars, k_png_header
+from pico_cart import load_image_of_size, get_res_path, pico_base64_decode, pico_base64_encode, k_png_header
 from pico_export import lz4_uncompress, lz4_compress
 from pico_defs import decode_luastr, encode_luastr
 from pico_compress import print_size, compress_code, uncompress_code, encode_p8str, decode_p8str
 from picotron_defs import get_default_picotron_version, Cart64Glob
-from picotron_fs import PicotronFile, PicotronDir
-import base64
+from picotron_fs import PicotronFile, PicotronDir, k_pod
 
 class Cart64Format(Enum):
     """An enum representing the supported cart formats"""
-    auto = p64 = png = rom = tiny_rom = lua = dir = fs = label = ...
+    auto = p64 = png = rom = tiny_rom = lua = pod = dir = fs = label = ...
 
     @property
     def is_input(m):
@@ -23,7 +22,7 @@ def is_ext(m):
         return m not in (m.auto, m.dir, m.fs, m.label, m.tiny_rom)
     @property
     def is_src(m):
-        return m in (m.p64, m.lua, m.dir, m.fs)
+        return m in (m.p64, m.lua, m.pod, m.dir, m.fs)
     @property
     def is_export(m):
         return False
@@ -338,7 +337,7 @@ def flush():
             else:
                 data = b"\n".join(lines)
                 if data.startswith(k_p64_b64_prefix):
-                    data = base64.b64decode(data[4:], k_base64_alt_chars)
+                    data = pico_base64_decode(data[4:])
 
                 cart.files[fspath] = PicotronFile(data, line_start)
 
@@ -394,7 +393,7 @@ def write_cart64_to_source(cart, avoid_base64=False, **opts):
         lines.append(k_p64_file_prefix + encode_luastr(fspath))
         if e(file.data):
             if bad_p64_char_re.search(file.data):
-                data = k_p64_b64_prefix + base64.b64encode(file.data, k_base64_alt_chars)
+                data = k_p64_b64_prefix + pico_base64_encode(file.data)
                 for line in str_chunk(data, k_p64_b64_line_size):
                     lines.append(line)
             else:
@@ -409,6 +408,23 @@ def write_cart64_to_raw_source(cart, **_):
     check(main, "{k_p64_main_path} not found in cart")
     return main.data
 
+k_fictive_main_pod = "main.pod" # not a real picotron concept
+
+def read_cart64_from_single_pod(data, path=None, **_):
+    cart = Cart64(path=path)
+    main = PicotronFile(data)
+    if main.raw_metadata is None:
+        main.raw_metadata = k_pod
+    cart.files[k_fictive_main_pod] = main
+    return cart
+
+def write_cart64_to_single_pod(cart, **_):
+    main = cart.files.get(k_fictive_main_pod)
+    check(main, "{k_fictive_main_pod} not found in cart")
+    if main.raw_metadata == k_pod:
+        main.raw_metadata = None
+    return main.data
+
 def read_cart64_from_fs(path, is_dir=None, target_cart=None, fspath=None, sections=None, **opts):
     if target_cart is None:
         target_cart = Cart64(path=path)
@@ -500,6 +516,8 @@ def read_cart64(path, format=None, **opts):
         return read_cart64_from_tiny_rom(file_read(path), path=path, **opts)
     elif format == Cart64Format.lua:
         return read_cart64_from_source(file_read(path), raw=True, path=path, **opts)
+    elif format == Cart64Format.pod:
+        return read_cart64_from_single_pod(file_read(path), path=path, **opts)
     elif format in (Cart64Format.dir, Cart64Format.fs):
         return read_cart64_from_fs(path, is_dir=True if format == Cart64Format.dir else None, **opts)
     elif format == Cart64Format.label:
@@ -519,6 +537,8 @@ def write_cart64(path, cart, format, **opts):
         file_write(path, write_cart64_to_tiny_rom(cart, **opts))
     elif format == Cart64Format.lua:
         file_write(path, write_cart64_to_raw_source(cart, **opts))
+    elif format == Cart64Format.pod:
+        file_write(path, write_cart64_to_single_pod(cart, **opts))
     elif format in (Cart64Format.dir, Cart64Format.fs):
         write_cart64_to_fs(cart, path, is_dir=True if format == Cart64Format.dir else None, **opts)
     elif format == Cart64Format.label:
@@ -532,7 +552,7 @@ def filter_cart64(cart, sections):
     for path in to_delete:
         del cart.files[path]
 
-def preproc_cart64(cart, delete_meta=None, uncompress_pods=False, keep_pod_compression=False, need_pod_compression=False):
+def preproc_cart64(cart, delete_meta=None, uncompress_pods=False, base64_pods=False, keep_pod_compression=False, need_pod_compression=False):
     if delete_meta:
         to_delete = []
 
@@ -553,11 +573,11 @@ def preproc_cart64(cart, delete_meta=None, uncompress_pods=False, keep_pod_compr
         for path, file in cart.files.items():
             if not file.is_raw and not file.is_dir:
                 if uncompress_pods:
-                    file.set_payload(file.payload, compress=False, use_pxu=False)
+                    file.set_payload(file.payload, compress=False, use_pxu=False, use_base64=base64_pods)
                 elif need_pod_compression:
-                    file.set_payload(file.payload, compress=True, use_pxu=True)
+                    file.set_payload(file.payload, compress=True, use_pxu=True, use_base64=base64_pods)
                 else:
-                    file.set_payload(file.payload, compress=False, use_pxu=True)
+                    file.set_payload(file.payload, compress=False, use_pxu=True, use_base64=base64_pods)
 
 def merge_cart64(dest, src, sections):
     glob = Cart64Glob(sections) if e(sections) else None

picotron_fs.py

@@ -1,5 +1,6 @@
 from utils import *
 from pico_defs import Language, encode_luastr, decode_luastr
+from pico_cart import pico_base64_decode, pico_base64_encode
 from pico_export import lz4_compress, lz4_uncompress
 from pico_compress import update_mtf
 
@@ -78,15 +79,27 @@ def node_to_value(node):
                         table[index] = value
                     index += 1
             return table
+
+        elif node.type == NodeType.call and node.func.type == NodeType.var and node.func.name == "unpod" and len(node.args) == 1:
+            value = node_to_value(node.args[0])
+            if isinstance(value, str):
+                return read_pod(encode_luastr(value))
+
+            add_error(f"unknown unpod param: {value}")
+
         else:
             add_error(f"unknown pod syntax {node.type}", node)
 
     value = node_to_value(root) if root else None
 
     if token_errors or parse_errors or value_errors:
-        eprint(f"warning - errors while parsing pod: {pod}")
+        print(f"Parsing errors for POD:\n{pod}")
+
         for error in token_errors + parse_errors + value_errors:
-            eprint("  " + error.format())
+            print(error.format())
+
+        throw(f"Unrecognized POD format - please report issue and use --keep-pod-compression for now")
+
     return value
 
 def parse_meta_pod(pod):
@@ -162,6 +175,7 @@ def format_meta_pod(value):
 
 k_lz4_prefix = b"lz4\0"
 k_pxu_prefix = b"pxu\0"
+k_b64_prefix = b"b64:"
 
 class PxuFlags(Bitmask):
     unk_type = 0x3
@@ -184,13 +198,10 @@ def read_pxu(data, idx):
         size = width * height
 
         bits = r.u8()
-        check(bits == 4, "unsupported pxu bits") # TODO - allow more?
+        check(bits == 4, "unsupported pxu bits") # TODO - allow more? (entirely untested)
         mask = (1 << bits) - 1
         ext_count = 1 << (8 - bits)
 
-        # the general idea behind the complexity is that repeated pixels can
-        # take up spots from low-valued pixels.
-
         data = bytearray()
         mapping = [i for i in range(mask)]
         mtf = [i for i in range(mask)]
@@ -224,6 +235,9 @@ def read_pxu(data, idx):
 def read_pod(value):
     """Reads a picotron pod from possibly compressed bytes"""
 
+    if value.startswith(k_b64_prefix):
+        value = pico_base64_decode(value[4:])
+
     if value.startswith(k_lz4_prefix):
         with BinaryReader(BytesIO(value)) as r:
             r.addpos(4)
@@ -312,7 +326,7 @@ def write_pxu(ud):
 
         return w.f.getvalue()
 
-def write_pod(pod, compress=True, use_pxu=True):
+def write_pod(pod, compress=True, use_pxu=True, use_base64=False):
     """Writes a picotron pod into optionally compressed bytes"""
 
     pxu_datas = None
@@ -345,6 +359,9 @@ def handle_userdata(ud, str_data):
             w.bytes(compressed)
             value = w.f.getvalue()
 
+    if use_base64:
+        value = k_b64_prefix + pico_base64_encode(value)
+
     return value
 
 class PicotronFile:
@@ -421,11 +438,11 @@ def payload(m):
     def payload(m, value):
         m.set_payload(value)
 
-    def set_payload(m, value, compress=True, use_pxu=True):
+    def set_payload(m, value, compress=True, use_pxu=True, use_base64=False):
         if m.is_raw:
             m.raw_payload = value
         else:
-            m.raw_payload = write_pod(value, compress=compress, use_pxu=use_pxu)
+            m.raw_payload = write_pod(value, compress=compress, use_pxu=use_pxu, use_base64=use_base64)
 
     is_dir = False
 

shrinko.py

@@ -169,15 +169,16 @@ def create_parser():
             pgroup.add_argument("--delete-meta", type=SplitBySeps, action="extend",
                                 help=f"specify a {sections_desc} to delete metadata of (default: * if minifying unsafely, else none)")
             pgroup.add_argument("--keep-pod-compression", action="store_true", help="keep compression of all pod files as-is")
-            pgroup.add_argument("--uncompress-pods", action="store_true", help="uncompress all pod files to plain text")
+            pgroup.add_argument("-u", "--uncompress-pods", action="store_true", help="uncompress all pod files to plain text")
+            pgroup.add_argument("--base64-pods", action="store_true", help="base64 all pod files")
             pgroup.add_argument("--list", action="store_true", help="list all files inside the cart")
             pgroup.add_argument("--filter", type=SplitBySeps, action="extend", help=f"specify a {sections_desc} to keep in the output")
             pgroup.add_argument("--insert", nargs='+', action="append", metavar=(f"INPUT [FSPATH] [FILES_FILTER]", ""),
                                 help=f"insert the specified INPUT file or directory at FSPATH")
             pgroup.add_argument("--extract", nargs='+', action="append", metavar=(f"FSPATH [OUTPUT]", ""),
                                 help=f"extract the specified file or directory from FSPATH to OUTPUT ")
         else:
-            pgroup.set_defaults(code_sections=None, delete_meta=None, uncompress_pods=None, keep_pod_compression=None,
+            pgroup.set_defaults(code_sections=None, delete_meta=None, uncompress_pods=None, keep_pod_compression=None, base64_pods=None,
                                 filter=None, insert=None, extract=None)
 
         pgroup.add_argument("--merge", nargs='+', action="append", metavar=(f"INPUT {sections_meta} [FORMAT]", ""),
@@ -518,7 +519,7 @@ def handle_processing(args, main_cart, extra_carts):
             if is_picotron:
                 preproc_cart_func(cart, delete_meta=args.delete_meta,
                                   keep_pod_compression=args.keep_pod_compression,
-                                  uncompress_pods=args.uncompress_pods,
+                                  uncompress_pods=args.uncompress_pods, base64_pods=args.base64_pods,
                                   # binary formats are already compressed, so pod compression just hurts
                                   need_pod_compression=args.format and args.format.is_src)