From 11ea564bba0d624dbbaf7a2dc9ea3b5e2f140453 Mon Sep 17 00:00:00 2001 From: BigtoC Date: Sat, 6 Dec 2025 02:27:16 +0800 Subject: [PATCH 1/5] Add toon-format dependency to Cargo.toml --- Cargo.lock | 969 +++++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + 2 files changed, 942 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b8a7ae..ecda4f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -91,6 +97,26 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +[[package]] +name = "arboard" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +dependencies = [ + "clipboard-win", + "image", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation", + "parking_lot", + "percent-encoding", + "windows-sys 0.60.2", + "x11rb", +] + [[package]] name = "as-any" version = "0.3.2" @@ -163,6 +189,30 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "2.9.4" @@ -178,18 +228,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.37" @@ -217,7 +305,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -260,12 +348,46 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "comfy-table" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +dependencies = [ + "crossterm 0.29.0", + "unicode-segmentation", + "unicode-width 0.2.0", +] + +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -282,6 +404,60 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags", + "crossterm_winapi", + "document-features", + "parking_lot", + "rustix 1.1.2", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -292,6 +468,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "deepwiki-rs" version = "1.2.6" @@ -315,10 +526,20 @@ dependencies = [ "thiserror", "tokio", "toml", + "toon-format", "uuid", "walkdir", ] +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -329,6 +550,16 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -340,12 +571,27 @@ dependencies = [ "syn", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -371,6 +617,12 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + [[package]] name = "eventsource-stream" version = "0.2.3" @@ -382,24 +634,80 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +[[package]] +name = "flate2" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2152dbcb980c05735e2a651d96011320a949eb31a0c8b38b72645ce97dec676" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -529,6 +837,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.2", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -583,11 +901,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -825,6 +1159,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -846,6 +1186,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "tiff", +] + [[package]] name = "indexmap" version = "2.11.1" @@ -856,6 +1210,28 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "io-uring" version = "0.7.10" @@ -889,6 +1265,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -905,12 +1290,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -923,6 +1326,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.13" @@ -939,6 +1348,15 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown", +] + [[package]] name = "markdown" version = "1.0.0" @@ -993,6 +1411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1002,10 +1421,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", + "log", "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] +[[package]] +name = "moxcms" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80986bbbcf925ebd3be54c26613d861255284584501595cf418320c078945608" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -1024,22 +1454,101 @@ dependencies = [ ] [[package]] -name = "nom" -version = "7.1.3" +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-graphics", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "memchr", - "minimal-lexical", + "bitflags", + "objc2", + "objc2-core-foundation", ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "objc2-io-surface" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "autocfg", + "bitflags", + "objc2", + "objc2-core-foundation", ] [[package]] @@ -1063,6 +1572,28 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "openssl" version = "0.10.73" @@ -1139,6 +1670,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pathdiff" version = "0.2.3" @@ -1189,6 +1726,32 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plist" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" +dependencies = [ + "base64", + "indexmap", + "quick-xml", + "serde", + "time", +] + +[[package]] +name = "png" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -1198,6 +1761,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1216,6 +1785,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3502d6155304a4173a5f2c34b52b7ed0dd085890326cb50fd625fdf39e86b3b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.40" @@ -1260,6 +1853,27 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags", + "cassowary", + "compact_str", + "crossterm 0.28.1", + "indoc", + "instability", + "itertools", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -1411,6 +2025,25 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.2" @@ -1420,7 +2053,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.11.0", "windows-sys 0.61.0", ] @@ -1543,18 +2176,28 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -1574,14 +2217,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ + "indexmap", "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -1611,6 +2256,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -1620,6 +2286,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.11" @@ -1648,12 +2320,40 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1691,6 +2391,27 @@ dependencies = [ "syn", ] +[[package]] +name = "syntect" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" +dependencies = [ + "bincode", + "flate2", + "fnv", + "once_cell", + "onig", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "thiserror", + "walkdir", + "yaml-rust", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -1721,30 +2442,90 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix", + "rustix 1.1.2", "windows-sys 0.61.0", ] [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "tiktoken-rs" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d" +dependencies = [ + "anyhow", + "base64", + "bstr", + "fancy-regex", + "lazy_static", + "regex", + "rustc-hash", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -1858,6 +2639,29 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +[[package]] +name = "toon-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784bdf955ee36b9feacf11ef9063555963853e4f15aeaa3b036549948f43096e" +dependencies = [ + "anyhow", + "arboard", + "chrono", + "clap", + "comfy-table", + "crossterm 0.28.1", + "indexmap", + "ratatui", + "serde", + "serde_json", + "syntect", + "thiserror", + "tiktoken-rs", + "tui-textarea", + "unicode-width 0.2.0", +] + [[package]] name = "tower" version = "0.5.2" @@ -1952,6 +2756,17 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tui-textarea" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5318dd619ed73c52a9417ad19046724effc1287fb75cdcc4eca1d6ac1acbae" +dependencies = [ + "crossterm 0.28.1", + "ratatui", + "unicode-width 0.2.0", +] + [[package]] name = "typenum" version = "1.18.0" @@ -1976,6 +2791,35 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "untrusted" version = "0.9.0" @@ -2168,6 +3012,28 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2177,6 +3043,12 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.0" @@ -2185,7 +3057,7 @@ checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.0", + "windows-link 0.2.1", "windows-result 0.4.0", "windows-strings 0.5.0", ] @@ -2220,9 +3092,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-registry" @@ -2250,7 +3122,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2268,7 +3140,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2304,7 +3176,7 @@ version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2454,6 +3326,32 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix 1.1.2", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yoke" version = "0.8.0" @@ -2557,3 +3455,18 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 5ecf823..dfa82f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,3 +52,4 @@ markdown = "1.0" futures = "0.3" uuid = { version = "1.0", features = ["v4", "serde"] } glob = "0.3" +toon-format = "0.4.0" From b9915e7a6769b15cb866982ffc1e9fe89b4d0d53 Mon Sep 17 00:00:00 2001 From: BigtoC Date: Sat, 6 Dec 2025 02:27:25 +0800 Subject: [PATCH 2/5] feat: enhance token usage estimation with TOON format support --- src/generator/agent_executor.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/generator/agent_executor.rs b/src/generator/agent_executor.rs index 4b61fba..cf30825 100644 --- a/src/generator/agent_executor.rs +++ b/src/generator/agent_executor.rs @@ -1,6 +1,7 @@ use anyhow::Result; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use toon_format::encode_default as toon_encode; use crate::generator::context::GeneratorContext; use crate::llm::client::utils::estimate_token_usage; @@ -138,15 +139,24 @@ where // Estimate token usage let input_text = format!("{} {}", prompt_sys, prompt_user); - let output_text = serde_json::to_string(&reply).unwrap_or_default(); - let token_usage = estimate_token_usage(&input_text, &output_text); + let output_text_json = serde_json::to_string(&reply).unwrap_or_default(); + let output_text_toon = toon_encode(&reply).unwrap_or_default(); + let token_usage = estimate_token_usage(&input_text, &output_text_json); + let token_usage_toon = estimate_token_usage(&input_text, &output_text_toon); + let token_saved = token_usage.total_tokens.saturating_sub(token_usage_toon.total_tokens); + let token_saved_percent = if token_usage.total_tokens > 0 { + (token_saved as f64 / token_usage.total_tokens as f64) * 100.0 + } else { + 0.0 + }; + println!("Estimated token usage - JSON: {}, TOON: {}, Saved: {} ({:.6}%)", token_usage.total_tokens, token_usage_toon.total_tokens, token_saved, token_saved_percent); // Cache result - Use method with token information context .cache_manager .write() .await - .set_with_tokens(cache_scope, &prompt_key, &reply, token_usage) + .set_with_tokens(cache_scope, &prompt_key, &reply, token_usage_toon) .await?; Ok(reply) From d4629c5008a7a471b3b8abb9d0af7e6f2e6f46ba Mon Sep 17 00:00:00 2001 From: BigtoC Date: Sun, 7 Dec 2025 23:54:21 +0800 Subject: [PATCH 3/5] feat: simplify token usage estimation by removing JSON output and focusing on TOON format --- src/generator/agent_executor.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/generator/agent_executor.rs b/src/generator/agent_executor.rs index cf30825..c00a9a3 100644 --- a/src/generator/agent_executor.rs +++ b/src/generator/agent_executor.rs @@ -139,24 +139,15 @@ where // Estimate token usage let input_text = format!("{} {}", prompt_sys, prompt_user); - let output_text_json = serde_json::to_string(&reply).unwrap_or_default(); - let output_text_toon = toon_encode(&reply).unwrap_or_default(); - let token_usage = estimate_token_usage(&input_text, &output_text_json); - let token_usage_toon = estimate_token_usage(&input_text, &output_text_toon); - let token_saved = token_usage.total_tokens.saturating_sub(token_usage_toon.total_tokens); - let token_saved_percent = if token_usage.total_tokens > 0 { - (token_saved as f64 / token_usage.total_tokens as f64) * 100.0 - } else { - 0.0 - }; - println!("Estimated token usage - JSON: {}, TOON: {}, Saved: {} ({:.6}%)", token_usage.total_tokens, token_usage_toon.total_tokens, token_saved, token_saved_percent); + let output_text = toon_encode(&reply).unwrap_or_default(); + let token_usage = estimate_token_usage(&input_text, &output_text); // Cache result - Use method with token information context .cache_manager .write() .await - .set_with_tokens(cache_scope, &prompt_key, &reply, token_usage_toon) + .set_with_tokens(cache_scope, &prompt_key, &reply, token_usage) .await?; Ok(reply) From d1647f5de815498c0645db7760c8c4bd12072974 Mon Sep 17 00:00:00 2001 From: BigtoC Date: Mon, 8 Dec 2025 21:19:56 +0800 Subject: [PATCH 4/5] feat: enhance Ollama client with custom reqwest client and optional authorization header --- src/llm/client/providers.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/llm/client/providers.rs b/src/llm/client/providers.rs index 0852ff9..2b44367 100644 --- a/src/llm/client/providers.rs +++ b/src/llm/client/providers.rs @@ -1,6 +1,7 @@ //! LLM Provider support module use anyhow::Result; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use rig::{ agent::Agent, client::CompletionClient, @@ -28,7 +29,7 @@ pub enum ProviderClient { OpenRouter(rig::providers::openrouter::Client), Anthropic(rig::providers::anthropic::Client), Gemini(rig::providers::gemini::Client), - Ollama(rig::providers::ollama::Client), + Ollama(rig::providers::ollama::Client), } impl ProviderClient { @@ -72,8 +73,24 @@ impl ProviderClient { Ok(ProviderClient::Gemini(client)) } LLMProvider::Ollama => { + // Create custom reqwest client with Authorization header + let mut headers = HeaderMap::new(); + if !config.api_key.is_empty() { + let auth_value = format!("Bearer {}", config.api_key); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&auth_value) + .map_err(|e| anyhow::anyhow!("Invalid API key format: {}", e))?, + ); + } + let http_client = reqwest::Client::builder() + .default_headers(headers) + .build() + .map_err(|e| anyhow::anyhow!("Failed to build HTTP client: {}", e))?; + let client = rig::providers::ollama::Client::builder() .base_url(&config.api_base_url) + .with_client(http_client) .build(); Ok(ProviderClient::Ollama(client)) } From 98c03a918436aeb1bf845390eeb9b551200d8df9 Mon Sep 17 00:00:00 2001 From: BigtoC Date: Tue, 9 Dec 2025 12:36:12 +0800 Subject: [PATCH 5/5] feat: enhance OllamaExtractorWrapper to support TOON format and improve parsing strategies --- src/llm/client/ollama_extractor.rs | 168 +++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 30 deletions(-) diff --git a/src/llm/client/ollama_extractor.rs b/src/llm/client/ollama_extractor.rs index 97ac74f..1db4cd6 100644 --- a/src/llm/client/ollama_extractor.rs +++ b/src/llm/client/ollama_extractor.rs @@ -1,7 +1,8 @@ //! Ollama Structured Output Wrapper //! //! Ollama does not support native structured output (unlike OpenAI), so this module -//! provides a wrapper to parse JSON from Ollama's text responses and validate against schemas +//! provides a wrapper to parse TOON/JSON from Ollama's text responses and validate against schemas. +//! TOON (Token-Oriented Object Notation) is used to reduce token usage in prompts. use anyhow::{Context, Result}; use regex::Regex; @@ -10,11 +11,24 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::sync::LazyLock; +use toon_format::{decode_default as toon_decode, encode_default as toon_encode}; /// JSON code block regex pattern static JSON_CODE_BLOCK_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"```(?:json)?\s*(\{[\s\S]*?\})\s*```").unwrap()); +/// TOON code block regex pattern +static TOON_CODE_BLOCK_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"```toon\s*([\s\S]*?)\s*```").unwrap()); + +/// Empty array pattern in TOON (e.g., "functions[0]:" with no content) +static EMPTY_ARRAY_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^\s*(\w+)\[0\]:\s*$").unwrap()); + +/// Malformed array pattern (e.g., "items[0]{{...}}:" with no content) +static EMPTY_ARRAY_WITH_SCHEMA_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^\s*(\w+)\[0\]\{\{[^}]*\}\}:\s*$").unwrap()); + /// Ollama structured output extractor pub struct OllamaExtractorWrapper { agent: Agent>, @@ -27,7 +41,10 @@ where T: JsonSchema + Serialize + for<'de> Deserialize<'de>, { /// Create a new Ollama extractor - pub fn new(agent: Agent>, max_retries: u32) -> Self { + pub fn new( + agent: Agent>, + max_retries: u32, + ) -> Self { Self { agent, max_retries, @@ -60,27 +77,74 @@ where )) } - /// Build enhanced prompt with schema and instructions + /// Build enhanced prompt with schema and instructions using TOON format for token efficiency + /// Falls back to JSON instructions on retry attempts for better compatibility fn build_prompt(&self, base_prompt: &str, previous_error: Option<&str>) -> String { let schema = schemars::schema_for!(T); - let schema_json = serde_json::to_string_pretty(&schema) - .unwrap_or_else(|_| "{}".to_string()); + let schema_json = serde_json::to_string_pretty(&schema).unwrap_or_else(|_| "{}".to_string()); + + // On retry attempts (when there's a previous error), fall back to JSON format + // as it's more reliable for models that struggle with TOON + let use_json_fallback = previous_error.is_some(); + + let mut prompt = if use_json_fallback { + format!( + r#"{} + +**IMPORTANT: Return your response as valid JSON only.** + +Follow this JSON schema: +```json +{} +``` - let mut prompt = format!( - "{}\n\n**CRITICAL: YOU MUST RETURN VALID JSON**\n\nYou MUST return the result as a valid JSON object that strictly follows this schema:\n\n```json\n{}\n```\n\n", - base_prompt, schema_json - ); +Requirements: +1. Return ONLY valid JSON, no other text +2. All required fields must be present +3. Use null for optional fields with no value +4. Use empty arrays [] for arrays with no items +5. Wrap your response in ```json and ``` markers - prompt.push_str("Requirements:\n"); - prompt.push_str("1. Return pure JSON object, do not add any extra text\n"); - prompt.push_str("2. All required fields must be present\n"); - prompt.push_str("3. Field types must match schema exactly\n"); - prompt.push_str("4. Arrays and nested objects must be correctly formatted\n\n"); +"#, + base_prompt, schema_json + ) + } else { + // Convert schema to TOON format to save tokens + let schema_toon = toon_encode(&schema).unwrap_or_else(|_| schema_json.clone()); + + format!( + r#"{} + +**Return your response in TOON format (a compact YAML-like notation).** + +TOON quick reference: +- Key-value: `key: value` +- Nested objects: use 2-space indentation +- Empty arrays: omit the field or use `field: []` +- Arrays with objects: `items[count]{{field1,field2}}:` then values per line + +Schema: +```toon +{} +``` + +Requirements: +1. Return ONLY valid TOON format, no extra text +2. All required fields must be present +3. Use null for optional fields with no value +4. For empty arrays, omit the field entirely +5. Wrap your response in ```toon and ``` markers + +"#, + base_prompt, schema_toon + ) + }; if let Some(error) = previous_error { prompt.push_str(&format!( - "**Previous attempt failed with error: {}**\nPlease fix these issues and regenerate.\n\n", - error + "**Previous attempt failed: {}**\nPlease return valid {} format.\n\n", + error, + if use_json_fallback { "JSON" } else { "TOON" } )); } @@ -102,8 +166,8 @@ where self.validate_json(&parsed)?; let result: T = serde_json::from_value(parsed.clone()).with_context(|| { - let json_str = serde_json::to_string_pretty(&parsed) - .unwrap_or_else(|_| "invalid".to_string()); + let json_str = + serde_json::to_string_pretty(&parsed).unwrap_or_else(|_| "invalid".to_string()); format!( "Failed to deserialize JSON to target type on attempt {}. JSON structure: {}", attempt, json_str @@ -113,40 +177,73 @@ where Ok(result) } - /// Parse JSON response using multiple strategies + /// Parse response using multiple strategies (TOON first, then JSON fallback) fn parse_json_response(&self, response: &str, attempt: usize) -> Result { - // Strategy 1: Try direct parsing + // Strategy 1: Try TOON from code block (preferred) + if let Some(toon_str) = self.extract_from_toon_code_block(response) { + // Preprocess to fix common Ollama TOON output issues + let preprocessed = self.preprocess_toon(&toon_str); + if let Ok(parsed) = toon_decode::(&preprocessed) { + return Ok(parsed); + } + } + + // Strategy 2: Try direct TOON parsing (if response looks like TOON) + if response.contains(": ") && !response.trim_start().starts_with('{') { + let preprocessed = self.preprocess_toon(response.trim()); + if let Ok(parsed) = toon_decode::(&preprocessed) { + return Ok(parsed); + } + } + + // Strategy 3: Try direct JSON parsing (fallback) if let Ok(json) = serde_json::from_str::(response) { return Ok(json); } - // Strategy 2: Extract from markdown code blocks - if let Some(json_str) = self.extract_from_code_block(response) { + // Strategy 4: Extract from JSON markdown code blocks (fallback) + if let Some(json_str) = self.extract_from_json_code_block(response) { if let Ok(parsed) = serde_json::from_str::(&json_str) { return Ok(parsed); } } - // Strategy 3: Extract first JSON object + // Strategy 5: Extract first JSON object (fallback) if let Some(json_str) = self.extract_first_json_object(response) { if let Ok(parsed) = serde_json::from_str::(&json_str) { return Ok(parsed); } } - // Strategy 4: Clean and try parsing + // Strategy 6: Clean, preprocess, and try parsing let cleaned = self.clean_response(response); + let preprocessed = self.preprocess_toon(&cleaned); + + // Try TOON first on cleaned and preprocessed response + if let Ok(parsed) = toon_decode::(&preprocessed) { + return Ok(parsed); + } + + // Finally try JSON serde_json::from_str::(&cleaned).with_context(|| { - let preview = response.chars().take(200).collect::(); + let preview = response.chars().take(500).collect::(); format!( - "Failed to parse JSON from Ollama response (attempt {}). Response preview: {}", + "Failed to parse TOON/JSON from Ollama response (attempt {}). Preview (500 chars): {}", attempt, preview ) }) } - /// Extract JSON from markdown code blocks - fn extract_from_code_block(&self, text: &str) -> Option { + /// Extract TOON from markdown code blocks + fn extract_from_toon_code_block(&self, text: &str) -> Option { + TOON_CODE_BLOCK_REGEX + .captures(text) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str().to_string()) + } + + /// Extract JSON from markdown code blocks (fallback) + fn extract_from_json_code_block(&self, text: &str) -> Option { JSON_CODE_BLOCK_REGEX .captures(text) .and_then(|cap| cap.get(1)) @@ -180,9 +277,10 @@ where } } - /// Clean response text + /// Clean response text (handles both TOON and JSON markers) fn clean_response(&self, text: &str) -> String { text.trim() + .trim_start_matches("```toon") .trim_start_matches("```json") .trim_start_matches("```") .trim_end_matches("```") @@ -190,6 +288,16 @@ where .to_string() } + /// Preprocess TOON response to fix common Ollama output issues + /// - Removes empty array declarations like "functions[0]:" or "items[0]{{field}}:" + /// - These are invalid TOON syntax that Ollama sometimes generates + fn preprocess_toon(&self, text: &str) -> String { + // Remove lines with empty arrays (e.g., "functions[0]:" or "items[0]{{...}}:") + let result = EMPTY_ARRAY_PATTERN.replace_all(text, ""); + let result = EMPTY_ARRAY_WITH_SCHEMA_PATTERN.replace_all(&result, ""); + result.to_string() + } + /// Validate basic JSON structure fn validate_json(&self, json: &Value) -> Result<()> { if !json.is_object() { @@ -197,4 +305,4 @@ where } Ok(()) } -} \ No newline at end of file +}