From fda07537b18001466e032bea04767863f3a5504c Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:01 -0700 Subject: [PATCH 01/35] Ignore staged native sidecar copies Keep fetch/stage outputs local so binding native/ dirs and dist/native are not published as prebuilts. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 776b6a2..2e24d74 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,10 @@ bench/ *.dylib *.bin +# Staged native sidecar copies (local/CI fetch only; not published prebuilts) +bindings/*/native/STAGED.txt +dist/native/ + # Ignore release info and test result files release-info.txt test_results.json From 0d37600322c7c994edd90a8279f971a3bfc87488 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:08 -0700 Subject: [PATCH 02/35] Add fixed-point half-ULP quantization proofs Prove cleared-denominator Nat/Int round and L1 bounds so Float claims can stay honest markers instead of axioms. --- src/lean/ModelAssetGuard/Quant/Fixed.lean | 243 ++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 src/lean/ModelAssetGuard/Quant/Fixed.lean diff --git a/src/lean/ModelAssetGuard/Quant/Fixed.lean b/src/lean/ModelAssetGuard/Quant/Fixed.lean new file mode 100644 index 0000000..57fdc74 --- /dev/null +++ b/src/lean/ModelAssetGuard/Quant/Fixed.lean @@ -0,0 +1,243 @@ +/- +Fixed-point / integer quantization bounds proveable in Lean 4 without Mathlib. + +Runtime Float paths in `Core.lean` use obligation markers (not axioms); these +theorems are the honest half-ULP core on `Nat`/`Int`. Wave 28 aborted the +Mathlib / IEEE Float bridge (see `docs/float-obligation.md`); markers stay. +-/ + +namespace ModelAssetGuard.Quant.Fixed + +/-- Nearest-integer rounding of the nonnegative rational `n / d` (ties round up). -/ +def roundDivNat (n d : Nat) (_hd : d > 0) : Nat := + (2 * n + d) / (2 * d) + +/-- `|r - d|` as Nat. -/ +def absDiffNat (r d : Nat) : Nat := + if r ≥ d then r - d else d - r + +theorem absDiffNat_eq_natAbs (r d : Nat) : + absDiffNat r d = Int.natAbs ((r : Int) - (d : Int)) := by + unfold absDiffNat + by_cases h : r ≥ d + · rw [if_pos h] + have : (r : Int) - (d : Int) = ((r - d : Nat) : Int) := (Int.ofNat_sub h).symm + rw [this, Int.natAbs_natCast] + · rw [if_neg h] + have hlt : r < d := Nat.lt_of_not_ge h + have hle : r ≤ d := Nat.le_of_lt hlt + have hsub : (d : Int) - (r : Int) = ((d - r : Nat) : Int) := (Int.ofNat_sub hle).symm + have hneg : (r : Int) - (d : Int) = - ((d : Int) - (r : Int)) := by + rw [Int.sub_eq_add_neg, Int.sub_eq_add_neg, Int.neg_add, Int.neg_neg, Int.add_comm] + rw [hneg, Int.natAbs_neg, hsub, Int.natAbs_natCast] + +theorem absDiffNat_le_of_lt_two_mul (r d : Nat) (hr : r < 2 * d) : + absDiffNat r d ≤ d := by + unfold absDiffNat + by_cases h : r ≥ d + · rw [if_pos h] + have hle : r ≤ 2 * d := Nat.le_of_lt hr + have hle' : r ≤ d + d := by + rw [Nat.two_mul] at hle + exact hle + exact Nat.sub_le_iff_le_add.mpr hle' + · rw [if_neg h] + exact Nat.sub_le d r + +private theorem two_mul_sub_eq + (n q d r : Nat) (h : 2 * n + d = 2 * d * q + r) : + (2 : Int) * ((n : Int) - (q : Int) * (d : Int)) = (r : Int) - (d : Int) := by + have hI : + (2 : Int) * (n : Int) + (d : Int) = + (2 : Int) * (d : Int) * (q : Int) + (r : Int) := by + have := congrArg (fun x : Nat => (x : Int)) h + simpa [Int.natCast_add, Int.natCast_mul] using this + -- Goal equivalent to 2n - 2qd = r - d + apply Eq.symm + -- r - d = 2n + d - d - 2dq = 2n - 2dq from hI + calc + (r : Int) - (d : Int) + = (2 : Int) * (d : Int) * (q : Int) + (r : Int) - (d : Int) - + (2 : Int) * (d : Int) * (q : Int) := by + simp [Int.add_sub_cancel, Int.sub_eq_add_neg, Int.add_assoc, Int.add_left_comm, + Int.add_comm, Int.add_left_neg, Int.add_right_neg, Int.zero_add, Int.add_zero] + _ = (2 : Int) * (n : Int) + (d : Int) - (d : Int) - + (2 : Int) * (d : Int) * (q : Int) := by + rw [← hI] + _ = (2 : Int) * (n : Int) - (2 : Int) * (d : Int) * (q : Int) := by + simp [Int.add_sub_cancel] + _ = (2 : Int) * (n : Int) - (2 : Int) * ((q : Int) * (d : Int)) := by + rw [show (2 : Int) * (d : Int) * (q : Int) = (2 : Int) * ((q : Int) * (d : Int)) by + rw [Int.mul_assoc, Int.mul_comm (d : Int) (q : Int), ← Int.mul_assoc]] + _ = (2 : Int) * ((n : Int) - (q : Int) * (d : Int)) := by + rw [← Int.mul_sub] + +/-- Half-ULP bound: `2 * |n - round(n/d) * d| ≤ d`. -/ +theorem roundDivNat_error (n d : Nat) (hd : d > 0) : + 2 * Int.natAbs ((n : Int) - (roundDivNat n d hd : Int) * (d : Int)) ≤ d := by + let q := roundDivNat n d hd + let m := 2 * n + d + let denom := 2 * d + have hdivmod : denom * (m / denom) + m % denom = m := Nat.div_add_mod m denom + have hdenpos : 0 < denom := Nat.mul_pos (by decide : 0 < 2) hd + have hrlt : m % denom < denom := Nat.mod_lt m hdenpos + have hq : q = m / denom := rfl + have hNat : 2 * n + d = 2 * d * q + m % denom := by + have h1 : denom * q + m % denom = m := by + simpa [hq] using hdivmod + -- unfold lets: m = 2*n+d, denom = 2*d + have h2 : (2 * d) * q + m % (2 * d) = 2 * n + d := by + simpa [denom, m] using h1 + -- rewrite RHS→LHS form expected by two_mul_sub_eq + have h3 := h2.symm + -- normalize (2*d)*q to 2*d*q + simpa [Nat.mul_assoc] using h3 + have hDiff := two_mul_sub_eq n q d (m % denom) hNat + have hmul : + Int.natAbs ((2 : Int) * ((n : Int) - (q : Int) * (d : Int))) = + 2 * Int.natAbs ((n : Int) - (q : Int) * (d : Int)) := by + rw [Int.natAbs_mul] + rfl + have hAbs : + 2 * Int.natAbs ((n : Int) - (q : Int) * (d : Int)) = + absDiffNat (m % denom) d := by + have h1 := congrArg Int.natAbs hDiff + -- h1: natAbs(2*(n-qd)) = natAbs(r-d) + have h2 : + 2 * Int.natAbs ((n : Int) - (q : Int) * (d : Int)) = + Int.natAbs (((m % denom : Nat) : Int) - (d : Int)) := by + rw [← hmul, h1] + rw [h2, absDiffNat_eq_natAbs] + have hLe : absDiffNat (m % denom) d ≤ d := + absDiffNat_le_of_lt_two_mul (m % denom) d (by + change m % denom < 2 * d + simpa [denom] using hrlt) + exact hAbs ▸ hLe + +/-- Signed rounding of `n / d` (unsigned round on `|n|`, restore sign). -/ +def roundDivInt (n : Int) (d : Nat) (hd : d > 0) : Int := + let q := roundDivNat (Int.natAbs n) d hd + if n ≥ 0 then (q : Int) else - (q : Int) + +theorem roundDivInt_error (n : Int) (d : Nat) (hd : d > 0) : + 2 * Int.natAbs (n - roundDivInt n d hd * (d : Int)) ≤ d := by + unfold roundDivInt + by_cases hpos : n ≥ 0 + · rw [if_pos hpos] + have hn : n = (Int.natAbs n : Int) := Int.eq_natAbs_of_nonneg hpos + rw [hn] + simpa using roundDivNat_error (Int.natAbs n) d hd + · rw [if_neg hpos] + let a := Int.natAbs n + have hn : n = - (a : Int) := + match Int.natAbs_eq n with + | Or.inl hEq => False.elim (hpos (hEq ▸ Int.natCast_nonneg _)) + | Or.inr hEq => by simpa [a] using hEq + have h := roundDivNat_error a d hd + let q := roundDivNat a d hd + have hparse : + n - (-(q : Int)) * (d : Int) = n + (q : Int) * (d : Int) := by + have : (-(q : Int)) * (d : Int) = - ((q : Int) * (d : Int)) := by + rw [Int.neg_mul] + rw [this, Int.sub_neg] + have hgoal : + n + (q : Int) * (d : Int) = + - ((a : Int) - (q : Int) * (d : Int)) := by + rw [hn] + rw [Int.sub_eq_add_neg, Int.neg_add, Int.neg_neg, Int.add_comm] + rw [hparse, hgoal, Int.natAbs_neg] + exact h + +/-- Dot product of equal-length integer lists. -/ +def dot (es xs : List Int) : Int := + match es, xs with + | [], [] => 0 + | e :: es, x :: xs => e * x + dot es xs + | _, _ => 0 + +/-- L1 norm of an integer list. -/ +def l1Norm : List Int → Nat + | [] => 0 + | x :: xs => Int.natAbs x + l1Norm xs + +/-- Cleared-denominator triangle bound for a row·vector product. +If every entry satisfies `2*|e| ≤ scale`, then +`2 * |∑ eᵢ xᵢ| ≤ scale * ∑ |xᵢ|`. -/ +theorem dot_abs_bound_of_entrywise + (es xs : List Int) (scale : Nat) (_hs : scale > 0) + (hlen : es.length = xs.length) + (hbound : ∀ e ∈ es, 2 * Int.natAbs e ≤ scale) : + 2 * Int.natAbs (dot es xs) ≤ scale * l1Norm xs := by + induction es generalizing xs with + | nil => + cases xs with + | nil => simp [dot, l1Norm] + | cons _ _ => cases hlen + | cons e es ih => + cases xs with + | nil => cases hlen + | cons x xs => + simp only [List.length_cons] at hlen + have hlen' : es.length = xs.length := Nat.succ.inj hlen + have he : 2 * Int.natAbs e ≤ scale := + hbound e (List.mem_cons.mpr (Or.inl rfl)) + have hes : ∀ e' ∈ es, 2 * Int.natAbs e' ≤ scale := fun e' he' => + hbound e' (List.mem_cons.mpr (Or.inr he')) + have ih := ih xs hlen' hes + simp only [dot, l1Norm] + have htri : + Int.natAbs (e * x + dot es xs) ≤ + Int.natAbs (e * x) + Int.natAbs (dot es xs) := + Int.natAbs_add_le _ _ + have hprod : 2 * Int.natAbs (e * x) ≤ scale * Int.natAbs x := by + have hm : Int.natAbs (e * x) = Int.natAbs e * Int.natAbs x := + Int.natAbs_mul e x + calc + 2 * Int.natAbs (e * x) + = 2 * (Int.natAbs e * Int.natAbs x) := by rw [hm] + _ = (2 * Int.natAbs e) * Int.natAbs x := by rw [Nat.mul_assoc] + _ ≤ scale * Int.natAbs x := Nat.mul_le_mul_right _ he + have hsum : + 2 * Int.natAbs (e * x) + 2 * Int.natAbs (dot es xs) ≤ + scale * Int.natAbs x + scale * l1Norm xs := + Nat.add_le_add hprod ih + have hleft : + 2 * Int.natAbs (e * x + dot es xs) ≤ + 2 * Int.natAbs (e * x) + 2 * Int.natAbs (dot es xs) := by + calc + 2 * Int.natAbs (e * x + dot es xs) + ≤ 2 * (Int.natAbs (e * x) + Int.natAbs (dot es xs)) := + Nat.mul_le_mul_left 2 htri + _ = 2 * Int.natAbs (e * x) + 2 * Int.natAbs (dot es xs) := by + rw [Nat.mul_add] + have hright : + scale * Int.natAbs x + scale * l1Norm xs = + scale * (Int.natAbs x + l1Norm xs) := by + rw [Nat.mul_add] + exact Nat.le_trans hleft (by + rw [← hright] + exact hsum) + +/-- Half-ULP restated for the Float bridge narrative: the integer error is at +most half a denominator unit (`2 * |err| ≤ d` ⇔ `|err| ≤ d/2` when thinking in +rationals `n/d`). Replaces inventing a Mathlib Float theorem. -/ +theorem roundDivNat_error_half_unit (n d : Nat) (hd : d > 0) : + 2 * Int.natAbs ((n : Int) - (roundDivNat n d hd : Int) * (d : Int)) ≤ d := + roundDivNat_error n d hd + +/-- Same statement for signed `roundDivInt`. -/ +theorem roundDivInt_error_half_unit (n : Int) (d : Nat) (hd : d > 0) : + 2 * Int.natAbs (n - roundDivInt n d hd * (d : Int)) ≤ d := + roundDivInt_error n d hd + +/-- Every row that meets the entrywise scale hypothesis is L1-bounded against `xs`. -/ +theorem rows_dot_abs_bound + (rows : List (List Int)) (xs : List Int) (scale : Nat) (hs : scale > 0) + (hrows : ∀ es ∈ rows, + es.length = xs.length ∧ ∀ e ∈ es, 2 * Int.natAbs e ≤ scale) : + ∀ es ∈ rows, 2 * Int.natAbs (dot es xs) ≤ scale * l1Norm xs := by + intro es hes + have h := hrows es hes + exact dot_abs_bound_of_entrywise es xs scale hs h.1 h.2 + +end ModelAssetGuard.Quant.Fixed From 74c0ad1166870bfe35afaef8f78325186b3dc36e Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:16 -0700 Subject: [PATCH 03/35] Replace Float quant axioms with obligation markers Point Core/LayerBound at Fixed Int theorems and keep Float L2 claims as True markers after aborting IEEE discharge. --- src/lean/ModelAssetGuard/Quant/Core.lean | 64 ++++++++++++++----- .../ModelAssetGuard/Quant/LayerBound.lean | 36 +++++++---- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/src/lean/ModelAssetGuard/Quant/Core.lean b/src/lean/ModelAssetGuard/Quant/Core.lean index e642344..fb498c3 100644 --- a/src/lean/ModelAssetGuard/Quant/Core.lean +++ b/src/lean/ModelAssetGuard/Quant/Core.lean @@ -1,5 +1,6 @@ import Init.Data.Array.Basic import Init.Data.List.Basic +import ModelAssetGuard.Quant.Fixed namespace ModelAssetGuard.Quant @@ -23,10 +24,21 @@ def quant_error (x : Float) : Float := /-- ULP (Unit in Last Place) for int8 -/ def ulp_int8 : Float := 1.0 -/-- Rounding lemma: quantization error is bounded by 1/2 ULP. -This is currently imported as an external formal assumption until the complete proof lands. -/ -axiom round_int8_error_bound (x : Float) : - Float.abs (quant_error x) ≤ 0.5 * ulp_int8 +/-- Float half-ULP claim for `round_int8` is **not** an axiom (Wave 6). + +Lean 4 `Float` ops are opaque externals; a real proof is not available without +a Float model. Honest status: unproven obligation marker (same pattern as +`layer_error_bound_float_l2_obligation`). Proven analogue on cleared-denominator +rationals is `fixed_round_half_ulp` / `Fixed.roundDivInt_error`. +Wave 28 aborted Mathlib/IEEE discharge; see `docs/float-obligation.md`. +Runtime L2 checks live in Rust `guardd_verify_quant_128_vectors`. -/ +def round_int8_error_bound_float_obligation (_x : Float) : Prop := + True + +/-- Proven half-ULP bound on fixed-point rationals (see `Fixed`). -/ +theorem fixed_round_half_ulp (n : Int) (d : Nat) (hd : d > 0) : + 2 * Int.natAbs (n - Fixed.roundDivInt n d hd * (d : Int)) ≤ d := + Fixed.roundDivInt_error n d hd /-- Layer weight matrix type -/ abbrev WeightMatrix := Array (Array Float) @@ -42,10 +54,16 @@ def quantize_matrix (W : WeightMatrix) : QuantizedMatrix := def dequantize_matrix (Wq : QuantizedMatrix) : WeightMatrix := Wq.map (fun row => row.map int8_to_float) -/-- Quantization error matrix -/ +/-- Elementwise matrix subtraction `A - B` (row-major, truncated to shared shape). -/ +def matrix_sub (A B : WeightMatrix) : WeightMatrix := + A.zip B |>.map fun (rowA, rowB) => + rowA.zip rowB |>.map fun (a, b) => a - b + +/-- Quantization error matrix: `W_q - W` after int8 quantize/dequantize. +Previously (F015) this incorrectly returned dequantized weights alone. -/ def quantization_error_matrix (W : WeightMatrix) : WeightMatrix := let Wq := dequantize_matrix (quantize_matrix W) - Wq + matrix_sub Wq W /-- L2 norm of a vector -/ def l2_norm (x : Array Float) : Float := @@ -57,16 +75,32 @@ def matvec_mul (W : WeightMatrix) (x : Array Float) : Array Float := let pairs := row.zip x pairs.foldl (fun acc (a, b) => acc + (a * b)) 0.0 -/-- Layer error bound: ||Wq x - W x||₂ ≤ ε||x||₂. -Tracked as formal debt and enforced as explicit axiom until full derivation is completed. -/ -axiom layer_error_bound (W : WeightMatrix) (x : Array Float) : - let Wq := quantization_error_matrix W - let ε := ulp_int8 - l2_norm (matvec_mul Wq x) ≤ ε * l2_norm x +/-- Float L2 claim `||Ex||₂ ≤ (1/2)||x||₂` is **not** an axiom (Wave 5). -/-- Per-channel quantization error bound. -Tracked as formal debt and enforced as explicit axiom until full derivation is completed. -/ -axiom per_channel_error_bound (W : WeightMatrix) (x : Array Float) : +Honest status: unproven on opaque `Float`. Proven analogue under cleared-denominator +Int hypotheses is `layer_row_l1_bound` / `Fixed.rows_dot_abs_bound`. Runtime +authority for the L2 metric remains Rust `guardd_verify_quant_128_vectors`. -/ +def layer_error_bound_float_l2_obligation (_W : WeightMatrix) (_x : Array Float) : Prop := True +/-- Proven entrywise Int bound: if `2*|eⱼ| ≤ scale` then `2*|e·x| ≤ scale * ||x||₁`. -/ +theorem layer_row_l1_bound + (es xs : List Int) (scale : Nat) (hs : scale > 0) + (hlen : es.length = xs.length) + (hbound : ∀ e ∈ es, 2 * Int.natAbs e ≤ scale) : + 2 * Int.natAbs (Fixed.dot es xs) ≤ scale * Fixed.l1Norm xs := + Fixed.dot_abs_bound_of_entrywise es xs scale hs hlen hbound + +/-- Row-wise Int layer bound: every row satisfying the entrywise scale is L1-bounded. -/ +theorem layer_rows_l1_bound + (rows : List (List Int)) (xs : List Int) (scale : Nat) (hs : scale > 0) + (hrows : ∀ es ∈ rows, + es.length = xs.length ∧ ∀ e ∈ es, 2 * Int.natAbs e ≤ scale) : + ∀ es ∈ rows, 2 * Int.natAbs (Fixed.dot es xs) ≤ scale * Fixed.l1Norm xs := + Fixed.rows_dot_abs_bound rows xs scale hs hrows + +/-- Vacuous placeholder retained for API stability; not a contentful bound. -/ +theorem per_channel_error_bound (_W : WeightMatrix) (_x : Array Float) : True := + trivial + end ModelAssetGuard.Quant diff --git a/src/lean/ModelAssetGuard/Quant/LayerBound.lean b/src/lean/ModelAssetGuard/Quant/LayerBound.lean index 48aa9cf..3f26e84 100644 --- a/src/lean/ModelAssetGuard/Quant/LayerBound.lean +++ b/src/lean/ModelAssetGuard/Quant/LayerBound.lean @@ -10,10 +10,10 @@ structure LayerConfig where quant_type : String -- "int8", "fp16", etc. deriving Repr -/-- Compute ε bound for a layer -/ +/-- Compute ε bound for a layer (aligned with Rust `compute_epsilon_bound`). -/ def compute_epsilon_bound (config : LayerConfig) : Float := match config.quant_type with - | "int8" => ulp_int8 * Float.sqrt config.fan_in.toFloat + | "int8" => 0.5 * ulp_int8 * Float.sqrt config.fan_in.toFloat | "fp16" => 2.0 * Float.sqrt config.fan_in.toFloat | _ => 1.0 * Float.sqrt config.fan_in.toFloat @@ -118,9 +118,10 @@ def verify_model_128_vectors (layers : List (String × WeightMatrix × LayerConf worst_layer := none } -/-- Proof obligations are tracked separately; this placeholder keeps API stable. -/ -axiom verify_128_vectors_sound (layer_name : String) (W : WeightMatrix) - (config : LayerConfig) (verification : LayerVerification128) : Prop +/-- Unproven proof obligation marker (not an axiom claiming soundness). -/ +def verify_128_vectors_sound (_layer_name : String) (_W : WeightMatrix) + (_config : LayerConfig) (_verification : LayerVerification128) : Prop := + True /-- Legacy verification for backward compatibility -/ structure LayerVerification where @@ -163,12 +164,23 @@ def verify_model (layers : List (String × WeightMatrix × LayerConfig)) : Model passed_layers := passed_count } -/-- Proof that ε bound is positive -/ -axiom epsilon_bound_positive (config : LayerConfig) : - compute_epsilon_bound config ≥ 0 - -/-- Proof obligations are tracked separately; this placeholder keeps API stable. -/ -axiom layer_verification_sound (layer_name : String) (W : WeightMatrix) - (config : LayerConfig) (verification : LayerVerification) : Prop +/-- Nonnegative Nat scale aligned with `compute_epsilon_bound` (squared units): +int8 → `fan_in` stands for `(2ε)²` when ULP=1; fp16 → `16 * fan_in`; else `4 * fan_in`. +Float positivity of `compute_epsilon_bound` is not claimed formally. -/ +def epsilonBoundSquaredUnits (config : LayerConfig) : Nat := + match config.quant_type with + | "int8" => config.fan_in + | "fp16" => 16 * config.fan_in + | _ => 4 * config.fan_in + +/-- Honest nonnegative quantity for ε-scale (replaces the old Float axiom). -/ +theorem epsilonBoundSquaredUnits_nonneg (config : LayerConfig) : + 0 ≤ epsilonBoundSquaredUnits config := + Nat.zero_le _ + +/-- Unproven proof obligation marker (not an axiom claiming soundness). -/ +def layer_verification_sound (_layer_name : String) (_W : WeightMatrix) + (_config : LayerConfig) (_verification : LayerVerification) : Prop := + True end ModelAssetGuard.Quant From 84d091112e861d7a5ca43adb19807f278b77c63e Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:23 -0700 Subject: [PATCH 04/35] Add HashSeeded slot modular arithmetic lemmas Provide shared CHD hash-mod bounds used by builders and well-sized placement proofs. --- .../ModelAssetGuard/Token/HashSeeded.lean | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/lean/ModelAssetGuard/Token/HashSeeded.lean diff --git a/src/lean/ModelAssetGuard/Token/HashSeeded.lean b/src/lean/ModelAssetGuard/Token/HashSeeded.lean new file mode 100644 index 0000000..f61dda3 --- /dev/null +++ b/src/lean/ModelAssetGuard/Token/HashSeeded.lean @@ -0,0 +1,73 @@ +import Init.Data.List.Basic +import Init.Data.Nat.Basic + +/-! +Lean model of Rust `PerfectHashVocab::hash_seeded` (Wave 20). + +Uses `Nat` arithmetic mod `2^64` to mirror wrapping `u64`. This is an +executable refinement of the CHD hash — not a cryptographic claim. +-/ + +namespace ModelAssetGuard.Token.HashSeeded + +/-- `2^64` as the UInt64 modulus. -/ +def u64Mod : Nat := 18446744073709551616 -- 2^64 + +@[inline] def wrap (x : Nat) : Nat := x % u64Mod + +/-- Default CHD seeds (must match Rust `DEFAULT_SEED0` / `DEFAULT_SEED1`). -/ +def defaultSeed0 : Nat := 0x9e3779b97f4a7c15 +def defaultSeed1 : Nat := 0xbf58476d1ce4e5b9 + +/-- Mix step: `h = h * 33 + b` (wrapping). -/ +def mix33 (h b : Nat) : Nat := + wrap (wrap h * 33 + (b % 256)) + +/-- Fold bytes into the accumulator (pre-finalizer). -/ +def foldBytes (seed : Nat) (bytes : List Nat) : Nat := + bytes.foldl (fun h b => mix33 h b) (wrap (seed + 5381)) + +/-- SplitMix64 finalizer (wrapping Nat). -/ +def splitMix64 (h0 : Nat) : Nat := + let h1 := wrap ((wrap h0 ^^^ (wrap h0 / 2 ^ 30)) * 0xbf58476d1ce4e5b9) + let h2 := wrap ((wrap h1 ^^^ (wrap h1 / 2 ^ 27)) * 0x94d049bb133111eb) + wrap (wrap h2 ^^^ (wrap h2 / 2 ^ 31)) + +/-- Full seeded hash over a byte list (values truncated to 8 bits). -/ +def hashSeededBytes (seed : Nat) (bytes : List Nat) : Nat := + splitMix64 (foldBytes seed bytes) + +/-- Hash ASCII / UTF-8 code units given as Nats. -/ +def hashSeededAscii (seed : Nat) (s : String) : Nat := + hashSeededBytes seed (s.toList.map Char.toNat) + +/-- Slot index used by CHD: `hash % n` when `n > 0`. -/ +def slotMod (h n : Nat) (_hn : n > 0) : Nat := + h % n + +theorem slotMod_lt (h n : Nat) (hn : n > 0) : slotMod h n hn < n := + Nat.mod_lt h hn + +/-- Determinism: same inputs ⇒ same hash. -/ +theorem hashSeededBytes_deterministic (seed : Nat) (bytes : List Nat) : + hashSeededBytes seed bytes = hashSeededBytes seed bytes := rfl + +/-- CHD bucket index from seed0. -/ +def bucketOf (seed0 nBuckets : Nat) (hn : nBuckets > 0) (bytes : List Nat) : Nat := + slotMod (hashSeededBytes seed0 bytes) nBuckets hn + +/-- CHD slot from seed1 + displace `d`. -/ +def slotOf (seed1 d n : Nat) (hn : n > 0) (bytes : List Nat) : Nat := + slotMod (hashSeededBytes (wrap (seed1 + d)) bytes) n hn + +/-- Candidate slots for a bucket's keys under displace `d`. -/ +def candidateSlots (seed1 d n : Nat) (hn : n > 0) (keys : List (List Nat)) : List Nat := + keys.map (fun k => slotOf seed1 d n hn k) + +/-- Goldens checked against Rust/Python mirrors (Wave 20). -/ +def goldenEmptySeed0 : Nat := 9862407849072414892 +def goldenASeed0 : Nat := 13358874647675253293 +def goldenAbSeed0 : Nat := 10513868738868360859 +def goldenHelloSeed0 : Nat := 5468378884496047854 + +end ModelAssetGuard.Token.HashSeeded From 515d150da60a50718326fea58391ebfc5b9ad8a5 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:30 -0700 Subject: [PATCH 05/35] Formalize CHD placement and bucket merge predicates Capture injective key-to-slot post-conditions and disjoint bucket merge so MPH success is checkable without the Rust search loop. --- src/lean/ModelAssetGuard/Token/CHD.lean | 333 ++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 src/lean/ModelAssetGuard/Token/CHD.lean diff --git a/src/lean/ModelAssetGuard/Token/CHD.lean b/src/lean/ModelAssetGuard/Token/CHD.lean new file mode 100644 index 0000000..25cce22 --- /dev/null +++ b/src/lean/ModelAssetGuard/Token/CHD.lean @@ -0,0 +1,333 @@ +import Init.Data.List.Basic + +/-! +Abstract CHD / MPH placement + construction fragment (Waves 18–19). + +Runtime builds a Compress-Hash-Displace table (`kind: "chd_mph"`) in Rust. +This module does **not** formalize `hash_seeded` or the displace search loop. +It captures: + +1. Post-condition of a successful MPH: injective key→slot placement +2. Combinatorial merge of successful bucket places (Rust post-conditions) +-/ + +namespace ModelAssetGuard.Token.CHD + +/-- Placement of vocab keys into MPH slots (parallel lists). -/ +structure Placement where + keys : List String + slots : List Nat + deriving Repr + +namespace Placement + +/-- Number of keys (intended table size). -/ +def n (p : Placement) : Nat := p.keys.length + +/-- Pairwise distinctness on a list (first-occurrence order). -/ +def pairwiseDistinct {α : Type} : List α → Prop + | [] => True + | x :: xs => (∀ y ∈ xs, x ≠ y) ∧ pairwiseDistinct xs + +theorem pairwiseDistinct_of_mem_ne {α : Type} + {xs : List α} (h : pairwiseDistinct xs) {i j : Nat} + (hi : i < xs.length) (hj : j < xs.length) (hne : i ≠ j) : + xs[i] ≠ xs[j] := by + induction xs generalizing i j with + | nil => cases hi + | cons x xs ih => + obtain ⟨hxdist, hrest⟩ := h + match i, j with + | 0, 0 => exact (hne rfl).elim + | 0, j' + 1 => + have hj' : j' < xs.length := Nat.lt_of_succ_lt_succ hj + have mem : xs[j'] ∈ xs := List.getElem_mem hj' + exact fun heq => (hxdist xs[j'] mem) heq + | i' + 1, 0 => + have hi' : i' < xs.length := Nat.lt_of_succ_lt_succ hi + have mem : xs[i'] ∈ xs := List.getElem_mem hi' + exact fun heq => (hxdist xs[i'] mem) heq.symm + | i' + 1, j' + 1 => + have hi' : i' < xs.length := Nat.lt_of_succ_lt_succ hi + have hj' : j' < xs.length := Nat.lt_of_succ_lt_succ hj + have hne' : i' ≠ j' := mt (congrArg Nat.succ) hne + exact ih hrest hi' hj' hne' + +/-- Append preserves pairwise distinctness when sides are distinct and cross-disjoint. -/ +theorem pairwiseDistinct_append {α : Type} {xs ys : List α} + (hx : pairwiseDistinct xs) (hy : pairwiseDistinct ys) + (hdisj : ∀ x ∈ xs, ∀ y ∈ ys, x ≠ y) : + pairwiseDistinct (xs ++ ys) := by + induction xs with + | nil => simpa using hy + | cons x xs ih => + obtain ⟨hxhead, hxrest⟩ := hx + refine ⟨?_, ih hxrest (fun z hz y hy => hdisj z (List.mem_cons_of_mem x hz) y hy)⟩ + intro z hz + have hz' : z ∈ xs ++ ys := by simpa using hz + match List.mem_append.1 hz' with + | Or.inl hzx => exact hxhead z hzx + | Or.inr hzy => exact hdisj x (List.Mem.head xs) z hzy + +/-- Successful CHD/MPH build post-condition (abstract). -/ +def wellFormed (p : Placement) : Prop := + p.slots.length = p.keys.length ∧ + (∀ i, (h : i < p.slots.length) → p.slots[i] < p.keys.length) ∧ + pairwiseDistinct p.slots ∧ + pairwiseDistinct p.keys + +/-- Constructor interface used by construction lemmas. -/ +theorem wellFormed_mk + (keys : List String) (slots : List Nat) + (hlen : slots.length = keys.length) + (hbound : ∀ i, (h : i < slots.length) → slots[i] < keys.length) + (hslots : pairwiseDistinct slots) + (hkeys : pairwiseDistinct keys) : + wellFormed { keys := keys, slots := slots } := + ⟨hlen, hbound, hslots, hkeys⟩ + +/-- Distinct key indices receive distinct slots. -/ +theorem slots_injective_of_wellFormed + (p : Placement) (hw : p.wellFormed) + {i j : Nat} (hi : i < p.slots.length) (hj : j < p.slots.length) + (heq : p.slots[i] = p.slots[j]) : i = j := by + have hdist := hw.2.2.1 + match decEq i j with + | isTrue h => exact h + | isFalse hne => exact absurd heq (pairwiseDistinct_of_mem_ne hdist hi hj hne) + +/-- Distinct key strings at distinct indices. -/ +theorem keys_injective_of_wellFormed + (p : Placement) (hw : p.wellFormed) + {i j : Nat} (hi : i < p.keys.length) (hj : j < p.keys.length) + (heq : p.keys[i] = p.keys[j]) : i = j := by + have hdist := hw.2.2.2 + match decEq i j with + | isTrue h => exact h + | isFalse hne => exact absurd heq (pairwiseDistinct_of_mem_ne hdist hi hj hne) + +/-- Same slot ↔ same key index under well-formedness. -/ +theorem slot_eq_implies_index_eq + (p : Placement) (hw : p.wellFormed) + {i j : Nat} (hi : i < p.slots.length) (hj : j < p.slots.length) : + p.slots[i] = p.slots[j] ↔ i = j := + ⟨fun h => slots_injective_of_wellFormed p hw hi hj h, fun h => h ▸ rfl⟩ + +/-- Pairwise `≠` as a Bool for smoke tests. -/ +def pairwiseNeBool {α : Type} [BEq α] : List α → Bool + | [] => true + | x :: xs => xs.all (fun y => y != x) && pairwiseNeBool xs + +/-- Bool gate matches Prop for lawful `BEq` types (Wave 23). -/ +theorem pairwiseNeBool_eq_true_iff_pairwiseDistinct {α : Type} + [BEq α] [LawfulBEq α] (xs : List α) : + pairwiseNeBool xs = true ↔ pairwiseDistinct xs := by + induction xs with + | nil => simp [pairwiseNeBool, pairwiseDistinct] + | cons x xs ih => + simp only [pairwiseNeBool, pairwiseDistinct, Bool.and_eq_true] + constructor + · intro ⟨hall, hrest⟩ + refine ⟨?_, ih.mp hrest⟩ + intro y hy + exact (bne_iff_ne.mp ((List.all_eq_true.1 hall) y hy)).symm + · intro ⟨hne, hrest⟩ + refine ⟨?_, ih.mpr hrest⟩ + exact List.all_eq_true.2 fun y hy => bne_iff_ne.mpr (hne y hy).symm + +theorem pairwiseDistinct_of_pairwiseNeBool {α : Type} + [BEq α] [LawfulBEq α] {xs : List α} (h : pairwiseNeBool xs = true) : + pairwiseDistinct xs := + (pairwiseNeBool_eq_true_iff_pairwiseDistinct xs).1 h + +/-- Executable well-formedness check for smoke tests. -/ +def wellFormedBool (p : Placement) : Bool := + let n := p.keys.length + (p.slots.length == n) && + p.slots.all (fun s => decide (s < n)) && + pairwiseNeBool p.slots && + pairwiseNeBool p.keys + +/-- `wellFormedBool = true` implies abstract `wellFormed` (Wave 23). -/ +theorem wellFormed_of_wellFormedBool (p : Placement) + (h : wellFormedBool p = true) : wellFormed p := by + simp only [wellFormedBool, Bool.and_eq_true, beq_iff_eq] at h + obtain ⟨⟨⟨hlen, hboundB⟩, hslotsB⟩, hkeysB⟩ := h + refine ⟨hlen, ?_, pairwiseDistinct_of_pairwiseNeBool hslotsB, + pairwiseDistinct_of_pairwiseNeBool hkeysB⟩ + intro i hi + have hmem : p.slots[i] ∈ p.slots := List.getElem_mem hi + have := (List.all_eq_true.1 hboundB) p.slots[i] hmem + exact of_decide_eq_true this + +/-- Tiny example: two keys → slots 0,1 (abstract MPH). -/ +def exampleTwo : Placement := + { keys := ["a", "b"], slots := [0, 1] } + +/-- Collision example (not well-formed): two keys share slot 0. -/ +def exampleCollision : Placement := + { keys := ["a", "b"], slots := [0, 0] } + +/-- Three-key MPH example (slots are a permutation of `0..2`). -/ +def exampleThree : Placement := + { keys := ["x", "y", "z"], slots := [2, 0, 1] } + +end Placement + +/-! ## Construction fragment (Wave 19) + +Models successful CHD bucket places as parallel `(keyIndex, slot)` lists. +Does not claim the Rust search finds such places — only that merging +successful places preserves injectivity. -/ + +/-- One successful bucket displace: parallel key indices → slots. -/ +structure BucketPlace where + keyIndices : List Nat + slots : List Nat + deriving Repr + +namespace BucketPlace + +/-- Combinatorial success condition for one CHD bucket (no hash model). -/ +def wellSized (bp : BucketPlace) (tableSize : Nat) : Prop := + bp.keyIndices.length = bp.slots.length ∧ + Placement.pairwiseDistinct bp.slots ∧ + Placement.pairwiseDistinct bp.keyIndices ∧ + (∀ i, (h : i < bp.slots.length) → bp.slots[i] < tableSize) ∧ + (∀ i, (h : i < bp.keyIndices.length) → bp.keyIndices[i] < tableSize) + +/-- Concatenate two bucket places (sequential CHD bucket processing). -/ +def append (p q : BucketPlace) : BucketPlace := + { keyIndices := p.keyIndices ++ q.keyIndices + slots := p.slots ++ q.slots } + +/-- Cross-disjoint slots and key indices. -/ +def disjoint (p q : BucketPlace) : Prop := + (∀ s ∈ p.slots, ∀ t ∈ q.slots, s ≠ t) ∧ + (∀ a ∈ p.keyIndices, ∀ b ∈ q.keyIndices, a ≠ b) + +/-- Merging two well-sized, cross-disjoint places yields a well-sized place. -/ +theorem wellSized_append + (p q : BucketPlace) (n : Nat) + (hp : p.wellSized n) (hq : q.wellSized n) (hd : disjoint p q) : + (append p q).wellSized n := by + obtain ⟨hlenp, hsp, hkp, hsbp, hkbp⟩ := hp + obtain ⟨hlenq, hsq, hkq, hsbq, hkbq⟩ := hq + refine ⟨?_, ?_, ?_, ?_, ?_⟩ + · simp [append, hlenp, hlenq] + · exact Placement.pairwiseDistinct_append hsp hsq hd.1 + · exact Placement.pairwiseDistinct_append hkp hkq hd.2 + · intro i hi + simp only [append] at hi ⊢ + have hi' : i < p.slots.length + q.slots.length := by + simpa [List.length_append] using hi + by_cases hlt : i < p.slots.length + · have : (p.slots ++ q.slots)[i] = p.slots[i] := List.getElem_append_left hlt + rw [this] + exact hsbp i hlt + · have hge : p.slots.length ≤ i := Nat.le_of_not_gt hlt + have hj : i - p.slots.length < q.slots.length := by + have := Nat.sub_lt_left_of_lt_add hge hi' + simpa using this + have : (p.slots ++ q.slots)[i] = q.slots[i - p.slots.length] := + List.getElem_append_right hge + rw [this] + exact hsbq (i - p.slots.length) hj + · intro i hi + simp only [append] at hi ⊢ + have hi' : i < p.keyIndices.length + q.keyIndices.length := by + simpa [List.length_append] using hi + by_cases hlt : i < p.keyIndices.length + · have : (p.keyIndices ++ q.keyIndices)[i] = p.keyIndices[i] := + List.getElem_append_left hlt + rw [this] + exact hkbp i hlt + · have hge : p.keyIndices.length ≤ i := Nat.le_of_not_gt hlt + have hj : i - p.keyIndices.length < q.keyIndices.length := by + have := Nat.sub_lt_left_of_lt_add hge hi' + simpa using this + have : (p.keyIndices ++ q.keyIndices)[i] = + q.keyIndices[i - p.keyIndices.length] := List.getElem_append_right hge + rw [this] + exact hkbq (i - p.keyIndices.length) hj + +/-- Empty bucket place is well-sized. -/ +theorem wellSized_nil (n : Nat) : + wellSized { keyIndices := [], slots := [] } n := by + refine ⟨rfl, trivial, trivial, ?_, ?_⟩ + · intro i hi; cases hi + · intro i hi; cases hi + +/-- Executable wellSized check for smoke tests. -/ +def wellSizedBool (bp : BucketPlace) (tableSize : Nat) : Bool := + (bp.keyIndices.length == bp.slots.length) && + Placement.pairwiseNeBool bp.slots && + Placement.pairwiseNeBool bp.keyIndices && + bp.slots.all (fun s => decide (s < tableSize)) && + bp.keyIndices.all (fun k => decide (k < tableSize)) + +/-- `wellSizedBool = true` implies `wellSized` (Wave 23). -/ +theorem wellSized_of_wellSizedBool (bp : BucketPlace) (tableSize : Nat) + (h : wellSizedBool bp tableSize = true) : wellSized bp tableSize := by + simp only [wellSizedBool, Bool.and_eq_true, beq_iff_eq] at h + obtain ⟨⟨⟨⟨hlen, hslotsB⟩, hkeysB⟩, hsb⟩, hkb⟩ := h + refine ⟨hlen, Placement.pairwiseDistinct_of_pairwiseNeBool hslotsB, + Placement.pairwiseDistinct_of_pairwiseNeBool hkeysB, ?_, ?_⟩ + · intro i hi + have hmem : bp.slots[i] ∈ bp.slots := List.getElem_mem hi + exact of_decide_eq_true ((List.all_eq_true.1 hsb) bp.slots[i] hmem) + · intro i hi + have hmem : bp.keyIndices[i] ∈ bp.keyIndices := List.getElem_mem hi + exact of_decide_eq_true ((List.all_eq_true.1 hkb) bp.keyIndices[i] hmem) + +/-- Cross-disjoint Bool check (smoke). -/ +def disjointBool (p q : BucketPlace) : Bool := + p.slots.all (fun s => q.slots.all (fun t => s != t)) && + p.keyIndices.all (fun a => q.keyIndices.all (fun b => a != b)) + +end BucketPlace + +/-- Displace-search correctness for a *found* displace (Wave 20). + +If `candidateSlots` under `(seed1,d,n)` are pairwise distinct and free in +`occupied`, the resulting `BucketPlace` is `wellSized`. This discharges the +combinatorial half of CHD placement once Rust (or an oracle) supplies `d`. +Hash correctness is `HashSeeded`; finding `d` remains `chd_displace_search_obligation`. -/ +def displaceCandidatesOk + (slots : List Nat) (tableSize : Nat) (occupied : List Bool) : Prop := + Placement.pairwiseDistinct slots ∧ + occupied.length = tableSize ∧ + (∀ s ∈ slots, s < tableSize) ∧ + (∀ s ∈ slots, ∃ h : s < occupied.length, occupied[s] = false) + +/-- Build a bucket place from key indices + candidate slots of equal length. -/ +def placeFromCandidates (keyIndices slots : List Nat) : BucketPlace := + { keyIndices := keyIndices, slots := slots } + +theorem wellSized_of_displaceCandidatesOk + (keyIndices slots : List Nat) (tableSize : Nat) (occupied : List Bool) + (hlen : keyIndices.length = slots.length) + (hk : Placement.pairwiseDistinct keyIndices) + (hkeyBound : ∀ i, (h : i < keyIndices.length) → keyIndices[i] < tableSize) + (hc : displaceCandidatesOk slots tableSize occupied) : + (placeFromCandidates keyIndices slots).wellSized tableSize := by + obtain ⟨hsdist, _, hsbound, _⟩ := hc + refine ⟨hlen, hsdist, hk, ?_, hkeyBound⟩ + intro i hi + have hsmem : slots[i] ∈ slots := List.getElem_mem hi + exact hsbound slots[i] hsmem + +/-- Runtime CHD verifies the stored string after slot lookup; Lean models +injective placement + merge of bucket places + hash-seeded slot candidates. -/ +def runtime_string_verify_obligation : Prop := True + +/-- Finding a displace `d` (search loop) remains an obligation (Rust + tests). + +Wave 24 shrinks—not discharges—this marker: Lean proves +`findDisplace_returns_zero_of_candidatesOk_at_zero` (gate already true at `d = 0`) +and executable 2-key / nonempty-occupied smokes. A full hash-family completeness +theorem is still not claimed. Once a displace is found, +`wellSized_of_displaceCandidatesOk` / `wellSized_of_findDisplace_success` apply. -/ +def chd_displace_search_obligation : Prop := True + +end ModelAssetGuard.Token.CHD From 4515c431497f776c60e7f237cc5fc0c4b03ec563 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:40 -0700 Subject: [PATCH 06/35] Prove CHD displace search and bucketed assemble gates Executable findDisplace/assembleBucketed now carry wellFormed implications and stable bucket ordering aligned with the Rust builder. --- src/lean/ModelAssetGuard/Token/CHDBuild.lean | 682 +++++++++++++++++++ 1 file changed, 682 insertions(+) create mode 100644 src/lean/ModelAssetGuard/Token/CHDBuild.lean diff --git a/src/lean/ModelAssetGuard/Token/CHDBuild.lean b/src/lean/ModelAssetGuard/Token/CHDBuild.lean new file mode 100644 index 0000000..d4a7fac --- /dev/null +++ b/src/lean/ModelAssetGuard/Token/CHDBuild.lean @@ -0,0 +1,682 @@ +import Init.Data.List.Basic +import ModelAssetGuard.Token.CHD +import ModelAssetGuard.Token.HashSeeded + +/-! +Executable CHD displace search + small builder (Waves 21-24). + +**Proven:** singleton `wellSized`; `candidateSlots` length + bounds; +`findDisplace_eq_some_imp`; Bool/Prop gates; compositional +`wellSized_of_findDisplace_success` (gate chain, no mega-unfold); +`wellSized_merge_bucket_places`; singleton free-`occupied` completeness; +`findDisplace_returns_zero_of_candidatesOk_at_zero` (restricted ∃d); +`finalizePlacement` / `assembleBucketed` success ⇒ `wellFormed`. + +**Executable:** `findDisplace` / `buildAsciiMph` / `buildAsciiMphBucketed` +(Rust-aligned λ-plans, stable bucket order + pilots). Multi-key hash-family +completeness remains `chd_displace_search_obligation`. +-/ + +namespace ModelAssetGuard.Token.CHDBuild + +open ModelAssetGuard.Token.CHD +open ModelAssetGuard.Token.CHD.Placement +open ModelAssetGuard.Token.CHD.BucketPlace +open ModelAssetGuard.Token.HashSeeded + +/-- Singleton bucket place is well-sized when indices are in range. -/ +theorem singleton_wellSized (i s n : Nat) (hi : i < n) (hs : s < n) : + wellSized { keyIndices := [i], slots := [s] } n := by + refine ⟨rfl, ?slotsDist, ?keysDist, ?slotsBound, ?keysBound⟩ + case slotsDist => + change pairwiseDistinct [s] + refine And.intro ?_ trivial + intro y hy; cases hy + case keysDist => + change pairwiseDistinct [i] + refine And.intro ?_ trivial + intro y hy; cases hy + case slotsBound => + intro j hj + have hj0 : j = 0 := Nat.lt_one_iff.mp hj + subst hj0 + exact hs + case keysBound => + intro j hj + have hj0 : j = 0 := Nat.lt_one_iff.mp hj + subst hj0 + exact hi + +/-- `candidateSlots` preserves length. -/ +theorem candidateSlots_length + (seed1 d n : Nat) (hn : n > 0) (keys : List (List Nat)) : + (candidateSlots seed1 d n hn keys).length = keys.length := by + simp [candidateSlots, List.length_map] + +/-- Every candidate slot is `< n`. -/ +theorem candidateSlots_lt + (seed1 d n : Nat) (hn : n > 0) (keys : List (List Nat)) : + ∀ s ∈ candidateSlots seed1 d n hn keys, s < n := by + intro s hs + simp only [candidateSlots, List.mem_map] at hs + rcases hs with ⟨_, _, rfl⟩ + exact slotMod_lt _ n hn + +/-- Occupied mask of length `n`, all free. -/ +def freeOccupied (n : Nat) : List Bool := + List.replicate n false + +/-- Is slot `s` free in `occupied`? -/ +def slotFreeBool (occupied : List Bool) (s : Nat) : Bool := + if h : s < occupied.length then !(occupied[s]) else false + +/-- Bool check: slots pairwise distinct, in-range, and free. -/ +def candidatesOkBool (slots : List Nat) (tableSize : Nat) (occupied : List Bool) : Bool := + (occupied.length == tableSize) && + pairwiseNeBool slots && + slots.all (fun s => decide (s < tableSize) && slotFreeBool occupied s) + +/-- Production Rust `MAX_DISPLACE` in `try_chd`. Lean smokes use `leanSmokeMaxD`. -/ +def rustMaxDisplace : Nat := 4_000_000 + +/-- Default Lean search budget for builders / goldens (≪ `rustMaxDisplace`). -/ +def leanSmokeMaxD : Nat := 10_000 + +/-- Search displace `d` in `[0, maxD]`. + +Rust production uses `rustMaxDisplace` (4_000_000). Lean executable goldens and +smokes typically pass `leanSmokeMaxD` (10_000); callers may raise `maxD` for +parity with the Rust budget when needed. -/ +def findDisplace + (seed1 n : Nat) (hn : n > 0) + (keyBytes : List (List Nat)) + (occupied : List Bool) + (maxD : Nat) : Option Nat := + (List.range (maxD + 1)).findSome? fun d => + let slots := candidateSlots seed1 d n hn keyBytes + if candidatesOkBool slots n occupied then some d else none + +/-- Mark slots occupied (executable). -/ +def markOccupied (occupied : List Bool) (slots : List Nat) : List Bool := + slots.foldl + (fun occ s => if _h : s < occ.length then occ.set s true else occ) + occupied + +/-- Finalize placement only when the Bool gate holds. -/ +def finalizePlacement (keys : List String) (slots : List Nat) : Option Placement := + let p : Placement := { keys := keys, slots := slots } + if wellFormedBool p then some p else none + +theorem finalizePlacement_eq_some_imp + (keys : List String) (slots : List Nat) (p : Placement) + (h : finalizePlacement keys slots = some p) : + p = { keys := keys, slots := slots } ∧ wellFormedBool p = true := by + simp only [finalizePlacement] at h + split at h + · next hw => + have : p = { keys := keys, slots := slots } := Option.some.inj h.symm + subst this + exact ⟨rfl, hw⟩ + · exact False.elim (Option.noConfusion h) + +theorem finalizePlacement_eq_some_imp_wellFormed + (keys : List String) (slots : List Nat) (p : Placement) + (h : finalizePlacement keys slots = some p) : wellFormed p := + wellFormed_of_wellFormedBool p (finalizePlacement_eq_some_imp keys slots p h).2 + +/-- Tiny ASCII CHD with singleton buckets (`n_buckets = n` processing order). + +Places keys `0 .. n-1` sequentially using `findDisplace` on each singleton. +Returns a well-formed `Placement` when every key finds a free slot. -/ +def buildAsciiMph (words : List String) (maxD : Nat := leanSmokeMaxD) : Option Placement := + let n := words.length + if n == 0 then + some { keys := [], slots := [] } + else if h : n > 0 then + let keyBytes := words.map (fun w => w.toList.map Char.toNat) + let step (st : Option (List Bool × List Nat)) (i : Nat) : + Option (List Bool × List Nat) := + match st with + | none => none + | some (occ, acc) => + match keyBytes[i]? with + | none => none + | some bytes => + match findDisplace defaultSeed1 n h [bytes] occ maxD with + | none => none + | some d => + let s := slotOf defaultSeed1 d n h bytes + some (markOccupied occ [s], acc ++ [s]) + match (List.range n).foldl step (some (freeOccupied n, [])) with + | none => none + | some (_, slotList) => + finalizePlacement words slotList + else + none + +/-- Smoke helper: build succeeds and is well-formed. -/ +def buildAsciiMphOk (words : List String) : Bool := + match buildAsciiMph words with + | some p => wellFormedBool p + | none => false + +/-! ## Wave 22 -- search soundness + n=1 completeness + bucketed builder -/ + +/-- `List.range n` starts with `0` when `n > 0`. -/ +theorem range_eq_cons_zero (n : Nat) (hn : n > 0) : + ∃ t, List.range n = 0 :: t := by + match n, hn with + | 0, h => exact (Nat.lt_irrefl 0 h).elim + | n + 1, _ => + induction n with + | zero => exact ⟨[], by simp [List.range_one]⟩ + | succ n ih => + have ⟨t, ht⟩ := ih (Nat.succ_pos _) + refine ⟨t ++ [n + 1], ?_⟩ + rw [List.range_succ, ht, List.cons_append] + +/-- If `findDisplace` returns `some d`, then `d ≤ maxD` and the Bool gate passed. -/ +theorem findDisplace_eq_some_imp + (seed1 n : Nat) (hn : n > 0) (keyBytes : List (List Nat)) + (occupied : List Bool) (maxD d : Nat) + (h : findDisplace seed1 n hn keyBytes occupied maxD = some d) : + d ≤ maxD ∧ + candidatesOkBool (candidateSlots seed1 d n hn keyBytes) n occupied = true := by + simp only [findDisplace] at h + rcases (List.findSome?_eq_some_iff).1 h with ⟨_l₁, a, _l₂, hl, hf, _⟩ + have ha_range : a ∈ List.range (maxD + 1) := by + have : a ∈ _l₁ ++ a :: _l₂ := List.mem_append_right _ (List.mem_cons_self) + simpa [← hl] using this + have hle : a ≤ maxD := Nat.lt_succ_iff.mp (List.mem_range.mp ha_range) + cases hif : (candidatesOkBool (candidateSlots seed1 a n hn keyBytes) n occupied) with + | true => + have heq : some a = some d := by + simp only [hif] at hf + exact hf + have : a = d := Option.some.inj heq + subst this + exact ⟨hle, hif⟩ + | false => + have heq : (none : Option Nat) = some d := by + simp only [hif] at hf + exact hf + exact False.elim (Option.noConfusion heq) + +/-- `hash % 1 = 0`. -/ +theorem slotMod_one (h : Nat) : slotMod h 1 (by decide : 1 > 0) = 0 := by + simp [slotMod, Nat.mod_one] + +/-- For table size 1, every candidate slot is `0`. -/ +theorem candidateSlots_tableSize_one + (seed1 d : Nat) (hn : (1 : Nat) > 0) (keys : List (List Nat)) : + candidateSlots seed1 d 1 hn keys = List.replicate keys.length 0 := by + induction keys with + | nil => simp [candidateSlots] + | cons k ks ih => + simp only [candidateSlots, List.map_cons, List.replicate_succ] + have hs : slotOf seed1 d 1 hn k = 0 := by + simp [slotOf, slotMod_one] + change slotOf seed1 d 1 hn k :: candidateSlots seed1 d 1 hn ks = + 0 :: List.replicate ks.length 0 + rw [hs, show candidateSlots seed1 d 1 hn ks = List.replicate ks.length 0 from ih] + +/-- Bool gate holds for a single free slot at table size 1. -/ +theorem candidatesOkBool_singleton_free_one : + candidatesOkBool [0] 1 (freeOccupied 1) = true := by + native_decide + +/-- Completeness for `n = 1` is checked executably (`findDisplace_n1_smoke` in Tests). + +A full forall-maxD `findDisplace` proof triggers deep `simp` recursion on +`findSome?`/`if` in Lean 4.21 without Mathlib helpers; the gate + `slotMod_one` facts +above are the reusable lemmas. Wave 23 proves singleton free-occupied completeness. -/ +def findDisplace_n1_ok (seed1 : Nat) (bytes : List Nat) (maxD : Nat := 8) : Bool := + findDisplace seed1 1 (by decide) [bytes] (freeOccupied 1) maxD == some 0 + +/-- Group key indices `0..n-1` by `bucketOf seed0 nBuckets`. -/ +def indicesByBucket + (seed0 nBuckets : Nat) (hn : nBuckets > 0) + (keyBytes : List (List Nat)) : List (List Nat) := + let n := keyBytes.length + let buckets : List (List Nat) := List.replicate nBuckets [] + (List.range n).foldl + (fun bucks i => + match keyBytes[i]? with + | none => bucks + | some bytes => + let b := bucketOf seed0 nBuckets hn bytes + if hb : b < bucks.length then + bucks.set b (bucks[b] ++ [i]) + else bucks) + buckets + +/-- Place one multi-key bucket via `findDisplace` (executable). -/ +def placeBucket + (seed1 n : Nat) (hn : n > 0) + (keyBytes : List (List Nat)) (idxs : List Nat) + (occ : List Bool) (maxD : Nat) : Option (List Bool × List (Nat × Nat) × Nat) := + let bytes := idxs.filterMap (fun i => keyBytes[i]?) + if bytes.length != idxs.length then none + else + match findDisplace seed1 n hn bytes occ maxD with + | none => none + | some d => + let slots := candidateSlots seed1 d n hn bytes + if slots.length != idxs.length then none + else + let pairs := List.zip idxs slots + some (markOccupied occ slots, pairs, d) + +/-- Bucketed CHD build result: placement + per-bucket pilots (Rust `pilots`). -/ +structure BucketedBuild where + placement : Placement + nBuckets : Nat + pilots : List Nat + deriving Repr + +/-- Assemble key-to-slot placement + pilots through the wellFormed Bool gate. -/ +def assembleBucketed + (words : List String) (nBuckets : Nat) + (pairs : List (Nat × Nat)) (pilots : List Nat) : Option BucketedBuild := + let n := words.length + let slots := + (List.range n).map fun i => + (pairs.find? fun p => p.1 == i).map (·.2) |>.getD 0 + match finalizePlacement words slots with + | none => none + | some p => some { placement := p, nBuckets := nBuckets, pilots := pilots } + +theorem assembleBucketed_eq_some_imp + (words : List String) (nBuckets : Nat) + (pairs : List (Nat × Nat)) (pilots : List Nat) (b : BucketedBuild) + (h : assembleBucketed words nBuckets pairs pilots = some b) : + wellFormed b.placement ∧ b.nBuckets = nBuckets ∧ b.pilots = pilots := by + simp only [assembleBucketed] at h + cases hf : finalizePlacement words + ((List.range words.length).map fun i => + (pairs.find? fun p => p.1 == i).map (·.2) |>.getD 0) with + | none => simp [hf] at h + | some p => + simp only [hf] at h + have hb : b = { placement := p, nBuckets := nBuckets, pilots := pilots } := + Option.some.inj h.symm + subst hb + exact ⟨finalizePlacement_eq_some_imp_wellFormed _ _ _ hf, rfl, rfl⟩ + +theorem assembleBucketed_eq_some_imp_wellFormed + (words : List String) (nBuckets : Nat) + (pairs : List (Nat × Nat)) (pilots : List Nat) (b : BucketedBuild) + (h : assembleBucketed words nBuckets pairs pilots = some b) : + wellFormed b.placement := + (assembleBucketed_eq_some_imp words nBuckets pairs pilots b h).1 + +/-- Largest length first; equal lengths ⇒ ascending bucket id (Rust Wave 24). -/ +def bucketLe (groups : List (List Nat)) (b1 b2 : Nat) : Bool := + let l1 := (groups[b1]?).map List.length |>.getD 0 + let l2 := (groups[b2]?).map List.length |>.getD 0 + decide (l1 > l2 ∨ (l1 = l2 ∧ b1 ≤ b2)) + +/-- Stable bucket processing order matching Rust `bucket_process_order`. -/ +def bucketProcessOrder (nBuckets : Nat) (groups : List (List Nat)) : List Nat := + (List.range nBuckets).mergeSort (bucketLe groups) + +/-- Core bucketed builder for a fixed `nBuckets` (Rust `try_chd` shape). + +Groups by `seed0`, largest-first + id tie-break, searches displace, records `pilots[b]`. -/ +def tryAsciiMphBucketed + (words : List String) (nBuckets : Nat) (maxD : Nat := leanSmokeMaxD) : + Option BucketedBuild := + let n := words.length + if n == 0 then + some { placement := { keys := [], slots := [] }, nBuckets := nBuckets, pilots := [] } + else if h : n > 0 then + if hnB : nBuckets > 0 then + let keyBytes := words.map (fun w => w.toList.map Char.toNat) + let groups := indicesByBucket defaultSeed0 nBuckets hnB keyBytes + let order := bucketProcessOrder nBuckets groups + let step (st : Option (List Bool × List (Nat × Nat) × List Nat)) (b : Nat) : + Option (List Bool × List (Nat × Nat) × List Nat) := + match st with + | none => none + | some (occ, acc, pilots) => + match groups[b]? with + | none => some (occ, acc, pilots) + | some idxs => + if idxs.isEmpty then some (occ, acc, pilots) + else + match placeBucket defaultSeed1 n h keyBytes idxs occ maxD with + | none => none + | some (occ', pairs, d) => + let pilots' := + if _hb : b < pilots.length then pilots.set b d else pilots + some (occ', acc ++ pairs, pilots') + let initPilots := List.replicate nBuckets 0 + match order.foldl step (some (freeOccupied n, [], initPilots)) with + | none => none + | some (_, pairs, pilots) => assembleBucketed words nBuckets pairs pilots + else none + else + none + +/-- Dedup preserving first occurrence (for Rust bucket plans). -/ +def natEraseDups : List Nat → List Nat + | [] => [] + | x :: xs => x :: (natEraseDups xs).filter (· != x) + +/-- Rust-aligned bucket plan: lambda~5, lambda~2, then `n` (always last). -/ +def rustBucketPlans (n : Nat) : List Nat := + if n == 0 then [0] + else + let a := ((n + 4) / 5).max 1 + let b := ((n + 1) / 2).max 1 + natEraseDups [a, b, n] + +/-- Bucketed ASCII MPH: try Rust bucket plans; return placement. -/ +def buildAsciiMphBucketed + (words : List String) (maxD : Nat := leanSmokeMaxD) : Option Placement := + let n := words.length + if n == 0 then + some { keys := [], slots := [] } + else + (rustBucketPlans n).findSome? fun nb => + (tryAsciiMphBucketed words nb maxD).map (·.placement) + +/-- Same as `buildAsciiMphBucketed` but keeps pilots (for Lean/Rust goldens). -/ +def buildAsciiMphBucketedFull + (words : List String) (maxD : Nat := leanSmokeMaxD) : Option BucketedBuild := + let n := words.length + if n == 0 then + some { placement := { keys := [], slots := [] }, nBuckets := 0, pilots := [] } + else + (rustBucketPlans n).findSome? fun nb => tryAsciiMphBucketed words nb maxD + +/-- Force `n_buckets = n` (Lean/Rust golden parity under lambda~1). -/ +def buildAsciiMphBucketedN + (words : List String) (maxD : Nat := leanSmokeMaxD) : Option BucketedBuild := + let n := words.length + if n == 0 then + some { placement := { keys := [], slots := [] }, nBuckets := 0, pilots := [] } + else + tryAsciiMphBucketed words n maxD + +def buildAsciiMphBucketedOk (words : List String) : Bool := + match buildAsciiMphBucketed words with + | some p => wellFormedBool p + | none => false + +/-! ## Wave 23 -- Bool gates, placeBucket soundness, free-occupied completeness -/ + +/-- `a != b = false` iff `a = b` for Nat. -/ +theorem nat_bne_eq_false_iff {a b : Nat} : ((a != b) = false) ↔ a = b := by + cases h : (a == b) with + | true => + have : a = b := beq_iff_eq.mp h + simp [bne, h, this] + | false => + have hne : a ≠ b := fun heq => by + have : (a == b) = true := beq_iff_eq.mpr heq + simp [h] at this + simp [bne, h] + exact hne + +/-- `not (b = true)` implies `b = false`. -/ +theorem bool_ne_true_eq_false {b : Bool} (h : ¬ b = true) : b = false := by + cases b with + | true => exact (h rfl).elim + | false => rfl + +/-- `candidatesOkBool` implies combinatorial `displaceCandidatesOk`. -/ +theorem displaceCandidatesOk_of_candidatesOkBool + (slots : List Nat) (tableSize : Nat) (occupied : List Bool) + (h : candidatesOkBool slots tableSize occupied = true) : + displaceCandidatesOk slots tableSize occupied := by + simp only [candidatesOkBool, Bool.and_eq_true, beq_iff_eq] at h + obtain ⟨⟨hlen, hdistB⟩, hall⟩ := h + refine ⟨pairwiseDistinct_of_pairwiseNeBool hdistB, hlen, ?_, ?_⟩ + · intro s hs + have := (List.all_eq_true.1 hall) s hs + simp only [Bool.and_eq_true, decide_eq_true_eq] at this + exact this.1 + · intro s hs + have := (List.all_eq_true.1 hall) s hs + simp only [Bool.and_eq_true, decide_eq_true_eq] at this + obtain ⟨hsBound, hfree⟩ := this + have hsOcc : s < occupied.length := by simpa [hlen] using hsBound + refine ⟨hsOcc, ?_⟩ + simp only [slotFreeBool, hsOcc, ↓reduceDIte] at hfree + cases hocc : occupied[s] with + | true => simp [hocc] at hfree + | false => rfl + +/-- Empty candidate list is always OK when the mask length matches. -/ +theorem candidatesOkBool_nil (tableSize : Nat) (occupied : List Bool) + (hlen : occupied.length = tableSize) : + candidatesOkBool [] tableSize occupied = true := by + simp [candidatesOkBool, pairwiseNeBool, hlen] + +/-- Free mask: every in-range slot reports free. -/ +theorem slotFreeBool_freeOccupied (n s : Nat) (hs : s < n) : + slotFreeBool (freeOccupied n) s = true := by + simp [slotFreeBool, freeOccupied, List.length_replicate, hs, List.getElem_replicate] + +/-- Singleton slot on a free mask passes the Bool gate. -/ +theorem candidatesOkBool_singleton_free (n s : Nat) (hs : s < n) : + candidatesOkBool [s] n (freeOccupied n) = true := by + simp [candidatesOkBool, pairwiseNeBool, slotFreeBool, freeOccupied, + List.length_replicate, hs, List.getElem_replicate, decide_eq_true hs, + List.all_cons, List.all_nil] + +/-- Singleton candidate slots under any displace are free-OK on an empty table. -/ +theorem candidatesOkBool_singleton_candidate_free + (seed1 d n : Nat) (hn : n > 0) (bytes : List Nat) : + candidatesOkBool (candidateSlots seed1 d n hn [bytes]) n (freeOccupied n) = true := by + simp only [candidateSlots, List.map_cons, List.map_nil] + exact candidatesOkBool_singleton_free n (slotOf seed1 d n hn bytes) (slotMod_lt _ n hn) + +/-- Empty key list: `findDisplace` returns `some 0` (first candidate). -/ +theorem findDisplace_nil_keys + (seed1 n : Nat) (hn : n > 0) (occupied : List Bool) (maxD : Nat) + (hlen : occupied.length = n) : + findDisplace seed1 n hn [] occupied maxD = some 0 := by + have ⟨t, ht⟩ := range_eq_cons_zero (maxD + 1) (Nat.succ_pos _) + simp only [findDisplace, ht, List.findSome?_cons] + have hok : candidatesOkBool (candidateSlots seed1 0 n hn []) n occupied = true := by + simpa [candidateSlots] using candidatesOkBool_nil n occupied hlen + simp [hok] + +/-- Singleton + free occupied: search always returns `d = 0` (Wave 23 completeness). -/ +theorem findDisplace_singleton_freeOccupied + (seed1 n : Nat) (hn : n > 0) (bytes : List Nat) (maxD : Nat) : + findDisplace seed1 n hn [bytes] (freeOccupied n) maxD = some 0 := by + have ⟨t, ht⟩ := range_eq_cons_zero (maxD + 1) (Nat.succ_pos _) + simp only [findDisplace, ht, List.findSome?_cons] + have hok := candidatesOkBool_singleton_candidate_free seed1 0 n hn bytes + simp [hok] + +/-- Table size 1 cannot place two-or-more keys (all candidates collide at slot 0). -/ +theorem candidatesOkBool_two_zeros_false : + candidatesOkBool [0, 0] 1 (freeOccupied 1) = false := by + native_decide + +/-! +`placeBucket` soundness is compositional (avoid deep `match`/`if` recursion): + +1. `findDisplace_eq_some_imp` — returned `d` passes `candidatesOkBool` +2. `displaceCandidatesOk_of_candidatesOkBool` — Bool gate → Prop +3. `wellSized_of_displaceCandidatesOk` — Prop → `wellSized` (under distinct idxs) + +A single mega-theorem unfolding `placeBucket`'s nested `match`/`if` hits the same +Lean 4.21 recursion traps as full `findDisplace` completeness (Wave 22). +Wave 24 packages the chain as `wellSized_of_findDisplace_success` and the merge +step as `wellSized_merge_bucket_places` (reuses `wellSized_append`). +-/ + +/-- Compositional: successful `findDisplace` ⇒ `wellSized` place (no `placeBucket` unfold). -/ +theorem wellSized_of_findDisplace_success + (seed1 n : Nat) (hn : n > 0) + (bytes : List (List Nat)) (idxs : List Nat) + (occupied : List Bool) (maxD d : Nat) + (hfind : findDisplace seed1 n hn bytes occupied maxD = some d) + (hlen : idxs.length = bytes.length) + (hk : Placement.pairwiseDistinct idxs) + (hkeyBound : ∀ i, (h : i < idxs.length) → idxs[i] < n) : + (placeFromCandidates idxs (candidateSlots seed1 d n hn bytes)).wellSized n := by + have ⟨_, hbool⟩ := findDisplace_eq_some_imp seed1 n hn bytes occupied maxD d hfind + have hprop := displaceCandidatesOk_of_candidatesOkBool + (candidateSlots seed1 d n hn bytes) n occupied hbool + have hlen' : idxs.length = (candidateSlots seed1 d n hn bytes).length := by + simpa [candidateSlots_length seed1 d n hn bytes] using hlen + exact wellSized_of_displaceCandidatesOk idxs + (candidateSlots seed1 d n hn bytes) n occupied hlen' hk hkeyBound hprop + +/-- Fold/append step: merging two successful places stays `wellSized` under disjointness. -/ +theorem wellSized_merge_bucket_places + (p q : BucketPlace) (n : Nat) + (hp : p.wellSized n) (hq : q.wellSized n) (hd : BucketPlace.disjoint p q) : + (BucketPlace.append p q).wellSized n := + BucketPlace.wellSized_append p q n hp hq hd + +/-- Restricted ∃d: if `d = 0` already passes the Bool gate, search returns `some 0`. + +Does **not** claim a full hash-family completeness theorem — only that the +executable search finds the first candidate when it is already OK. -/ +theorem findDisplace_returns_zero_of_candidatesOk_at_zero + (seed1 n : Nat) (hn : n > 0) (keyBytes : List (List Nat)) + (occupied : List Bool) (maxD : Nat) + (hok : candidatesOkBool (candidateSlots seed1 0 n hn keyBytes) n occupied = true) : + findDisplace seed1 n hn keyBytes occupied maxD = some 0 := by + have ⟨t, ht⟩ := range_eq_cons_zero (maxD + 1) (Nat.succ_pos _) + simp only [findDisplace, ht, List.findSome?_cons] + simp [hok] + +/-- Singleton `placeBucket` on a free table succeeds at `d = 0`. -/ +theorem placeBucket_singleton_freeOccupied + (seed1 n : Nat) (hn : n > 0) (keyBytes : List (List Nat)) + (i : Nat) (bytes : List Nat) (maxD : Nat) + (hi : keyBytes[i]? = some bytes) : + placeBucket seed1 n hn keyBytes [i] (freeOccupied n) maxD = + some ( + markOccupied (freeOccupied n) (candidateSlots seed1 0 n hn [bytes]), + [(i, slotOf seed1 0 n hn bytes)], + 0) := by + have hfind := findDisplace_singleton_freeOccupied seed1 n hn bytes maxD + -- Executable equality for the singleton filterMap/zip path. + simp [placeBucket, List.filterMap, hi, hfind, candidateSlots, + List.zip_cons_cons, List.zip_nil_right] + +theorem empty_placement_wellFormed : + wellFormed ({ keys := [], slots := [] } : Placement) := + wellFormed_of_wellFormedBool _ (by native_decide) + +/-- Lean/Rust golden: key-to-slot under `n_buckets = n` for `["a","b","c"]`. -/ +def goldenAbcSlots : List Nat := [0, 2, 1] +def goldenAbcPilots : List Nat := [0, 0, 0] + +/-- Lean/Rust golden: `["hello","world","test"]` with `n_buckets = n`. -/ +def goldenHelloSlots : List Nat := [0, 2, 1] +def goldenHelloPilots : List Nat := [2, 0, 1] + +/-- Lean/Rust golden: `["a","b","c","d"]` with `n_buckets = n`. -/ +def goldenAbcdSlots : List Nat := [2, 1, 0, 3] +def goldenAbcdPilots : List Nat := [0, 3, 0, 1] + +/-- Smoke: bucketed-N build matches shared CHD goldens. -/ +def goldenBucketedNOk : Bool := + match buildAsciiMphBucketedN ["a", "b", "c"] with + | none => false + | some b => + (b.placement.slots == goldenAbcSlots) && + (b.pilots == goldenAbcPilots) && + wellFormedBool b.placement && + match buildAsciiMphBucketedN ["hello", "world", "test"] with + | none => false + | some b2 => + (b2.placement.slots == goldenHelloSlots) && + (b2.pilots == goldenHelloPilots) && + match buildAsciiMphBucketedN ["a", "b", "c", "d"] with + | none => false + | some b3 => + (b3.placement.slots == goldenAbcdSlots) && + (b3.pilots == goldenAbcdPilots) + +/-! ## Wave 24 -- stable order, λ-plan goldens, compositional gates, restricted ∃d -/ + +/-- Artificial equal-length order: three size-2 buckets ⇒ `[0,1,2]`. -/ +def bucketOrderEqualLengthOk : Bool := + bucketProcessOrder 3 [[0, 1], [2, 3], [4, 5]] == [0, 1, 2] && + bucketProcessOrder 4 [[0, 1, 2], [3], [4, 5, 6], [7]] == [0, 2, 1, 3] && + bucketProcessOrder 4 [[], [0, 1], [], [2, 3]] == [1, 3, 0, 2] + +/-- Real-key equal-length stresses (shared with Rust `test_chd_bucket_order_parity_stress`). -/ +def bucketOrderStressOk : Bool := + let words4 := ["one", "two", "three", "four"] + let kb4 := words4.map (fun w => w.toList.map Char.toNat) + let g4 := indicesByBucket defaultSeed0 2 (by decide) kb4 + let words8 := ["w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"] + let kb8 := words8.map (fun w => w.toList.map Char.toNat) + let g8 := indicesByBucket defaultSeed0 4 (by decide) kb8 + (g4.map List.length == [2, 2]) && + (bucketProcessOrder 2 g4 == [0, 1]) && + (g8.map List.length == [2, 0, 4, 2]) && + (bucketProcessOrder 4 g8 == [2, 0, 3, 1]) + +/-- First successful λ-plan goldens (Rust `test_chd_lean_golden_lambda_plans`). -/ +def goldenPlanAbcSlots : List Nat := [0, 2, 1] +def goldenPlanAbcPilots : List Nat := [0] +def goldenPlanHelloSlots : List Nat := [1, 2, 0] +def goldenPlanHelloPilots : List Nat := [5] +def goldenPlanAbcdSlots : List Nat := [2, 3, 1, 0] +def goldenPlanAbcdPilots : List Nat := [22] +def goldenPlanW8Slots : List Nat := [1, 3, 2, 0, 5, 4, 7, 6] +def goldenPlanW8Pilots : List Nat := [1, 92] +def goldenPlanXyzSlots : List Nat := [0, 4, 3, 5, 1, 2] +def goldenPlanXyzPilots : List Nat := [41, 5] +def goldenPlanOneSlots : List Nat := [0, 1, 3, 2] +def goldenPlanOnePilots : List Nat := [5] + +/-- Check one λ-plan golden (nBuckets + slots + pilots + wellFormed). -/ +def goldenPlanCaseOk + (words : List String) (nBuckets : Nat) + (slots pilots : List Nat) : Bool := + match buildAsciiMphBucketedFull words with + | none => false + | some b => + (b.nBuckets == nBuckets) && + (b.placement.slots == slots) && + (b.pilots == pilots) && + wellFormedBool b.placement + +/-- Smoke: first successful Rust λ-plan matches Lean `buildAsciiMphBucketedFull`. -/ +def goldenBucketedPlanOk : Bool := + goldenPlanCaseOk ["a", "b", "c"] 1 goldenPlanAbcSlots goldenPlanAbcPilots && + goldenPlanCaseOk ["hello", "world", "test"] 1 goldenPlanHelloSlots goldenPlanHelloPilots && + goldenPlanCaseOk ["a", "b", "c", "d"] 1 goldenPlanAbcdSlots goldenPlanAbcdPilots && + goldenPlanCaseOk ["w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"] 2 + goldenPlanW8Slots goldenPlanW8Pilots && + goldenPlanCaseOk ["x", "y", "z", "w", "v", "u"] 2 + goldenPlanXyzSlots goldenPlanXyzPilots && + goldenPlanCaseOk ["one", "two", "three", "four"] 1 + goldenPlanOneSlots goldenPlanOnePilots + +/-- Two-key free-table smoke: search finds some displace (not a hash-family theorem). -/ +def findDisplace_twoKey_free_smoke : Bool := + match findDisplace defaultSeed1 2 (by decide) [[97], [98]] (freeOccupied 2) 64 with + | some _ => true + | none => false + +/-- Nonempty occupied: second singleton must take the free slot. -/ +def findDisplace_singleton_occupied_smoke : Bool := + let occ := markOccupied (freeOccupied 2) [0] + match findDisplace defaultSeed1 2 (by decide) [[98]] occ 64 with + | none => false + | some d => slotOf defaultSeed1 d 2 (by decide) [98] == 1 + +/-- Wave 24 aggregate executable checks. -/ +def wave24ChdOk : Bool := + bucketOrderEqualLengthOk && + bucketOrderStressOk && + goldenBucketedPlanOk && + findDisplace_twoKey_free_smoke && + findDisplace_singleton_occupied_smoke + +end ModelAssetGuard.Token.CHDBuild From 9fd6f245e23e710e1eb330f24cffdee5c8fcc050 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:07:52 -0700 Subject: [PATCH 07/35] Add Lean Rust sidecar discovery helpers Locate optional guardd CLI binaries so tests can exercise SHA-256 and related sidecars when present. --- src/lean/ModelAssetGuard/RustSidecar.lean | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/lean/ModelAssetGuard/RustSidecar.lean diff --git a/src/lean/ModelAssetGuard/RustSidecar.lean b/src/lean/ModelAssetGuard/RustSidecar.lean new file mode 100644 index 0000000..25b3064 --- /dev/null +++ b/src/lean/ModelAssetGuard/RustSidecar.lean @@ -0,0 +1,62 @@ +import Init.System.IO +import Init.System.FilePath + +namespace ModelAssetGuard.RustSidecar + +/-- `lake exe foo -- args` may forward a literal `--` as argv[0]; drop it. -/ +def stripLakeDashDash (args : List String) : List String := + match args with + | "--" :: rest => rest + | other => other + +/-- Candidate paths for a Rust `guardd_*` helper binary. -/ +def candidates (binName : String) : List System.FilePath := + [ s!"src/rust/guardd/target/release/{binName}", + s!"src/rust/guardd/target/debug/{binName}", + s!"src/rust/guardd/target/release/{binName}.exe", + s!"src/rust/guardd/target/debug/{binName}.exe" ] + +/-- Locate a built helper binary by name, if present. -/ +def findBin (binName : String) : IO (Option System.FilePath) := do + for candidate in candidates binName do + if (← candidate.pathExists) then + return some candidate + return none + +/-- Run a Rust helper; returns stdout on exit 0, else none. -/ +def runCapturing (binName : String) (args : Array String) : IO (Option String) := do + match (← findBin binName) with + | none => return none + | some bin => do + let out ← IO.Process.output { cmd := bin.toString, args } + if out.exitCode != 0 then + if out.stderr.trim != "" then + IO.eprintln out.stderr.trim + if out.stdout.trim != "" then + IO.eprintln out.stdout.trim + return none + return some out.stdout.trim + +/-- Run a Rust helper and return its exit code (255 if binary missing). -/ +def runExitCode (binName : String) (args : Array String) : IO UInt32 := do + match (← findBin binName) with + | none => do + IO.eprintln s!"Error: Rust helper `{binName}` not found." + IO.eprintln "Build with:" + IO.eprintln s!" cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin {binName}" + return 255 + | some bin => do + let out ← IO.Process.output { cmd := bin.toString, args } + if out.stdout.trim != "" then + IO.println out.stdout.trim + if out.stderr.trim != "" then + IO.eprintln out.stderr.trim + return out.exitCode + +/-- Print a missing-binary hint for `binName`. -/ +def missingBinHint (binName : String) : IO Unit := do + IO.println s!"Error: Rust helper `{binName}` not found or failed." + IO.println "Build with:" + IO.println s!" cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin {binName}" + +end ModelAssetGuard.RustSidecar From cbe341295349edbb1d33dc3c6c0e0b6f3610f504 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:24 -0700 Subject: [PATCH 08/35] Route Lean SHA-256 verification through guardd Clarify that simpleDigest is a Lean placeholder and prefer the Rust guardd_sha256 sidecar for real hashes. --- src/lean/ModelAssetGuard/Weights.lean | 46 ++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/lean/ModelAssetGuard/Weights.lean b/src/lean/ModelAssetGuard/Weights.lean index ad141b3..72af093 100644 --- a/src/lean/ModelAssetGuard/Weights.lean +++ b/src/lean/ModelAssetGuard/Weights.lean @@ -1,5 +1,6 @@ import Init.System.IO import Init.Data.String.Basic +import Init.System.FilePath namespace ModelAssetGuard @@ -10,30 +11,65 @@ structure Checkpoint where digest : String deriving Repr -/-- A lightweight deterministic digest placeholder for Lean-side tooling. -/ +/-- +Lean-only placeholder digest (NOT SHA-256). + +Used only for in-process Lean unit experiments. Production / CLI verification +must use the Rust `guardd_sha256` binary (real SHA-256). +-/ def simpleDigest (content : String) : String := let folded := content.foldl (fun acc c => (acc * 131 + c.toNat) % 1000000007) 0 toString folded -/-- Verify that a checkpoint's digest matches its file content -/ +/-- Candidate locations for the Rust SHA-256 helper. -/ +def guarddSha256Candidates : List System.FilePath := + [ "src/rust/guardd/target/release/guardd_sha256", + "src/rust/guardd/target/debug/guardd_sha256", + "src/rust/guardd/target/release/guardd_sha256.exe", + "src/rust/guardd/target/debug/guardd_sha256.exe" ] + +/-- Locate a built `guardd_sha256` binary, if present. -/ +def findGuarddSha256 : IO (Option System.FilePath) := do + for candidate in guarddSha256Candidates do + if (← candidate.pathExists) then + return some candidate + return none + +/-- +Compute lowercase hex SHA-256 via the Rust sidecar CLI. + +Returns `none` if the binary is missing or the process fails. +-/ +def sha256HexViaRust (path : String) : IO (Option String) := do + match (← findGuarddSha256) with + | none => return none + | some bin => do + let out ← IO.Process.output { + cmd := bin.toString + args := #[path] + } + if out.exitCode != 0 then + return none + return some out.stdout.trim + +/-- Verify that a checkpoint's digest matches its file content (placeholder digest). -/ def verify (checkpoint : Checkpoint) : IO Bool := do let content ← IO.FS.readFile checkpoint.path let computedDigest := simpleDigest content return computedDigest == checkpoint.digest -/-- Create a checkpoint from a file path -/ +/-- Create a checkpoint from a file path using the Lean placeholder digest. -/ def createCheckpoint (path : String) : IO Checkpoint := do let content ← IO.FS.readFile path let digest := simpleDigest content let size := content.length return { path, size, digest } -/-- Verify checkpoint with size validation -/ +/-- Verify checkpoint with size validation (placeholder digest). -/ def verifyWithSize (checkpoint : Checkpoint) : IO Bool := do let content ← IO.FS.readFile checkpoint.path let computedDigest := simpleDigest content let size := content.length - return computedDigest == checkpoint.digest && size == checkpoint.size end ModelAssetGuard From 680c6b2a6b428819b0763dcb03854415aa9404fb Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:24 -0700 Subject: [PATCH 09/35] Strengthen Lean tokenizer models and round-trip props Align PerfectHashTokenizer with CHD runtime vocabulary and fix vocab_size to depend on the tokenizer instance. --- src/lean/ModelAssetGuard/Token/Tokenizer.lean | 560 +++++++++++++++--- 1 file changed, 484 insertions(+), 76 deletions(-) diff --git a/src/lean/ModelAssetGuard/Token/Tokenizer.lean b/src/lean/ModelAssetGuard/Token/Tokenizer.lean index 74a70a0..516f3e5 100644 --- a/src/lean/ModelAssetGuard/Token/Tokenizer.lean +++ b/src/lean/ModelAssetGuard/Token/Tokenizer.lean @@ -1,6 +1,9 @@ import Init.Data.String.Basic -import Init.Data.ByteArray +import Init.Data.Char.Lemmas import Init.Data.List.Basic +import Init.Data.List.Lemmas +import Init.Data.List.Find +import Init.Data.List.Pairwise namespace ModelAssetGuard.Token @@ -11,7 +14,7 @@ abbrev Token := Nat class Tokenizer (α : Type) where encode : α → String → List Token decode : α → List Token → String - vocab_size : Nat + vocab_size : α → Nat /-- BPE (Byte Pair Encoding) tokenizer -/ structure BPETokenizer where @@ -25,67 +28,56 @@ structure SentencePieceTokenizer where model_proto : List UInt8 deriving Repr -/-- Perfect-Hash tokenizer -/ +/-- Hash-vocab tokenizer (runtime: CHD MPH by default; Lean model uses assoc list). +Legacy open-address JSON still loads in Rust; see `Token.CHD` for MPH placement lemmas. -/ structure PerfectHashTokenizer where vocab : List (String × Token) - hash_table : List Nat -- Precomputed perfect hash table (maps string hash to token index) + hash_table : List Nat deriving Repr -/-- Determinism property: encode ∘ decode = id -/ +/-- Determinism property: encode ∘ decode = id on token lists -/ def is_deterministic {α : Type} [Tokenizer α] (t : α) : Prop := ∀ tokens : List Token, Tokenizer.encode t (Tokenizer.decode t tokens) = tokens -/-- Surjectivity property for UTF-8 -/ +/-- Surjectivity property for UTF-8 strings -/ def is_surjective_utf8 {α : Type} [Tokenizer α] (t : α) : Prop := ∀ s : String, ∃ tokens : List Token, Tokenizer.decode t tokens = s -/-- BPE encode implementation -/ +/-- String round-trip: decode ∘ encode = id -/ +def is_string_roundtrip {α : Type} [Tokenizer α] (t : α) : Prop := + ∀ s : String, Tokenizer.decode t (Tokenizer.encode t s) = s + +/-- Decode one token to a character (shared by BPE/SP stubs). -/ +def decode_token_char (vocab : List (String × Token)) (token : Token) : Char := + match vocab.find? (fun (_str, tok) => tok == token) with + | some (str, _) => str.toList.headD (Char.ofNat token) + | none => Char.ofNat token + +/-- BPE encode: character codes (merges not applied in this stub). -/ def bpe_encode (_t : BPETokenizer) (text : String) : List Token := - -- Simplified BPE encoding - -- In practice, this would implement the full BPE algorithm - let chars := text.toList - let initial_tokens := chars.map (fun c => c.toNat) - -- Apply merges - initial_tokens - -/-- BPE decode implementation -/ + text.toList.map (fun c => c.toNat) + +/-- BPE decode: vocab lookup head char, else `Char.ofNat`. -/ def bpe_decode (t : BPETokenizer) (tokens : List Token) : String := - let chars := tokens.map (fun token => - match t.vocab.find? (fun (_str, tok) => tok == token) with - | some (str, _) => str.toList.headD (Char.ofNat token) - | none => Char.ofNat token - ) - String.ofList chars + String.mk (tokens.map (decode_token_char t.vocab)) -/-- SentencePiece encode implementation -/ +/-- SentencePiece encode (char-level stub). -/ def sp_encode (_t : SentencePieceTokenizer) (text : String) : List Token := - -- Simplified SentencePiece encoding - -- In practice, this would use the actual SentencePiece library - let chars := text.toList - chars.map (fun c => c.toNat) + text.toList.map (fun c => c.toNat) -/-- SentencePiece decode implementation -/ +/-- SentencePiece decode (char-level stub). -/ def sp_decode (t : SentencePieceTokenizer) (tokens : List Token) : String := - let chars := tokens.map (fun token => - match t.vocab.find? (fun (_str, tok) => tok == token) with - | some (str, _) => str.toList.headD (Char.ofNat token) - | none => Char.ofNat token - ) - String.ofList chars + String.mk (tokens.map (decode_token_char t.vocab)) -/-- Perfect-hash encode implementation -/ +/-- Hash-vocab encode: whole-string lookup. -/ def perfect_hash_encode (t : PerfectHashTokenizer) (text : String) : List Token := - -- Simplified placeholder: treat full input as one token candidate - let tokens := [text].map (fun word => - match t.vocab.find? (fun (str, _) => str == word) with - | some (_, tok) => tok - | none => 0 -- Unknown token - ) - tokens + match t.vocab.find? (fun (str, _) => str == text) with + | some (_, tok) => [tok] + | none => [0] -/-- Perfect-hash decode implementation -/ +/-- Hash-vocab decode. -/ def perfect_hash_decode (t : PerfectHashTokenizer) (tokens : List Token) : String := let words := tokens.map (fun token => match t.vocab.find? (fun (_, tok) => tok == token) with @@ -94,47 +86,466 @@ def perfect_hash_decode (t : PerfectHashTokenizer) (tokens : List Token) : Strin ) String.intercalate " " words -/-- BPE Tokenizer instance -/ instance : Tokenizer BPETokenizer where encode := fun t text => bpe_encode t text decode := fun t tokens => bpe_decode t tokens - vocab_size := 0 -- Placeholder + vocab_size := fun t => t.vocab.length -/-- SentencePiece Tokenizer instance -/ instance : Tokenizer SentencePieceTokenizer where encode := fun t text => sp_encode t text decode := fun t tokens => sp_decode t tokens - vocab_size := 0 -- Placeholder + vocab_size := fun t => t.vocab.length -/-- PerfectHash Tokenizer instance -/ instance : Tokenizer PerfectHashTokenizer where encode := fun t text => perfect_hash_encode t text decode := fun t tokens => perfect_hash_decode t tokens - vocab_size := 0 -- Placeholder - -/-- Proof that BPE is deterministic for valid vocab. -This remains an explicit assumption while the full algorithmic proof is completed. -/ -axiom bpe_deterministic (t : BPETokenizer) (h_valid : t.vocab.length > 0) : - is_deterministic t - -/-- Proof that SentencePiece is surjective for UTF-8. -This remains an explicit assumption while the full algorithmic proof is completed. -/ -axiom sp_surjective_utf8 (t : SentencePieceTokenizer) (h_valid : t.vocab.length > 0) : - is_surjective_utf8 t - -/-- Perfect-hash property: injective mapping from string to token -/ -def is_perfect_hash (t : PerfectHashTokenizer) : Prop := - ∀ (s₁ s₂ : String), s₁ ≠ s₂ → - (∃ tok₁ tok₂, (t.vocab.find? (fun (str, _tok) => str == s₁) = some (s₁, tok₁)) ∧ - (t.vocab.find? (fun (str, _tok) => str == s₂) = some (s₂, tok₂)) ∧ - tok₁ ≠ tok₂) - -/-- Proof that perfect-hash tokenizer is injective for valid vocab. -This remains an explicit assumption while collision-freedom proof obligations are formalized. -/ -axiom perfect_hash_injective (t : PerfectHashTokenizer) (h_valid : t.vocab.length > 0) : - is_perfect_hash t - -/-- Test tokenizer with random strings -/ + vocab_size := fun t => t.vocab.length + +/-- Vocab decode is faithful when every token decodes as `Char.ofNat`. +Empty vocab is faithful; nonempty single-char identity vocabs also qualify. -/ +def vocab_decode_faithful (vocab : List (String × Token)) : Prop := + ∀ tok : Token, decode_token_char vocab tok = Char.ofNat tok + +private theorem decode_chars_of_faithful + (vocab : List (String × Token)) (hfaith : vocab_decode_faithful vocab) + (tokens : List Token) : + tokens.map (decode_token_char vocab) = tokens.map Char.ofNat := by + induction tokens with + | nil => rfl + | cons t ts ih => + simp only [List.map_cons, hfaith t, ih] + +private theorem chars_ofNat_toNat (cs : List Char) : + (cs.map Char.toNat).map Char.ofNat = cs := by + induction cs with + | nil => rfl + | cons c cs ih => simp [Char.ofNat_toNat, ih] + +/-- Char-level BPE: decode∘encode = id when vocab decode is faithful (merges unused). -/ +theorem bpe_string_roundtrip_of_faithful_vocab + (t : BPETokenizer) (hfaith : vocab_decode_faithful t.vocab) : + is_string_roundtrip t := by + intro s + simp only [Tokenizer.decode, Tokenizer.encode, bpe_decode, bpe_encode] + have hmap := decode_chars_of_faithful t.vocab hfaith (s.toList.map Char.toNat) + rw [hmap, chars_ofNat_toNat] + exact String.ext rfl + +/-- Empty vocab is decode-faithful. -/ +theorem vocab_decode_faithful_empty : + vocab_decode_faithful ([] : List (String × Token)) := by + intro tok + rfl + +/-- Nonempty vocabs whose every entry is the single char `Char.ofNat tok` are decode-faithful. -/ +theorem vocab_decode_faithful_of_forall_single_char + (vocab : List (String × Token)) + (h : ∀ p ∈ vocab, p.1.toList = [Char.ofNat p.2]) : + vocab_decode_faithful vocab := by + intro tok + match vocab with + | [] => rfl + | p :: ps => + have hp : p.1.toList = [Char.ofNat p.2] := + h p (List.mem_cons.mpr (Or.inl rfl)) + have hps : ∀ q ∈ ps, q.1.toList = [Char.ofNat q.2] := fun q hq => + h q (List.mem_cons.mpr (Or.inr hq)) + have ihtail : vocab_decode_faithful ps := + vocab_decode_faithful_of_forall_single_char ps hps + show decode_token_char (p :: ps) tok = Char.ofNat tok + unfold decode_token_char + -- `find?` on cons: match on head predicate, else recurse. + cases hbeq : (p.2 == tok) with + | true => + have htok : p.2 = tok := by simpa [beq_iff_eq] using hbeq + simp only [List.find?, hbeq] + have : p.1.toList.headD (Char.ofNat tok) = Char.ofNat tok := by + rw [hp, htok] + rfl + exact this + | false => + simp only [List.find?, hbeq] + simpa [decode_token_char] using ihtail tok + +/-- Singleton single-char identity vocab is decode-faithful. -/ +theorem vocab_decode_faithful_singleton (c : Char) : + vocab_decode_faithful [(String.mk [c], c.toNat)] := by + apply vocab_decode_faithful_of_forall_single_char + intro p hp + simp only [List.mem_singleton] at hp + cases hp + simp [Char.ofNat_toNat] + +/-- Char-level BPE empty-vocab round-trip (corollary). -/ +theorem bpe_char_level_string_roundtrip + (t : BPETokenizer) (hempty : t.vocab = []) : + is_string_roundtrip t := by + have hfaith : vocab_decode_faithful t.vocab := by + rw [hempty] + exact vocab_decode_faithful_empty + exact bpe_string_roundtrip_of_faithful_vocab t hfaith + +/-- Nonempty single-char identity vocab ⇒ BPE string round-trip. -/ +theorem bpe_string_roundtrip_of_single_char_vocab + (t : BPETokenizer) + (h : ∀ p ∈ t.vocab, p.1.toList = [Char.ofNat p.2]) : + is_string_roundtrip t := + bpe_string_roundtrip_of_faithful_vocab t + (vocab_decode_faithful_of_forall_single_char t.vocab h) + +/-- Char-level SentencePiece surjectivity when vocab is empty. -/ +theorem sp_surjective_of_empty_vocab + (t : SentencePieceTokenizer) (hempty : t.vocab = []) : + is_surjective_utf8 t := by + intro s + refine ⟨s.toList.map Char.toNat, ?_⟩ + simp only [Tokenizer.decode, sp_decode] + have hfaith : vocab_decode_faithful t.vocab := by + rw [hempty] + exact vocab_decode_faithful_empty + have hmap := decode_chars_of_faithful t.vocab hfaith (s.toList.map Char.toNat) + rw [hmap, chars_ofNat_toNat] + exact String.ext rfl + +/-- Char-level SP: decode∘encode = id under decode-faithful vocab. -/ +theorem sp_string_roundtrip_of_faithful_vocab + (t : SentencePieceTokenizer) (hfaith : vocab_decode_faithful t.vocab) : + is_string_roundtrip t := by + intro s + simp only [Tokenizer.decode, Tokenizer.encode, sp_decode, sp_encode] + have hmap := decode_chars_of_faithful t.vocab hfaith (s.toList.map Char.toNat) + rw [hmap, chars_ofNat_toNat] + exact String.ext rfl + +/-- Char-level SP string round-trip when vocab is empty (corollary). -/ +theorem sp_char_level_string_roundtrip + (t : SentencePieceTokenizer) (hempty : t.vocab = []) : + is_string_roundtrip t := by + have hfaith : vocab_decode_faithful t.vocab := by + rw [hempty] + exact vocab_decode_faithful_empty + exact sp_string_roundtrip_of_faithful_vocab t hfaith + +/-- One BPE merge step: replace the first adjacent pair `(a,b)` with `m`. -/ +def apply_merge_once : List Token → Token → Token → Token → List Token + | [], _, _, _ => [] + | [x], _, _, _ => [x] + | x :: y :: rest, a, b, m => + if x == a && y == b then m :: rest + else x :: apply_merge_once (y :: rest) a b m + +/-- A merge step never lengthens the token list. -/ +theorem apply_merge_once_length_le (tokens : List Token) (a b m : Token) : + (apply_merge_once tokens a b m).length ≤ tokens.length := by + induction tokens with + | nil => simp [apply_merge_once] + | cons x xs ih => + cases xs with + | nil => simp [apply_merge_once] + | cons y rest => + simp only [apply_merge_once] + by_cases h : (x == a && y == b) = true + · rw [if_pos h] + -- m::rest vs x::y::rest + simp only [List.length_cons] + exact Nat.le_succ _ + · rw [if_neg h] + simp only [List.length_cons] + exact Nat.succ_le_succ ih + +/-- Pair `(a,b)` occurs as adjacent tokens (propositional equality). -/ +def adjacent_pair_occurs : List Token → Token → Token → Prop + | [], _, _ => False + | [_], _, _ => False + | x :: y :: rest, a, b => (x = a ∧ y = b) ∨ adjacent_pair_occurs (y :: rest) a b + +/-- When the merge pair occurs, the token list strictly shortens. -/ +theorem apply_merge_once_length_lt_of_occurs + (tokens : List Token) (a b m : Token) + (hocc : adjacent_pair_occurs tokens a b) : + (apply_merge_once tokens a b m).length < tokens.length := by + induction tokens with + | nil => exact False.elim hocc + | cons x xs ih => + cases xs with + | nil => exact False.elim hocc + | cons y rest => + simp only [adjacent_pair_occurs, apply_merge_once] at hocc ⊢ + cases hocc with + | inl heq => + have hx : x == a := by simp [heq.1, beq_iff_eq] + have hy : y == b := by simp [heq.2, beq_iff_eq] + have h : (x == a && y == b) = true := by simp [hx, hy] + rw [if_pos h] + exact Nat.lt_succ_self _ + | inr hrest => + by_cases h : (x == a && y == b) = true + · rw [if_pos h] + exact Nat.lt_succ_self _ + · rw [if_neg h] + have hlt := ih hrest + simp only [List.length_cons] at hlt ⊢ + exact Nat.succ_lt_succ hlt + +/-- When the merge pair occurs, length decreases by exactly one (additive form). -/ +theorem apply_merge_once_length_eq_pred_of_occurs' + (tokens : List Token) (a b m : Token) + (hocc : adjacent_pair_occurs tokens a b) : + (apply_merge_once tokens a b m).length + 1 = tokens.length := by + induction tokens with + | nil => exact False.elim hocc + | cons x xs ih => + cases xs with + | nil => exact False.elim hocc + | cons y rest => + simp only [adjacent_pair_occurs, apply_merge_once] at hocc ⊢ + cases hocc with + | inl heq => + have hx : x == a := by simp [heq.1, beq_iff_eq] + have hy : y == b := by simp [heq.2, beq_iff_eq] + have h : (x == a && y == b) = true := by simp [hx, hy] + rw [if_pos h] + simp [List.length_cons] + | inr hrest => + by_cases h : (x == a && y == b) = true + · rw [if_pos h] + simp [List.length_cons] + · rw [if_neg h] + have ih' := ih hrest + simp only [List.length_cons] at ih' ⊢ + omega + +/-- When the merge pair occurs, length decreases by exactly one. -/ +theorem apply_merge_once_length_eq_pred_of_occurs + (tokens : List Token) (a b m : Token) + (hocc : adjacent_pair_occurs tokens a b) : + (apply_merge_once tokens a b m).length = tokens.length - 1 := by + have h := apply_merge_once_length_eq_pred_of_occurs' tokens a b m hocc + have hpos : 0 < tokens.length := by + cases tokens with + | nil => exact False.elim hocc + | cons x xs => + cases xs with + | nil => exact False.elim hocc + | cons _ _ => simp [List.length_cons] + omega + +/-- Apply a token-level merge schedule left-to-right (structural multi-merge). -/ +def apply_merge_table (tokens : List Token) (merges : List (Token × Token × Token)) : List Token := + match merges with + | [] => tokens + | (a, b, m) :: rest => apply_merge_table (apply_merge_once tokens a b m) rest + +/-- Empty merge schedule is identity. -/ +theorem apply_merge_table_nil (tokens : List Token) : + apply_merge_table tokens [] = tokens := + rfl + +/-- Multi-merge never lengthens the token list. -/ +theorem apply_merge_table_length_le + (tokens : List Token) (merges : List (Token × Token × Token)) : + (apply_merge_table tokens merges).length ≤ tokens.length := by + induction merges generalizing tokens with + | nil => simp [apply_merge_table] + | cons head rest ih => + simp only [apply_merge_table] + have honce := + apply_merge_once_length_le tokens head.1 head.2.1 head.2.2 + have hrest := ih (apply_merge_once tokens head.1 head.2.1 head.2.2) + exact Nat.le_trans hrest honce + +/-- Char-level encode then apply a token merge schedule. -/ +def bpe_encode_with_merge_table (text : String) (merges : List (Token × Token × Token)) : List Token := + apply_merge_table (text.toList.map Char.toNat) merges + +/-- Encode-with-empty-table agrees with char-level encode. -/ +theorem bpe_encode_with_merge_table_nil (text : String) : + bpe_encode_with_merge_table text [] = text.toList.map Char.toNat := by + simp [bpe_encode_with_merge_table, apply_merge_table] + +/-- Encode-with-merge-table length is at most the character count. -/ +theorem bpe_encode_with_merge_table_length_le + (text : String) (merges : List (Token × Token × Token)) : + (bpe_encode_with_merge_table text merges).length ≤ text.toList.length := by + simpa [bpe_encode_with_merge_table, List.length_map] using + apply_merge_table_length_le (text.toList.map Char.toNat) merges + +/-- A firing first merge on the char stream leaves room of at most `chars - 1`. -/ +theorem bpe_encode_with_merge_table_cons_length_of_occurs + (text : String) (a b m : Token) (rest : List (Token × Token × Token)) + (hocc : adjacent_pair_occurs (text.toList.map Char.toNat) a b) : + (bpe_encode_with_merge_table text ((a, b, m) :: rest)).length ≤ + text.toList.length - 1 := by + simp only [bpe_encode_with_merge_table, apply_merge_table] + have hstep := + apply_merge_once_length_eq_pred_of_occurs' (text.toList.map Char.toNat) a b m hocc + have hrest := + apply_merge_table_length_le + (apply_merge_once (text.toList.map Char.toNat) a b m) rest + have hlen : + (apply_merge_once (text.toList.map Char.toNat) a b m).length = + text.toList.length - 1 := by + have hmap : (text.toList.map Char.toNat).length = text.toList.length := by + simp [List.length_map] + omega + omega + +/-- When the merge pair does not occur, one step is the identity. -/ +theorem apply_merge_once_eq_of_not_occurs + (tokens : List Token) (a b m : Token) + (h : ¬ adjacent_pair_occurs tokens a b) : + apply_merge_once tokens a b m = tokens := by + induction tokens with + | nil => rfl + | cons x xs ih => + cases xs with + | nil => rfl + | cons y rest => + simp only [adjacent_pair_occurs, apply_merge_once, not_or] at h ⊢ + have hxneq : ¬ (x = a ∧ y = b) := h.1 + have hrest : ¬ adjacent_pair_occurs (y :: rest) a b := h.2 + have hbeq : (x == a && y == b) = false := by + by_cases hx : x = a + · by_cases hy : y = b + · exact False.elim (hxneq ⟨hx, hy⟩) + · simp [hx, hy, beq_iff_eq] + · simp [hx, beq_iff_eq] + rw [if_neg (by simp [hbeq])] + simp only [List.cons.injEq, true_and] + exact ih hrest + +/-- Absent merge pair ⇒ length unchanged (corollary of identity). -/ +theorem apply_merge_once_length_eq_of_not_occurs + (tokens : List Token) (a b m : Token) + (h : ¬ adjacent_pair_occurs tokens a b) : + (apply_merge_once tokens a b m).length = tokens.length := by + rw [apply_merge_once_eq_of_not_occurs tokens a b m h] + +/-- Cons form of multi-merge is one step then the remainder (definitional). -/ +theorem apply_merge_table_cons + (tokens : List Token) (a b m : Token) (rest : List (Token × Token × Token)) : + apply_merge_table tokens ((a, b, m) :: rest) = + apply_merge_table (apply_merge_once tokens a b m) rest := + rfl + +/-- Merge schedules compose under list append. -/ +theorem apply_merge_table_append + (tokens : List Token) (m₁ m₂ : List (Token × Token × Token)) : + apply_merge_table tokens (m₁ ++ m₂) = + apply_merge_table (apply_merge_table tokens m₁) m₂ := by + induction m₁ generalizing tokens with + | nil => simp [apply_merge_table] + | cons head rest ih => + cases head with + | mk a bm => + cases bm with + | mk b m => + simp only [List.cons_append, apply_merge_table] + exact ih (apply_merge_once tokens a b m) + +/-- Encode-with-merge-table schedules compose under append (runtime twin of table append). -/ +theorem bpe_encode_with_merge_table_append + (text : String) (m₁ m₂ : List (Token × Token × Token)) : + bpe_encode_with_merge_table text (m₁ ++ m₂) = + apply_merge_table (bpe_encode_with_merge_table text m₁) m₂ := by + simp only [bpe_encode_with_merge_table] + exact apply_merge_table_append (text.toList.map Char.toNat) m₁ m₂ + +/-- Empty merge schedule leaves encode length equal to character count. -/ +theorem bpe_encode_with_merge_table_nil_length (text : String) : + (bpe_encode_with_merge_table text []).length = text.toList.length := by + simp [bpe_encode_with_merge_table, apply_merge_table, List.length_map] + +/-- Correct vocab token-injectivity (distinct keys ⇒ distinct token ids when both found). -/ +def is_vocab_token_injective (t : PerfectHashTokenizer) : Prop := + ∀ (s₁ s₂ : String) (tok₁ tok₂ : Token), + t.vocab.find? (fun (str, _) => str == s₁) = some (s₁, tok₁) → + t.vocab.find? (fun (str, _) => str == s₂) = some (s₂, tok₂) → + s₁ ≠ s₂ → tok₁ ≠ tok₂ + +/-- Distinct keys in an assoc list carry distinct mapped tokens. -/ +def vocab_keys_token_unique (vocab : List (String × Token)) : Prop := + ∀ s₁ s₂ t₁ t₂, + (s₁, t₁) ∈ vocab → (s₂, t₂) ∈ vocab → s₁ ≠ s₂ → t₁ ≠ t₂ + +/-- Structural injectivity from unique key→token association (proven; not an axiom). -/ +theorem perfect_hash_injective_of_unique + (t : PerfectHashTokenizer) + (huniq : vocab_keys_token_unique t.vocab) : + is_vocab_token_injective t := by + intro s₁ s₂ tok₁ tok₂ h1 h2 hs + have mem1 : (s₁, tok₁) ∈ t.vocab := List.mem_of_find?_eq_some h1 + have mem2 : (s₂, tok₂) ∈ t.vocab := List.mem_of_find?_eq_some h2 + exact huniq s₁ s₂ tok₁ tok₂ mem1 mem2 hs + +/-- Encode recovers the token when key `find?` succeeds. -/ +theorem perfect_hash_encode_eq_of_find + (t : PerfectHashTokenizer) (s : String) (tok : Token) + (h : t.vocab.find? (fun (str, _) => str == s) = some (s, tok)) : + perfect_hash_encode t s = [tok] := by + simp [perfect_hash_encode, h] + +/-- Distinct tokens map to at most one key (decode direction). -/ +def vocab_token_keys_unique (vocab : List (String × Token)) : Prop := + ∀ s₁ s₂ t₁ t₂, + (s₁, t₁) ∈ vocab → (s₂, t₂) ∈ vocab → t₁ = t₂ → s₁ = s₂ + +/-- Decode of a singleton recovers the string when token `find?` succeeds. -/ +theorem perfect_hash_decode_singleton_eq_of_find + (t : PerfectHashTokenizer) (s : String) (tok : Token) + (h : t.vocab.find? (fun (_, t') => t' == tok) = some (s, tok)) : + perfect_hash_decode t [tok] = s := by + simp [perfect_hash_decode, h, String.intercalate] + rfl + +/-- Under token uniqueness, a successful key lookup implies the matching token lookup. -/ +theorem find_token_of_find_key_of_token_unique + (vocab : List (String × Token)) + (huniq : vocab_token_keys_unique vocab) + (s : String) (tok : Token) + (h : vocab.find? (fun (str, _) => str == s) = some (s, tok)) : + vocab.find? (fun (_, t') => t' == tok) = some (s, tok) := by + have hdecomp := (List.find?_eq_some_iff_append).1 h + rcases hdecomp with ⟨_, as, bs, hv, hpre⟩ + have smem : (s, tok) ∈ vocab := by + rw [hv] + exact List.mem_append_right _ (List.mem_cons_self) + rw [List.find?_eq_some_iff_append] + refine ⟨by simp [beq_iff_eq], as, bs, hv, ?_⟩ + intro a ha + cases htok : (a.snd == tok) with + | true => + have htokeq : a.snd = tok := by simpa [beq_iff_eq] using htok + have amem : a ∈ vocab := by + rw [hv] + exact List.mem_append_left _ ha + have akey : a.fst = s := by + cases a with + | mk s₀ t₀ => + change (s₀, t₀) ∈ vocab at amem + have ht₀ : t₀ = tok := htokeq + exact huniq s₀ s t₀ tok amem smem ht₀ + have hkey : (!(a.fst == s)) = true := hpre a ha + simp [akey, beq_iff_eq] at hkey + | false => + simp [htok] + +/-- Key round-trip under unique key→token and unique token→key (assoc-list semantics). + +Companion to CHD placement injectivity: Lean encode/decode is the string-verify +view after MPH slot lookup; Rust CHD supplies the O(1) placement. -/ +theorem perfect_hash_key_roundtrip_of_unique + (t : PerfectHashTokenizer) + (_hkt : vocab_keys_token_unique t.vocab) + (htk : vocab_token_keys_unique t.vocab) + (s : String) (tok : Token) + (hfind : t.vocab.find? (fun (str, _) => str == s) = some (s, tok)) : + perfect_hash_decode t (perfect_hash_encode t s) = s := by + rw [perfect_hash_encode_eq_of_find t s tok hfind] + have htok := find_token_of_find_key_of_token_unique t.vocab htk s tok hfind + exact perfect_hash_decode_singleton_eq_of_find t s tok htok + +/-- Test tokenizer with strings -/ def test_tokenizer {α : Type} [Tokenizer α] (t : α) (test_strings : List String) : Bool := test_strings.all fun s => let tokens := Tokenizer.encode t s @@ -143,10 +554,7 @@ def test_tokenizer {α : Type} [Tokenizer α] (t : α) (test_strings : List Stri /-- Fuzz test tokenizer -/ def fuzz_test_tokenizer {α : Type} [Tokenizer α] (t : α) (num_tests : Nat) : IO Bool := do - -- Generate random UTF-8 strings and test determinism - let test_strings := List.range num_tests |>.map fun _ => - "test" -- Placeholder for random string generation - + let test_strings := List.range num_tests |>.map fun _ => "test" return test_tokenizer t test_strings end ModelAssetGuard.Token From 3fad47e4a32becfef2f16653bee5d9f14ed8eb63 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:30 -0700 Subject: [PATCH 10/35] Wire Fixed, CHD, and RustSidecar into root Lean package Export the new formal modules from ModelAssetGuard so lake builds and Tests can import them. --- src/lean/ModelAssetGuard.lean | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lean/ModelAssetGuard.lean b/src/lean/ModelAssetGuard.lean index ab3fc66..d9df504 100644 --- a/src/lean/ModelAssetGuard.lean +++ b/src/lean/ModelAssetGuard.lean @@ -1,4 +1,9 @@ import ModelAssetGuard.Weights +import ModelAssetGuard.RustSidecar import ModelAssetGuard.Quant.Core +import ModelAssetGuard.Quant.Fixed import ModelAssetGuard.Quant.LayerBound import ModelAssetGuard.Token.Tokenizer +import ModelAssetGuard.Token.CHD +import ModelAssetGuard.Token.HashSeeded +import ModelAssetGuard.Token.CHDBuild From 551d62bc389ac8dfd15cfce3af3d5249288aad5f Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:30 -0700 Subject: [PATCH 11/35] Expand Lean tests for CHD gates and quant error matrix Cover Fixed/CHDBuild goldens, quantization_error_matrix correctness, and optional guardd_sha256 when built. --- src/lean/Tests.lean | 343 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 341 insertions(+), 2 deletions(-) diff --git a/src/lean/Tests.lean b/src/lean/Tests.lean index cb25abb..5618302 100644 --- a/src/lean/Tests.lean +++ b/src/lean/Tests.lean @@ -1,5 +1,344 @@ +import ModelAssetGuard.Weights +import ModelAssetGuard.Quant.Core +import ModelAssetGuard.Quant.LayerBound +import ModelAssetGuard.Token.Tokenizer +import ModelAssetGuard.Token.CHD +import ModelAssetGuard.Token.HashSeeded +import ModelAssetGuard.Token.CHDBuild +import ModelAssetGuard.RustSidecar import Init.System.IO +open ModelAssetGuard +open ModelAssetGuard.Quant +open ModelAssetGuard.Quant.Fixed +open ModelAssetGuard.Token +open ModelAssetGuard.Token.CHD +open ModelAssetGuard.Token.HashSeeded +open ModelAssetGuard.Token.CHDBuild +open ModelAssetGuard.RustSidecar + +/-- Fail the harness with a message. -/ +def fail (msg : String) : IO Bool := do + IO.println s!"FAIL: {msg}" + return false + +/-- Pass with a message. -/ +def pass (msg : String) : IO Bool := do + IO.println s!"PASS: {msg}" + return true + +/-- simpleDigest must be deterministic for the same input. -/ +def testSimpleDigestDeterministic : IO Bool := do + let a := simpleDigest "checkpoint-bytes" + let b := simpleDigest "checkpoint-bytes" + let c := simpleDigest "other-bytes" + if a != b then + fail "simpleDigest not deterministic" + else if a == c then + fail "simpleDigest collided on distinct inputs (unlikely but treated as fail)" + else + pass "simpleDigest deterministic" + +/-- F015: quantization_error_matrix must be Wq - W, not dequantized Wq. -/ +def testQuantizationErrorMatrix : IO Bool := do + let W : WeightMatrix := #[#[0.4, -0.6]] + let Wq := dequantize_matrix (quantize_matrix W) + let E := quantization_error_matrix W + -- Element (0,0): error = dequant(round(0.4)) - 0.4 + let e00 := E[0]![0]! + let expected := Wq[0]![0]! - W[0]![0]! + if Float.abs (e00 - expected) > 1e-9 then + fail s!"quantization_error_matrix[0][0]={e00} expected {expected}" + else if Float.abs e00 < 1e-12 && Float.abs (W[0]![0]! - Wq[0]![0]!) > 1e-9 then + -- If there is a nonzero quant error, E must not be identical to Wq alone + -- (this branch is a secondary sanity check). + fail "quantization_error_matrix looks like dequantized weights" + else + pass "quantization_error_matrix is Wq - W" + +/-- F027: vocab_size must reflect vocab length. -/ +def testVocabSize : IO Bool := do + let t : PerfectHashTokenizer := { + vocab := [("hello", 1), ("world", 2)] + hash_table := [] + } + let n := Tokenizer.vocab_size t + if n != 2 then + fail s!"PerfectHashTokenizer.vocab_size={n} expected 2" + else + pass "Tokenizer.vocab_size uses vocab.length" + +/-- Optional: if guardd_sha256 is built, a tiny file must hash successfully. -/ +def testGuarddSha256IfPresent : IO Bool := do + match (← findBin "guardd_sha256") with + | none => do + IO.println "SKIP: guardd_sha256 not built (optional)" + return true + | some _ => do + let path := "lean_tests_sha256.tmp" + IO.FS.writeFile path "model-asset-guard-test" + match (← sha256HexViaRust path) with + | none => do + try IO.FS.removeFile path catch _ => pure () + fail "guardd_sha256 present but sha256HexViaRust failed" + | some hex => do + try IO.FS.removeFile path catch _ => pure () + if hex.length != 64 then + fail s!"unexpected digest length {hex.length}" + else + pass s!"guardd_sha256 digest len=64" + +/-- Shared fixture formula: int8 ε = 0.5 * sqrt(fan_in); fan_in=16 → 2.0. -/ +def testEpsilonBoundMatchesSharedFixture : IO Bool := do + let config : LayerConfig := { fan_in := 16, fan_out := 4, quant_type := "int8" } + let eps := compute_epsilon_bound config + let expected : Float := 2.0 + if Float.abs (eps - expected) > 1e-6 then + fail s!"int8 epsilon for fan_in=16 is {eps}, expected {expected} (shared fixture)" + else + pass "compute_epsilon_bound matches shared int8 fixture (0.5*sqrt(16)=2)" + +/-- Fixed-point half-ULP smoke: round(5/2)=3 and error form holds. -/ +def testFixedRoundHalfUlp : IO Bool := do + let hd : (2 : Nat) > 0 := by decide + let q := roundDivNat 5 2 hd + let err := 2 * Int.natAbs ((5 : Int) - (q : Int) * (2 : Int)) + -- theorem roundDivNat_error guarantees err ≤ 2 + if q != 3 then + fail s!"roundDivNat 5/2 = {q}, expected 3" + else if err > 2 then + fail s!"half-ULP residual {err} > 2" + else + pass "Fixed.roundDivNat half-ULP smoke (5/2 → 3)" + +/-- Empty-vocab BPE stub round-trips a simple string. -/ +def testBpeCharLevelRoundtrip : IO Bool := do + let t : BPETokenizer := { vocab := [], merges := [] } + let s := "ab" + let tokens := bpe_encode t s + let decoded := bpe_decode t tokens + if decoded != s then + fail s!"bpe roundtrip {s} → {tokens} → {decoded}" + else + pass "bpe_char_level empty-vocab roundtrip" + +/-- Nonempty faithful single-char vocab still round-trips (merges unused). -/ +def testBpeFaithfulNonemptyRoundtrip : IO Bool := do + let t : BPETokenizer := { + vocab := [("a", 'a'.toNat), ("b", 'b'.toNat)] + merges := [] + } + let s := "ab" + let tokens := bpe_encode t s + let decoded := bpe_decode t tokens + if decoded != s then + fail s!"bpe faithful nonempty roundtrip {s} → {tokens} → {decoded}" + else + pass "bpe_faithful nonempty single-char vocab roundtrip" + +/-- Merge step shortens when the pair occurs. -/ +def testBpeMergeOnceShortens : IO Bool := do + let tokens : List Token := [1, 2, 3] + let merged := apply_merge_once tokens 1 2 9 + if merged != [9, 3] then + fail s!"apply_merge_once expected [9,3], got {merged}" + else if merged.length >= tokens.length then + fail "apply_merge_once should shorten on hit" + else + pass "apply_merge_once shortens on adjacent pair" + +/-- Multi-merge table applies left-to-right and never lengthens. -/ +def testBpeMergeTable : IO Bool := do + let tokens : List Token := [1, 2, 3, 4] + let merges : List (Token × Token × Token) := [(1, 2, 9), (9, 3, 8)] + let merged := apply_merge_table tokens merges + if merged != [8, 4] then + fail s!"apply_merge_table expected [8,4], got {merged}" + else if merged.length > tokens.length then + fail "apply_merge_table lengthened tokens" + else + pass "apply_merge_table multi-merge shortens stepwise" + +/-- No-pair merge is identity; append composes schedules. -/ +def testBpeMergeIdempotentAndCompose : IO Bool := do + let tokens : List Token := [1, 3, 2] + let once := apply_merge_once tokens 1 2 9 + if once != tokens then + fail s!"expected identity when pair absent, got {once}" + else + let m₁ : List (Token × Token × Token) := [(1, 3, 7)] + let m₂ : List (Token × Token × Token) := [(7, 2, 8)] + let composed := apply_merge_table tokens (m₁ ++ m₂) + let stepwise := apply_merge_table (apply_merge_table tokens m₁) m₂ + if composed != stepwise then + fail s!"append composition mismatch {composed} vs {stepwise}" + else if composed != [8] then + fail s!"expected [8], got {composed}" + else + pass "apply_merge_once identity + apply_merge_table append compose" + +/-- Encode-with-merge-table append composition (Wave 15). -/ +def testBpeEncodeMergeTableAppend : IO Bool := do + let text := "ab" + let m₁ : List (Token × Token × Token) := [('a'.toNat, 'b'.toNat, 9)] + let m₂ : List (Token × Token × Token) := [] + let composed := bpe_encode_with_merge_table text (m₁ ++ m₂) + let stepwise := apply_merge_table (bpe_encode_with_merge_table text m₁) m₂ + if composed != stepwise then + fail s!"encode merge append mismatch {composed} vs {stepwise}" + else if composed != [9] then + fail s!"expected [9], got {composed}" + else + pass "bpe_encode_with_merge_table append compose" + +/-- Wave 18: abstract CHD placement well-formedness Bool check. -/ +def testChdPlacementWellFormed : IO Bool := do + if !Placement.wellFormedBool Placement.exampleTwo then + fail "exampleTwo should be well-formed" + else if !Placement.wellFormedBool Placement.exampleThree then + fail "exampleThree should be well-formed" + else if Placement.wellFormedBool Placement.exampleCollision then + fail "exampleCollision should not be well-formed" + else + pass "CHD Placement.wellFormedBool accepts MPH examples, rejects collision" + +/-- Wave 19: bucket-place merge preserves wellSized under disjointness. -/ +def testChdBucketPlaceMerge : IO Bool := do + let p : BucketPlace := { keyIndices := [0], slots := [1] } + let q : BucketPlace := { keyIndices := [1], slots := [0] } + if !BucketPlace.wellSizedBool p 2 then + fail "p should be wellSized" + else if !BucketPlace.wellSizedBool q 2 then + fail "q should be wellSized" + else if !BucketPlace.disjointBool p q then + fail "p,q should be disjoint" + else + let m := BucketPlace.append p q + if !BucketPlace.wellSizedBool m 2 then + fail "merged place should be wellSized" + else if m.keyIndices != [0, 1] || m.slots != [1, 0] then + fail s!"unexpected merge {m.keyIndices} / {m.slots}" + else + pass "CHD BucketPlace.append merges disjoint wellSized places" + +/-- Wave 20: Lean hashSeeded matches Rust/Python goldens (ASCII). -/ +def testHashSeededGoldens : IO Bool := do + let h0 := hashSeededAscii defaultSeed0 "" + let ha := hashSeededAscii defaultSeed0 "a" + let hab := hashSeededAscii defaultSeed0 "ab" + let hh := hashSeededAscii defaultSeed0 "hello" + if h0 != goldenEmptySeed0 then + fail s!"empty hash {h0} expected {goldenEmptySeed0}" + else if ha != goldenASeed0 then + fail s!"'a' hash {ha} expected {goldenASeed0}" + else if hab != goldenAbSeed0 then + fail s!"'ab' hash {hab} expected {goldenAbSeed0}" + else if hh != goldenHelloSeed0 then + fail s!"'hello' hash {hh} expected {goldenHelloSeed0}" + else if !(slotMod ha 7 (by decide : 7 > 0) < 7) then + fail "slotMod not < n" + else + pass "HashSeeded goldens match Rust/Python (seed0 ASCII)" + +/-- Wave 21: executable ASCII MPH builder finds a well-formed placement. -/ +def testChdBuildAsciiMph : IO Bool := do + if !buildAsciiMphOk ["a", "b", "c"] then + fail "buildAsciiMphOk failed for [a,b,c]" + else if !buildAsciiMphOk ["hello", "world"] then + fail "buildAsciiMphOk failed for [hello,world]" + else if !buildAsciiMphOk [] then + fail "empty build should succeed" + else + match buildAsciiMph ["x", "y"] with + | none => fail "expected some placement" + | some p => + if p.keys != ["x", "y"] then + fail "keys mismatch" + else if p.slots.length != 2 then + fail "slots length mismatch" + else if Placement.pairwiseNeBool p.slots != true then + fail "slots not distinct" + else + pass "CHDBuild.buildAsciiMph places ASCII keys injectively" + +/-- Wave 22: n=1 displace completeness smoke + bucketed builder. -/ +def testChdBuildWave22 : IO Bool := do + if !findDisplace_n1_ok defaultSeed1 [97] then + fail "findDisplace n=1 should return some 0" + else if !buildAsciiMphBucketedOk ["a", "b", "c", "d"] then + fail "buildAsciiMphBucketedOk failed" + else if !buildAsciiMphBucketedOk ["hello", "world", "test"] then + fail "buildAsciiMphBucketedOk failed for 3 words" + else + pass "CHDBuild Wave22 n=1 smoke + bucketed ASCII MPH" + +/-- Wave 23: free-occupied completeness smoke + Lean/Rust CHD goldens. -/ +def testChdBuildWave23 : IO Bool := do + if !findDisplace_n1_ok defaultSeed1 [97] then + fail "n=1 smoke still required" + else if findDisplace defaultSeed1 3 (by decide) [[97]] (freeOccupied 3) 8 != some 0 then + fail "singleton freeOccupied should return d=0 for any n>0" + else if findDisplace defaultSeed1 2 (by decide) [] (freeOccupied 2) 4 != some 0 then + fail "empty key list should return d=0" + else if candidatesOkBool [0, 0] 1 (freeOccupied 1) != false then + fail "tableSize=1 multi-key gate should fail" + else if !goldenBucketedNOk then + fail "Lean/Rust goldenBucketedNOk mismatch (slots/pilots)" + else if !buildAsciiMphBucketedOk ["x", "y", "z", "w"] then + fail "bucketed plans failed for 4 words" + else + match buildAsciiMphBucketedN ["a", "b", "c"] with + | none => fail "expected bucketed-N build" + | some b => + if !Placement.wellFormedBool b.placement then + fail "golden placement not wellFormedBool" + else if b.nBuckets != 3 then + fail "expected n_buckets = 3" + else + pass "CHDBuild Wave23 freeOccupied completeness + Lean/Rust goldens" + +/-- Wave 24: stable bucket order, λ-plan goldens, 2-key / occupied smokes. -/ +def testChdBuildWave24 : IO Bool := do + if !bucketOrderEqualLengthOk then + fail "bucketProcessOrder equal-length tie-break mismatch" + else if !bucketOrderStressOk then + fail "bucket order stress (one/two/…, w0..w7) mismatch vs Rust" + else if !goldenBucketedPlanOk then + fail "λ-plan goldenBucketedPlanOk mismatch (slots/pilots/n_buckets)" + else if !findDisplace_twoKey_free_smoke then + fail "two-key free-table findDisplace smoke failed" + else if !findDisplace_singleton_occupied_smoke then + fail "singleton nonempty-occupied findDisplace smoke failed" + else if !wave24ChdOk then + fail "wave24ChdOk aggregate failed" + else + pass "CHDBuild Wave24 stable order + λ-plans + restricted ∃d smokes" + def main (_args : List String) : IO UInt32 := do - IO.println "Lean test harness placeholder" - return 0 + IO.println "Model Asset Guard Lean tests" + let r1 ← testSimpleDigestDeterministic + let r2 ← testQuantizationErrorMatrix + let r3 ← testVocabSize + let r4 ← testGuarddSha256IfPresent + let r5 ← testEpsilonBoundMatchesSharedFixture + let r6 ← testFixedRoundHalfUlp + let r7 ← testBpeCharLevelRoundtrip + let r8 ← testBpeFaithfulNonemptyRoundtrip + let r9 ← testBpeMergeOnceShortens + let r10 ← testBpeMergeTable + let r11 ← testBpeMergeIdempotentAndCompose + let r12 ← testBpeEncodeMergeTableAppend + let r13 ← testChdPlacementWellFormed + let r14 ← testChdBucketPlaceMerge + let r15 ← testHashSeededGoldens + let r16 ← testChdBuildAsciiMph + let r17 ← testChdBuildWave22 + let r18 ← testChdBuildWave23 + let r19 ← testChdBuildWave24 + let ok := r1 && r2 && r3 && r4 && r5 && r6 && r7 && r8 && r9 && r10 && r11 && r12 && r13 && r14 && r15 && r16 && r17 && r18 && r19 + if ok then + IO.println "All Lean tests passed" + return 0 + else + IO.println "Lean tests failed" + return 1 From 287114dbff8bb3dc906288de393c617d15afbf0b Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:43 -0700 Subject: [PATCH 12/35] Delegate Lean CLIs to Rust guardd sidecars Replace simulated Lean CLI success paths with RustSidecar runners so missing binaries fail closed. --- src/lean/cli/Benchmarks/Main.lean | 152 ++++++---------------- src/lean/cli/BitFlipCorpus/Main.lean | 151 ++++----------------- src/lean/cli/PerfectHash/Main.lean | 133 ++++++------------- src/lean/cli/QuantBound/Main.lean | 112 ++++++---------- src/lean/cli/QuantVerify128/Main.lean | 180 ++++---------------------- src/lean/cli/TokenizerTest/Main.lean | 121 +++++++++-------- src/lean/cli/VerifyWeights/Main.lean | 70 ++++++---- 7 files changed, 270 insertions(+), 649 deletions(-) diff --git a/src/lean/cli/Benchmarks/Main.lean b/src/lean/cli/Benchmarks/Main.lean index a4adf6a..9ca50ba 100644 --- a/src/lean/cli/Benchmarks/Main.lean +++ b/src/lean/cli/Benchmarks/Main.lean @@ -1,122 +1,42 @@ -import ModelAssetGuard.Weights -import ModelAssetGuard.Quant.Core +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.ByteArray -def benchmarkSHA256 (fileSize : Nat) : IO Float := do - IO.println s!"Benchmarking SHA-256 for {fileSize} byte file..." +open ModelAssetGuard.RustSidecar - -- Create test data - let testData := List.replicate fileSize 0x42 |>.map Char.ofNat |>.join "" - let testPath := "benchmark_test.bin" - IO.FS.writeFile testPath testData +/-- +Measure real SHA-256 throughput via `guardd_sha256` on a temporary file. - -- Time the operation - let startTime ← IO.monoMsNow - let checkpoint ← createCheckpoint testPath - let endTime ← IO.monoMsNow +Does not print invented MB/s targets. Exits non-zero if the helper is missing. +-/ +def main (_args : List String) : IO UInt32 := do + IO.println "Model Asset Guard benchmarks" + IO.println "----------------------------" - let duration := endTime - startTime - let throughput := (fileSize.toFloat / 1024.0 / 1024.0) / (duration.toFloat / 1000.0) -- MB/s - - IO.println s!" Duration: {duration}ms" - IO.println s!" Throughput: {throughput} MB/s" - - -- Cleanup - IO.FS.removeFile testPath - - return throughput - -def benchmarkQuantization (matrixSize : Nat) : IO Float := do - IO.println s!"Benchmarking quantization for {matrixSize}x{matrixSize} matrix..." - - -- Create test matrix (simplified) - let testMatrix := List.replicate matrixSize (List.replicate matrixSize 0.5) - - -- Time the operation - let startTime ← IO.monoMsNow - - -- Simulate quantization operations - let mut totalOperations := 0 - for i in List.range 1000 do - totalOperations := totalOperations + 1 - -- In practice, this would perform actual quantization - - let endTime ← IO.monoMsNow - - let duration := endTime - startTime - let operationsPerSecond := (totalOperations.toFloat * 1000.0) / duration.toFloat - - IO.println s!" Duration: {duration}ms" - IO.println s!" Operations/sec: {operationsPerSecond}" - - return operationsPerSecond - -def benchmarkTokenizer (numStrings : Nat) : IO Float := do - IO.println s!"Benchmarking tokenizer with {numStrings} strings..." - - -- Create test tokenizer - let tokenizer := BPETokenizer.mk - [("hello", 1), ("world", 2), ("test", 3), ("benchmark", 4)] - [("he", "llo", "hello")] - - -- Create test strings - let testStrings := List.range numStrings |>.map fun i => - s!"test string {i} for benchmarking" - - -- Time the operation - let startTime ← IO.monoMsNow - - let mut totalTokens := 0 - for str in testStrings do - let tokens := encode tokenizer str - totalTokens := totalTokens + tokens.length - - let endTime ← IO.monoMsNow - - let duration := endTime - startTime - let stringsPerSecond := (numStrings.toFloat * 1000.0) / duration.toFloat - - IO.println s!" Duration: {duration}ms" - IO.println s!" Strings/sec: {stringsPerSecond}" - IO.println s!" Total tokens: {totalTokens}" - - return stringsPerSecond - -def main (args : List String) : IO UInt32 := do - IO.println "Running Model Asset Guard benchmarks..." - IO.println "" - - -- SHA-256 benchmark - let sha256Throughput ← benchmarkSHA256 (1024 * 1024) -- 1MB - IO.println "" - - -- Quantization benchmark - let quantThroughput ← benchmarkQuantization 512 - IO.println "" - - -- Tokenizer benchmark - let tokenizerThroughput ← benchmarkTokenizer 10000 - IO.println "" - - -- Performance targets - let sha256Target := 400.0 -- MB/s - let quantTarget := 50000.0 -- ops/s - let tokenizerTarget := 1000000.0 -- strings/s - - IO.println "Performance Summary:" - IO.println s!" SHA-256: {sha256Throughput} MB/s (target: {sha256Target} MB/s)" - IO.println s!" Quantization: {quantThroughput} ops/s (target: {quantTarget} ops/s)" - IO.println s!" Tokenizer: {tokenizerThroughput} strings/s (target: {tokenizerTarget} strings/s)" - - let sha256Ok := sha256Throughput ≥ sha256Target - let quantOk := quantThroughput ≥ quantTarget - let tokenizerOk := tokenizerThroughput ≥ tokenizerTarget - - if sha256Ok && quantOk && tokenizerOk then - IO.println "✓ All benchmarks meet targets" - return 0 - else - IO.println "✗ Some benchmarks below targets" + match (← findBin "guardd_sha256") with + | none => do + missingBinHint "guardd_sha256" return 1 + | some bin => do + let tmp := "bench_guardd_sha256.tmp" + -- 256 KiB deterministic payload (honest timing, not a claimed target) + let body := String.mk (List.replicate (256 * 1024) 'x') + IO.FS.writeFile tmp body + let t0 ← IO.monoMsNow + let out ← IO.Process.output { cmd := bin.toString, args := #[tmp] } + let t1 ← IO.monoMsNow + try IO.FS.removeFile tmp catch _ => pure () + if out.exitCode != 0 then + IO.println "Error: guardd_sha256 failed during benchmark" + return 1 + let elapsedMs := t1 - t0 + let bytes := body.length + let mb := bytes.toFloat / (1024.0 * 1024.0) + let secs := elapsedMs.toFloat / 1000.0 + let mbs := if secs > 0.0 then mb / secs else 0.0 + IO.println s!"op: sha256 via {bin}" + IO.println s!"bytes: {bytes}" + IO.println s!"elapsed_ms: {elapsedMs}" + IO.println s!"throughput_MBps: {mbs}" + IO.println s!"digest: {out.stdout.trim}" + IO.println "status: measured_ok" + return 0 diff --git a/src/lean/cli/BitFlipCorpus/Main.lean b/src/lean/cli/BitFlipCorpus/Main.lean index 3b1af8e..8a6893d 100644 --- a/src/lean/cli/BitFlipCorpus/Main.lean +++ b/src/lean/cli/BitFlipCorpus/Main.lean @@ -1,134 +1,29 @@ +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.String.Basic +open ModelAssetGuard.RustSidecar + +/-- +BitFlipCorpus Lean CLI. + +Delegates to Rust `guardd_bitflip`. Default size is 1 MiB; GB-scale requires +`--allow-large` (never invents a fake rejection rate). +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with - | [] => do - IO.println "Usage: bitflip-corpus [--size ] [--corruptions ] [--temp-dir ] [--comprehensive] [--quick]" + | ["--help"] | ["-h"] => do + IO.println "Usage: bitflipcorpus [--size-mb ] [--size-gb ] [--corruptions ] [--temp-dir ] [--allow-large]" IO.println "" - IO.println "Options:" - IO.println " --size File size in GB (default: 100)" - IO.println " --corruptions Number of corruptions to test (default: 20)" - IO.println " --temp-dir Temporary directory for test files" - IO.println " --comprehensive Run full comprehensive test suite" - IO.println " --quick Run quick test with 1GB file" + IO.println "Runs the Rust bit-flip corpus via guardd_bitflip." + IO.println "Default (no flags): --size-mb 1 --corruptions 5" + IO.println "Sizes above 64 MiB require --allow-large." IO.println "" - IO.println "Examples:" - IO.println " lake exe bitflipcorpus --quick" - IO.println " lake exe bitflipcorpus --size 10 --corruptions 10" - IO.println " lake exe bitflipcorpus --comprehensive" - return 1 - | _ => do - let mut fileSize := 100 - let mut numCorruptions := 20 - let mut tempDir := "/tmp/bitflip_corpus" - let mut comprehensive := false - let mut quick := false - - -- Parse arguments - let mut i := 0 - while i < args.length do - match args.get? i with - | some "--size" => - match args.get? (i + 1) with - | some size => - match String.toNat? size with - | some s => - fileSize := s - i := i + 2 - | none => do - IO.println "Error: --size requires a numeric value" - return 1 - | none => do - IO.println "Error: --size requires a value" - return 1 - | some "--corruptions" => - match args.get? (i + 1) with - | some num => - match String.toNat? num with - | some n => - numCorruptions := n - i := i + 2 - | none => do - IO.println "Error: --corruptions requires a numeric value" - return 1 - | none => do - IO.println "Error: --corruptions requires a value" - return 1 - | some "--temp-dir" => - match args.get? (i + 1) with - | some dir => - tempDir := dir - i := i + 2 - | none => do - IO.println "Error: --temp-dir requires a value" - return 1 - | some "--comprehensive" => - comprehensive := true - i := i + 1 - | some "--quick" => - quick := true - i := i + 1 - | some arg => do - IO.println s!"Unknown argument: {arg}" - return 1 - | none => break - - -- Apply quick mode override - if quick then - fileSize := 1 - numCorruptions := 5 - - IO.println "=" * 80 - IO.println "100GB Bit-Flip Corpus Test for Model Asset Guard" - IO.println "=" * 80 - IO.println s!"File size: {fileSize}GB" - IO.println s!"Corruptions: {numCorruptions}" - IO.println s!"Temp directory: {tempDir}" - IO.println s!"Comprehensive: {comprehensive}" - IO.println s!"Quick mode: {quick}" - IO.println "" - - -- Create temp directory - IO.FS.createDirAll tempDir - - if comprehensive then - IO.println "Running comprehensive test suite..." - -- This would call the Rust comprehensive test - IO.println "✓ Comprehensive test completed (simulated)" - return 0 - else - IO.println s!"Running {fileSize}GB bit-flip test with {numCorruptions} corruptions..." - - -- Simulate the test process - IO.println "Creating test file..." - IO.println "Computing original hash..." - IO.println "Testing original file integrity..." - - -- Simulate corruption tests - let mut passedTests := 0 - for i in List.range numCorruptions do - IO.println s!" Corruption {i + 1}/{numCorruptions}: Applying bit flips..." - IO.println s!" Result: Rejected ✓" - passedTests := passedTests + 1 - - let rejectionRate := (passedTests.toFloat * 100.0) / numCorruptions.toFloat - let testPassed := rejectionRate == 100.0 - - IO.println "" - IO.println "Test Results:" - IO.println s!" File size: {fileSize}GB" - IO.println s!" Corruptions tested: {numCorruptions}" - IO.println s!" Rejections: {passedTests}/{numCorruptions}" - IO.println s!" Rejection rate: {rejectionRate:.1f}%" - IO.println s!" Test passed: {if testPassed then '✓' else '✗'}" - - if testPassed then - IO.println "" - IO.println "✓ All bit-flip corpus tests passed!" - return 0 - else - IO.println "" - IO.println "✗ Some bit-flip corpus tests failed!" - return 1 + IO.println "Build helper:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_bitflip" + return 0 + | [] => + -- Safe default: 1 MiB corpus (real work, not a simulated pass). + runExitCode "guardd_bitflip" #["--size-mb", "1", "--corruptions", "5"] + | _ => + runExitCode "guardd_bitflip" args.toArray diff --git a/src/lean/cli/PerfectHash/Main.lean b/src/lean/cli/PerfectHash/Main.lean index 6872b12..b9390a7 100644 --- a/src/lean/cli/PerfectHash/Main.lean +++ b/src/lean/cli/PerfectHash/Main.lean @@ -1,103 +1,50 @@ -import ModelAssetGuard.Token.Tokenizer +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.String.Basic -import Init.Data.Nat.Basic +open ModelAssetGuard.RustSidecar + +/-- +Perfect-hash Lean CLI. + +Encode/decode/test/generate delegate to the Rust `guardd_perfect_hash` sidecar. +Missing binary or sidecar failure exits non-zero (no simulated success). +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with | [] => do IO.println "Usage: perfecthash [options]" IO.println "" - IO.println "Commands:" + IO.println "Commands (via Rust guardd_perfect_hash):" IO.println " encode - Encode text using perfect hash tokenizer" IO.println " decode - Decode token using perfect hash tokenizer" - IO.println " test [--fuzz ] - Test perfect hash tokenizer determinism" - IO.println " generate - Generate perfect hash vocabulary from file" + IO.println " test - Validate encode/decode round-trips" + IO.println " generate - Generate perfect hash vocabulary JSON" + IO.println "" + IO.println "Build helper:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash" + return 1 + | "encode" :: vocab :: text :: _ => + runExitCode "guardd_perfect_hash" #["encode", vocab, text] + | "encode" :: _ => do + IO.println "Error: encode requires " + return 1 + | "decode" :: vocab :: token :: _ => + runExitCode "guardd_perfect_hash" #["decode", vocab, token] + | "decode" :: _ => do + IO.println "Error: decode requires " + return 1 + | "test" :: vocab :: _ => + runExitCode "guardd_perfect_hash" #["test", vocab] + | "test" :: _ => do + IO.println "Error: test requires " + return 1 + | "generate" :: vocab :: output :: _ => + runExitCode "guardd_perfect_hash" #["generate", vocab, output] + | "generate" :: _ => do + IO.println "Error: generate requires " + return 1 + | command :: _ => do + IO.println s!"Unknown command: {command}" + IO.println "Use 'perfecthash' without arguments for help" return 1 - | command :: rest => do - match command with - | "encode" => do - match rest with - | [vocabFile, text] => do - IO.println s!"Encoding text: '{text}'" - -- In practice, this would load the vocab file and call the Rust sidecar - IO.println "Perfect hash encoding would be implemented here" - IO.println s!"Result: [1, 2, 3] -- Placeholder tokens" - return 0 - | _ => do - IO.println "Error: encode requires vocab_file and text arguments" - return 1 - | "decode" => do - match rest with - | [vocabFile, tokenStr] => do - match String.toNat? tokenStr with - | some token => do - IO.println s!"Decoding token: {token}" - -- In practice, this would load the vocab file and call the Rust sidecar - IO.println "Perfect hash decoding would be implemented here" - IO.println s!"Result: 'decoded_word' -- Placeholder" - return 0 - | none => do - IO.println "Error: token must be a number" - return 1 - | _ => do - IO.println "Error: decode requires vocab_file and token arguments" - return 1 - | "test" => do - match rest with - | [vocabFile] => do - IO.println s!"Testing perfect hash tokenizer with vocab: {vocabFile}" - -- Test basic determinism - let testStrings := ["hello world", "perfect hash", "determinism test"] - let mut allPassed := true - - for testStr in testStrings do - IO.println s!"Testing: '{testStr}'" - -- In practice, this would encode and decode using the Rust sidecar - let encoded := [1, 2] -- Placeholder - let decoded := "decoded text" -- Placeholder - let passed := decoded == testStr - - if passed then - IO.println s!"✓ '{testStr}' → {encoded} → '{decoded}'" - else - IO.println s!"✗ '{testStr}' → {encoded} → '{decoded}'" - allPassed := false - - if allPassed then - IO.println "All tests passed" - return 0 - else - IO.println "Some tests failed" - return 1 - | [vocabFile, "--fuzz", numStr] => do - match String.toNat? numStr with - | some numTests => do - IO.println s!"Running fuzz test with {numTests} random strings..." - -- In practice, this would generate random strings and test them - IO.println "Fuzz testing would be implemented here" - IO.println s!"✓ Fuzz test passed ({numTests} tests)" - return 0 - | none => do - IO.println "Error: --fuzz requires a numeric value" - return 1 - | _ => do - IO.println "Error: test requires vocab_file argument" - return 1 - | "generate" => do - match rest with - | [vocabFile, outputFile] => do - IO.println s!"Generating perfect hash vocabulary from: {vocabFile}" - IO.println s!"Output to: {outputFile}" - -- In practice, this would call the Rust gen_perfect_hash tool - IO.println "Perfect hash generation would be implemented here" - IO.println "✓ Vocabulary generated successfully" - return 0 - | _ => do - IO.println "Error: generate requires vocab_file and output_file arguments" - return 1 - | _ => do - IO.println s!"Unknown command: {command}" - IO.println "Use 'perfecthash' without arguments for help" - return 1 diff --git a/src/lean/cli/QuantBound/Main.lean b/src/lean/cli/QuantBound/Main.lean index 3da10f8..bfe703a 100644 --- a/src/lean/cli/QuantBound/Main.lean +++ b/src/lean/cli/QuantBound/Main.lean @@ -1,43 +1,50 @@ -import ModelAssetGuard.Quant.LayerBound +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.String.Basic +open ModelAssetGuard.RustSidecar + +/-- +QuantBound Lean CLI. + +Requires a real LayerSpec JSON describing model layers. Delegates to +`guardd_quant128` — does not substitute hardcoded sample layers. +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with | [] => do - IO.println "Usage: quant-bound [--output ] [--verify] [--tolerance ]" + IO.println "Usage: quantbound [--output ] [--verify]" + IO.println "" + IO.println "layers.json must contain real layer payloads:" + IO.println " [{\"name\", \"weights\", \"fan_in\", \"fan_out\", \"quant_type\"}, ...]" + IO.println "" + IO.println "This CLI no longer invents sample layers. Provide weights from your model" + IO.println "(e.g. export via Python ModelAssetGuard / state_dict dump)." + IO.println "" + IO.println "Build helper:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_quant128" return 1 - | modelPath :: rest => do - let mut outputFile := none - let mut verify := false - let mut tolerance := 1e-6 - - -- Parse arguments - let mut i := 0 + | path :: rest => do + -- Accept and ignore legacy --verify / --tolerance flags after validating them. + let mut outArgs : Array String := #[path] + let mut i : Nat := 0 while i < rest.length do - match rest.get? i with - | some "--output" => - match rest.get? (i + 1) with - | some file => - outputFile := some file + match rest[i]? with + | some "--output" | some "-o" => + match rest[i + 1]? with + | some p => + outArgs := outArgs.push "--output" |>.push p i := i + 2 | none => do - IO.println "Error: --output requires a value" + IO.println "Error: --output requires a path" return 1 | some "--verify" => - verify := true i := i + 1 | some "--tolerance" => - match rest.get? (i + 1) with - | some tol => - match String.toFloat? tol with - | some t => - tolerance := t - i := i + 2 - | none => do - IO.println "Error: --tolerance requires a numeric value" - return 1 + match rest[i + 1]? with + | some _ => + IO.println "Warning: --tolerance is ignored; Rust epsilon bounds are used." + i := i + 2 | none => do IO.println "Error: --tolerance requires a value" return 1 @@ -46,48 +53,9 @@ def main (args : List String) : IO UInt32 := do return 1 | none => break - -- Create sample layer configurations (in practice, this would parse the model) - let layers := [ - ("layer1", LayerConfig.mk 512 512 "int8"), - ("layer2", LayerConfig.mk 512 512 "int8"), - ("layer3", LayerConfig.mk 512 512 "fp16") - ] - - -- Generate bounds - let bounds := layers.map fun (name, config) => - (name, compute_epsilon_bound config) - - -- Output results - if verify then - IO.println "Verifying quantization bounds..." - let mut allPassed := true - for (name, bound) in bounds do - let passed := bound ≤ tolerance - if passed then - IO.println s!"✓ {name}: {bound} ≤ {tolerance}" - else - IO.println s!"✗ {name}: {bound} > {tolerance}" - allPassed := false - - if allPassed then - IO.println "All layers passed verification" - return 0 - else - IO.println "Some layers failed verification" - return 1 - else - IO.println "Generated quantization bounds:" - for (name, bound) in bounds do - IO.println s!" {name}: ε ≤ {bound}" - - -- Write to file if specified - match outputFile with - | some file => do - let content := bounds.map fun (name, bound) => - s!"\"{name}\": {bound}" |>.join ",\n" - let json := s!"{{\n{content}\n}}" - IO.FS.writeFile file json - IO.println s!"Bounds written to {file}" - | none => pure () - - return 0 + -- Fail closed if the layers file is missing (do not fall back to samples). + let fp : System.FilePath := path + if !(← fp.pathExists) then + IO.println s!"Error: layers file not found: {path}" + return 1 + runExitCode "guardd_quant128" outArgs diff --git a/src/lean/cli/QuantVerify128/Main.lean b/src/lean/cli/QuantVerify128/Main.lean index 1d04502..a2bedec 100644 --- a/src/lean/cli/QuantVerify128/Main.lean +++ b/src/lean/cli/QuantVerify128/Main.lean @@ -1,169 +1,41 @@ -import ModelAssetGuard.Quant.LayerBound +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.String.Basic -import Init.Data.ByteArray +open ModelAssetGuard.RustSidecar + +/-- +QuantVerify128 Lean CLI. + +Requires a LayerSpec JSON file and delegates to Rust `guardd_quant128`. +Does not invent sample layers or print fake pass rates. +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with | [] => do - IO.println "Usage: quant-verify-128 [--layer ] [--fan-in ] [--fan-out ] [--quant-type ] [--output ] [--benchmark] [--comprehensive]" + IO.println "Usage: quantverify128 [--output ]" IO.println "" - IO.println "Options:" - IO.println " --layer Layer name for verification" - IO.println " --fan-in Input dimension (default: 512)" - IO.println " --fan-out Output dimension (default: 512)" - IO.println " --quant-type Quantization type: int8, fp16 (default: int8)" - IO.println " --output Output file for results (JSON)" - IO.println " --benchmark Run performance benchmark" - IO.println " --comprehensive Run comprehensive test suite" + IO.println "layers.json: JSON array of {name, weights, fan_in, fan_out, quant_type}" + IO.println "Verification uses Rust 128-vector sampling (vector error ||Wq x - W x||/||x||)." IO.println "" - IO.println "Examples:" - IO.println " lake exe quantverify128 --layer layer1 --fan-in 512 --fan-out 512 --quant-type int8" - IO.println " lake exe quantverify128 --comprehensive --output results.json" - IO.println " lake exe quantverify128 --benchmark" + IO.println "Build helper:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_quant128" return 1 - | _ => do - let mut layerName := "test_layer" - let mut fanIn := 512 - let mut fanOut := 512 - let mut quantType := "int8" - let mut outputFile := none - let mut benchmark := false - let mut comprehensive := false - - -- Parse arguments - let mut i := 0 - while i < args.length do - match args.get? i with - | some "--layer" => - match args.get? (i + 1) with - | some name => - layerName := name - i := i + 2 - | none => do - IO.println "Error: --layer requires a value" - return 1 - | some "--fan-in" => - match args.get? (i + 1) with - | some n => - match String.toNat? n with - | some num => - fanIn := num - i := i + 2 - | none => do - IO.println "Error: --fan-in requires a numeric value" - return 1 - | none => do - IO.println "Error: --fan-in requires a value" - return 1 - | some "--fan-out" => - match args.get? (i + 1) with - | some n => - match String.toNat? n with - | some num => - fanOut := num - i := i + 2 - | none => do - IO.println "Error: --fan-out requires a numeric value" - return 1 - | none => do - IO.println "Error: --fan-out requires a value" - return 1 - | some "--quant-type" => - match args.get? (i + 1) with - | some qtype => - quantType := qtype + | path :: rest => do + let mut outArgs : Array String := #[path] + let mut i : Nat := 0 + while i < rest.length do + match rest[i]? with + | some "--output" | some "-o" => + match rest[i + 1]? with + | some p => + outArgs := outArgs.push "--output" |>.push p i := i + 2 | none => do - IO.println "Error: --quant-type requires a value" + IO.println "Error: --output requires a path" return 1 - | some "--output" => - match args.get? (i + 1) with - | some file => - outputFile := some file - i := i + 2 - | none => do - IO.println "Error: --output requires a value" - return 1 - | some "--benchmark" => - benchmark := true - i := i + 1 - | some "--comprehensive" => - comprehensive := true - i := i + 1 | some arg => do IO.println s!"Unknown argument: {arg}" return 1 | none => break - - IO.println "=" * 80 - IO.println "128 Vectors/Layer Quantization Verification (Q-4 Requirement)" - IO.println "=" * 80 - IO.println s!"Layer: {layerName}" - IO.println s!"Fan-in: {fanIn}" - IO.println s!"Fan-out: {fanOut}" - IO.println s!"Quantization type: {quantType}" - IO.println s!"Benchmark mode: {benchmark}" - IO.println s!"Comprehensive mode: {comprehensive}" - IO.println "" - - if comprehensive then - IO.println "Running comprehensive test suite..." - -- This would call the Rust comprehensive test - IO.println "✓ Comprehensive 128 vectors verification completed (simulated)" - return 0 - else if benchmark then - IO.println "Running performance benchmark..." - -- Simulate benchmark results - IO.println " Layer verification: 128 vectors/layer" - IO.println " Throughput: > 50k layers/s (target met)" - IO.println " Memory usage: < 100MB for 128 vectors" - IO.println " Computation time: < 10ms per layer" - IO.println "✓ Performance benchmark completed" - return 0 - else - -- Create sample layer configuration - let config := LayerConfig.mk fanIn fanOut quantType - - IO.println s!"Running 128 vectors verification for {layerName}..." - IO.println s!"Configuration: {config}" - - -- Simulate the verification process - IO.println " Generating 128 random activation vectors..." - IO.println " Computing quantization error bounds..." - IO.println " Testing all 128 vectors..." - IO.println " Computing error statistics..." - - -- Simulate results - let epsilonBound := compute_epsilon_bound config - let maxError := 0.3 -- Simulated max error - let meanError := 0.15 -- Simulated mean error - let stdDeviation := 0.05 -- Simulated std deviation - let passed := maxError ≤ epsilonBound - - IO.println "" - IO.println "Verification Results:" - IO.println s!" Epsilon bound: {epsilonBound}" - IO.println s!" Max error (128 vectors): {maxError}" - IO.println s!" Mean error (128 vectors): {meanError}" - IO.println s!" Error std deviation: {stdDeviation}" - IO.println s!" Passed verification: {if passed then '✓' else '✗'}" - IO.println s!" Pass rate: {if passed then '100.0' else '0.0'}%" - - -- Write results to file if specified - match outputFile with - | some file => do - let jsonContent := s!"{{\n \"layer_name\": \"{layerName}\",\n \"fan_in\": {fanIn},\n \"fan_out\": {fanOut},\n \"quant_type\": \"{quantType}\",\n \"epsilon_bound\": {epsilonBound},\n \"max_error_128_vectors\": {maxError},\n \"mean_error_128_vectors\": {meanError},\n \"error_std_deviation\": {stdDeviation},\n \"passed_128_vectors\": {passed},\n \"verification_method\": \"128_random_activation_vectors\"\n}}" - IO.FS.writeFile file jsonContent - IO.println s!"Results written to {file}" - | none => pure () - - if passed then - IO.println "" - IO.println "✓ 128 vectors verification passed!" - return 0 - else - IO.println "" - IO.println "✗ 128 vectors verification failed!" - return 1 + runExitCode "guardd_quant128" outArgs diff --git a/src/lean/cli/TokenizerTest/Main.lean b/src/lean/cli/TokenizerTest/Main.lean index b338b4b..268157c 100644 --- a/src/lean/cli/TokenizerTest/Main.lean +++ b/src/lean/cli/TokenizerTest/Main.lean @@ -1,38 +1,54 @@ -import ModelAssetGuard.Token.Tokenizer +import ModelAssetGuard.RustSidecar import Init.System.IO -import Init.Data.List.Basic -import Init.Data.String.Basic +open ModelAssetGuard.RustSidecar + +/-- +TokenizerTest Lean CLI. + +Runs real vocab round-trip checks via `guardd_perfect_hash test`. +Supports open-addressed hash vocab JSON/txt and SentencePiece `.model` +(ModelProto pieces; CHAR/BPE/UNIGRAM encode is `guardd_perfect_hash sp-encode`). +Does not claim full official SentencePiece parity. +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with | [] => do - IO.println "Usage: tokenizer-test [--type ] [--fuzz ]" + IO.println "Usage: tokenizertest [--type ] [--fuzz ]" + IO.println "" + IO.println "Runtime checks go through Rust guardd_perfect_hash:" + IO.println " - JSON/txt: open-addressed hash vocab round-trip" + IO.println " - .model: ModelProto pieces + vocab round-trip (reports encode gate)" + IO.println "For CHAR/BPE/UNIGRAM segmentation: guardd_perfect_hash sp-encode (docs/sentencepiece-bpe.md)." + IO.println "Lean structural multi-merge lemmas are separate from SP score-BPE." + IO.println "" + IO.println "Build helper:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash" return 1 - | tokenizerPath :: rest => do - let mut tokenizerType := "bpe" - let mut fuzzTests := none - - -- Parse arguments - let mut i := 0 + | path :: rest => do + let mut tokType : String := "auto" + let mut fuzz : Option Nat := none + let mut i : Nat := 0 while i < rest.length do - match rest.get? i with + match rest[i]? with | some "--type" => - match rest.get? (i + 1) with + match rest[i + 1]? with | some t => - tokenizerType := t + tokType := t i := i + 2 | none => do IO.println "Error: --type requires a value" return 1 | some "--fuzz" => - match rest.get? (i + 1) with - | some num => - match String.toNat? num with + match rest[i + 1]? with + | some nStr => + match nStr.toNat? with | some n => - fuzzTests := some n + fuzz := some n i := i + 2 | none => do - IO.println "Error: --fuzz requires a numeric value" + IO.println "Error: --fuzz requires a non-negative integer" return 1 | none => do IO.println "Error: --fuzz requires a value" @@ -42,51 +58,32 @@ def main (args : List String) : IO UInt32 := do return 1 | none => break - -- Create sample tokenizer (in practice, this would load from file) - let tokenizer := match tokenizerType with - | "bpe" => BPETokenizer.mk - [("hello", 1), ("world", 2), ("test", 3)] - [("he", "llo", "hello")] - | "sentencepiece" => SentencePieceTokenizer.mk - [("hello", 1), ("world", 2), ("test", 3)] - #[] -- Empty protobuf for demo - | _ => do - IO.println s!"Unknown tokenizer type: {tokenizerType}" - return 1 - - -- Test basic determinism - IO.println s!"Testing {tokenizerType} tokenizer determinism..." - let testStrings := ["hello world", "test string", "determinism test"] - - let mut allPassed := true - for testStr in testStrings do - let tokens := encode tokenizer testStr - let decoded := decode tokenizer tokens - let passed := decoded == testStr - - if passed then - IO.println s!"✓ '{testStr}' → {tokens} → '{decoded}'" - else - IO.println s!"✗ '{testStr}' → {tokens} → '{decoded}'" - allPassed := false + let lower := path.toLower + let looksModel := lower.endsWith ".model" || lower.endsWith ".proto" + match tokType with + | "auto" | "perfecthash" | "perfect-hash" | "mph" | "sentencepiece" | "sp" => + if tokType == "sentencepiece" || tokType == "sp" then + if !looksModel then + IO.println "Error: --type sentencepiece requires a .model / .proto ModelProto file." + return 1 + pure () + | "bpe" => do + IO.println "Error: tokenizer type 'bpe' is not a tokenizertest mode." + IO.println "Structural Lean merges: lake tests. SP CHAR/BPE/UNIGRAM: guardd_perfect_hash sp-encode." + return 1 + | other => do + IO.println s!"Error: unknown tokenizer type '{other}'" + return 1 - -- Fuzz testing - match fuzzTests with - | some numTests => do - IO.println s!"Running fuzz test with {numTests} random strings..." - let fuzzResult ← fuzz_test_tokenizer tokenizer numTests + let fp : System.FilePath := path + if !(← fp.pathExists) then + IO.println s!"Error: vocab file not found: {path}" + return 1 - if fuzzResult then - IO.println s!"✓ Fuzz test passed ({numTests} tests)" - else - IO.println s!"✗ Fuzz test failed ({numTests} tests)" - allPassed := false + match fuzz with + | some n => + if n > 0 then + IO.println s!"Note: --fuzz {n} requests extra trials; Rust validate_roundtrip covers the vocab." | none => pure () - -- Final result - if allPassed then - IO.println "All tests passed" - return 0 - else - IO.println "Some tests failed" - return 1 + runExitCode "guardd_perfect_hash" #["test", path] diff --git a/src/lean/cli/VerifyWeights/Main.lean b/src/lean/cli/VerifyWeights/Main.lean index d851472..3bc274d 100644 --- a/src/lean/cli/VerifyWeights/Main.lean +++ b/src/lean/cli/VerifyWeights/Main.lean @@ -1,21 +1,35 @@ import ModelAssetGuard.Weights +import ModelAssetGuard.RustSidecar import Init.System.IO import Init.Data.List.Basic +open ModelAssetGuard +open ModelAssetGuard.RustSidecar + +/-- +VerifyWeights CLI. + +When `--digest` is provided, verification uses real SHA-256 via the Rust +`guardd_sha256` binary. If that binary is unavailable, the process exits +non-zero instead of claiming success with the Lean placeholder digest. +-/ def main (args : List String) : IO UInt32 := do + let args := stripLakeDashDash args match args with | [] => do - IO.println "Usage: verify-weights [--digest ]" + IO.println "Usage: verifyweights [--digest ]" + IO.println "" + IO.println "SHA-256 is computed by the Rust helper `guardd_sha256`." + IO.println "Build it with:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_sha256" return 1 | path :: rest => do - let mut expectedDigest := none - - -- Parse arguments - let mut i := 0 + let mut expectedDigest : Option String := none + let mut i : Nat := 0 while i < rest.length do - match rest.get? i with + match rest[i]? with | some "--digest" => - match rest.get? (i + 1) with + match rest[i + 1]? with | some digest => expectedDigest := some digest i := i + 2 @@ -27,21 +41,29 @@ def main (args : List String) : IO UInt32 := do return 1 | none => break - -- Create checkpoint - let checkpoint ← createCheckpoint path - - -- Verify checkpoint - let isValid ← verify checkpoint - - if isValid then - IO.println s!"✓ Checkpoint verified successfully" - IO.println s!" Path: {checkpoint.path}" - IO.println s!" Size: {checkpoint.size} bytes" - IO.println s!" Digest: {checkpoint.digest}" - return 0 - else - IO.println s!"✗ Checkpoint verification failed" - IO.println s!" Path: {checkpoint.path}" - IO.println s!" Expected digest: {expectedDigest}" - IO.println s!" Actual digest: {checkpoint.digest}" + match (← sha256HexViaRust path) with + | none => do + IO.println "Error: could not compute SHA-256 via Rust `guardd_sha256`." + IO.println "Build the helper and retry:" + IO.println " cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_sha256" + IO.println "Lean `simpleDigest` is a placeholder and is NOT used for CLI verification." return 1 + | some actual => do + match expectedDigest with + | none => do + IO.println "SHA-256 (via Rust guardd_sha256):" + IO.println s!" Path: {path}" + IO.println s!" Digest: {actual}" + return 0 + | some expected => do + if actual == expected then + IO.println "Checkpoint SHA-256 verified successfully" + IO.println s!" Path: {path}" + IO.println s!" Digest: {actual}" + return 0 + else + IO.println "Checkpoint SHA-256 verification failed" + IO.println s!" Path: {path}" + IO.println s!" Expected: {expected}" + IO.println s!" Actual: {actual}" + return 1 From e52e7c0c2f30969f4b581fb883b8bc8fa602bd9b Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:08:52 -0700 Subject: [PATCH 13/35] Add guardd CLI binaries and rlib crate type Expose sha256, perfect_hash, bitflip, and quant128 sidecars for Lean/CI and drop unused libc/winapi deps. --- src/rust/guardd/Cargo.lock | 24 - src/rust/guardd/Cargo.toml | 28 +- src/rust/guardd/src/bin/guardd_bitflip.rs | 139 ++++ .../guardd/src/bin/guardd_perfect_hash.rs | 675 ++++++++++++++++++ src/rust/guardd/src/bin/guardd_quant128.rs | 109 +++ src/rust/guardd/src/bin/guardd_sha256.rs | 42 ++ 6 files changed, 982 insertions(+), 35 deletions(-) create mode 100644 src/rust/guardd/src/bin/guardd_bitflip.rs create mode 100644 src/rust/guardd/src/bin/guardd_perfect_hash.rs create mode 100644 src/rust/guardd/src/bin/guardd_quant128.rs create mode 100644 src/rust/guardd/src/bin/guardd_sha256.rs diff --git a/src/rust/guardd/Cargo.lock b/src/rust/guardd/Cargo.lock index e496526..0f181a2 100644 --- a/src/rust/guardd/Cargo.lock +++ b/src/rust/guardd/Cargo.lock @@ -125,14 +125,12 @@ name = "guardd" version = "0.1.0" dependencies = [ "hex", - "libc", "memmap2", "rand", "serde", "serde_json", "sha2", "tempfile", - "winapi", ] [[package]] @@ -483,28 +481,6 @@ dependencies = [ "semver", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-link" version = "0.2.1" diff --git a/src/rust/guardd/Cargo.toml b/src/rust/guardd/Cargo.toml index f62a124..07157e8 100644 --- a/src/rust/guardd/Cargo.toml +++ b/src/rust/guardd/Cargo.toml @@ -5,31 +5,37 @@ edition = "2021" [lib] name = "guardd" -crate-type = ["cdylib", "staticlib"] +crate-type = ["cdylib", "staticlib", "rlib"] [[bin]] name = "gen_perfect_hash" path = "src/bin/gen_perfect_hash.rs" +[[bin]] +name = "guardd_sha256" +path = "src/bin/guardd_sha256.rs" + +[[bin]] +name = "guardd_perfect_hash" +path = "src/bin/guardd_perfect_hash.rs" + +[[bin]] +name = "guardd_bitflip" +path = "src/bin/guardd_bitflip.rs" + +[[bin]] +name = "guardd_quant128" +path = "src/bin/guardd_quant128.rs" + [dependencies] sha2 = "0.10" hex = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" memmap2 = "0.9" -libc = "0.2" rand = "0.8" tempfile = "3.8" -[target.'cfg(target_os = "linux")'.dependencies] -libc = "0.2" - -[target.'cfg(target_os = "macos")'.dependencies] -libc = "0.2" - -[target.'cfg(target_os = "windows")'.dependencies] -winapi = { version = "0.3", features = ["memoryapi", "handleapi", "fileapi"] } - [profile.release] opt-level = "z" lto = true diff --git a/src/rust/guardd/src/bin/guardd_bitflip.rs b/src/rust/guardd/src/bin/guardd_bitflip.rs new file mode 100644 index 0000000..4ad73c9 --- /dev/null +++ b/src/rust/guardd/src/bin/guardd_bitflip.rs @@ -0,0 +1,139 @@ +//! CLI helper for bit-flip corpus tests (used by Lean bitflipcorpus). +//! +//! Defaults are small (MB-scale). GB-scale sizes require `--allow-large`. + +use guardd::bitflip_corpus::{BitFlipCorpusTester, MAX_DEFAULT_CORPUS_BYTES}; +use std::env; +use std::process; + +fn print_usage() { + eprintln!( + "Usage: guardd_bitflip [--size-mb ] [--size-gb ] [--corruptions ] [--temp-dir ] [--allow-large] +Defaults: --size-mb 1, --corruptions 5 +Without --allow-large, size is capped at {} bytes ({} MB).", + MAX_DEFAULT_CORPUS_BYTES, + MAX_DEFAULT_CORPUS_BYTES / (1024 * 1024) + ); +} + +fn main() { + let args: Vec = env::args().skip(1).collect(); + let mut size_mb: Option = None; + let mut size_gb: Option = None; + let mut corruptions: usize = 5; + let mut temp_dir: Option = None; + let mut allow_large = false; + + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--size-mb" => { + let v = args.get(i + 1).and_then(|s| s.parse().ok()); + match v { + Some(n) => { + size_mb = Some(n); + i += 2; + } + None => { + eprintln!("Error: --size-mb requires a non-negative integer"); + process::exit(2); + } + } + } + "--size-gb" => { + let v = args.get(i + 1).and_then(|s| s.parse().ok()); + match v { + Some(n) => { + size_gb = Some(n); + i += 2; + } + None => { + eprintln!("Error: --size-gb requires a non-negative integer"); + process::exit(2); + } + } + } + "--corruptions" => { + let v = args.get(i + 1).and_then(|s| s.parse().ok()); + match v { + Some(n) => { + corruptions = n; + i += 2; + } + None => { + eprintln!("Error: --corruptions requires a positive integer"); + process::exit(2); + } + } + } + "--temp-dir" => match args.get(i + 1) { + Some(p) => { + temp_dir = Some(p.clone()); + i += 2; + } + None => { + eprintln!("Error: --temp-dir requires a path"); + process::exit(2); + } + }, + "--allow-large" => { + allow_large = true; + i += 1; + } + "-h" | "--help" => { + print_usage(); + process::exit(0); + } + other => { + eprintln!("Error: unknown argument '{other}'"); + print_usage(); + process::exit(2); + } + } + } + + if size_mb.is_some() && size_gb.is_some() { + eprintln!("Error: specify only one of --size-mb or --size-gb"); + process::exit(2); + } + + let size_bytes = if let Some(gb) = size_gb { + if gb == 0 { + 1024 * 1024 + } else { + gb.saturating_mul(1024 * 1024 * 1024) + } + } else { + let mb = size_mb.unwrap_or(1); + mb.saturating_mul(1024 * 1024).max(1) + }; + + let temp = temp_dir.unwrap_or_else(|| { + std::env::temp_dir() + .join("bitflip_corpus") + .to_string_lossy() + .into_owned() + }); + if let Err(e) = std::fs::create_dir_all(&temp) { + eprintln!("Error: cannot create temp dir '{temp}': {e}"); + process::exit(1); + } + + let tester = BitFlipCorpusTester::new(1, Some(temp)); + match tester.run_bitflip_test_bytes(size_bytes, corruptions, allow_large) { + Ok(result) => match serde_json::to_string_pretty(&result) { + Ok(json) => { + println!("{json}"); + process::exit(if result.test_passed { 0 } else { 1 }); + } + Err(e) => { + eprintln!("Error: serialize result: {e}"); + process::exit(1); + } + }, + Err(e) => { + eprintln!("Error: bitflip corpus failed: {e}"); + process::exit(1); + } + } +} diff --git a/src/rust/guardd/src/bin/guardd_perfect_hash.rs b/src/rust/guardd/src/bin/guardd_perfect_hash.rs new file mode 100644 index 0000000..476a051 --- /dev/null +++ b/src/rust/guardd/src/bin/guardd_perfect_hash.rs @@ -0,0 +1,675 @@ +//! CLI helper for perfect-hash encode/decode/test/generate (used by Lean perfecthash / tokenizertest). +//! +//! Also accepts SentencePiece `.model` (`ModelProto`) files: +//! - vocab piece extraction + hash-vocab round-trip +//! - real CHAR/BPE/UNIGRAM encode when the model passes the fail-closed support gate +//! (`sp-encode` / `sp-decode` / `sp-nbest` / `sp-sample`) + +use guardd::perfect_hash::PerfectHashVocab; +use guardd::sentencepiece::{ + self, EncodeOptions, SampleEncodeAndScoreOptions, SampleEncodeOptions, SentencePieceModel, +}; +use serde::Deserialize; +use std::env; +use std::fs; +use std::path::Path; +use std::process; + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum VocabFile { + Pairs(Vec<(String, u32)>), + Words { words: Vec }, + Table(PerfectHashVocab), +} + +fn print_usage() { + eprintln!( + "Usage: + guardd_perfect_hash encode + guardd_perfect_hash decode + guardd_perfect_hash test + guardd_perfect_hash generate + guardd_perfect_hash sp-encode <.model> [--silent-unk] [--no-merge-unk] [--reverse] [--bos] [--eos] [--extra reverse:bos:eos] [--forced ]... + guardd_perfect_hash sp-decode <.model> [--reverse] [token_id...] + guardd_perfect_hash sp-normalize <.model> + guardd_perfect_hash sp-nbest <.model> [--silent-unk] [--no-merge-unk] [--reverse] [--bos] [--eos] [--extra ...] [--nbest-stock-compat] + guardd_perfect_hash sp-sample <.model> [--nbest N] [--alpha A] [--seed S] [--silent-unk] [--no-merge-unk] [--reverse] [--bos] [--eos] [--extra ...] + guardd_perfect_hash sp-sample-score <.model> [--samples N] [--alpha A] [--seed S] [--silent-unk] [--no-merge-unk] [--reverse] [--bos] [--eos] [--extra ...] [--wor] [--include-best] + +SentencePiece .model: + - encode/decode/test/generate: piece extraction → hash vocab (whitespace-split encode) + - sp-encode/sp-decode: CHAR, BPE, UNIGRAM, or WORD segmentation when supported; else fail closed + - sp-normalize: charsmap + whitespace (+ treat_whitespace_as_suffix) + USER_DEFINED PrefixMatcher + - sp-nbest: UNIGRAM lattice only + - sp-sample: UNIGRAM lattice sample or BPE-dropout (alpha = dropout prob) + - sp-sample-score: UNIGRAM SampleEncodeAndScore (WR or WOR / Gumbel inclusion) + See docs/sentencepiece-bpe.md for the support matrix." + ); +} + +fn is_model_proto_path(path: &str) -> bool { + let p = Path::new(path); + matches!( + p.extension().and_then(|e| e.to_str()).map(|e| e.to_ascii_lowercase()), + Some(ext) if ext == "model" || ext == "proto" + ) +} + +fn load_vocab(path: &str) -> Result { + if is_model_proto_path(path) { + let model = sentencepiece::load_model_proto_file(path)?; + return Ok(model.vocab); + } + + let content = fs::read_to_string(path).map_err(|e| format!("read '{path}': {e}"))?; + if path.ends_with(".json") { + let parsed: VocabFile = + serde_json::from_str(&content).map_err(|e| format!("invalid vocab JSON: {e}"))?; + Ok(match parsed { + VocabFile::Pairs(pairs) => PerfectHashVocab::build(pairs), + VocabFile::Words { words } => PerfectHashVocab::from_words(words), + VocabFile::Table(table) => table, + }) + } else { + let words: Vec = content + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(str::to_string) + .collect(); + Ok(PerfectHashVocab::from_words(words)) + } +} + +fn load_sp_model(path: &str) -> Result { + if !is_model_proto_path(path) { + return Err("sp-* commands require a .model / .proto ModelProto file".into()); + } + sentencepiece::load_model_proto_file(path) +} + +fn parse_sp_encode_flags(args: &[String]) -> Result<(EncodeOptions, usize), String> { + // Returns (opts, index of first non-flag after model+text consumed by caller). + // Caller passes args starting at the first optional flag (after text). + let mut opts = EncodeOptions::default(); + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--silent-unk" => { + opts.allow_silent_unk = true; + i += 1; + } + "--no-merge-unk" => { + opts.merge_consecutive_unk = false; + i += 1; + } + "--reverse" => { + opts.reverse = true; + i += 1; + } + "--bos" => { + opts.add_bos = true; + i += 1; + } + "--eos" => { + opts.add_eos = true; + i += 1; + } + "--extra" => { + i += 1; + let spec = args.get(i).ok_or_else(|| { + "--extra requires a colon-separated ExtraOptions string".to_string() + })?; + opts.apply_extra_options_str(spec)?; + i += 1; + } + "--nbest-stock-compat" => { + opts.nbest_stock_compat = true; + i += 1; + } + "--forced" => { + i += 1; + let piece = args + .get(i) + .ok_or_else(|| "--forced requires a piece argument".to_string())?; + opts.forced_pieces.push(piece.clone()); + i += 1; + } + other if other.starts_with('-') => { + return Err(format!("unknown flag '{other}'")); + } + _ => break, + } + } + Ok((opts, i)) +} + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + print_usage(); + process::exit(2); + } + + match args[1].as_str() { + "encode" => { + if args.len() < 4 { + eprintln!("Error: encode requires "); + print_usage(); + process::exit(2); + } + let vocab = match load_vocab(&args[2]) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let text = &args[3]; + let tokens: Vec = text.split_whitespace().map(|w| vocab.encode(w)).collect(); + match serde_json::to_string(&tokens) { + Ok(j) => println!("{j}"), + Err(e) => { + eprintln!("Error: serialize tokens: {e}"); + process::exit(1); + } + } + } + "decode" => { + if args.len() < 4 { + eprintln!("Error: decode requires "); + print_usage(); + process::exit(2); + } + let vocab = match load_vocab(&args[2]) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let token: u32 = match args[3].parse() { + Ok(t) => t, + Err(_) => { + eprintln!("Error: token must be a non-negative integer"); + process::exit(2); + } + }; + println!("{}", vocab.decode(token)); + } + "sp-encode" => { + if args.len() < 4 { + eprintln!( + "Error: sp-encode requires <.model> [--silent-unk] [--forced p]..." + ); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let opts = match parse_sp_encode_flags(&args[4..]) { + Ok((o, consumed)) if consumed == args[4..].len() => o, + Ok(_) => { + eprintln!("Error: unexpected positional args after flags"); + process::exit(2); + } + Err(e) => { + eprintln!("Error: {e}"); + process::exit(2); + } + }; + let tokens = match sentencepiece::encode_with_options(&model, &args[3], opts) { + Ok(t) => t, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + match serde_json::to_string(&tokens) { + Ok(j) => println!("{j}"), + Err(e) => { + eprintln!("Error: serialize tokens: {e}"); + process::exit(1); + } + } + } + "sp-decode" => { + if args.len() < 4 { + eprintln!( + "Error: sp-decode requires <.model> [--reverse] [token_id...]" + ); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let mut reverse = false; + let mut tokens = Vec::new(); + for arg in &args[3..] { + if arg == "--reverse" { + reverse = true; + continue; + } + match arg.parse::() { + Ok(t) => tokens.push(t), + Err(_) => { + eprintln!("Error: token ids must be non-negative integers (got '{arg}')"); + process::exit(2); + } + } + } + if tokens.is_empty() { + eprintln!("Error: sp-decode requires at least one token id"); + process::exit(2); + } + match sentencepiece::decode_pieces_with_options( + &model, + &tokens, + sentencepiece::DecodeOptions { reverse }, + ) { + Ok(s) => println!("{s}"), + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + } + } + "sp-normalize" => { + if args.len() < 4 { + eprintln!("Error: sp-normalize requires <.model> "); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + match sentencepiece::normalize(&model, &args[3]) { + Ok(s) => println!("{s}"), + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + } + } + "sp-nbest" => { + if args.len() < 5 { + eprintln!("Error: sp-nbest requires <.model> "); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let n: usize = match args[4].parse() { + Ok(v) if v >= 1 => v, + _ => { + eprintln!("Error: n must be an integer >= 1"); + process::exit(2); + } + }; + let opts = match parse_sp_encode_flags(&args[5..]) { + Ok((o, consumed)) if consumed == args[5..].len() => o, + Ok(_) => { + eprintln!("Error: unexpected positional args after flags"); + process::exit(2); + } + Err(e) => { + eprintln!("Error: {e}"); + process::exit(2); + } + }; + match sentencepiece::nbest_encode_with_options(&model, &args[3], n, opts) { + Ok(nbests) => { + let rows: Vec = nbests + .iter() + .map( + |(tokens, score)| serde_json::json!({"tokens": tokens, "score": score}), + ) + .collect(); + println!("{}", serde_json::json!(rows)); + } + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + } + } + "sp-sample" => { + if args.len() < 4 { + eprintln!("Error: sp-sample requires <.model> [flags]"); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let mut sample = SampleEncodeOptions::default(); + let mut i = 4; + while i < args.len() { + match args[i].as_str() { + "--silent-unk" => { + sample.encode.allow_silent_unk = true; + i += 1; + } + "--no-merge-unk" => { + sample.encode.merge_consecutive_unk = false; + i += 1; + } + "--reverse" => { + sample.encode.reverse = true; + i += 1; + } + "--bos" => { + sample.encode.add_bos = true; + i += 1; + } + "--eos" => { + sample.encode.add_eos = true; + i += 1; + } + "--extra" => { + i += 1; + let spec = match args.get(i) { + Some(s) => s, + None => { + eprintln!( + "Error: --extra requires a colon-separated ExtraOptions string" + ); + process::exit(2); + } + }; + if let Err(e) = sample.encode.apply_extra_options_str(spec) { + eprintln!("Error: {e}"); + process::exit(2); + } + i += 1; + } + "--nbest" => { + i += 1; + let v: i32 = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) => v, + None => { + eprintln!("Error: --nbest requires an integer"); + process::exit(2); + } + }; + sample.nbest_size = v; + i += 1; + } + "--alpha" => { + i += 1; + let v: f32 = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) => v, + None => { + eprintln!("Error: --alpha requires a float"); + process::exit(2); + } + }; + sample.alpha = v; + i += 1; + } + "--seed" => { + i += 1; + let v: u64 = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) => v, + None => { + eprintln!("Error: --seed requires a u64"); + process::exit(2); + } + }; + sample.seed = Some(v); + i += 1; + } + other => { + eprintln!("Error: unknown flag '{other}'"); + process::exit(2); + } + } + } + match sentencepiece::sample_encode_with_options(&model, &args[3], sample) { + Ok(tokens) => println!("{}", serde_json::json!(tokens)), + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + } + } + "sp-sample-score" => { + if args.len() < 4 { + eprintln!("Error: sp-sample-score requires <.model> [flags]"); + print_usage(); + process::exit(2); + } + let model = match load_sp_model(&args[2]) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + let mut sample = SampleEncodeAndScoreOptions::default(); + let mut i = 4; + while i < args.len() { + match args[i].as_str() { + "--silent-unk" => { + sample.encode.allow_silent_unk = true; + i += 1; + } + "--no-merge-unk" => { + sample.encode.merge_consecutive_unk = false; + i += 1; + } + "--reverse" => { + sample.encode.reverse = true; + i += 1; + } + "--bos" => { + sample.encode.add_bos = true; + i += 1; + } + "--eos" => { + sample.encode.add_eos = true; + i += 1; + } + "--extra" => { + i += 1; + let spec = match args.get(i) { + Some(s) => s, + None => { + eprintln!( + "Error: --extra requires a colon-separated ExtraOptions string" + ); + process::exit(2); + } + }; + if let Err(e) = sample.encode.apply_extra_options_str(spec) { + eprintln!("Error: {e}"); + process::exit(2); + } + i += 1; + } + "--samples" => { + i += 1; + let v: usize = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) if v >= 1 => v, + _ => { + eprintln!("Error: --samples requires an integer >= 1"); + process::exit(2); + } + }; + sample.num_samples = v; + i += 1; + } + "--alpha" => { + i += 1; + let v: f32 = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) => v, + None => { + eprintln!("Error: --alpha requires a float"); + process::exit(2); + } + }; + sample.alpha = v; + i += 1; + } + "--seed" => { + i += 1; + let v: u64 = match args.get(i).and_then(|s| s.parse().ok()) { + Some(v) => v, + None => { + eprintln!("Error: --seed requires a u64"); + process::exit(2); + } + }; + sample.seed = Some(v); + i += 1; + } + "--wor" => { + sample.wor = true; + i += 1; + } + "--include-best" => { + sample.include_best = true; + i += 1; + } + other => { + eprintln!("Error: unknown flag '{other}'"); + process::exit(2); + } + } + } + match sentencepiece::sample_encode_and_score_with_options(&model, &args[3], sample) { + Ok(rows) => { + let out: Vec = rows + .iter() + .map( + |(tokens, score)| serde_json::json!({"tokens": tokens, "score": score}), + ) + .collect(); + println!("{}", serde_json::json!(out)); + } + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + } + } + "test" => { + if args.len() < 3 { + eprintln!("Error: test requires "); + print_usage(); + process::exit(2); + } + let path = &args[2]; + if is_model_proto_path(path) { + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + if model.vocab.vocab.is_empty() { + eprintln!("Error: empty vocabulary"); + process::exit(1); + } + if !model.vocab.validate_roundtrip() { + eprintln!("Error: round-trip validation failed"); + process::exit(1); + } + let encode_status = if model.encode_supported { + "supported" + } else { + "unsupported" + }; + println!( + "{{\"ok\":true,\"kind\":\"sentencepiece_model_proto\",\"model_type\":\"{}\",\ + \"encode\":\"{}\",\"vocab_size\":{},\"table_size\":{}}}", + model.model_type.as_str(), + encode_status, + model.vocab.vocab.len(), + model.vocab.table_size + ); + } else { + let vocab = match load_vocab(path) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + if vocab.vocab.is_empty() { + eprintln!("Error: empty vocabulary"); + process::exit(1); + } + if !vocab.validate_roundtrip() { + eprintln!("Error: round-trip validation failed"); + process::exit(1); + } + println!( + "{{\"ok\":true,\"kind\":\"hash_vocab\",\"vocab_size\":{},\"table_size\":{}}}", + vocab.vocab.len(), + vocab.table_size + ); + } + } + "generate" => { + if args.len() < 4 { + eprintln!("Error: generate requires "); + print_usage(); + process::exit(2); + } + let vocab = match load_vocab(&args[2]) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {e}"); + process::exit(1); + } + }; + if !vocab.validate_roundtrip() { + eprintln!("Error: generated vocabulary failed encode/decode validation"); + process::exit(1); + } + let json = match serde_json::to_string_pretty(&vocab) { + Ok(j) => j, + Err(e) => { + eprintln!("Error: serialize: {e}"); + process::exit(1); + } + }; + if let Err(e) = fs::write(&args[3], &json) { + eprintln!("Error: write '{}': {e}", args[3]); + process::exit(1); + } + println!("Perfect hash vocab written to {}", args[3]); + } + other => { + eprintln!("Error: unknown command '{other}'"); + print_usage(); + process::exit(2); + } + } +} diff --git a/src/rust/guardd/src/bin/guardd_quant128.rs b/src/rust/guardd/src/bin/guardd_quant128.rs new file mode 100644 index 0000000..8ec7001 --- /dev/null +++ b/src/rust/guardd/src/bin/guardd_quant128.rs @@ -0,0 +1,109 @@ +//! CLI helper: 128-vector quantization verification from a LayerSpec JSON file +//! (used by Lean quantverify128 / quantbound). + +use guardd::quant_verification::{verify_model_128_vectors, LayerSpec}; +use std::env; +use std::fs; +use std::process; + +fn print_usage() { + eprintln!( + "Usage: guardd_quant128 [--output ] +layers.json must be a JSON array of objects: + [{{\"name\",\"weights\",\"fan_in\",\"fan_out\",\"quant_type\"}}, ...] +Exit 0 only if every layer passes." + ); +} + +fn main() { + let args: Vec = env::args().skip(1).collect(); + if args.is_empty() { + print_usage(); + process::exit(2); + } + + let mut layers_path: Option = None; + let mut output_path: Option = None; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--output" | "-o" => match args.get(i + 1) { + Some(p) => { + output_path = Some(p.clone()); + i += 2; + } + None => { + eprintln!("Error: --output requires a path"); + process::exit(2); + } + }, + "-h" | "--help" => { + print_usage(); + process::exit(0); + } + other if !other.starts_with('-') && layers_path.is_none() => { + layers_path = Some(other.to_string()); + i += 1; + } + other => { + eprintln!("Error: unknown argument '{other}'"); + print_usage(); + process::exit(2); + } + } + } + + let layers_path = match layers_path { + Some(p) => p, + None => { + eprintln!("Error: missing layers.json path"); + print_usage(); + process::exit(2); + } + }; + + let content = match fs::read_to_string(&layers_path) { + Ok(c) => c, + Err(e) => { + eprintln!("Error: read '{layers_path}': {e}"); + process::exit(1); + } + }; + + let specs: Vec = match serde_json::from_str(&content) { + Ok(s) => s, + Err(e) => { + eprintln!("Error: invalid layers JSON: {e}"); + process::exit(1); + } + }; + + if specs.is_empty() { + eprintln!("Error: layers JSON array is empty"); + process::exit(1); + } + + let layers: Vec<_> = specs.into_iter().map(LayerSpec::into_tuple).collect(); + let verification = verify_model_128_vectors(layers); + let json = match serde_json::to_string_pretty(&verification) { + Ok(j) => j, + Err(e) => { + eprintln!("Error: serialize result: {e}"); + process::exit(1); + } + }; + + if let Some(path) = output_path { + if let Err(e) = fs::write(&path, &json) { + eprintln!("Error: write '{path}': {e}"); + process::exit(1); + } + println!("Wrote verification results to {path}"); + } else { + println!("{json}"); + } + + let all_passed = + verification.passed_layers == verification.total_layers && verification.total_layers > 0; + process::exit(if all_passed { 0 } else { 1 }); +} diff --git a/src/rust/guardd/src/bin/guardd_sha256.rs b/src/rust/guardd/src/bin/guardd_sha256.rs new file mode 100644 index 0000000..3381c8d --- /dev/null +++ b/src/rust/guardd/src/bin/guardd_sha256.rs @@ -0,0 +1,42 @@ +//! CLI helper: print lowercase hex SHA-256 of a file (used by Lean verifyweights). + +use sha2::{Digest, Sha256}; +use std::env; +use std::fs::File; +use std::io::Read; +use std::process; + +fn main() { + let mut args = env::args().skip(1); + let path = match args.next() { + Some(p) => p, + None => { + eprintln!("Usage: guardd_sha256 "); + process::exit(2); + } + }; + + let mut file = match File::open(&path) { + Ok(f) => f, + Err(e) => { + eprintln!("Error: cannot open '{path}': {e}"); + process::exit(1); + } + }; + + let mut hasher = Sha256::new(); + let mut buf = [0u8; 1024 * 64]; + loop { + match file.read(&mut buf) { + Ok(0) => break, + Ok(n) => hasher.update(&buf[..n]), + Err(e) => { + eprintln!("Error: read failed for '{path}': {e}"); + process::exit(1); + } + } + } + + let digest = hasher.finalize(); + println!("{}", hex::encode(digest)); +} From b5163431fbcd4338920712cc3994a675e2815641 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:00 -0700 Subject: [PATCH 14/35] Align Rust CHD builder with Lean bucket ordering Stabilize length-then-id bucket sort and LambdaPlanCase so MPH tables match Lean assembleBucketed and satisfy clippy. --- src/rust/guardd/src/bin/gen_perfect_hash.rs | 143 +++-- src/rust/guardd/src/perfect_hash.rs | 619 +++++++++++++++++++- 2 files changed, 691 insertions(+), 71 deletions(-) diff --git a/src/rust/guardd/src/bin/gen_perfect_hash.rs b/src/rust/guardd/src/bin/gen_perfect_hash.rs index 474d6e5..a432189 100644 --- a/src/rust/guardd/src/bin/gen_perfect_hash.rs +++ b/src/rust/guardd/src/bin/gen_perfect_hash.rs @@ -1,66 +1,129 @@ +use guardd::perfect_hash::PerfectHashVocab; +use serde::Deserialize; use std::env; use std::fs; use std::io::{self, Write}; +use std::process; -// Define the PerfectHashVocab structure locally for the binary -#[derive(serde::Serialize, serde::Deserialize)] -struct PerfectHashVocab { - words: Vec, - hash_table: Vec, +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum VocabFile { + /// List of `[word, token]` pairs. + Pairs(Vec<(String, u32)>), + /// Object with a `words` array (tokens assigned 1..N). + Words { words: Vec }, } -impl PerfectHashVocab { - fn build(vocab: Vec<(String, u32)>) -> Self { - let words: Vec = vocab.iter().map(|(word, _)| word.clone()).collect(); - let hash_table: Vec = vocab.iter().map(|(_, token)| *token).collect(); - PerfectHashVocab { words, hash_table } +fn print_usage() { + eprintln!( + "Usage: gen_perfect_hash [|--output ]" + ); +} + +fn parse_args(args: &[String]) -> Result<(String, Option), String> { + if args.len() < 2 { + return Err("missing vocab path".into()); + } + let vocab_path = args[1].clone(); + let mut output_path = None; + let mut i = 2; + while i < args.len() { + match args[i].as_str() { + "--output" | "-o" => { + let path = args + .get(i + 1) + .ok_or_else(|| "--output requires a path".to_string())?; + output_path = Some(path.clone()); + i += 2; + } + other if !other.starts_with('-') && output_path.is_none() => { + output_path = Some(other.to_string()); + i += 1; + } + other => return Err(format!("unknown argument: {other}")), + } + } + Ok((vocab_path, output_path)) +} + +fn load_vocab(vocab_path: &str, content: &str) -> Result { + if vocab_path.ends_with(".json") { + let parsed: VocabFile = + serde_json::from_str(content).map_err(|e| format!("invalid vocab JSON: {e}"))?; + let mph = match parsed { + VocabFile::Pairs(pairs) => PerfectHashVocab::build(pairs), + VocabFile::Words { words } => PerfectHashVocab::from_words(words), + }; + Ok(mph) + } else { + let words: Vec = content + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(str::to_string) + .collect(); + Ok(PerfectHashVocab::from_words(words)) } } fn main() { let args: Vec = env::args().collect(); - if args.len() < 2 { - eprintln!("Usage: gen_perfect_hash [output.json]"); - std::process::exit(1); - } - let vocab_path = &args[1]; - let output_path = if args.len() > 2 { Some(&args[2]) } else { None }; + let (vocab_path, output_path) = match parse_args(&args) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {e}"); + print_usage(); + process::exit(1); + } + }; - // Check if file exists - if fs::metadata(vocab_path).is_err() { - eprintln!("Error: File '{}' not found", vocab_path); - std::process::exit(1); + if fs::metadata(&vocab_path).is_err() { + eprintln!("Error: File '{vocab_path}' not found"); + process::exit(1); } - // Read vocab file - let content = match fs::read_to_string(vocab_path) { - Ok(content) => content, + let content = match fs::read_to_string(&vocab_path) { + Ok(c) => c, Err(e) => { - eprintln!("Failed to read vocab file '{}': {}", vocab_path, e); - std::process::exit(1); + eprintln!("Failed to read vocab file '{vocab_path}': {e}"); + process::exit(1); } }; - let vocab: Vec<(String, u32)> = if vocab_path.ends_with(".json") { - serde_json::from_str(&content).expect("Invalid vocab JSON format") - } else { - // Assume newline-separated, assign tokens 1..N - content - .lines() - .enumerate() - .map(|(i, line)| (line.trim().to_string(), (i + 1) as u32)) - .collect() + + let mph = match load_vocab(&vocab_path, &content) { + Ok(v) => v, + Err(e) => { + eprintln!("{e}"); + process::exit(1); + } }; - let mph = PerfectHashVocab::build(vocab); - let json = serde_json::to_string_pretty(&mph).expect("Failed to serialize perfect hash vocab"); + if !mph.validate_roundtrip() { + eprintln!("Error: generated vocabulary failed encode/decode validation"); + process::exit(1); + } + + let json = match serde_json::to_string_pretty(&mph) { + Ok(j) => j, + Err(e) => { + eprintln!("Failed to serialize perfect hash vocab: {e}"); + process::exit(1); + } + }; match output_path { Some(path) => { - fs::write(path, &json).expect("Failed to write output file"); - println!("Perfect hash vocab written to {}", path); + if let Err(e) = fs::write(&path, &json) { + eprintln!("Failed to write output file '{path}': {e}"); + process::exit(1); + } + println!("CHD perfect-hash vocab written to {path}"); } None => { - io::stdout().write_all(json.as_bytes()).unwrap(); + if let Err(e) = io::stdout().write_all(json.as_bytes()) { + eprintln!("Failed to write to stdout: {e}"); + process::exit(1); + } } } -} \ No newline at end of file +} diff --git a/src/rust/guardd/src/perfect_hash.rs b/src/rust/guardd/src/perfect_hash.rs index b01582e..efc7703 100644 --- a/src/rust/guardd/src/perfect_hash.rs +++ b/src/rust/guardd/src/perfect_hash.rs @@ -1,33 +1,280 @@ -use serde::{Serialize, Deserialize}; -use std::collections::HashMap; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Hash vocabulary used by FFI encode/decode. +/// +/// Two on-disk forms (detected at load): +/// 1. **CHD minimal perfect hash** (`kind: "chd_mph"`) — default for new builds. +/// Fields: `vocab`, `pilots`, `n_buckets`, `indices`, `seed0`, `seed1`. +/// 2. **Legacy open-addressing** — `vocab`, `hash_table`, `table_size` +/// (linear probing). Still loaded for older JSON; not produced by +/// [`PerfectHashVocab::build`] anymore. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PerfectHashVocab { - pub vocab: Vec<(String, u32)>, // (string, token) - pub hash_table: Vec, // Maps hash(string) % table_size -> vocab index + pub vocab: Vec<(String, u32)>, + /// `"chd_mph"` for CHD; omitted / `"open_address"` for legacy tables. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kind: Option, + /// Per-bucket displace values (CHD). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pilots: Vec, + /// CHD bucket count. + #[serde(default, skip_serializing_if = "is_zero_usize")] + pub n_buckets: usize, + /// `indices[slot] = vocab_index` for CHD slot in `0..n`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub indices: Vec, + #[serde(default, skip_serializing_if = "is_zero_u64")] + pub seed0: u64, + #[serde(default, skip_serializing_if = "is_zero_u64")] + pub seed1: u64, + /// Legacy open-addressed slots (`usize::MAX` = empty). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hash_table: Vec, + #[serde(default, skip_serializing_if = "is_zero_usize")] pub table_size: usize, } +fn is_zero_usize(v: &usize) -> bool { + *v == 0 +} +fn is_zero_u64(v: &u64) -> bool { + *v == 0 +} + +const KIND_CHD: &str = "chd_mph"; +const KIND_OPEN: &str = "open_address"; + +/// Default CHD seeds (fixed for deterministic cross-platform tables). +const DEFAULT_SEED0: u64 = 0x9e37_79b9_7f4a_7c15; +const DEFAULT_SEED1: u64 = 0xbf58_476d_1ce4_e5b9; + impl PerfectHashVocab { - /// Build a minimal perfect hash table for the given vocab - #[allow(dead_code)] + /// Build a **CHD minimal perfect hash** for the given vocab. + /// + /// Maps `n` keys injectively into `0..n` slots (true MPH). Lookup always + /// verifies the stored string so unknown keys still map to `0`. pub fn build(vocab: Vec<(String, u32)>) -> Self { - let table_size = vocab.len(); + match Self::build_chd_mph(vocab.clone()) { + Ok(v) => v, + Err(_) => { + // Extremely pathological key sets: fall back to open-addressing + // with an honest kind tag (not claimed as MPH). + let mut open = Self::build_open_address(vocab); + open.kind = Some(KIND_OPEN.to_string()); + open + } + } + } + + /// Explicit open-addressed table (legacy / fallback). Not an MPH. + pub fn build_open_address(vocab: Vec<(String, u32)>) -> Self { + if vocab.is_empty() { + return Self::empty_open(); + } + + let table_size = (vocab.len() * 2).next_power_of_two().max(4); let mut hash_table = vec![usize::MAX; table_size]; - let mut string_to_index = HashMap::new(); + for (i, (word, _)) in vocab.iter().enumerate() { - let mut hash = Self::hash(word) % table_size; - // Linear probing for simplicity (replace with a real MPH if needed) - while hash_table[hash] != usize::MAX { - hash = (hash + 1) % table_size; + let mut slot = Self::hash(word) % table_size; + for _ in 0..table_size { + if hash_table[slot] == usize::MAX { + hash_table[slot] = i; + break; + } + slot = (slot + 1) % table_size; + } + } + + Self { + vocab, + kind: Some(KIND_OPEN.to_string()), + pilots: Vec::new(), + n_buckets: 0, + indices: Vec::new(), + seed0: 0, + seed1: 0, + hash_table, + table_size, + } + } + + fn empty_open() -> Self { + Self { + vocab: Vec::new(), + kind: Some(KIND_OPEN.to_string()), + pilots: Vec::new(), + n_buckets: 0, + indices: Vec::new(), + seed0: 0, + seed1: 0, + hash_table: Vec::new(), + table_size: 0, + } + } + + fn empty_mph() -> Self { + Self { + vocab: Vec::new(), + kind: Some(KIND_CHD.to_string()), + pilots: Vec::new(), + n_buckets: 0, + indices: Vec::new(), + seed0: DEFAULT_SEED0, + seed1: DEFAULT_SEED1, + hash_table: Vec::new(), + table_size: 0, + } + } + + /// CHD (Compress, Hash, Displace) construction. + /// + /// Tries compact bucket counts first, then `n_buckets = n` if needed. + pub fn build_chd_mph(vocab: Vec<(String, u32)>) -> Result { + let n = vocab.len(); + if n == 0 { + return Ok(Self::empty_mph()); + } + + // Reject duplicate keys — MPH cannot exist for multisets. + { + let mut seen = HashSet::new(); + for (w, _) in &vocab { + if !seen.insert(w.as_str()) { + return Err(format!("duplicate vocab key {w:?}")); + } + } + } + + let seed0 = DEFAULT_SEED0; + let seed1 = DEFAULT_SEED1; + + // λ ≈ 5, λ ≈ 2, then `n` (always constructible for distinct keys with enough tries). + for n_buckets in Self::rust_bucket_plans(n) { + if let Ok(v) = Self::try_chd(&vocab, n_buckets, seed0, seed1) { + return Ok(v); } - hash_table[hash] = i; - string_to_index.insert(word, i); } - Self { vocab, hash_table, table_size } + Err("CHD could not find displace values for this key set".into()) } - /// Hash function (can be replaced with a better one) + /// Bucket processing order: descending length, then ascending bucket id. + /// + /// Shared with Lean `CHDBuild.bucketProcessOrder` (Wave 24). + fn bucket_process_order(buckets: &[Vec]) -> Vec { + let mut order: Vec = (0..buckets.len()).collect(); + order.sort_by_key(|&b| (std::cmp::Reverse(buckets[b].len()), b)); + order + } + + /// Rust λ-plans: `((n+4)/5).max(1)`, `((n+1)/2).max(1)`, then `n` (deduped). + fn rust_bucket_plans(n: usize) -> Vec { + if n == 0 { + return vec![0]; + } + let a = ((n + 4) / 5).max(1); + let b = ((n + 1) / 2).max(1); + let mut out = Vec::with_capacity(3); + for x in [a, b, n] { + if !out.contains(&x) { + out.push(x); + } + } + out + } + + fn try_chd( + vocab: &[(String, u32)], + n_buckets: usize, + seed0: u64, + seed1: u64, + ) -> Result { + let n = vocab.len(); + let mut buckets: Vec> = vec![Vec::new(); n_buckets]; + for (i, (word, _)) in vocab.iter().enumerate() { + let b = (Self::hash_seeded(word, seed0) as usize) % n_buckets; + buckets[b].push(i); + } + + // Largest bucket first; equal lengths tie-break by ascending bucket id. + // Stable + explicit key so Lean `bucketProcessOrder` matches exactly. + let order = Self::bucket_process_order(&buckets); + + let mut occupied = vec![false; n]; + let mut indices = vec![usize::MAX; n]; + let mut pilots = vec![0u32; n_buckets]; + + // Production displace search budget (Lean smokes may use smaller `maxD`). + const MAX_DISPLACE: u32 = 4_000_000; + + for &b in &order { + let keys = &buckets[b]; + if keys.is_empty() { + continue; + } + let mut placed = false; + for d in 0..=MAX_DISPLACE { + let mut slots: Vec = Vec::with_capacity(keys.len()); + let mut local = HashSet::with_capacity(keys.len()); + let mut ok = true; + for &ki in keys { + let slot = (Self::hash_seeded(&vocab[ki].0, seed1.wrapping_add(u64::from(d))) + as usize) + % n; + if occupied[slot] || !local.insert(slot) { + ok = false; + break; + } + slots.push(slot); + } + if !ok { + continue; + } + for (j, &slot) in slots.iter().enumerate() { + occupied[slot] = true; + indices[slot] = keys[j]; + } + pilots[b] = d; + placed = true; + break; + } + if !placed { + return Err(format!( + "no displace for bucket {b} (size {}) with {n_buckets} buckets", + keys.len() + )); + } + } + + if indices.iter().any(|&i| i == usize::MAX) { + return Err("CHD left unfilled slots".into()); + } + + Ok(Self { + vocab: vocab.to_vec(), + kind: Some(KIND_CHD.to_string()), + pilots, + n_buckets, + indices, + seed0, + seed1, + hash_table: Vec::new(), + table_size: 0, + }) + } + + /// Build from a word list, assigning tokens `1..N` in order (CHD MPH). + pub fn from_words(words: Vec) -> Self { + let vocab = words + .into_iter() + .enumerate() + .map(|(i, w)| (w, (i + 1) as u32)) + .collect(); + Self::build(vocab) + } + + /// djb2-style hash (legacy open-addressing; stable across platforms). pub fn hash(s: &str) -> usize { let mut hash = 5381usize; for b in s.bytes() { @@ -36,23 +283,81 @@ impl PerfectHashVocab { hash } - /// Encode a string to a token using the perfect hash + /// Seeded 64-bit hash for CHD (mix after djb2-style accumulation). + pub fn hash_seeded(s: &str, seed: u64) -> u64 { + let mut h = seed.wrapping_add(5381); + for b in s.bytes() { + h = h.wrapping_mul(33).wrapping_add(u64::from(b)); + } + // SplitMix64 finalizer + h = (h ^ (h >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + h = (h ^ (h >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + h ^ (h >> 31) + } + + pub fn is_chd_mph(&self) -> bool { + self.kind.as_deref() == Some(KIND_CHD) + || (!self.pilots.is_empty() && !self.indices.is_empty() && self.n_buckets > 0) + } + + pub fn is_open_address(&self) -> bool { + !self.is_chd_mph() && self.table_size > 0 && !self.hash_table.is_empty() + } + + /// Encode a string to a token. Unknown words map to `0`. pub fn encode(&self, word: &str) -> u32 { - let mut hash = Self::hash(word) % self.table_size; + if self.vocab.is_empty() { + return 0; + } + if self.is_chd_mph() { + return self.encode_chd(word); + } + self.encode_open(word) + } + + fn encode_chd(&self, word: &str) -> u32 { + let n = self.vocab.len(); + if n == 0 || self.n_buckets == 0 || self.indices.len() != n { + return 0; + } + let b = (Self::hash_seeded(word, self.seed0) as usize) % self.n_buckets; + let d = match self.pilots.get(b) { + Some(&d) => d, + None => return 0, + }; + let slot = (Self::hash_seeded(word, self.seed1.wrapping_add(u64::from(d))) as usize) % n; + let idx = match self.indices.get(slot).copied() { + Some(i) if i < n => i, + _ => return 0, + }; + match self.vocab.get(idx) { + Some((w, tok)) if w == word => *tok, + _ => 0, + } + } + + fn encode_open(&self, word: &str) -> u32 { + if self.table_size == 0 || self.vocab.is_empty() { + return 0; + } + + let mut slot = Self::hash(word) % self.table_size; for _ in 0..self.table_size { - let idx = self.hash_table[hash]; + let idx = self.hash_table[slot]; if idx == usize::MAX { - return 0; // Unknown token + return 0; } - if self.vocab[idx].0 == word { - return self.vocab[idx].1; + if let Some((w, tok)) = self.vocab.get(idx) { + if w == word { + return *tok; + } } - hash = (hash + 1) % self.table_size; + slot = (slot + 1) % self.table_size; } - 0 // Unknown token + 0 } - /// Decode a token to a string + /// Decode a token to a string. Unknown tokens map to `""`. pub fn decode(&self, token: u32) -> &str { for (word, tok) in &self.vocab { if *tok == token { @@ -61,12 +366,56 @@ impl PerfectHashVocab { } "" } + + /// Validate that every vocab entry round-trips through the table. + pub fn validate_roundtrip(&self) -> bool { + if self.is_chd_mph() { + // Structural MPH checks. + let n = self.vocab.len(); + if self.indices.len() != n || self.n_buckets == 0 || self.pilots.len() != self.n_buckets + { + return false; + } + let mut seen_slots = HashSet::new(); + for &idx in &self.indices { + if idx >= n || !seen_slots.insert(idx) { + return false; + } + } + } + let mut seen_tokens = HashMap::new(); + for (word, token) in &self.vocab { + if self.encode(word) != *token { + return false; + } + if self.decode(*token) != word { + return false; + } + if seen_tokens.insert(*token, word).is_some() { + return false; + } + } + true + } } #[cfg(test)] mod tests { use super::*; + #[test] + fn test_hash_seeded_goldens_wave20() { + // Shared with Lean `Token.HashSeeded` / Python mirror. + const SEED0: u64 = 0x9e37_79b9_7f4a_7c15; + assert_eq!(PerfectHashVocab::hash_seeded("", SEED0), 9862407849072414892); + assert_eq!(PerfectHashVocab::hash_seeded("a", SEED0), 13358874647675253293); + assert_eq!(PerfectHashVocab::hash_seeded("ab", SEED0), 10513868738868360859); + assert_eq!(PerfectHashVocab::hash_seeded("hello", SEED0), 5468378884496047854); + const SEED1: u64 = 0xbf58_476d_1ce4_e5b9; + assert_eq!(PerfectHashVocab::hash_seeded("", SEED1), 2747279002426353127); + assert_eq!(PerfectHashVocab::hash_seeded("a", SEED1), 13232201968939760250); + } + #[test] fn test_perfect_hash_encode_decode() { let vocab = vec![ @@ -75,24 +424,232 @@ mod tests { ("test".to_string(), 3), ]; let mph = PerfectHashVocab::build(vocab.clone()); + assert!(mph.is_chd_mph(), "default build must be CHD MPH"); + assert!(mph.validate_roundtrip()); for (word, token) in &vocab { assert_eq!(mph.encode(word), *token); - assert_eq!(mph.decode(*token), word); + assert_eq!(mph.decode(*token), word.as_str()); } - // Unknown word assert_eq!(mph.encode("unknown"), 0); assert_eq!(mph.decode(999), ""); } #[test] - fn test_no_collisions() { + fn test_json_roundtrip_schema() { + let mph = PerfectHashVocab::from_words(vec!["hello".into(), "world".into(), "test".into()]); + assert!(mph.is_chd_mph()); + let json = serde_json::to_string(&mph).unwrap(); + assert!(json.contains("chd_mph")); + assert!(!json.contains("hash_table") || !json.contains("\"hash_table\":[")); + let loaded: PerfectHashVocab = serde_json::from_str(&json).unwrap(); + assert_eq!(loaded, mph); + assert!(loaded.validate_roundtrip()); + assert_eq!(loaded.encode("hello"), 1); + assert_eq!(loaded.decode(2), "world"); + } + + #[test] + fn test_chd_is_minimal() { + let words: Vec = (0..200).map(|i| format!("w{i}")).collect(); + let mph = PerfectHashVocab::from_words(words); + assert!(mph.is_chd_mph()); + assert_eq!(mph.indices.len(), mph.vocab.len()); + assert!(mph.validate_roundtrip()); + // Minimal: exactly n filled slots, no open-address over-allocation. + assert!(mph.hash_table.is_empty()); + assert_eq!(mph.table_size, 0); + } + + #[test] + fn test_legacy_open_address_still_loads() { + let open = + PerfectHashVocab::build_open_address(vec![("hello".into(), 1), ("world".into(), 2)]); + assert!(open.is_open_address()); + assert!(!open.is_chd_mph()); + let json = serde_json::to_string(&open).unwrap(); + let loaded: PerfectHashVocab = serde_json::from_str(&json).unwrap(); + assert!(loaded.validate_roundtrip()); + assert_eq!(loaded.encode("hello"), 1); + + // Untagged legacy JSON without kind field. + let legacy = + r#"{"vocab":[["a",1],["b",2]],"hash_table":[0,18446744073709551615,1],"table_size":3}"#; + // table_size 3 may not match — build a real open table and strip kind. + let mut legacy_v = + PerfectHashVocab::build_open_address(vec![("a".into(), 1), ("b".into(), 2)]); + legacy_v.kind = None; + let legacy_json = serde_json::to_string(&legacy_v).unwrap(); + let loaded2: PerfectHashVocab = serde_json::from_str(&legacy_json).unwrap(); + assert!(loaded2.is_open_address()); + assert_eq!(loaded2.encode("a"), 1); + let _ = legacy; // silence if unused shape + } + + #[test] + fn test_mph_slots_are_bijective() { + let vocab: Vec<(String, u32)> = (0..64).map(|i| (format!("k{i}"), i as u32 + 1)).collect(); + let mph = PerfectHashVocab::build(vocab); + assert!(mph.is_chd_mph()); + let mut seen = HashSet::new(); + for (word, _) in &mph.vocab { + let b = (PerfectHashVocab::hash_seeded(word, mph.seed0) as usize) % mph.n_buckets; + let d = mph.pilots[b]; + let slot = (PerfectHashVocab::hash_seeded(word, mph.seed1.wrapping_add(u64::from(d))) + as usize) + % mph.vocab.len(); + assert!(seen.insert(slot), "slot collision for {word}"); + } + assert_eq!(seen.len(), mph.vocab.len()); + } + + fn slots_by_key(mph: &PerfectHashVocab) -> Vec { + let n = mph.vocab.len(); + let mut slots = vec![0usize; n]; + for (slot, &ki) in mph.indices.iter().enumerate() { + slots[ki] = slot; + } + slots + } + + /// Shared with Lean `CHDBuild.golden*` under forced `n_buckets = n` (Wave 23). + #[test] + fn test_chd_lean_golden_n_buckets_eq_n() { + let cases: &[(&[&str], &[usize], &[u32])] = &[ + (&["a", "b", "c"], &[0, 2, 1], &[0, 0, 0]), + (&["hello", "world", "test"], &[0, 2, 1], &[2, 0, 1]), + (&["a", "b", "c", "d"], &[2, 1, 0, 3], &[0, 3, 0, 1]), + ]; + for &(words, expect_slots, expect_pilots) in cases { + let vocab: Vec<(String, u32)> = words + .iter() + .enumerate() + .map(|(i, w)| ((*w).to_string(), (i + 1) as u32)) + .collect(); + let n = vocab.len(); + let mph = PerfectHashVocab::try_chd(&vocab, n, DEFAULT_SEED0, DEFAULT_SEED1) + .unwrap_or_else(|e| panic!("try_chd failed for {words:?}: {e}")); + assert_eq!(mph.n_buckets, n, "words={words:?}"); + assert_eq!(&mph.pilots[..], expect_pilots, "pilots words={words:?}"); + assert_eq!(slots_by_key(&mph), expect_slots, "slots words={words:?}"); + assert!(mph.validate_roundtrip()); + } + } + + /// Wave 24: equal-length buckets process in ascending bucket-id order. + #[test] + fn test_chd_bucket_order_equal_length_tiebreak() { + // Artificial equal lengths: three buckets of size 2. + let buckets = vec![vec![0, 1], vec![2, 3], vec![4, 5]]; + assert_eq!( + PerfectHashVocab::bucket_process_order(&buckets), + vec![0, 1, 2], + "equal lengths ⇒ ascending bucket id" + ); + // Mixed: size 3, size 1, size 3, size 1 → [0, 2, 1, 3] + let buckets2 = vec![vec![0, 1, 2], vec![3], vec![4, 5, 6], vec![7]]; + assert_eq!( + PerfectHashVocab::bucket_process_order(&buckets2), + vec![0, 2, 1, 3] + ); + // Empty buckets sort last among equals by id among zeros. + let buckets3 = vec![vec![], vec![0, 1], vec![], vec![2, 3]]; + assert_eq!( + PerfectHashVocab::bucket_process_order(&buckets3), + vec![1, 3, 0, 2] + ); + } + + /// Shared with Lean `CHDBuild.goldenPlan*` — first successful λ-plan (Wave 24). + #[test] + fn test_chd_lean_golden_lambda_plans() { + type LambdaPlanCase<'a> = (&'a [&'a str], usize, &'a [usize], &'a [u32]); + let cases: &[LambdaPlanCase<'_>] = &[ + (&["a", "b", "c"], 1, &[0, 2, 1], &[0]), + (&["hello", "world", "test"], 1, &[1, 2, 0], &[5]), + (&["a", "b", "c", "d"], 1, &[2, 3, 1, 0], &[22]), + ( + &["w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"], + 2, + &[1, 3, 2, 0, 5, 4, 7, 6], + &[1, 92], + ), + ( + &["x", "y", "z", "w", "v", "u"], + 2, + &[0, 4, 3, 5, 1, 2], + &[41, 5], + ), + (&["one", "two", "three", "four"], 1, &[0, 1, 3, 2], &[5]), + ]; + for &(words, expect_nb, expect_slots, expect_pilots) in cases { + let vocab: Vec<(String, u32)> = words + .iter() + .enumerate() + .map(|(i, w)| ((*w).to_string(), (i + 1) as u32)) + .collect(); + let n = vocab.len(); + let plans = PerfectHashVocab::rust_bucket_plans(n); + let mut mph = None; + for &nb in &plans { + if let Ok(v) = PerfectHashVocab::try_chd(&vocab, nb, DEFAULT_SEED0, DEFAULT_SEED1) { + mph = Some(v); + break; + } + } + let mph = mph.unwrap_or_else(|| panic!("no λ-plan for {words:?}")); + assert_eq!(mph.n_buckets, expect_nb, "n_buckets words={words:?}"); + assert_eq!(&mph.pilots[..], expect_pilots, "pilots words={words:?}"); + assert_eq!(slots_by_key(&mph), expect_slots, "slots words={words:?}"); + assert!(mph.validate_roundtrip()); + // Cross-check: `build_chd_mph` selects the same first plan. + let built = PerfectHashVocab::build_chd_mph(vocab).expect("build_chd_mph"); + assert_eq!(built.n_buckets, expect_nb); + assert_eq!(built.pilots, mph.pilots); + assert_eq!(built.indices, mph.indices); + } + } + + /// Wave 24: real key sets where multiple buckets share a length. + #[test] + fn test_chd_bucket_order_parity_stress() { + // nb=2: both buckets length 2 → order [0, 1] by ascending id. + let words = ["one", "two", "three", "four"]; + let nb = 2usize; + let mut buckets: Vec> = vec![Vec::new(); nb]; + for (i, w) in words.iter().enumerate() { + let b = (PerfectHashVocab::hash_seeded(w, DEFAULT_SEED0) as usize) % nb; + buckets[b].push(i); + } + assert_eq!(buckets.iter().map(|b| b.len()).collect::>(), vec![2, 2]); + assert_eq!(PerfectHashVocab::bucket_process_order(&buckets), vec![0, 1]); + + // nb=4: lengths [2,0,4,2] → process size-4 then the two size-2 (ids 0 before 3). + let words8 = ["w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"]; + let nb4 = 4usize; + let mut buckets4: Vec> = vec![Vec::new(); nb4]; + for (i, w) in words8.iter().enumerate() { + let b = (PerfectHashVocab::hash_seeded(w, DEFAULT_SEED0) as usize) % nb4; + buckets4[b].push(i); + } + assert_eq!( + buckets4.iter().map(|b| b.len()).collect::>(), + vec![2, 0, 4, 2] + ); + assert_eq!( + PerfectHashVocab::bucket_process_order(&buckets4), + vec![2, 0, 3, 1] + ); + } + + #[test] + fn test_no_collisions_resolve_open() { let vocab = vec![ ("a".to_string(), 1), ("b".to_string(), 2), ("c".to_string(), 3), ("d".to_string(), 4), ]; - let mph = PerfectHashVocab::build(vocab.clone()); + let mph = PerfectHashVocab::build_open_address(vocab.clone()); let mut seen = std::collections::HashSet::new(); for (word, _) in &vocab { let mut hash = PerfectHashVocab::hash(word) % mph.table_size; @@ -109,4 +666,4 @@ mod tests { assert!(found); } } -} \ No newline at end of file +} From 9c7c966b370bbe2ee53561aa479ed213097678b7 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:07 -0700 Subject: [PATCH 15/35] Add SentencePiece BPE, lattice, and NFKC normalizer Implement stock-aligned encode paths with honest nbest/MT19937 limits and curated NFKC support rather than claiming full Unicode parity. --- src/rust/guardd/src/bpe.rs | 85 + src/rust/guardd/src/sentencepiece.rs | 3053 ++++++++++++++++++++++++++ src/rust/guardd/src/sp_lattice.rs | 1026 +++++++++ src/rust/guardd/src/sp_normalizer.rs | 570 +++++ 4 files changed, 4734 insertions(+) create mode 100644 src/rust/guardd/src/bpe.rs create mode 100644 src/rust/guardd/src/sentencepiece.rs create mode 100644 src/rust/guardd/src/sp_lattice.rs create mode 100644 src/rust/guardd/src/sp_normalizer.rs diff --git a/src/rust/guardd/src/bpe.rs b/src/rust/guardd/src/bpe.rs new file mode 100644 index 0000000..7b54f81 --- /dev/null +++ b/src/rust/guardd/src/bpe.rs @@ -0,0 +1,85 @@ +//! Lean-aligned structural BPE merge operators. +//! +//! These match `ModelAssetGuard.Token.apply_merge_once` / +//! `apply_merge_table` in `Tokenizer.lean`: leftmost first-occurrence merge, +//! then a left-to-right merge schedule. This is **not** SentencePiece score- +//! based BPE (see `sentencepiece::encode_bpe`); it is the formal/runtime twin +//! of the Lean structural lemmas. + +/// One merge step: replace the first adjacent pair `(a, b)` with `m`. +pub fn apply_merge_once(tokens: &[u32], a: u32, b: u32, m: u32) -> Vec { + if tokens.is_empty() { + return Vec::new(); + } + if tokens.len() == 1 { + return vec![tokens[0]]; + } + let mut out = Vec::with_capacity(tokens.len()); + let mut i = 0usize; + let mut merged = false; + while i < tokens.len() { + if !merged && i + 1 < tokens.len() && tokens[i] == a && tokens[i + 1] == b { + out.push(m); + i += 2; + merged = true; + } else { + out.push(tokens[i]); + i += 1; + } + } + out +} + +/// Apply a token-level merge schedule left-to-right. +pub fn apply_merge_table(tokens: &[u32], merges: &[(u32, u32, u32)]) -> Vec { + let mut cur = tokens.to_vec(); + for &(a, b, m) in merges { + cur = apply_merge_once(&cur, a, b, m); + } + cur +} + +/// Char-level seed encode then structural merge table (Lean `bpe_encode_with_merge_table`). +pub fn encode_with_merge_table(text: &str, merges: &[(u32, u32, u32)]) -> Vec { + let seeds: Vec = text.chars().map(|c| c as u32).collect(); + apply_merge_table(&seeds, merges) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merge_once_shortens_on_hit() { + assert_eq!(apply_merge_once(&[1, 2, 3], 1, 2, 9), vec![9, 3]); + } + + #[test] + fn merge_once_idempotent_when_pair_absent() { + let tokens = vec![1u32, 3, 2]; + assert_eq!(apply_merge_once(&tokens, 1, 2, 9), tokens); + } + + #[test] + fn merge_table_multi_step() { + let tokens = [1u32, 2, 3, 4]; + let merges = [(1, 2, 9), (9, 3, 8)]; + assert_eq!(apply_merge_table(&tokens, &merges), vec![8, 4]); + } + + #[test] + fn merge_table_never_lengthens() { + let tokens = [5u32, 5, 5, 5]; + let merges = [(5, 5, 1), (1, 5, 2), (2, 5, 3)]; + let out = apply_merge_table(&tokens, &merges); + assert!(out.len() <= tokens.len()); + } + + #[test] + fn encode_with_merge_table_char_seed() { + // 'a'=97, 'b'=98 → merge to 1 + let merges = [(97u32, 98, 1)]; + assert_eq!(encode_with_merge_table("ab", &merges), vec![1]); + assert_eq!(encode_with_merge_table("a", &merges), vec![97]); + } +} diff --git a/src/rust/guardd/src/sentencepiece.rs b/src/rust/guardd/src/sentencepiece.rs new file mode 100644 index 0000000..cefa1e5 --- /dev/null +++ b/src/rust/guardd/src/sentencepiece.rs @@ -0,0 +1,3053 @@ +//! SentencePiece `ModelProto` support for Model Asset Guard. +//! +//! ## Supported (fail-closed otherwise) +//! - **Vocab integrity**: extract `pieces`, build open-addressed hash vocab +//! - **CHAR / BPE / UNIGRAM / WORD encode** after SP-faithful normalization +//! - **`precompiled_charsmap`**: Darts double-array leftmost-longest rewrite +//! (official SP algorithm; see `sp_normalizer`) +//! - **`treat_whitespace_as_suffix`**: dummy `▁` as suffix; WORD split attaches `▁` +//! - **USER_DEFINED / PrefixMatcher forced pieces**: model type-4 pieces + optional +//! runtime `forced_pieces` (must already be in vocab); normalize protect + CHAR/BPE +//! freeze + UNIGRAM score bonus +//! - **UNIGRAM lattice n-best** and **sample-encode** (real A* / forward-filter; +//! refuse for BPE/CHAR) +//! - **UNIGRAM SampleEncodeAndScore** with-replacement (scores = path log-prob − +//! marginal) and without-replacement (Gumbel-perturbed A* n-best + inclusion probs) +//! - **`byte_fallback`**: OOV surfaces → `<0xXX>` UTF-8 byte pieces +//! - **Decode**: ▁→space with dummy-prefix strip, BYTE reassembly, UNKNOWN→unk_surface, +//! CONTROL→empty, then optional `denormalizer_spec` charsmap (official SP path) +//! - **Optional silent ``**: `EncodeOptions.allow_silent_unk` (default false / +//! fail-closed) matches stock SP OOV without `byte_fallback` +//! - **Consecutive `` merge**: default matches stock SP *processor* +//! (`sentencepiece_processor.cc`); CONTROL ids reset the consecutive-unk run; +//! disable via `merge_consecutive_unk=false` for piece-level / fail-closed id lists +//! - **ExtraOptions `bos`/`eos`/`reverse`**: after consecutive-unk merge, flags compose +//! as stock documented `reverse:bos:eos` — reverse, then CONTROL wrap. Decode +//! `reverse` reverses ids before detokenize. ExtraOption `unk`/`unk_piece` is a +//! no-op on id APIs (stock only rewrites piece *strings* in SentencePieceText). +//! - **Reserved ids**: `unk` / `bos` / `eos` / `pad` resolved from piece names + types +//! - **BPE-dropout sample-encode**: priority-queue BPE + optional UNUSED resegmentation +//! +//! ## Explicitly unsupported (refuse) +//! - BPE/CHAR n-best / lattice sample (clear error; no stub algorithms) +//! - Pathological UNIGRAM n-best agenda shrink / Viterbi fallback by **default** +//! (refuse rather than silently diverge); opt-in `nbest_stock_compat` matches +//! stock `Lattice::NBest` shrink + timeout→Viterbi +//! - Bit-identical sample RNG vs stock SP process-global C++ MT19937 — **permanent +//! non-goal** unless a real C++-compatible MT19937 + identical call-order model +//! is implemented (do not stub). Cross-checks stay structural only. +//! +//! Without `byte_fallback` and without `allow_silent_unk`, missing coverage fails closed. +//! Reverse NFKC only runs when `denormalizer_spec.precompiled_charsmap` is present +//! (most stock models leave denorm empty — same permanent gap as official SP). +//! See `docs/sentencepiece-bpe.md`. + +use crate::perfect_hash::PerfectHashVocab; +use crate::sp_lattice::{ + effective_matcher, encode_unigram_lattice, nbest_encode_unigram, + sample_encode_and_score_unigram, sample_encode_and_score_unigram_wor, sample_encode_from_nbest, + sample_encode_unigram_lattice, +}; +pub use crate::sp_normalizer::LiteNormalizer; +use crate::sp_normalizer::{ + byte_to_piece, piece_to_byte, sp_denormalize, sp_normalize_with_matcher, PrecompiledCharsMap, + PrefixMatcher, +}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashMap, HashSet}; + +/// SentencePiece piece type (`ModelProto.SentencePiece.Type`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum PieceType { + Normal = 1, + Unknown = 2, + Control = 3, + UserDefined = 4, + Unused = 5, + Byte = 6, +} + +impl PieceType { + fn from_i32(v: i32) -> Result { + match v { + 1 => Ok(Self::Normal), + 2 => Ok(Self::Unknown), + 3 => Ok(Self::Control), + 4 => Ok(Self::UserDefined), + 5 => Ok(Self::Unused), + 6 => Ok(Self::Byte), + other => Err(format!("unknown SentencePiece.Type {other}")), + } + } +} + +/// TrainerSpec model type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum ModelType { + Unigram = 1, + Bpe = 2, + Word = 3, + Char = 4, +} + +impl ModelType { + fn from_i32(v: i32) -> Result { + match v { + 1 => Ok(Self::Unigram), + 2 => Ok(Self::Bpe), + 3 => Ok(Self::Word), + 4 => Ok(Self::Char), + other => Err(format!("unknown TrainerSpec.ModelType {other}")), + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Unigram => "UNIGRAM", + Self::Bpe => "BPE", + Self::Word => "WORD", + Self::Char => "CHAR", + } + } +} + +/// One vocab piece with optional score/type. +#[derive(Debug, Clone, PartialEq)] +pub struct SpPiece { + pub piece: String, + pub score: f32, + pub piece_type: PieceType, +} + +/// Default SP `unk_surface` (`" ⁇ "` = space + U+2047 + space). +pub const DEFAULT_UNK_SURFACE: &str = " \u{2047} "; + +/// Encode knobs. Default remains fail-closed on OOV without `byte_fallback`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodeOptions { + /// When true and `byte_fallback` is false, emit the UNKNOWN piece id for + /// uncovered seeds / Viterbi gaps (stock SentencePiece behavior). + pub allow_silent_unk: bool, + /// Extra forced pieces for this call (must already exist in vocab). + /// Merged with model `USER_DEFINED` PrefixMatcher; empty string refused. + pub forced_pieces: Vec, + /// When true, WORD `SplitIntoWords` ignores `TrainerSpec.treat_whitespace_as_suffix` + /// (matches stock `word_model.cc`, which always passes `false`). Default false + /// honors the trainer flag (suffix vocab works at encode time). + pub word_official_split: bool, + /// When true (default) and `byte_fallback` is false, merge consecutive + /// UNKNOWN ids into one (stock SP *processor* `Encode` / `PopulateSentencePieceText`). + /// Set false to keep model-level per-span ids (useful for span-aligned audits). + pub merge_consecutive_unk: bool, + /// Reverse piece ids after merge and **before** bos/eos. + /// Application order is always stock `reverse → bos → eos` (see + /// [`EncodeOptions::apply_extra_options_flags`]). CLI `--extra a:b:c` accepts + /// those tokens in **any order** (stock ExtraOptions string is a flag set). + pub reverse: bool, + /// Prepend CONTROL `bos_id` after consecutive-unk merge (stock ExtraOptions `bos`). + /// Refuses when the model has no CONTROL bos piece. + pub add_bos: bool, + /// Append CONTROL `eos_id` after consecutive-unk merge (stock ExtraOptions `eos`). + /// Refuses when the model has no CONTROL eos piece. + pub add_eos: bool, + /// UNIGRAM n-best only: when true, match stock `Lattice::NBest` pathological + /// agenda shrink (`min(512, nbest*10)`) and timeout→Viterbi fallback. + /// Default false refuses rather than silently truncating the search. + pub nbest_stock_compat: bool, +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self { + allow_silent_unk: false, + forced_pieces: Vec::new(), + word_official_split: false, + // Match stock SentencePieceProcessor public encode output. + merge_consecutive_unk: true, + reverse: false, + add_bos: false, + add_eos: false, + nbest_stock_compat: false, + } + } +} + +impl EncodeOptions { + /// Apply ExtraOptions-style flag tokens. Application order is always + /// `reverse → bos → eos` regardless of `flags` order (stock semantics: + /// the ExtraOptions string is a set of switches). + /// + /// Accepted tokens (case-insensitive): `reverse`, `bos`, `eos`. + /// Unknown tokens → `Err`. + pub fn apply_extra_options_flags(&mut self, flags: &[&str]) -> Result<(), String> { + let mut reverse = false; + let mut bos = false; + let mut eos = false; + for raw in flags { + let t = raw.trim(); + if t.is_empty() { + continue; + } + match t.to_ascii_lowercase().as_str() { + "reverse" => reverse = true, + "bos" => bos = true, + "eos" => eos = true, + other => { + return Err(format!( + "unknown ExtraOptions token '{other}' (supported: reverse, bos, eos)" + )); + } + } + } + // Fixed application order (not input order). + if reverse { + self.reverse = true; + } + if bos { + self.add_bos = true; + } + if eos { + self.add_eos = true; + } + Ok(()) + } + + /// Parse a colon-separated ExtraOptions string (`bos:eos:reverse` etc.). + pub fn apply_extra_options_str(&mut self, s: &str) -> Result<(), String> { + let parts: Vec<&str> = s.split(':').collect(); + self.apply_extra_options_flags(&parts) + } +} + +/// Decode knobs. Default matches id→text without ExtraOptions. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct DecodeOptions { + /// Reverse token ids before detokenize (stock decode ExtraOptions `reverse`). + pub reverse: bool, +} + +/// Merge consecutive UNKNOWN ids (processor-level; skipped when `byte_fallback`). +/// +/// Stock `SentencePieceProcessor` resets the consecutive-unk run when a CONTROL +/// id appears (BOS/EOS/PAD or other control pieces). Segmentation encoders do +/// not emit CONTROL as matchable edges; ExtraOptions `add_bos`/`add_eos` wrap +/// **after** merge (stock `ApplyExtraOptions` order). Injected CONTROL ids in +/// an id list still reset the run via `control_ids`. +pub fn merge_consecutive_unk_ids(ids: &[u32], unk_id: u32) -> Vec { + merge_consecutive_unk_ids_with_controls(ids, unk_id, &[]) +} + +/// Like [`merge_consecutive_unk_ids`], but CONTROL ids in `control_ids` reset +/// the consecutive-unk run (official `is_prev_unk = false` on `IsControl`). +pub fn merge_consecutive_unk_ids_with_controls( + ids: &[u32], + unk_id: u32, + control_ids: &[u32], +) -> Vec { + let mut out = Vec::with_capacity(ids.len()); + let mut prev_unk = false; + for &id in ids { + if control_ids.contains(&id) { + out.push(id); + prev_unk = false; + continue; + } + let is_unk = id == unk_id; + if is_unk && prev_unk { + continue; + } + out.push(id); + prev_unk = is_unk; + } + out +} + +fn model_control_ids(model: &SentencePieceModel) -> Vec { + let mut ids: Vec = model + .piece_records + .iter() + .enumerate() + .filter_map(|(i, p)| { + if p.piece_type == PieceType::Control { + Some(i as u32) + } else { + None + } + }) + .collect(); + // Reserved CONTROL slots are already typed Control in records; keep explicit + // bos/eos/pad as a belt-and-suspenders for incomplete piece typing. + for id in [model.bos_id, model.eos_id, model.pad_id] + .into_iter() + .flatten() + { + if !ids.contains(&id) { + ids.push(id); + } + } + ids +} + +fn finalize_encode_ids( + model: &SentencePieceModel, + opts: &EncodeOptions, + ids: Vec, +) -> Result, String> { + // Stock order: consecutive-unk merge inside PopulateSentencePieceText, then + // ApplyExtraOptions. Our bool flags compose as documented CLI + // `reverse:bos:eos` (reverse, then bos, then eos) — not arbitrary colon order. + let mut ids = if opts.merge_consecutive_unk && !model.byte_fallback { + let controls = model_control_ids(model); + merge_consecutive_unk_ids_with_controls(&ids, model.unk_id, &controls) + } else { + ids + }; + if opts.reverse { + ids.reverse(); + } + if opts.add_bos { + let bos = model.bos_id.ok_or_else(|| { + "add_bos: model has no CONTROL bos_id (bos_piece missing or not CONTROL)".to_string() + })?; + ids.insert(0, bos); + } + if opts.add_eos { + let eos = model.eos_id.ok_or_else(|| { + "add_eos: model has no CONTROL eos_id (eos_piece missing or not CONTROL)".to_string() + })?; + ids.push(eos); + } + Ok(ids) +} + +fn finalize_encode_nbest( + model: &SentencePieceModel, + opts: &EncodeOptions, + nbests: Vec<(Vec, f32)>, +) -> Result, f32)>, String> { + let mut out = Vec::with_capacity(nbests.len()); + for (ids, score) in nbests { + out.push((finalize_encode_ids(model, opts, ids)?, score)); + } + Ok(out) +} + +fn finalize_encode_scored( + model: &SentencePieceModel, + opts: &EncodeOptions, + rows: Vec<(Vec, f32)>, +) -> Result, f32)>, String> { + let mut out = Vec::with_capacity(rows.len()); + for (ids, score) in rows { + out.push((finalize_encode_ids(model, opts, ids)?, score)); + } + Ok(out) +} + +/// Sample-encode knobs (UNIGRAM lattice / BPE refused). +#[derive(Debug, Clone)] +pub struct SampleEncodeOptions { + /// Official SP: `<0` lattice sample; `0|1` Viterbi; `>1` sample from n-best. + pub nbest_size: i32, + /// Temperature / smoothing (`alpha` in SP). Must be finite and `>= 0`. + pub alpha: f32, + /// Optional RNG seed for reproducible tests. `None` → `StdRng::from_entropy()`. + /// Not stock SP's process-global C++ MT19937 (permanent non-goal; see module docs). + pub seed: Option, + pub encode: EncodeOptions, +} + +impl Default for SampleEncodeOptions { + fn default() -> Self { + Self { + nbest_size: -1, + alpha: 0.0, + seed: None, + encode: EncodeOptions::default(), + } + } +} + +/// SampleEncodeAndScore knobs (UNIGRAM only). +#[derive(Debug, Clone)] +pub struct SampleEncodeAndScoreOptions { + /// Number of samples to draw (`samples` / `num_samples` in SP). + pub num_samples: usize, + /// Inverse temperature (`alpha` in SP). Must be finite and `>= 0`. + pub alpha: f32, + /// Without-replacement via Gumbel-perturbed A* n-best + inclusion probs. + pub wor: bool, + /// Force-include Viterbi best (requires `wor=true` in official SP). + pub include_best: bool, + /// Optional RNG seed for reproducible tests. `None` → `StdRng::from_entropy()`. + /// Not stock SP's process-global C++ MT19937 (permanent non-goal; see module docs). + pub seed: Option, + pub encode: EncodeOptions, +} + +impl Default for SampleEncodeAndScoreOptions { + fn default() -> Self { + Self { + num_samples: 1, + alpha: 0.0, + wor: false, + include_best: false, + seed: None, + encode: EncodeOptions::default(), + } + } +} + +/// Default reserved piece spellings (TrainerSpec defaults). +pub const DEFAULT_UNK_PIECE: &str = ""; +pub const DEFAULT_BOS_PIECE: &str = ""; +pub const DEFAULT_EOS_PIECE: &str = ""; +pub const DEFAULT_PAD_PIECE: &str = ""; + +/// Parsed SentencePiece model. +#[derive(Debug, Clone)] +pub struct SentencePieceModel { + /// Open-addressed table over `pieces` with token id = piece index. + pub vocab: PerfectHashVocab, + /// Raw piece strings in model order (index = token id). + pub pieces: Vec, + /// Full piece records (scores / types). + pub piece_records: Vec, + /// Model type from TrainerSpec (default UNIGRAM if field absent). + pub model_type: ModelType, + /// Lite normalizer settings used by encode. + pub normalizer: LiteNormalizer, + /// Parsed charsmap when present and valid. + pub charsmap: Option, + /// Denormalizer lite flags (from `denormalizer_spec`). + pub denormalizer: LiteNormalizer, + /// Parsed denormalizer charsmap when present and valid. + pub denorm_charsmap: Option, + /// `TrainerSpec.byte_fallback`. + pub byte_fallback: bool, + /// `TrainerSpec.treat_whitespace_as_suffix`. + pub treat_whitespace_as_suffix: bool, + /// `TrainerSpec.allow_whitespace_only_pieces` (WORD split). + pub allow_whitespace_only_pieces: bool, + /// Surface emitted for UNKNOWN piece ids on decode. + pub unk_surface: String, + /// Token id of the UNKNOWN piece (default 0 when present). + pub unk_id: u32, + /// BOS id when piece exists and is CONTROL; else `None` (`-1` in SP). + pub bos_id: Option, + /// EOS id when piece exists and is CONTROL; else `None`. + pub eos_id: Option, + /// PAD id when piece exists and is CONTROL; else `None`. + pub pad_id: Option, + /// PrefixMatcher over `USER_DEFINED` pieces (may be empty). + pub prefix_matcher: PrefixMatcher, + /// True when encode path is allowed for this model. + pub encode_supported: bool, + /// Human-readable reason when encode is refused. + pub encode_unsupported_reason: Option, +} + +fn read_varint(data: &[u8], i: &mut usize) -> Result { + let mut result: u64 = 0; + let mut shift = 0u32; + loop { + if *i >= data.len() { + return Err("truncated protobuf varint".into()); + } + let byte = data[*i]; + *i += 1; + result |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok(result); + } + shift += 7; + if shift > 63 { + return Err("protobuf varint overflow".into()); + } + } +} + +fn read_fixed32(data: &[u8], i: &mut usize) -> Result { + if *i + 4 > data.len() { + return Err("truncated protobuf fixed32".into()); + } + let v = u32::from_le_bytes(data[*i..*i + 4].try_into().unwrap()); + *i += 4; + Ok(v) +} + +fn skip_field(data: &[u8], i: &mut usize, wire_type: u64) -> Result<(), String> { + match wire_type { + 0 => { + let _ = read_varint(data, i)?; + Ok(()) + } + 1 => { + if *i + 8 > data.len() { + return Err("truncated protobuf fixed64".into()); + } + *i += 8; + Ok(()) + } + 2 => { + let len = read_varint(data, i)? as usize; + if *i + len > data.len() { + return Err("truncated protobuf length-delimited field".into()); + } + *i += len; + Ok(()) + } + 5 => { + let _ = read_fixed32(data, i)?; + Ok(()) + } + other => Err(format!("unsupported protobuf wire type {other}")), + } +} + +fn parse_sentence_piece(msg: &[u8]) -> Result { + let mut i = 0usize; + let mut piece: Option = None; + let mut score: f32 = 0.0; + let mut piece_type = PieceType::Normal; + while i < msg.len() { + let key = read_varint(msg, &mut i)?; + let field = key >> 3; + let wire = key & 0x7; + match (field, wire) { + (1, 2) => { + let len = read_varint(msg, &mut i)? as usize; + if i + len > msg.len() { + return Err("truncated SentencePiece.piece".into()); + } + let s = std::str::from_utf8(&msg[i..i + len]) + .map_err(|_| "SentencePiece.piece is not valid UTF-8".to_string())? + .to_string(); + i += len; + if s.is_empty() { + return Err("SentencePiece.piece must be non-empty".into()); + } + piece = Some(s); + } + (2, 5) => { + let bits = read_fixed32(msg, &mut i)?; + score = f32::from_bits(bits); + if !score.is_finite() { + return Err("SentencePiece.score must be finite".into()); + } + } + (3, 0) => { + let v = read_varint(msg, &mut i)?; + if v > i32::MAX as u64 { + return Err("SentencePiece.type out of range".into()); + } + piece_type = PieceType::from_i32(v as i32)?; + } + _ => skip_field(msg, &mut i, wire)?, + } + } + let piece = piece.ok_or_else(|| "SentencePiece message missing field piece".to_string())?; + Ok(SpPiece { + piece, + score, + piece_type, + }) +} + +#[derive(Debug, Default)] +struct TrainerFlags { + model_type: Option, + byte_fallback: bool, + treat_whitespace_as_suffix: bool, + allow_whitespace_only_pieces: bool, + unk_surface: Option, + unk_piece: Option, + bos_piece: Option, + eos_piece: Option, + pad_piece: Option, +} + +fn parse_trainer_spec(msg: &[u8]) -> Result { + let mut i = 0usize; + let mut flags = TrainerFlags::default(); + while i < msg.len() { + let key = read_varint(msg, &mut i)?; + let field = key >> 3; + let wire = key & 0x7; + match (field, wire) { + (3, 0) => { + let v = read_varint(msg, &mut i)?; + if v > i32::MAX as u64 { + return Err("TrainerSpec.model_type out of range".into()); + } + flags.model_type = Some(ModelType::from_i32(v as i32)?); + } + (24, 0) => { + flags.treat_whitespace_as_suffix = read_varint(msg, &mut i)? != 0; + } + (26, 0) => { + flags.allow_whitespace_only_pieces = read_varint(msg, &mut i)? != 0; + } + (35, 0) => { + flags.byte_fallback = read_varint(msg, &mut i)? != 0; + } + (44, 2) => { + let len = read_varint(msg, &mut i)? as usize; + if i + len > msg.len() { + return Err("truncated TrainerSpec.unk_surface".into()); + } + let s = std::str::from_utf8(&msg[i..i + len]) + .map_err(|_| "TrainerSpec.unk_surface is not valid UTF-8".to_string())? + .to_string(); + i += len; + flags.unk_surface = Some(s); + } + (45, 2) => { + flags.unk_piece = Some(read_trainer_string(msg, &mut i, "unk_piece")?); + } + (46, 2) => { + flags.bos_piece = Some(read_trainer_string(msg, &mut i, "bos_piece")?); + } + (47, 2) => { + flags.eos_piece = Some(read_trainer_string(msg, &mut i, "eos_piece")?); + } + (48, 2) => { + flags.pad_piece = Some(read_trainer_string(msg, &mut i, "pad_piece")?); + } + _ => skip_field(msg, &mut i, wire)?, + } + } + Ok(flags) +} + +fn read_trainer_string(msg: &[u8], i: &mut usize, field: &str) -> Result { + let len = read_varint(msg, i)? as usize; + if *i + len > msg.len() { + return Err(format!("truncated TrainerSpec.{field}")); + } + let s = std::str::from_utf8(&msg[*i..*i + len]) + .map_err(|_| format!("TrainerSpec.{field} is not valid UTF-8"))? + .to_string(); + *i += len; + Ok(s) +} + +#[derive(Debug, Default)] +struct NormalizerFlags { + lite: LiteNormalizer, + charsmap_bytes: Option>, +} + +fn parse_normalizer_spec(msg: &[u8]) -> Result { + let mut i = 0usize; + let mut flags = NormalizerFlags::default(); + while i < msg.len() { + let key = read_varint(msg, &mut i)?; + let field = key >> 3; + let wire = key & 0x7; + match (field, wire) { + (2, 2) => { + let len = read_varint(msg, &mut i)? as usize; + if i + len > msg.len() { + return Err("truncated NormalizerSpec.precompiled_charsmap".into()); + } + if len > 0 { + flags.charsmap_bytes = Some(msg[i..i + len].to_vec()); + } + i += len; + } + (3, 0) => { + flags.lite.add_dummy_prefix = read_varint(msg, &mut i)? != 0; + } + (4, 0) => { + flags.lite.remove_extra_whitespaces = read_varint(msg, &mut i)? != 0; + } + (5, 0) => { + flags.lite.escape_whitespaces = read_varint(msg, &mut i)? != 0; + } + _ => skip_field(msg, &mut i, wire)?, + } + } + Ok(flags) +} + +struct ParsedModelProto { + pieces: Vec, + trainer: TrainerFlags, + normalizer: NormalizerFlags, + has_normalizer_spec: bool, + denormalizer: NormalizerFlags, + has_denormalizer_spec: bool, +} + +fn parse_model_proto(data: &[u8]) -> Result { + let mut i = 0usize; + let mut pieces = Vec::new(); + let mut trainer = TrainerFlags::default(); + let mut normalizer = NormalizerFlags::default(); + let mut has_normalizer_spec = false; + let mut denormalizer = NormalizerFlags::default(); + let mut has_denormalizer_spec = false; + while i < data.len() { + let key = read_varint(data, &mut i)?; + let field = key >> 3; + let wire = key & 0x7; + match (field, wire) { + (1, 2) => { + let len = read_varint(data, &mut i)? as usize; + if i + len > data.len() { + return Err("truncated ModelProto.pieces entry".into()); + } + let piece = parse_sentence_piece(&data[i..i + len])?; + i += len; + pieces.push(piece); + } + (2, 2) => { + let len = read_varint(data, &mut i)? as usize; + if i + len > data.len() { + return Err("truncated ModelProto.trainer_spec".into()); + } + trainer = parse_trainer_spec(&data[i..i + len])?; + i += len; + } + (3, 2) => { + let len = read_varint(data, &mut i)? as usize; + if i + len > data.len() { + return Err("truncated ModelProto.normalizer_spec".into()); + } + normalizer = parse_normalizer_spec(&data[i..i + len])?; + has_normalizer_spec = true; + i += len; + } + (5, 2) => { + let len = read_varint(data, &mut i)? as usize; + if i + len > data.len() { + return Err("truncated ModelProto.denormalizer_spec".into()); + } + denormalizer = parse_normalizer_spec(&data[i..i + len])?; + has_denormalizer_spec = true; + i += len; + } + _ => skip_field(data, &mut i, wire)?, + } + } + Ok(ParsedModelProto { + pieces, + trainer, + normalizer, + has_normalizer_spec, + denormalizer, + has_denormalizer_spec, + }) +} + +fn encode_support_gate( + _model_type: ModelType, + trainer: &TrainerFlags, + normalizer: &NormalizerFlags, + charsmap: &Option, +) -> Result<(), String> { + // All ModelType variants are encodeable when structural gates pass. + if normalizer.charsmap_bytes.is_some() && charsmap.is_none() { + return Err( + "encode unsupported: NormalizerSpec.precompiled_charsmap present but invalid".into(), + ); + } + if !normalizer.lite.escape_whitespaces { + return Err("encode unsupported when escape_whitespaces=false".into()); + } + let _ = trainer.byte_fallback; // validated later once pieces known + Ok(()) +} + +fn validate_byte_fallback_vocab(pieces: &[SpPiece]) -> Result<(), String> { + let mut seen = [false; 256]; + for p in pieces { + if p.piece_type == PieceType::Byte { + if let Some(b) = piece_to_byte(&p.piece) { + seen[b as usize] = true; + } + } + } + // Also accept NORMAL pieces named <0xXX> (some crafted fixtures). + for p in pieces { + if let Some(b) = piece_to_byte(&p.piece) { + seen[b as usize] = true; + } + } + if seen.iter().any(|&x| !x) { + return Err( + "encode unsupported: byte_fallback=true but vocab missing some <0xXX> pieces".into(), + ); + } + Ok(()) +} + +/// Encode a u64 as a protobuf varint (test / fixture helper). +pub fn encode_varint(mut v: u64, out: &mut Vec) { + loop { + let mut byte = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + byte |= 0x80; + } + out.push(byte); + if v == 0 { + break; + } + } +} + +fn encode_len_delim(field: u64, payload: &[u8], out: &mut Vec) { + encode_varint((field << 3) | 2, out); + encode_varint(payload.len() as u64, out); + out.extend_from_slice(payload); +} + +#[inline] +fn tag_varint(field: u64) -> u64 { + field << 3 +} + +#[inline] +fn tag_fixed32(field: u64) -> u64 { + (field << 3) | 5 +} + +fn encode_sp_piece(piece: &str, score: Option, piece_type: Option) -> Vec { + let mut sp = Vec::new(); + encode_varint((1 << 3) | 2, &mut sp); + encode_varint(piece.len() as u64, &mut sp); + sp.extend_from_slice(piece.as_bytes()); + if let Some(s) = score { + encode_varint(tag_fixed32(2), &mut sp); + sp.extend_from_slice(&s.to_bits().to_le_bytes()); + } + if let Some(t) = piece_type { + encode_varint(tag_varint(3), &mut sp); + encode_varint(t as u64, &mut sp); + } + sp +} + +/// Build a minimal `ModelProto` bytes blob with the given pieces (for tests). +pub fn encode_model_proto_pieces(pieces: &[&str]) -> Vec { + let mut out = Vec::new(); + for piece in pieces { + encode_len_delim(1, &encode_sp_piece(piece, None, None), &mut out); + } + out +} + +/// Fixture builder: pieces + optional trainer/normalizer/denormalizer fields. +#[derive(Default)] +pub struct ModelProtoFixture { + pub pieces: Vec<(String, Option, Option)>, + pub model_type: Option, + pub byte_fallback: Option, + pub treat_whitespace_as_suffix: Option, + pub allow_whitespace_only_pieces: Option, + pub unk_surface: Option, + pub unk_piece: Option, + pub bos_piece: Option, + pub eos_piece: Option, + pub pad_piece: Option, + pub add_dummy_prefix: Option, + pub remove_extra_whitespaces: Option, + pub escape_whitespaces: Option, + pub precompiled_charsmap: Option>, + pub denorm_precompiled_charsmap: Option>, + pub denorm_add_dummy_prefix: Option, + pub denorm_remove_extra_whitespaces: Option, + pub denorm_escape_whitespaces: Option, +} + +impl ModelProtoFixture { + pub fn encode(&self) -> Vec { + let mut out = Vec::new(); + for (piece, score, ty) in &self.pieces { + encode_len_delim(1, &encode_sp_piece(piece, *score, *ty), &mut out); + } + + let mut trainer = Vec::new(); + if let Some(mt) = self.model_type { + encode_varint(tag_varint(3), &mut trainer); + encode_varint(mt as u64, &mut trainer); + } + if let Some(v) = self.treat_whitespace_as_suffix { + encode_varint(tag_varint(24), &mut trainer); + encode_varint(u64::from(v), &mut trainer); + } + if let Some(v) = self.allow_whitespace_only_pieces { + encode_varint(tag_varint(26), &mut trainer); + encode_varint(u64::from(v), &mut trainer); + } + if let Some(v) = self.byte_fallback { + encode_varint(tag_varint(35), &mut trainer); + encode_varint(u64::from(v), &mut trainer); + } + if let Some(ref s) = self.unk_surface { + encode_len_delim(44, s.as_bytes(), &mut trainer); + } + if let Some(ref s) = self.unk_piece { + encode_len_delim(45, s.as_bytes(), &mut trainer); + } + if let Some(ref s) = self.bos_piece { + encode_len_delim(46, s.as_bytes(), &mut trainer); + } + if let Some(ref s) = self.eos_piece { + encode_len_delim(47, s.as_bytes(), &mut trainer); + } + if let Some(ref s) = self.pad_piece { + encode_len_delim(48, s.as_bytes(), &mut trainer); + } + if !trainer.is_empty() { + encode_len_delim(2, &trainer, &mut out); + } + + let mut norm = Vec::new(); + if let Some(ref map) = self.precompiled_charsmap { + encode_len_delim(2, map, &mut norm); + } + if let Some(v) = self.add_dummy_prefix { + encode_varint(tag_varint(3), &mut norm); + encode_varint(u64::from(v), &mut norm); + } + if let Some(v) = self.remove_extra_whitespaces { + encode_varint(tag_varint(4), &mut norm); + encode_varint(u64::from(v), &mut norm); + } + if let Some(v) = self.escape_whitespaces { + encode_varint(tag_varint(5), &mut norm); + encode_varint(u64::from(v), &mut norm); + } + if !norm.is_empty() { + encode_len_delim(3, &norm, &mut out); + } + + let mut denorm = Vec::new(); + if let Some(ref map) = self.denorm_precompiled_charsmap { + encode_len_delim(2, map, &mut denorm); + } + if let Some(v) = self.denorm_add_dummy_prefix { + encode_varint(tag_varint(3), &mut denorm); + encode_varint(u64::from(v), &mut denorm); + } + if let Some(v) = self.denorm_remove_extra_whitespaces { + encode_varint(tag_varint(4), &mut denorm); + encode_varint(u64::from(v), &mut denorm); + } + if let Some(v) = self.denorm_escape_whitespaces { + encode_varint(tag_varint(5), &mut denorm); + encode_varint(u64::from(v), &mut denorm); + } + if !denorm.is_empty() { + encode_len_delim(5, &denorm, &mut out); + } + out + } +} + +/// Parse SentencePiece `ModelProto` bytes and build a vocab table. +pub fn load_model_proto(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("empty model_proto".into()); + } + let parsed = parse_model_proto(bytes)?; + if parsed.pieces.is_empty() { + return Err("ModelProto contains no pieces; refuse empty SentencePiece vocab".into()); + } + { + let mut seen = HashSet::new(); + for p in &parsed.pieces { + if !seen.insert(p.piece.as_str()) { + return Err(format!( + "duplicate SentencePiece piece {:?}; refuse ambiguous vocab", + p.piece + )); + } + } + } + + let model_type = parsed.trainer.model_type.unwrap_or(ModelType::Unigram); + let normalizer = if parsed.has_normalizer_spec { + parsed.normalizer.lite.clone() + } else { + LiteNormalizer::default() + }; + + let (charsmap, charsmap_parse_err) = match &parsed.normalizer.charsmap_bytes { + Some(blob) => match PrecompiledCharsMap::parse(blob) { + Ok(m) => (Some(m), None), + Err(e) => (None, Some(e)), + }, + None => (None, None), + }; + + // Official SP only constructs a denormalizer when charsmap is non-empty. + let denormalizer = if parsed.has_denormalizer_spec { + parsed.denormalizer.lite.clone() + } else { + LiteNormalizer { + add_dummy_prefix: false, + remove_extra_whitespaces: false, + escape_whitespaces: false, + } + }; + let (denorm_charsmap, denorm_parse_err) = match &parsed.denormalizer.charsmap_bytes { + Some(blob) if !blob.is_empty() => match PrecompiledCharsMap::parse(blob) { + Ok(m) => (Some(m), None), + Err(e) => (None, Some(e)), + }, + _ => (None, None), + }; + if let Some(e) = denorm_parse_err { + return Err(format!( + "invalid denormalizer_spec.precompiled_charsmap ({e})" + )); + } + + let mut encode_gate = + encode_support_gate(model_type, &parsed.trainer, &parsed.normalizer, &charsmap); + if let Some(ref e) = charsmap_parse_err { + encode_gate = Err(format!( + "encode unsupported: invalid precompiled_charsmap ({e})" + )); + } + // Invalid denorm refuses decode later; encode still allowed (matches stock SP load). + if encode_gate.is_ok() && parsed.trainer.byte_fallback { + if let Err(e) = validate_byte_fallback_vocab(&parsed.pieces) { + encode_gate = Err(e); + } + } + + let (encode_supported, encode_unsupported_reason) = match encode_gate { + Ok(()) => (true, None), + Err(reason) => (false, Some(reason)), + }; + + let unk_id = parsed + .pieces + .iter() + .position(|p| p.piece_type == PieceType::Unknown) + .unwrap_or(0) as u32; + + let unk_piece_name = parsed + .trainer + .unk_piece + .as_deref() + .unwrap_or(DEFAULT_UNK_PIECE); + let bos_piece_name = parsed + .trainer + .bos_piece + .as_deref() + .unwrap_or(DEFAULT_BOS_PIECE); + let eos_piece_name = parsed + .trainer + .eos_piece + .as_deref() + .unwrap_or(DEFAULT_EOS_PIECE); + let pad_piece_name = parsed + .trainer + .pad_piece + .as_deref() + .unwrap_or(DEFAULT_PAD_PIECE); + + // Official SP: bos/eos/pad only valid when PieceToId hits a CONTROL piece. + let bos_id = resolve_control_id(&parsed.pieces, bos_piece_name); + let eos_id = resolve_control_id(&parsed.pieces, eos_piece_name); + let pad_id = resolve_control_id(&parsed.pieces, pad_piece_name); + // Prefer typed UNKNOWN; fall back to named unk_piece if typed id missing. + let unk_id = if parsed + .pieces + .get(unk_id as usize) + .is_some_and(|p| p.piece_type == PieceType::Unknown) + { + unk_id + } else { + parsed + .pieces + .iter() + .position(|p| p.piece == unk_piece_name && p.piece_type == PieceType::Unknown) + .unwrap_or(unk_id as usize) as u32 + }; + + let user_defined: Vec = parsed + .pieces + .iter() + .filter(|p| p.piece_type == PieceType::UserDefined) + .map(|p| p.piece.clone()) + .collect(); + let prefix_matcher = PrefixMatcher::from_pieces(user_defined)?; + + let piece_strings: Vec = parsed.pieces.iter().map(|p| p.piece.clone()).collect(); + let pairs: Vec<(String, u32)> = piece_strings + .iter() + .enumerate() + .map(|(i, p)| (p.clone(), i as u32)) + .collect(); + let vocab = PerfectHashVocab::build(pairs); + if !vocab.validate_roundtrip() { + return Err("parsed ModelProto vocab failed encode/decode round-trip".into()); + } + + Ok(SentencePieceModel { + vocab, + pieces: piece_strings, + piece_records: parsed.pieces, + model_type, + normalizer, + charsmap, + denormalizer, + denorm_charsmap, + byte_fallback: parsed.trainer.byte_fallback, + treat_whitespace_as_suffix: parsed.trainer.treat_whitespace_as_suffix, + allow_whitespace_only_pieces: parsed.trainer.allow_whitespace_only_pieces, + unk_surface: parsed + .trainer + .unk_surface + .unwrap_or_else(|| DEFAULT_UNK_SURFACE.to_string()), + unk_id, + bos_id, + eos_id, + pad_id, + prefix_matcher, + encode_supported, + encode_unsupported_reason, + }) +} + +fn resolve_control_id(pieces: &[SpPiece], name: &str) -> Option { + pieces.iter().enumerate().find_map(|(i, p)| { + if p.piece == name && p.piece_type == PieceType::Control { + Some(i as u32) + } else { + None + } + }) +} + +/// Load a `.model` file from disk. +pub fn load_model_proto_file(path: &str) -> Result { + let bytes = std::fs::read(path).map_err(|e| format!("read '{path}': {e}"))?; + load_model_proto(&bytes) +} + +/// Real runtime path: build an open-addressed vocab table from JSON text. +pub fn load_vocab_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| format!("invalid vocab JSON: {e}")) +} + +const SP_SPACE: char = '\u{2581}'; + +/// SP-faithful normalization (charsmap + whitespace + USER_DEFINED matcher). +pub fn normalize(model: &SentencePieceModel, text: &str) -> Result { + normalize_with_options(model, text, &EncodeOptions::default()) +} + +pub fn normalize_with_options( + model: &SentencePieceModel, + text: &str, + opts: &EncodeOptions, +) -> Result { + let matcher = effective_matcher(model, opts)?; + let matcher_ref = if matcher.is_empty() { + None + } else { + Some(&matcher) + }; + sp_normalize_with_matcher( + text, + &model.normalizer, + model.charsmap.as_ref(), + matcher_ref, + model.treat_whitespace_as_suffix, + ) +} + +/// Backward-compatible alias: whitespace-only normalize (no charsmap / matcher). +pub fn lite_normalize(text: &str, norm: &LiteNormalizer) -> Result { + sp_normalize_with_matcher(text, norm, None, None, false) +} + +fn piece_index_map(model: &SentencePieceModel) -> HashMap<&str, (u32, f32, PieceType)> { + let mut map = HashMap::new(); + for (i, rec) in model.piece_records.iter().enumerate() { + // Match official SP: CONTROL / UNKNOWN / BYTE are reserved — not segmentation + // candidates. UNUSED is skipped at match time. BYTE looked up via byte_fallback. + if matches!( + rec.piece_type, + PieceType::Unused | PieceType::Control | PieceType::Unknown | PieceType::Byte + ) { + continue; + } + map.insert(rec.piece.as_str(), (i as u32, rec.score, rec.piece_type)); + } + map +} + +/// Full piece map including reserved types (for byte_fallback / explicit id lookups). +fn piece_index_map_all(model: &SentencePieceModel) -> HashMap<&str, (u32, f32, PieceType)> { + let mut map = HashMap::new(); + for (i, rec) in model.piece_records.iter().enumerate() { + if rec.piece_type == PieceType::Unused { + continue; + } + map.insert(rec.piece.as_str(), (i as u32, rec.score, rec.piece_type)); + } + map +} + +pub(crate) fn byte_piece_ids( + model: &SentencePieceModel, + surface: &str, +) -> Result, String> { + let map = piece_index_map_all(model); + let mut out = Vec::with_capacity(surface.len()); + for &b in surface.as_bytes() { + let key = byte_to_piece(b); + match map.get(key.as_str()) { + Some(&(id, _, _)) => out.push(id), + None => { + return Err(format!("byte_fallback: missing piece {key} in vocab")); + } + } + } + Ok(out) +} + +fn require_encode(model: &SentencePieceModel) -> Result<(), String> { + if model.encode_supported { + Ok(()) + } else { + Err(model + .encode_unsupported_reason + .clone() + .unwrap_or_else(|| "encode unsupported".into())) + } +} + +/// CHAR encode: PrefixMatcher seeds (USER_DEFINED freeze) then one id per seed. +pub fn encode_char_with_options( + model: &SentencePieceModel, + text: &str, + opts: EncodeOptions, +) -> Result, String> { + if model.model_type != ModelType::Char { + return Err(format!( + "encode_char requires CHAR model, got {}", + model.model_type.as_str() + )); + } + require_encode(model)?; + let matcher = effective_matcher(model, &opts)?; + let normalized = normalize_with_options(model, text, &opts)?; + let map = piece_index_map(model); + let mut out = Vec::new(); + let mut rest = normalized.as_str(); + while !rest.is_empty() { + let (mblen, _found) = matcher.prefix_match(rest); + let key = &rest[..mblen]; + match map.get(key) { + Some(&(id, _, _)) => out.push(id), + None if model.byte_fallback => { + out.extend(byte_piece_ids(model, key)?); + } + None if opts.allow_silent_unk => { + out.push(model.unk_id); + } + None => { + return Err(format!( + "CHAR encode: seed {key:?} not in vocab (fail-closed; no byte_fallback)" + )); + } + } + rest = &rest[mblen..]; + } + finalize_encode_ids(model, &opts, out) +} + +/// CHAR encode (fail-closed default). +pub fn encode_char(model: &SentencePieceModel, text: &str) -> Result, String> { + encode_char_with_options(model, text, EncodeOptions::default()) +} + +#[derive(Clone)] +struct BpeSymbol { + prev: i32, + next: i32, + freeze: bool, + /// Byte range into the normalized string (piece may span merges). + start: usize, + len: usize, +} + +#[derive(Clone)] +struct BpePair { + left: i32, + right: i32, + size: usize, +} + +#[derive(Clone)] +struct BpePairOrd { + idx: usize, + score: f32, + left: i32, +} + +impl PartialEq for BpePairOrd { + fn eq(&self, other: &Self) -> bool { + self.score == other.score && self.left == other.left + } +} +impl Eq for BpePairOrd {} +impl PartialOrd for BpePairOrd { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for BpePairOrd { + fn cmp(&self, other: &Self) -> Ordering { + // Max-heap by score; tie-break lower left index (official SP). + match self + .score + .partial_cmp(&other.score) + .unwrap_or(Ordering::Equal) + { + Ordering::Equal => other.left.cmp(&self.left), + ord => ord, + } + } +} + +/// BPE piece map: NORMAL / USER_DEFINED / UNUSED (official `pieces_` includes unused). +fn bpe_piece_map(model: &SentencePieceModel) -> HashMap<&str, (u32, f32, PieceType)> { + let mut map = HashMap::new(); + for (i, rec) in model.piece_records.iter().enumerate() { + if matches!( + rec.piece_type, + PieceType::Control | PieceType::Unknown | PieceType::Byte + ) { + continue; + } + map.insert(rec.piece.as_str(), (i as u32, rec.score, rec.piece_type)); + } + map +} + +fn bpe_resegment( + w: &str, + map: &HashMap<&str, (u32, f32, PieceType)>, + rev_merge: &HashMap, + out: &mut Vec, +) -> Result<(), String> { + match map.get(w) { + Some(&(id, _, ty)) if ty != PieceType::Unused => { + out.push(id); + Ok(()) + } + Some(_) | None => { + if let Some((left, right)) = rev_merge.get(w) { + bpe_resegment(left, map, rev_merge, out)?; + bpe_resegment(right, map, rev_merge, out) + } else if let Some(&(id, _, _)) = map.get(w) { + // UNUSED without rev_merge entry (should not happen for merges we recorded). + out.push(id); + Ok(()) + } else { + Err(format!("BPE encode: final symbol {w:?} missing from vocab")) + } + } + } +} + +/// Priority-queue BPE with optional dropout (`alpha` = skip-merge probability). +/// +/// Matches official `bpe_model.cc` `SampleEncode`: Encode is `alpha=0`. +/// UNUSED merges are recorded in `rev_merge` and resegmented at the end. +pub fn encode_bpe_sample_with_options( + model: &SentencePieceModel, + text: &str, + alpha: f32, + opts: EncodeOptions, + rng: &mut impl Rng, +) -> Result, String> { + if model.model_type != ModelType::Bpe { + return Err(format!( + "encode_bpe requires BPE model, got {}", + model.model_type.as_str() + )); + } + if !(alpha.is_finite() && alpha >= 0.0) { + return Err("BPE sample: alpha must be finite and >= 0".into()); + } + require_encode(model)?; + let matcher = effective_matcher(model, &opts)?; + let normalized = normalize_with_options(model, text, &opts)?; + if normalized.is_empty() { + return Ok(Vec::new()); + } + let map = bpe_piece_map(model); + let norm_bytes = normalized.as_bytes(); + + // Seed symbols via PrefixMatcher (USER_DEFINED freeze), else one Unicode scalar. + // Unk seeds are held as frozen spans that never merge (expanded at emit time). + let mut symbols: Vec = Vec::new(); + let mut unk_spans: HashMap = HashMap::new(); + let mut rest = normalized.as_str(); + let mut byte_off = 0usize; + while !rest.is_empty() { + let (mblen, found) = matcher.prefix_match(rest); + let key = &rest[..mblen]; + let idx = symbols.len() as i32; + if map.contains_key(key) { + symbols.push(BpeSymbol { + prev: if idx == 0 { -1 } else { idx - 1 }, + next: -1, // patched below + freeze: found, + start: byte_off, + len: mblen, + }); + } else if model.byte_fallback || opts.allow_silent_unk { + unk_spans.insert(symbols.len(), key.to_string()); + symbols.push(BpeSymbol { + prev: if idx == 0 { -1 } else { idx - 1 }, + next: -1, + freeze: true, + start: byte_off, + len: mblen, + }); + } else { + return Err(format!( + "BPE encode: seed symbol {key:?} not in vocab (fail-closed; no byte_fallback)" + )); + } + byte_off += mblen; + rest = &rest[mblen..]; + } + let n = symbols.len(); + for (i, sym) in symbols.iter_mut().enumerate() { + sym.next = if i + 1 < n { (i + 1) as i32 } else { -1 }; + } + + let mut pool: Vec = Vec::new(); + let mut agenda: BinaryHeap = BinaryHeap::new(); + let mut rev_merge: HashMap = HashMap::new(); + + let surface = |syms: &[BpeSymbol], i: usize| -> &str { + let s = &syms[i]; + std::str::from_utf8(&norm_bytes[s.start..s.start + s.len]).unwrap_or("") + }; + + let maybe_add = |syms: &[BpeSymbol], + pool: &mut Vec, + agenda: &mut BinaryHeap, + rev_merge: &mut HashMap, + left: i32, + right: i32| { + if left < 0 || right < 0 { + return; + } + let (l, r) = (left as usize, right as usize); + if syms[l].freeze || syms[r].freeze { + return; + } + let piece = format!("{}{}", surface(syms, l), surface(syms, r)); + let Some(&(_, score, ty)) = map.get(piece.as_str()) else { + return; + }; + let idx = pool.len(); + pool.push(BpePair { + left, + right, + size: piece.len(), + }); + agenda.push(BpePairOrd { idx, score, left }); + if ty == PieceType::Unused { + rev_merge.insert( + piece, + (surface(syms, l).to_string(), surface(syms, r).to_string()), + ); + } + }; + + for i in 1..n { + maybe_add( + &symbols, + &mut pool, + &mut agenda, + &mut rev_merge, + (i - 1) as i32, + i as i32, + ); + } + + while let Some(top) = agenda.pop() { + let pair = pool[top.idx].clone(); + let (li, ri) = (pair.left as usize, pair.right as usize); + if symbols[li].len == 0 + || symbols[ri].len == 0 + || symbols[li].len + symbols[ri].len != pair.size + { + continue; + } + // BPE-dropout: skip merge with probability `alpha`. + if alpha >= 1.0 || (alpha > 0.0 && rng.gen::() < f64::from(alpha)) { + continue; + } + + // Merge right into left (byte span grows; right cleared). + symbols[li].len += symbols[ri].len; + symbols[li].next = symbols[ri].next; + let right_next = symbols[ri].next; + if right_next >= 0 { + symbols[right_next as usize].prev = pair.left; + } + symbols[ri].len = 0; + + maybe_add( + &symbols, + &mut pool, + &mut agenda, + &mut rev_merge, + symbols[li].prev, + pair.left, + ); + maybe_add( + &symbols, + &mut pool, + &mut agenda, + &mut rev_merge, + pair.left, + symbols[li].next, + ); + } + + let mut out = Vec::new(); + let mut index = 0i32; + while index >= 0 { + let i = index as usize; + if let Some(unk) = unk_spans.get(&i) { + if model.byte_fallback { + out.extend(byte_piece_ids(model, unk)?); + } else if opts.allow_silent_unk { + out.push(model.unk_id); + } else { + return Err(format!( + "BPE encode: seed symbol {unk:?} not in vocab (fail-closed; no byte_fallback)" + )); + } + } else { + let w = surface(&symbols, i).to_string(); + bpe_resegment(&w, &map, &rev_merge, &mut out)?; + } + index = symbols[i].next; + } + finalize_encode_ids(model, &opts, out) +} + +/// SentencePiece-style BPE encode: PrefixMatcher seeds + priority-queue merges +/// (official `Encode` ≡ `SampleEncode(alpha=0)`). +pub fn encode_bpe_with_options( + model: &SentencePieceModel, + text: &str, + opts: EncodeOptions, +) -> Result, String> { + // Deterministic path: no RNG draws when alpha == 0. + let mut rng = rand::rngs::StdRng::seed_from_u64(0); + encode_bpe_sample_with_options(model, text, 0.0, opts, &mut rng) +} + +pub fn encode_bpe(model: &SentencePieceModel, text: &str) -> Result, String> { + encode_bpe_with_options(model, text, EncodeOptions::default()) +} + +/// Official SP `SplitIntoWords` over a normalized string (`▁` = whitespace meta). +pub fn split_into_words( + normalized: &str, + treat_ws_as_suffix: bool, + allow_ws_only_pieces: bool, +) -> Vec<&str> { + const SPACE: &str = "\u{2581}"; + let bytes = normalized.as_bytes(); + if bytes.is_empty() { + return Vec::new(); + } + let mut result: Vec<(usize, usize)> = Vec::new(); // (start, len) in bytes + let mut in_ws_sequence = false; + let mut offset = 0usize; + + let char_len = |at: usize| -> usize { + let b = bytes[at]; + match b { + 0x00..=0x7F => 1, + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF7 => 4, + _ => 1, + } + .min(bytes.len() - at) + }; + + if treat_ws_as_suffix { + result.push((0, 0)); + while offset < bytes.len() { + let mblen = char_len(offset); + let is_ws = normalized.get(offset..offset + mblen) == Some(SPACE); + if is_ws { + in_ws_sequence = true; + } else if in_ws_sequence { + if allow_ws_only_pieces { + result.push((offset, 0)); + } + in_ws_sequence = false; + } + let last = result.len() - 1; + result[last].1 += mblen; + offset += mblen; + if offset < bytes.len() && is_ws && !allow_ws_only_pieces { + result.push((offset, 0)); + } + } + } else { + while offset < bytes.len() { + let mblen = char_len(offset); + let is_ws = normalized.get(offset..offset + mblen) == Some(SPACE); + if offset == 0 || (is_ws && (!in_ws_sequence || !allow_ws_only_pieces)) { + result.push((offset, 0)); + in_ws_sequence = true; + } + if in_ws_sequence && !is_ws { + in_ws_sequence = false; + } + let last = result.len() - 1; + result[last].1 += mblen; + offset += mblen; + } + } + + result + .into_iter() + .filter(|(_, len)| *len > 0) + .map(|(start, len)| &normalized[start..start + len]) + .collect() +} + +/// WORD encode: whitespace-delimited pieces after normalize (`SplitIntoWords`). +pub fn encode_word_with_options( + model: &SentencePieceModel, + text: &str, + opts: EncodeOptions, +) -> Result, String> { + if model.model_type != ModelType::Word { + return Err(format!( + "encode_word requires WORD model, got {}", + model.model_type.as_str() + )); + } + require_encode(model)?; + let normalized = normalize_with_options(model, text, &opts)?; + if normalized.is_empty() { + return Ok(Vec::new()); + } + let map = piece_index_map(model); + let treat_suffix = if opts.word_official_split { + false + } else { + model.treat_whitespace_as_suffix + }; + let words = split_into_words( + &normalized, + treat_suffix, + model.allow_whitespace_only_pieces, + ); + let mut out = Vec::with_capacity(words.len()); + for w in words { + match map.get(w) { + Some(&(id, _, _)) => out.push(id), + None if model.byte_fallback => { + out.extend(byte_piece_ids(model, w)?); + } + None if opts.allow_silent_unk => { + out.push(model.unk_id); + } + None => { + return Err(format!( + "WORD encode: piece {w:?} not in vocab (fail-closed; no byte_fallback)" + )); + } + } + } + finalize_encode_ids(model, &opts, out) +} + +pub fn encode_word(model: &SentencePieceModel, text: &str) -> Result, String> { + encode_word_with_options(model, text, EncodeOptions::default()) +} + +/// UNIGRAM encode: lattice Viterbi over Unicode character positions. +pub fn encode_unigram_with_options( + model: &SentencePieceModel, + text: &str, + opts: EncodeOptions, +) -> Result, String> { + if model.model_type != ModelType::Unigram { + return Err(format!( + "encode_unigram requires UNIGRAM model, got {}", + model.model_type.as_str() + )); + } + require_encode(model)?; + let normalized = normalize_with_options(model, text, &opts)?; + let ids = encode_unigram_lattice(model, &normalized, opts.clone())?; + finalize_encode_ids(model, &opts, ids) +} + +pub fn encode_unigram(model: &SentencePieceModel, text: &str) -> Result, String> { + encode_unigram_with_options(model, text, EncodeOptions::default()) +} + +/// N-best encode (UNIGRAM lattice A* only). Returns `(ids, score)` best-first. +pub fn nbest_encode_with_options( + model: &SentencePieceModel, + text: &str, + nbest_size: usize, + opts: EncodeOptions, +) -> Result, f32)>, String> { + require_encode(model)?; + match model.model_type { + ModelType::Unigram => { + let normalized = normalize_with_options(model, text, &opts)?; + let nbests = nbest_encode_unigram(model, &normalized, nbest_size, opts.clone())?; + finalize_encode_nbest(model, &opts, nbests) + } + other => Err(format!( + "nbest_encode unsupported for model_type {} (UNIGRAM lattice only)", + other.as_str() + )), + } +} + +pub fn nbest_encode( + model: &SentencePieceModel, + text: &str, + nbest_size: usize, +) -> Result, f32)>, String> { + nbest_encode_with_options(model, text, nbest_size, EncodeOptions::default()) +} + +/// Sample-encode (UNIGRAM lattice sample / BPE-dropout). See [`SampleEncodeOptions`]. +pub fn sample_encode_with_options( + model: &SentencePieceModel, + text: &str, + sample: SampleEncodeOptions, +) -> Result, String> { + require_encode(model)?; + if !sample.alpha.is_finite() || sample.alpha < 0.0 { + return Err("sample_encode: alpha must be finite and >= 0".into()); + } + let opts = sample.encode.clone(); + match model.model_type { + ModelType::Unigram => { + let normalized = normalize_with_options(model, text, &opts)?; + let mut rng: StdRng = match sample.seed { + Some(s) => StdRng::seed_from_u64(s), + None => StdRng::from_entropy(), + }; + let ids = if sample.nbest_size < 0 { + sample_encode_unigram_lattice( + model, + &normalized, + sample.alpha, + opts.clone(), + &mut rng, + )? + } else if sample.nbest_size == 0 || sample.nbest_size == 1 { + encode_unigram_lattice(model, &normalized, opts.clone())? + } else { + sample_encode_from_nbest( + model, + &normalized, + sample.nbest_size as usize, + sample.alpha, + opts.clone(), + &mut rng, + )? + }; + finalize_encode_ids(model, &opts, ids) + } + ModelType::Bpe => { + // Official BPE: nbest_size ignored; alpha is dropout probability. + let mut rng: StdRng = match sample.seed { + Some(s) => StdRng::seed_from_u64(s), + None => StdRng::from_entropy(), + }; + // encode_bpe_sample_with_options already finalizes. + encode_bpe_sample_with_options(model, text, sample.alpha, opts, &mut rng) + } + other => Err(format!( + "sample_encode unsupported for model_type {} (UNIGRAM lattice or BPE-dropout only)", + other.as_str() + )), + } +} + +pub fn sample_encode( + model: &SentencePieceModel, + text: &str, + nbest_size: i32, + alpha: f32, +) -> Result, String> { + sample_encode_with_options( + model, + text, + SampleEncodeOptions { + nbest_size, + alpha, + seed: None, + encode: EncodeOptions::default(), + }, + ) +} + +/// SampleEncodeAndScore (UNIGRAM). With-replacement returns `(ids, log_prob − marginal)`. +/// Without-replacement returns Gumbel-Top-k draws with log inclusion probabilities. +pub fn sample_encode_and_score_with_options( + model: &SentencePieceModel, + text: &str, + sample: SampleEncodeAndScoreOptions, +) -> Result, f32)>, String> { + require_encode(model)?; + if model.model_type != ModelType::Unigram { + return Err(format!( + "sample_encode_and_score unsupported for model_type {} (UNIGRAM only)", + model.model_type.as_str() + )); + } + if !sample.alpha.is_finite() || sample.alpha < 0.0 { + return Err("sample_encode_and_score: alpha must be finite and >= 0".into()); + } + if sample.num_samples == 0 { + return Err("sample_encode_and_score: num_samples must be >= 1".into()); + } + if sample.num_samples > 1024 { + return Err("sample_encode_and_score: num_samples must be <= 1024".into()); + } + if sample.include_best && !sample.wor { + return Err( + "sample_encode_and_score: include_best requires wor=true (official SP constraint)" + .into(), + ); + } + let opts = sample.encode.clone(); + let normalized = normalize_with_options(model, text, &opts)?; + let mut rng: StdRng = match sample.seed { + Some(s) => StdRng::seed_from_u64(s), + None => StdRng::from_entropy(), + }; + let rows = if sample.wor { + sample_encode_and_score_unigram_wor( + model, + &normalized, + sample.num_samples, + sample.alpha, + sample.include_best, + opts.clone(), + &mut rng, + )? + } else { + sample_encode_and_score_unigram( + model, + &normalized, + sample.num_samples, + sample.alpha, + opts.clone(), + &mut rng, + )? + }; + finalize_encode_scored(model, &opts, rows) +} + +pub fn sample_encode_and_score( + model: &SentencePieceModel, + text: &str, + num_samples: usize, + alpha: f32, +) -> Result, f32)>, String> { + sample_encode_and_score_with_options( + model, + text, + SampleEncodeAndScoreOptions { + num_samples, + alpha, + wor: false, + include_best: false, + seed: None, + encode: EncodeOptions::default(), + }, + ) +} + +/// Encode using the model's declared type (BPE, CHAR, UNIGRAM, or WORD). +pub fn encode_with_options( + model: &SentencePieceModel, + text: &str, + opts: EncodeOptions, +) -> Result, String> { + require_encode(model)?; + match model.model_type { + ModelType::Bpe => encode_bpe_with_options(model, text, opts), + ModelType::Char => encode_char_with_options(model, text, opts), + ModelType::Unigram => encode_unigram_with_options(model, text, opts), + ModelType::Word => encode_word_with_options(model, text, opts), + } +} + +/// Encode with fail-closed OOV default (`EncodeOptions::default()`). +pub fn encode(model: &SentencePieceModel, text: &str) -> Result, String> { + encode_with_options(model, text, EncodeOptions::default()) +} + +/// Decode token ids toward official SP (`▁` strip, BYTE UTF-8, unk_surface, denorm). +/// +/// Permanent gap vs lossy NFKC: without a non-empty `denormalizer_spec` charsmap, +/// reverse NFKC is not applied (same as stock SP for most models). +pub fn decode_pieces(model: &SentencePieceModel, tokens: &[u32]) -> Result { + decode_pieces_with_options(model, tokens, DecodeOptions::default()) +} + +/// Decode with ExtraOptions-style knobs (currently `reverse` only). +pub fn decode_pieces_with_options( + model: &SentencePieceModel, + tokens: &[u32], + opts: DecodeOptions, +) -> Result { + let reversed; + let tokens = if opts.reverse { + reversed = { + let mut v = tokens.to_vec(); + v.reverse(); + v + }; + reversed.as_slice() + } else { + tokens + }; + decode_pieces_inner(model, tokens) +} + +fn decode_pieces_inner(model: &SentencePieceModel, tokens: &[u32]) -> Result { + let mut piece_strs: Vec<&str> = Vec::with_capacity(tokens.len()); + for &tok in tokens { + let idx = tok as usize; + let piece = model + .pieces + .get(idx) + .ok_or_else(|| format!("token id {tok} out of range"))?; + piece_strs.push(piece.as_str()); + } + + let mut text = String::new(); + let mut byte_buf: Vec = Vec::new(); + let mut is_bos_ws = true; + + let flush_bytes = |buf: &mut Vec, text: &mut String| -> Result<(), String> { + if buf.is_empty() { + return Ok(()); + } + let mut offset = 0; + while offset < buf.len() { + match std::str::from_utf8(&buf[offset..]) { + Ok(s) => { + text.push_str(s); + offset = buf.len(); + } + Err(e) => { + let valid = e.valid_up_to(); + if valid > 0 { + text.push_str(std::str::from_utf8(&buf[offset..offset + valid]).unwrap()); + offset += valid; + } + // Invalid sequence: emit U+FFFD and skip one byte (SP-like). + text.push('\u{FFFD}'); + offset += 1; + } + } + } + buf.clear(); + Ok(()) + }; + + for (i, &tok) in tokens.iter().enumerate() { + let idx = tok as usize; + let rec = &model.piece_records[idx]; + let piece = piece_strs[i]; + + if rec.piece_type == PieceType::Byte || piece_to_byte(piece).is_some() { + if let Some(b) = piece_to_byte(piece) { + byte_buf.push(b); + } else { + return Err(format!("BYTE piece {piece:?} is not <0xXX>")); + } + continue; + } + + flush_bytes(&mut byte_buf, &mut text)?; + + if rec.piece_type == PieceType::Control { + if !text.is_empty() { + is_bos_ws = false; + } + continue; + } + + if rec.piece_type == PieceType::Unknown { + // When decoding from ids, piece == IdToPiece → emit unk_surface. + text.push_str(&model.unk_surface); + is_bos_ws = false; + continue; + } + + let mut p = piece; + if is_bos_ws + && (model.normalizer.add_dummy_prefix || model.normalizer.remove_extra_whitespaces) + { + if let Some(rest) = p.strip_prefix(SP_SPACE) { + p = rest; + // With remove_extra_whitespaces, do not keep bos_ws sticky after consume. + if model.normalizer.remove_extra_whitespaces { + // matches SP: has_bos_ws forced false after consume + } + } + } + + let decoded = p.replace(SP_SPACE, " "); + if !decoded.is_empty() { + text.push_str(&decoded); + is_bos_ws = false; + } else if !text.is_empty() { + is_bos_ws = false; + } + // Empty after bos ▁ consume: keep is_bos_ws true when remove_extra_whitespaces + // (SP keeps expecting bos until a non-empty surface appears). + } + flush_bytes(&mut byte_buf, &mut text)?; + + // Official SP: if denormalizer_ present, text = denormalizer_->Normalize(text). + if model.denorm_charsmap.is_some() { + text = sp_denormalize(&text, &model.denormalizer, model.denorm_charsmap.as_ref())?; + } + Ok(text) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/fixtures") + .join(name) + } + + #[test] + fn empty_bytes_fail_closed() { + let err = load_model_proto(&[]).unwrap_err(); + assert!(err.contains("empty")); + } + + #[test] + fn garbage_bytes_fail_closed() { + let err = load_model_proto(&[0x0a, 0x7f, 0x01]).unwrap_err(); + assert!(!err.is_empty()); + } + + #[test] + fn no_pieces_fail_closed() { + let bytes = { + let mut v = Vec::new(); + encode_varint(2 << 3, &mut v); + encode_varint(1, &mut v); + v + }; + let err = load_model_proto(&bytes).unwrap_err(); + assert!(err.contains("no pieces")); + } + + #[test] + fn minimal_model_proto_roundtrip() { + let bytes = encode_model_proto_pieces(&["", "▁a", "b"]); + let model = load_model_proto(&bytes).unwrap(); + assert_eq!(model.pieces.len(), 3); + assert_eq!(model.vocab.encode(""), 0); + assert_eq!(model.vocab.encode("▁a"), 1); + assert_eq!(model.vocab.encode("b"), 2); + assert!(model.vocab.validate_roundtrip()); + assert_eq!(model.model_type, ModelType::Unigram); + assert!(model.encode_supported); + assert_eq!(encode(&model, "a").unwrap(), vec![1]); + } + + #[test] + fn duplicate_piece_fail_closed() { + let bytes = encode_model_proto_pieces(&["a", "a"]); + let err = load_model_proto(&bytes).unwrap_err(); + assert!(err.contains("duplicate")); + } + + #[test] + fn vocab_json_roundtrip_path() { + let mph = PerfectHashVocab::from_words(vec!["a".into(), "b".into()]); + let json = serde_json::to_string(&mph).unwrap(); + let loaded = load_vocab_json(&json).unwrap(); + assert!(loaded.validate_roundtrip()); + assert_eq!(loaded.encode("a"), mph.encode("a")); + } + + #[test] + fn skips_unknown_fields() { + let mut bytes = encode_model_proto_pieces(&["x", "y"]); + encode_varint((99 << 3) | 5, &mut bytes); + bytes.extend_from_slice(&[1, 2, 3, 4]); + let model = load_model_proto(&bytes).unwrap(); + assert_eq!(model.pieces, vec!["x".to_string(), "y".to_string()]); + } + + #[test] + fn non_utf8_piece_fail_closed() { + let mut sp = Vec::new(); + encode_varint((1 << 3) | 2, &mut sp); + encode_varint(2, &mut sp); + sp.extend_from_slice(&[0xff, 0xfe]); + let mut bytes = Vec::new(); + encode_len_delim(1, &sp, &mut bytes); + let err = load_model_proto(&bytes).unwrap_err(); + assert!(err.contains("UTF-8")); + } + + #[test] + fn oversized_length_fail_closed() { + let err = load_model_proto(&[0x0a, 0xff, 0xff, 0xff, 0xff, 0x0f]).unwrap_err(); + assert!(!err.is_empty()); + } + + #[test] + fn char_encode_path() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(0.0), Some(1)), + ("a".into(), Some(0.0), Some(1)), + ("b".into(), Some(0.0), Some(1)), + ], + model_type: Some(4), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(model.encode_supported); + assert_eq!(model.model_type, ModelType::Char); + let ids = encode(&model, "ab").unwrap(); + assert_eq!(ids, vec![1, 2, 3]); + let decoded = decode_pieces(&model, &ids).unwrap(); + assert_eq!(decoded, "ab"); + } + + #[test] + fn extra_options_reverse_bos_eos_composition() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("".into(), Some(0.0), Some(3)), + ("".into(), Some(0.0), Some(3)), + ("▁".into(), Some(0.0), Some(1)), + ("a".into(), Some(0.0), Some(1)), + ("b".into(), Some(0.0), Some(1)), + ], + model_type: Some(4), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let body = encode(&model, "ab").unwrap(); + assert_eq!(body, vec![3, 4, 5]); + // Stock documented `reverse:bos:eos`: reverse body, then bos, then eos. + let mut expected = body.clone(); + expected.reverse(); + expected.insert(0, model.bos_id.unwrap()); + expected.push(model.eos_id.unwrap()); + let got = encode_with_options( + &model, + "ab", + EncodeOptions { + reverse: true, + add_bos: true, + add_eos: true, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(got, expected); + assert_eq!(got, vec![1, 5, 4, 3, 2]); + let fwd = decode_pieces(&model, &body).unwrap(); + let rev = decode_pieces_with_options( + &model, + &{ + let mut r = body.clone(); + r.reverse(); + r + }, + DecodeOptions { reverse: true }, + ) + .unwrap(); + assert_eq!(fwd, rev); + } + + #[test] + fn extra_options_str_order_independent() { + let mut a = EncodeOptions::default(); + a.apply_extra_options_str("eos:bos:reverse").unwrap(); + let mut b = EncodeOptions::default(); + b.apply_extra_options_str("reverse:bos:eos").unwrap(); + assert!(a.reverse && a.add_bos && a.add_eos); + assert_eq!(a.reverse, b.reverse); + assert_eq!(a.add_bos, b.add_bos); + assert_eq!(a.add_eos, b.add_eos); + let mut bad = EncodeOptions::default(); + assert!(bad.apply_extra_options_str("bos:unk").is_err()); + } + + #[test] + fn bpe_encode_merges_by_score() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-5.0), Some(1)), + ("a".into(), Some(-4.0), Some(1)), + ("b".into(), Some(-4.0), Some(1)), + ("ab".into(), Some(-1.0), Some(1)), + ("▁a".into(), Some(-0.5), Some(1)), + ("▁ab".into(), Some(0.0), Some(1)), + ], + model_type: Some(2), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(model.encode_supported); + let ids = encode(&model, "ab").unwrap(); + assert_eq!(ids, vec![6]); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "ab"); + } + + #[test] + fn unigram_viterbi_prefers_highest_score_path() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ("b".into(), Some(-5.0), Some(1)), + ("ab".into(), Some(-3.0), Some(1)), + ("▁a".into(), Some(-2.0), Some(1)), + ("▁ab".into(), Some(-0.5), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(model.encode_supported); + assert_eq!(model.model_type, ModelType::Unigram); + let ids = encode(&model, "ab").unwrap(); + assert_eq!(ids, vec![6]); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "ab"); + } + + #[test] + fn unigram_viterbi_multi_piece_when_better() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ("b".into(), Some(-1.0), Some(1)), + ("▁a".into(), Some(-1.0), Some(1)), + ("▁ab".into(), Some(-20.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let ids = encode(&model, "ab").unwrap(); + assert_eq!(ids, vec![4, 3]); + } + + #[test] + fn unigram_incomplete_coverage_fail_closed() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("▁".into(), Some(-1.0), Some(1)), + ("a".into(), Some(-1.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let err = encode(&model, "ab").unwrap_err(); + assert!(err.contains("no covering segmentation"), "{err}"); + } + + #[test] + fn word_model_encodes_whitespace_words() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("\u{2581}hello".into(), Some(0.0), Some(1)), + ("\u{2581}world".into(), Some(0.0), Some(1)), + ], + model_type: Some(3), // WORD + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!( + model.encode_supported, + "{:?}", + model.encode_unsupported_reason + ); + assert_eq!(model.model_type, ModelType::Word); + let ids = encode(&model, "hello world").unwrap(); + assert_eq!(ids, vec![1, 2]); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "hello world"); + } + + #[test] + fn treat_whitespace_as_suffix_normalize_and_word() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("hello\u{2581}".into(), Some(0.0), Some(1)), + ("world\u{2581}".into(), Some(0.0), Some(1)), + ], + model_type: Some(3), + treat_whitespace_as_suffix: Some(true), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(model.encode_supported); + assert!(model.treat_whitespace_as_suffix); + assert_eq!( + normalize(&model, "hello world").unwrap(), + "hello\u{2581}world\u{2581}" + ); + let ids = encode(&model, "hello world").unwrap(); + assert_eq!(ids, vec![1, 2]); + } + + #[test] + fn control_symbols_resolve_and_decode_empty() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("".into(), Some(0.0), Some(3)), + ("".into(), Some(0.0), Some(3)), + ("".into(), Some(0.0), Some(3)), + ("\u{2581}a".into(), Some(-1.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert_eq!(model.bos_id, Some(1)); + assert_eq!(model.eos_id, Some(2)); + assert_eq!(model.pad_id, Some(3)); + assert_eq!(model.unk_id, 0); + // Control surfaces decode to empty. + assert_eq!(decode_pieces(&model, &[1, 4, 2]).unwrap(), "a"); + // Control piece string must not match as a NORMAL segmentation edge. + let ids = encode(&model, "a").unwrap(); + assert_eq!(ids, vec![4]); + // ExtraOptions bos/eos wrap after merge (stock ApplyExtraOptions order). + let wrapped = encode_with_options( + &model, + "a", + EncodeOptions { + add_bos: true, + add_eos: true, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(wrapped, vec![1, 4, 2]); + assert_eq!(decode_pieces(&model, &wrapped).unwrap(), "a"); + // reverse alone on a single body piece is a no-op before wrap. + let rev_wrapped = encode_with_options( + &model, + "a", + EncodeOptions { + reverse: true, + add_bos: true, + add_eos: true, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(rev_wrapped, vec![1, 4, 2]); + // reverse:bos:eos on a multi-id body (manual body + finalize composition). + let mut body = vec![4u32, 4u32]; + body.reverse(); + body.insert(0, 1); + body.push(2); + assert_eq!(body, vec![1, 4, 4, 2]); + // Decode ExtraOptions reverse undoes id order before detokenize. + let fwd = decode_pieces(&model, &[1, 4, 2]).unwrap(); + let rev = decode_pieces_with_options(&model, &[2, 4, 1], DecodeOptions { reverse: true }) + .unwrap(); + assert_eq!(fwd, rev); + // Refuse when CONTROL bos/eos missing. + let no_ctrl = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("\u{2581}a".into(), Some(-1.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let bare = load_model_proto(&no_ctrl.encode()).unwrap(); + assert!(bare.bos_id.is_none()); + let err = encode_with_options( + &bare, + "a", + EncodeOptions { + add_bos: true, + ..Default::default() + }, + ) + .unwrap_err(); + assert!(err.contains("add_bos"), "{err}"); + } + + #[test] + fn sample_encode_and_score_with_replacement() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("\u{2581}".into(), Some(-1.0), Some(1)), + ("a".into(), Some(-1.0), Some(1)), + ("b".into(), Some(-1.0), Some(1)), + ("ab".into(), Some(-0.5), Some(1)), + ("\u{2581}a".into(), Some(-0.5), Some(1)), + ("\u{2581}ab".into(), Some(-0.1), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let scored = sample_encode_and_score_with_options( + &model, + "ab", + SampleEncodeAndScoreOptions { + num_samples: 3, + alpha: 0.5, + wor: false, + include_best: false, + seed: Some(7), + encode: EncodeOptions::default(), + }, + ) + .unwrap(); + assert_eq!(scored.len(), 3); + for (ids, score) in &scored { + assert!(!ids.is_empty()); + assert!(score.is_finite(), "{score}"); + } + let wor = sample_encode_and_score_with_options( + &model, + "ab", + SampleEncodeAndScoreOptions { + num_samples: 2, + alpha: 0.5, + wor: true, + include_best: true, + seed: Some(1), + encode: EncodeOptions::default(), + }, + ) + .unwrap(); + assert!(!wor.is_empty()); + // include_best → first row is Viterbi with log-inclusion 0. + assert_eq!(wor[0].1, 0.0); + let best = encode(&model, "ab").unwrap(); + assert_eq!(wor[0].0, best); + let mut seen = HashSet::new(); + for (ids, score) in &wor { + assert!(score.is_finite(), "{score}"); + assert!( + seen.insert(ids.clone()), + "WOR samples must be unique: {ids:?}" + ); + } + let ib_err = sample_encode_and_score_with_options( + &model, + "ab", + SampleEncodeAndScoreOptions { + num_samples: 2, + alpha: 0.5, + wor: false, + include_best: true, + seed: Some(1), + encode: EncodeOptions::default(), + }, + ) + .unwrap_err(); + assert!(ib_err.contains("include_best"), "{ib_err}"); + } + + #[test] + fn refuses_invalid_precompiled_charsmap() { + let fixture = ModelProtoFixture { + pieces: vec![("▁".into(), Some(0.0), None), ("a".into(), Some(0.0), None)], + model_type: Some(2), + precompiled_charsmap: Some(vec![1, 2, 3]), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(!model.encode_supported); + assert!(encode(&model, "a") + .unwrap_err() + .contains("precompiled_charsmap")); + } + + #[test] + fn accepts_valid_custom_charsmap_fixture() { + let path = fixture("sp_tiny_custom_norm_unigram.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!( + model.encode_supported, + "{:?}", + model.encode_unsupported_reason + ); + assert!(model.charsmap.is_some()); + // Fullwidth A rewritten to ASCII A; A is OOV → fail closed without BF. + let err = encode(&model, "\u{FF21}bc").unwrap_err(); + assert!( + err.contains("no covering") || err.contains("not in vocab"), + "{err}" + ); + // Covered ASCII path still works. + let ids = encode(&model, "hello").unwrap(); + assert_eq!(ids, vec![1, 12, 4, 6, 6, 2]); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "hello"); + } + + #[test] + fn identity_trained_fixture_encode() { + let path = fixture("sp_tiny_identity_unigram.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!(model.encode_supported); + assert!(model.charsmap.is_none()); + assert_eq!(encode(&model, "hello").unwrap(), vec![1, 12, 4, 6, 6, 2]); + assert_eq!( + encode(&model, " foo bar ").unwrap(), + vec![1, 5, 2, 2, 1, 7, 3, 9] + ); + } + + #[test] + fn byte_fallback_bpe_fixture() { + let path = fixture("sp_tiny_identity_bpe_bytefallback.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!( + model.encode_supported, + "{:?}", + model.encode_unsupported_reason + ); + assert!(model.byte_fallback); + let ids = encode(&model, "Z").unwrap(); + // ▁ + <0x5A> + assert_eq!(decode_pieces(&model, &ids).unwrap(), "Z"); + let emoji = encode(&model, "😀").unwrap(); + assert_eq!(decode_pieces(&model, &emoji).unwrap(), "😀"); + let hello = encode(&model, "hello").unwrap(); + assert_eq!(decode_pieces(&model, &hello).unwrap(), "hello"); + } + + #[test] + fn refuses_byte_fallback_without_byte_pieces() { + let fixture = ModelProtoFixture { + pieces: vec![("a".into(), Some(0.0), None)], + model_type: Some(2), + byte_fallback: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(!model.encode_supported); + assert!(encode(&model, "a").unwrap_err().contains("byte_fallback")); + } + + #[test] + fn missing_seed_char_fail_closed() { + let fixture = ModelProtoFixture { + pieces: vec![("▁".into(), Some(0.0), None), ("a".into(), Some(0.0), None)], + model_type: Some(2), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let err = encode(&model, "z").unwrap_err(); + assert!(err.contains("not in vocab")); + } + + #[test] + fn decode_unk_uses_unk_surface() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(0.0), Some(1)), + ("a".into(), Some(0.0), Some(1)), + ], + model_type: Some(4), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert_eq!( + decode_pieces(&model, &[1, 0, 2]).unwrap(), + format!("{}a", DEFAULT_UNK_SURFACE) + ); + } + + #[test] + fn silent_unk_opt_in_matches_stock_shape() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-5.0), Some(1)), + ("a".into(), Some(-4.0), Some(1)), + ], + model_type: Some(4), // CHAR + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert!(encode(&model, "z").is_err()); + let ids = encode_with_options( + &model, + "z", + EncodeOptions { + allow_silent_unk: true, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(ids, vec![1, 0]); // ▁ + + assert_eq!(decode_pieces(&model, &ids).unwrap(), DEFAULT_UNK_SURFACE); + } + + #[test] + fn silent_unk_bpe_and_unigram() { + let bpe = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-5.0), Some(1)), + ("a".into(), Some(-4.0), Some(1)), + ], + model_type: Some(2), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&bpe.encode()).unwrap(); + let opts = EncodeOptions { + allow_silent_unk: true, + ..Default::default() + }; + assert_eq!( + encode_with_options(&model, "z", opts.clone()).unwrap(), + vec![1, 0] + ); + + let ug = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&ug.encode()).unwrap(); + assert_eq!(encode_with_options(&model, "z", opts).unwrap(), vec![1, 0]); + } + + #[test] + fn denorm_fixture_applies_reverse_charsmap() { + let path = fixture("sp_tiny_identity_denorm_unigram.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!(model.denorm_charsmap.is_some()); + let ids = encode(&model, "TM").unwrap(); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "\u{2122}"); + let ids = encode(&model, "xTMy").unwrap(); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "x\u{2122}y"); + // Covered identity path unchanged. + assert_eq!( + decode_pieces(&model, &encode(&model, "hello").unwrap()).unwrap(), + "hello" + ); + } + + #[test] + fn denorm_crafted_fixture_roundtrip() { + let blob = std::fs::read(fixture("sp_denorm_charsmap.bin")).unwrap(); + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-1.0), Some(1)), + ("T".into(), Some(-1.0), Some(1)), + ("M".into(), Some(-1.0), Some(1)), + ("▁TM".into(), Some(0.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + denorm_precompiled_charsmap: Some(blob), + denorm_add_dummy_prefix: Some(false), + denorm_remove_extra_whitespaces: Some(false), + denorm_escape_whitespaces: Some(false), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let ids = encode(&model, "TM").unwrap(); + assert_eq!(ids, vec![4]); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "\u{2122}"); + } + + #[test] + fn user_defined_forced_piece_unigram_and_bpe() { + let ug = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ("b".into(), Some(-5.0), Some(1)), + ("ab".into(), Some(-0.1), Some(1)), + ("▁a".into(), Some(-2.0), Some(1)), + ("".into(), Some(0.0), Some(4)), // USER_DEFINED + ("f".into(), Some(-5.0), Some(1)), + ("o".into(), Some(-5.0), Some(1)), + ("<".into(), Some(-5.0), Some(1)), + (">".into(), Some(-5.0), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&ug.encode()).unwrap(); + assert!(!model.prefix_matcher.is_empty()); + assert_eq!(encode(&model, "").unwrap(), vec![1, 6]); + assert_eq!(encode(&model, "ab").unwrap(), vec![1, 6, 4]); + // Runtime forced piece must exist in vocab. + let err = encode_with_options( + &model, + "a", + EncodeOptions { + forced_pieces: vec!["not_in_vocab".into()], + ..Default::default() + }, + ) + .unwrap_err(); + assert!(err.contains("forced_pieces"), "{err}"); + + let bpe = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-5.0), Some(1)), + ("a".into(), Some(-4.0), Some(1)), + ("b".into(), Some(-4.0), Some(1)), + ("ab".into(), Some(-1.0), Some(1)), + ("▁a".into(), Some(-0.5), Some(1)), + ("".into(), Some(0.0), Some(4)), + ("f".into(), Some(-4.0), Some(1)), + ("o".into(), Some(-4.0), Some(1)), + ("<".into(), Some(-4.0), Some(1)), + (">".into(), Some(-4.0), Some(1)), + ], + model_type: Some(2), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&bpe.encode()).unwrap(); + assert_eq!(encode(&model, "ab").unwrap(), vec![5, 6, 3]); + } + + #[test] + fn unigram_nbest_and_sample_seeded() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ("b".into(), Some(-5.0), Some(1)), + ("ab".into(), Some(-3.0), Some(1)), + ("▁a".into(), Some(-2.0), Some(1)), + ("▁ab".into(), Some(-0.5), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let nbest = nbest_encode(&model, "ab", 5).unwrap(); + assert!(!nbest.is_empty()); + assert_eq!(nbest[0].0, encode(&model, "ab").unwrap()); + // Distinct segmentations among top-n. + let uniq: HashSet> = nbest.iter().map(|(ids, _)| ids.clone()).collect(); + assert!(uniq.len() >= 2, "{nbest:?}"); + + let sampled = sample_encode_with_options( + &model, + "ab", + SampleEncodeOptions { + nbest_size: 5, + alpha: 1.0, + seed: Some(42), + encode: EncodeOptions::default(), + }, + ) + .unwrap(); + assert!(uniq.contains(&sampled), "{sampled:?} not in {uniq:?}"); + + let lattice_sample = sample_encode_with_options( + &model, + "ab", + SampleEncodeOptions { + nbest_size: -1, + alpha: 0.0, + seed: Some(7), + encode: EncodeOptions::default(), + }, + ) + .unwrap(); + assert!(!lattice_sample.is_empty()); + + let bpe = ModelProtoFixture { + pieces: vec![ + ("▁".into(), Some(-5.0), Some(1)), + ("a".into(), Some(-4.0), Some(1)), + ("b".into(), Some(-4.0), Some(1)), + ("ab".into(), Some(-1.0), Some(1)), + ("▁a".into(), Some(-0.5), Some(1)), + ("▁ab".into(), Some(0.0), Some(1)), + ], + model_type: Some(2), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let bpe_model = load_model_proto(&bpe.encode()).unwrap(); + assert!(nbest_encode(&bpe_model, "ab", 3) + .unwrap_err() + .contains("UNIGRAM")); + let det = encode(&bpe_model, "ab").unwrap(); + let mut variants = HashSet::new(); + for seed in 0..30u64 { + let sampled = sample_encode_with_options( + &bpe_model, + "ab", + SampleEncodeOptions { + nbest_size: -1, + alpha: 0.5, + seed: Some(seed), + encode: EncodeOptions::default(), + }, + ) + .unwrap(); + variants.insert(sampled); + } + assert!( + variants.len() >= 2, + "BPE-dropout should diversify, got {variants:?}" + ); + assert!( + variants.contains(&det), + "deterministic encode must remain reachable" + ); + } + + #[test] + fn word_official_split_ignores_trainer_suffix_flag() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("hello\u{2581}".into(), Some(0.0), Some(1)), + ("world\u{2581}".into(), Some(0.0), Some(1)), + ], + model_type: Some(3), + treat_whitespace_as_suffix: Some(true), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + assert_eq!(encode(&model, "hello world").unwrap(), vec![1, 2]); + let compat = encode_with_options( + &model, + "hello world", + EncodeOptions { + word_official_split: true, + allow_silent_unk: true, + ..Default::default() + }, + ) + .unwrap(); + // Flag-blind prefix SplitIntoWords → three OOV spans → three unks at + // model level; stock SP *processor* merges consecutive unks by default. + assert_eq!(compat, vec![0]); + let piece_level = encode_with_options( + &model, + "hello world", + EncodeOptions { + word_official_split: true, + allow_silent_unk: true, + merge_consecutive_unk: false, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(piece_level, vec![0, 0, 0]); + } + + #[test] + fn consecutive_unk_merge_matches_processor_default() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(0.0), Some(1)), + ("a".into(), Some(0.0), Some(1)), + ], + model_type: Some(4), // CHAR + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let silent = EncodeOptions { + allow_silent_unk: true, + ..Default::default() + }; + // xyz → ▁ + three OOVs → ▁ + one merged unk (stock processor). + assert_eq!( + encode_with_options(&model, "xyz", silent.clone()).unwrap(), + vec![1, 0] + ); + // xay → ▁ + unk + a + unk (non-adjacent unks preserved). + assert_eq!( + encode_with_options(&model, "xay", silent.clone()).unwrap(), + vec![1, 0, 2, 0] + ); + let piece = EncodeOptions { + allow_silent_unk: true, + merge_consecutive_unk: false, + ..Default::default() + }; + assert_eq!( + encode_with_options(&model, "xyz", piece).unwrap(), + vec![1, 0, 0, 0] + ); + } + + #[test] + fn consecutive_unk_merge_resets_on_control_ids() { + // Stock processor: CONTROL resets is_prev_unk so unk runs on either side + // of BOS/EOS do not merge across the control. + let unk = 0u32; + let bos = 1u32; + let eos = 2u32; + let ids = [unk, unk, bos, unk, unk, eos, unk]; + assert_eq!( + merge_consecutive_unk_ids_with_controls(&ids, unk, &[bos, eos]), + vec![unk, bos, unk, eos, unk] + ); + // Without control awareness, bos≠unk so runs already split — but when + // a control id equals neither unk nor a "normal" separator, reset still + // matters for id lists that inject CONTROL between unks. + assert_eq!(merge_consecutive_unk_ids(&[unk, unk, unk], unk), vec![unk]); + } + + #[test] + fn merge_consecutive_unk_skipped_with_byte_fallback() { + // byte_fallback never emits unk ids; merge flag is a no-op. + let path = fixture("sp_tiny_identity_bpe_bytefallback.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!(model.byte_fallback); + let ids = encode(&model, "z").unwrap(); + assert!(!ids.is_empty()); + assert!(!ids.contains(&model.unk_id)); + } + + #[test] + fn empty_denorm_is_permanent_gap_for_lossy_norm() { + // Custom charsmap rewrites ™→TM; without denorm, decode cannot restore ™. + let path = fixture("sp_tiny_custom_norm_unigram.model"); + let model = load_model_proto_file(path.to_str().unwrap()).unwrap(); + assert!(model.denorm_charsmap.is_none()); + // Covered ASCII still round-trips. + let ids = encode(&model, "hello").unwrap(); + assert_eq!(decode_pieces(&model, &ids).unwrap(), "hello"); + } +} diff --git a/src/rust/guardd/src/sp_lattice.rs b/src/rust/guardd/src/sp_lattice.rs new file mode 100644 index 0000000..a49d514 --- /dev/null +++ b/src/rust/guardd/src/sp_lattice.rs @@ -0,0 +1,1026 @@ +//! UNIGRAM lattice: Viterbi, A* n-best, Gumbel-perturbed n-best (WOR), and +//! forward-filter / backward sample. +//! +//! Mirrors official SentencePiece `unigram_model.cc` Lattice semantics closely +//! enough for cross-checks on crafted fixtures. Not a claim of bit-identical +//! floating-point. Pathological agenda growth **refuses by default**; opt-in +//! `EncodeOptions.nbest_stock_compat` matches stock shrink +//! (`min(512, nbest_size*10)`) and timeout→Viterbi fallback. + +use crate::sentencepiece::{byte_piece_ids, EncodeOptions, PieceType, SentencePieceModel}; +use rand::Rng; +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashMap}; +use std::time::Instant; + +const UNK_PENALTY: f32 = 10.0; +const MAX_NBEST: usize = 1024; +const MAX_AGENDA: usize = 10_000; +/// Official stock shrink floor (`kMinAgendaSize` in `unigram_model.cc`). +const MIN_AGENDA: usize = 512; +/// Official default n-best timeout (ms). Default mode refuses; stock-compat +/// falls back to Viterbi only when the agenda also overflows (stock policy). +const DEFAULT_NBEST_TIMEOUT_MS: u128 = 30_000; + +/// Limits for A* n-best (production defaults + test overrides). +#[derive(Debug, Clone, Copy)] +struct NbestLimits { + max_agenda: usize, + min_agenda: usize, + timeout_ms: u128, + stock_compat: bool, +} + +impl NbestLimits { + fn from_opts(opts: &EncodeOptions) -> Self { + Self { + max_agenda: MAX_AGENDA, + min_agenda: MIN_AGENDA, + timeout_ms: DEFAULT_NBEST_TIMEOUT_MS, + stock_compat: opts.nbest_stock_compat, + } + } +} + +/// Internal A* outcome: paths, or stock-compat timeout → Viterbi fallback. +#[derive(Debug)] +enum NbestAstarOutcome { + Paths(Vec<(Vec, f32)>), + ViterbiFallback, +} + +/// Official SP: USER_DEFINED pieces get `0.1 * (char_length - 1)` so they win Viterbi. +#[inline] +pub fn user_defined_score(char_len: usize) -> f32 { + 0.1 * (char_len.saturating_sub(1) as f32) +} + +#[derive(Debug, Clone)] +struct Node { + /// Vocab id; `u32::MAX` for BOS/EOS sentinels. + id: u32, + /// Start character position. + pos: usize, + /// Length in Unicode scalars. + length: usize, + score: f32, + #[allow(dead_code)] + node_id: usize, + /// Best path score from BOS through this node (filled by Viterbi). + backtrace_score: f32, + /// Prev node id in Viterbi path (`usize::MAX` = none). + prev: usize, + /// True when this is an unk / byte_fallback single-char edge. + is_unk: bool, +} + +struct Lattice { + /// Character count of the sentence. + len: usize, + nodes: Vec, + begin_nodes: Vec>, + end_nodes: Vec>, +} + +impl Lattice { + fn new(char_len: usize) -> Self { + let mut nodes = Vec::new(); + let mut begin_nodes = vec![Vec::new(); char_len + 1]; + let mut end_nodes = vec![Vec::new(); char_len + 1]; + + // BOS + let bos_id = 0; + nodes.push(Node { + id: u32::MAX, + pos: 0, + length: 0, + score: 0.0, + node_id: bos_id, + backtrace_score: 0.0, + prev: usize::MAX, + is_unk: false, + }); + end_nodes[0].push(bos_id); + + // EOS + let eos_id = 1; + nodes.push(Node { + id: u32::MAX, + pos: char_len, + length: 0, + score: 0.0, + node_id: eos_id, + backtrace_score: 0.0, + prev: usize::MAX, + is_unk: false, + }); + begin_nodes[char_len].push(eos_id); + + Self { + len: char_len, + nodes, + begin_nodes, + end_nodes, + } + } + + fn insert(&mut self, pos: usize, length: usize, id: u32, score: f32, is_unk: bool) -> usize { + let node_id = self.nodes.len(); + self.nodes.push(Node { + id, + pos, + length, + score, + node_id, + backtrace_score: 0.0, + prev: usize::MAX, + is_unk, + }); + self.begin_nodes[pos].push(node_id); + self.end_nodes[pos + length].push(node_id); + node_id + } + + fn bos(&self) -> usize { + 0 + } + + fn eos(&self) -> usize { + 1 + } + + /// Left-to-right Viterbi; fills `backtrace_score` / `prev`. + fn viterbi(&mut self) -> Result<(Vec, f32), String> { + let len = self.len; + for pos in 0..=len { + let begin = self.begin_nodes[pos].clone(); + for &r_id in &begin { + let mut best_score = f32::NEG_INFINITY; + let mut best_prev = usize::MAX; + for &l_id in &self.end_nodes[pos] { + let score = self.nodes[l_id].backtrace_score + self.nodes[r_id].score; + if score > best_score { + best_score = score; + best_prev = l_id; + } + } + if best_prev == usize::MAX { + return Err("UNIGRAM lattice: Viterbi failed to find a path".into()); + } + self.nodes[r_id].prev = best_prev; + self.nodes[r_id].backtrace_score = best_score; + } + } + + let eos = self.eos(); + let score = self.nodes[eos].backtrace_score; + let mut path = Vec::new(); + let mut cur = self.nodes[eos].prev; + while cur != self.bos() { + if cur == usize::MAX { + return Err("UNIGRAM lattice: corrupt Viterbi backpointer".into()); + } + path.push(cur); + cur = self.nodes[cur].prev; + } + path.reverse(); + Ok((path, score)) + } + + fn log_sum_exp(a: f32, b: f32, init: bool) -> f32 { + if init { + return b; + } + if a == f32::NEG_INFINITY { + return b; + } + if b == f32::NEG_INFINITY { + return a; + } + let (max_v, min_v) = if a > b { (a, b) } else { (b, a) }; + max_v + (min_v - max_v).exp().ln_1p() + } + + fn forward(&self, inv_theta: f32) -> Vec { + let mut alpha = vec![f32::NEG_INFINITY; self.nodes.len()]; + alpha[self.bos()] = 0.0; + for pos in 0..=self.len { + for &r_id in &self.begin_nodes[pos] { + let mut init = true; + let mut acc = f32::NEG_INFINITY; + for &l_id in &self.end_nodes[pos] { + let cand = inv_theta * self.nodes[l_id].score + alpha[l_id]; + acc = Self::log_sum_exp(acc, cand, init); + init = false; + } + alpha[r_id] = acc; + } + } + alpha + } +} + +#[derive(Clone)] +struct Hyp { + node: usize, + next: Option, // index into hyp pool + fx: f32, + gx: f32, +} + +#[derive(Clone)] +struct HypOrd { + idx: usize, + fx: f32, +} + +impl PartialEq for HypOrd { + fn eq(&self, other: &Self) -> bool { + self.fx == other.fx + } +} +impl Eq for HypOrd {} +impl PartialOrd for HypOrd { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for HypOrd { + fn cmp(&self, other: &Self) -> Ordering { + // Max-heap by fx + self.fx.partial_cmp(&other.fx).unwrap_or(Ordering::Equal) + } +} + +/// Official SP `Gumbel()`: `-log(-log(U(0,1) + eps))`. +#[inline] +fn gumbel_noise(rng: &mut impl Rng) -> f32 { + const EPS: f32 = 1e-7; + let u: f32 = rng.gen::(); + -(-(u + EPS).ln()).ln() +} + +/// Softplus-style numerically stable `x - softplus(x)` helper used by truncated Gumbel. +#[inline] +fn log_gumbel_truncated_adjust(top_fx: f32, perturbed: f32, max_perturbed: f32) -> f32 { + // Official: v = top.fx - pert + log1p(-exp(pert - max)) + // adjusted = top.fx - max(0, v) - log1p(exp(-|v|)) + let v = top_fx - perturbed + (-(perturbed - max_perturbed).exp()).ln_1p(); + top_fx - v.max(0.0) - (-v.abs()).exp().ln_1p() +} + +/// Log Gumbel survival `log(1 - exp(-exp(x)))` with the official series for small x. +#[inline] +fn log_gumbel_survival(x: f64) -> f64 { + let y = x.exp(); + if x <= -10.0 { + // Series expansion of the log Gumbel survival function up to eps. + x - (y / 2.0) + (y.powi(2) / 24.0) - y.powi(4) / 2880.0 + } else { + (-((-y).exp_m1())).ln() + } +} + +fn extract_path_from_hyp(pool: &[Hyp], hyp_idx: usize) -> Vec { + let mut path = Vec::new(); + let mut cur = pool[hyp_idx].next; + while let Some(i) = cur { + // Chain: BOS_hyp -> n1 -> n2 -> ... -> EOS_hyp (next=None). + if pool[i].next.is_none() { + break; + } + path.push(pool[i].node); + cur = pool[i].next; + } + path +} + +fn nbest_astar( + lattice: &Lattice, + nbest_size: usize, + sample: bool, + inv_theta: f32, + alpha_fwd: Option<&[f32]>, + mut rng: Option<&mut R>, + limits: NbestLimits, +) -> Result { + if nbest_size == 0 { + return Ok(NbestAstarOutcome::Paths(Vec::new())); + } + if nbest_size == 1 && !sample { + return Err("nbest_astar: use viterbi for nbest_size==1".into()); + } + + let mut pool: Vec = Vec::new(); + let mut agenda: BinaryHeap = BinaryHeap::new(); + let start = Instant::now(); + + let eos = lattice.eos(); + let eos_fx = if sample { + let r = rng + .as_deref_mut() + .ok_or_else(|| "nbest_astar: sample mode requires rng".to_string())?; + gumbel_noise(r) + } else { + lattice.nodes[eos].backtrace_score + }; + pool.push(Hyp { + node: eos, + next: None, + fx: eos_fx, + gx: 0.0, + }); + agenda.push(HypOrd { + idx: 0, + fx: pool[0].fx, + }); + + let mut results: Vec<(Vec, f32)> = Vec::new(); + + while let Some(top) = agenda.pop() { + let hyp_idx = top.idx; + let node = pool[hyp_idx].node; + + if node == lattice.bos() { + let path = extract_path_from_hyp(&pool, hyp_idx); + results.push((path, pool[hyp_idx].fx)); + if results.len() == nbest_size { + break; + } + continue; + } + + let ends = &lattice.end_nodes[lattice.nodes[node].pos]; + if sample { + let alpha = alpha_fwd + .ok_or_else(|| "nbest_astar: sample mode requires forward marginals".to_string())?; + let rng = match rng.as_deref_mut() { + Some(r) => r, + None => { + return Err("nbest_astar: sample mode requires rng".into()); + } + }; + let z = alpha[node]; + let top_gx = pool[hyp_idx].gx; + let top_fx = pool[hyp_idx].fx; + let mut probs = Vec::with_capacity(ends.len()); + let mut perturbed = Vec::with_capacity(ends.len()); + let mut max_score = f32::NEG_INFINITY; + for &l_id in ends { + let p = top_gx + alpha[l_id] + inv_theta * lattice.nodes[l_id].score - z; + let g = p + gumbel_noise(rng); + if g > max_score { + max_score = g; + } + probs.push(p); + perturbed.push(g); + } + for (i, &l_id) in ends.iter().enumerate() { + let fx = log_gumbel_truncated_adjust(top_fx, perturbed[i], max_score); + let new_idx = pool.len(); + pool.push(Hyp { + node: l_id, + next: Some(hyp_idx), + fx, + gx: probs[i], + }); + agenda.push(HypOrd { idx: new_idx, fx }); + } + } else { + for &l_id in ends { + let gx = lattice.nodes[l_id].score + pool[hyp_idx].gx; + let fx = lattice.nodes[l_id].backtrace_score + pool[hyp_idx].gx; + let new_idx = pool.len(); + pool.push(Hyp { + node: l_id, + next: Some(hyp_idx), + fx, + gx, + }); + agenda.push(HypOrd { idx: new_idx, fx }); + } + } + + // Official SP (`unigram_model.cc` Lattice::NBest): when agenda ≥ 10000, + // either timeout→Viterbi fallback (if timeout_ms > 0) or shrink to + // min(512, nbest_size*10). Default here refuses; stock_compat matches. + if agenda.len() >= limits.max_agenda { + if limits.stock_compat { + if limits.timeout_ms > 0 && start.elapsed().as_millis() >= limits.timeout_ms { + return Ok(NbestAstarOutcome::ViterbiFallback); + } + let keep = limits.min_agenda.min(nbest_size.saturating_mul(10)).max(1); + let mut kept: Vec = Vec::with_capacity(keep); + for _ in 0..keep { + if let Some(h) = agenda.pop() { + kept.push(h); + } else { + break; + } + } + agenda = BinaryHeap::from(kept); + } else { + return Err(format!( + "UNIGRAM nbest: agenda exceeded {} (pathological lattice); refuse rather than shrink (set nbest_stock_compat=true for stock shrink/timeout fallback)", + limits.max_agenda + )); + } + } else if !limits.stock_compat && start.elapsed().as_millis() >= limits.timeout_ms { + // Stricter default: refuse on wall-clock timeout even without + // agenda overflow (stock only checks timeout inside the overflow + // branch). Avoids silently returning a partial n-best list. + return Err(format!( + "UNIGRAM nbest: exceeded {}ms timeout; refuse rather than Viterbi fallback (set nbest_stock_compat=true for stock policy)", + limits.timeout_ms + )); + } + } + + Ok(NbestAstarOutcome::Paths(results)) +} + +fn expand_path_ids( + model: &SentencePieceModel, + chars: &[char], + lattice: &Lattice, + path: &[usize], + opts: EncodeOptions, +) -> Result, String> { + let mut out = Vec::new(); + for &nid in path { + let node = &lattice.nodes[nid]; + if node.is_unk { + let surface: String = chars[node.pos..node.pos + node.length].iter().collect(); + if model.byte_fallback { + out.extend(byte_piece_ids(model, &surface)?); + } else if opts.allow_silent_unk { + out.push(model.unk_id); + } else { + return Err("UNIGRAM lattice: unk edge without byte_fallback/silent_unk".into()); + } + } else { + out.push(node.id); + } + } + Ok(out) +} + +fn piece_index_map(model: &SentencePieceModel) -> HashMap<&str, (u32, f32, PieceType)> { + let mut map = HashMap::new(); + for (i, rec) in model.piece_records.iter().enumerate() { + // Official UNIGRAM trie excludes reserved CONTROL/UNKNOWN/BYTE. + if matches!( + rec.piece_type, + PieceType::Unused | PieceType::Control | PieceType::Unknown | PieceType::Byte + ) { + continue; + } + map.insert(rec.piece.as_str(), (i as u32, rec.score, rec.piece_type)); + } + map +} + +fn populate_lattice( + model: &SentencePieceModel, + chars: &[char], + opts: EncodeOptions, + forced: &HashMap<&str, ()>, +) -> Result { + let n = chars.len(); + let mut lattice = Lattice::new(n); + if n == 0 { + return Ok(lattice); + } + + let map = piece_index_map(model); + let max_piece_chars = map + .keys() + .map(|k| k.chars().count()) + .max() + .unwrap_or(0) + .max(1); + + let min_score = map + .values() + .map(|(_, s, _)| *s) + .fold(f32::INFINITY, f32::min); + let unk_score = if min_score.is_finite() { + min_score - UNK_PENALTY + } else { + -UNK_PENALTY + }; + let allow_unk_edge = model.byte_fallback || opts.allow_silent_unk; + + for pos in 0..n { + let mut cand = String::new(); + let max_len = max_piece_chars.min(n - pos); + let mut has_single = false; + for len in 1..=max_len { + cand.push(chars[pos + len - 1]); + if let Some(&(id, score, ty)) = map.get(cand.as_str()) { + if ty == PieceType::Unused { + continue; + } + let edge_score = + if ty == PieceType::UserDefined || forced.contains_key(cand.as_str()) { + user_defined_score(len) + } else { + score + }; + lattice.insert(pos, len, id, edge_score, false); + if len == 1 { + has_single = true; + } + } + } + if !has_single && allow_unk_edge { + lattice.insert(pos, 1, model.unk_id, unk_score, true); + } + // else: leave gap — Viterbi / covering check will fail closed + } + + // Ensure every position that has outgoing begin nodes can connect; if a + // position has no edges at all, covering will fail. + Ok(lattice) +} + +fn resolve_forced<'a>( + model: &'a SentencePieceModel, + opts: &EncodeOptions, +) -> Result, String> { + let mut forced: HashMap<&str, ()> = HashMap::new(); + for e in model.prefix_matcher.entries() { + forced.insert(e.as_str(), ()); + } + for p in &opts.forced_pieces { + if p.is_empty() { + return Err("forced_pieces: empty piece not allowed".into()); + } + if !model.pieces.iter().any(|x| x == p) { + return Err(format!("forced_pieces: {p:?} not in vocab (fail-closed)")); + } + // Lifetime: forced_pieces owned by opts; we need owned keys for map. + // Use piece_records string from model when equal. + if let Some(rec) = model.piece_records.iter().find(|r| r.piece == *p) { + forced.insert(rec.piece.as_str(), ()); + } + } + Ok(forced) +} + +/// Build effective PrefixMatcher (model USER_DEFINED ∪ runtime forced_pieces). +pub fn effective_matcher( + model: &SentencePieceModel, + opts: &EncodeOptions, +) -> Result { + if opts.forced_pieces.is_empty() { + return Ok(model.prefix_matcher.clone()); + } + for p in &opts.forced_pieces { + if p.is_empty() { + return Err("forced_pieces: empty piece not allowed".into()); + } + if !model.pieces.iter().any(|x| x == p) { + return Err(format!("forced_pieces: {p:?} not in vocab (fail-closed)")); + } + } + model.prefix_matcher.with_extra(opts.forced_pieces.clone()) +} + +/// UNIGRAM 1-best via lattice Viterbi (USER_DEFINED-aware). +pub fn encode_unigram_lattice( + model: &SentencePieceModel, + normalized: &str, + opts: EncodeOptions, +) -> Result, String> { + let chars: Vec = normalized.chars().collect(); + if chars.is_empty() { + return Ok(Vec::new()); + } + let forced = resolve_forced(model, &opts)?; + let mut lattice = populate_lattice(model, &chars, opts.clone(), &forced)?; + // Check coverage: every position must be reachable. + let (path, _) = lattice.viterbi().map_err(|_| { + "UNIGRAM encode: no covering segmentation (fail-closed; no unk/byte_fallback)".to_string() + })?; + expand_path_ids(model, &chars, &lattice, &path, opts) +} + +/// Lattice A* n-best. Returns `(token_ids, path_score)` ordered best-first. +pub fn nbest_encode_unigram( + model: &SentencePieceModel, + normalized: &str, + nbest_size: usize, + opts: EncodeOptions, +) -> Result, f32)>, String> { + let nbest_size = nbest_size.clamp(1, MAX_NBEST); + let chars: Vec = normalized.chars().collect(); + if chars.is_empty() { + return Ok(vec![(Vec::new(), 0.0)]); + } + let forced = resolve_forced(model, &opts)?; + let mut lattice = populate_lattice(model, &chars, opts.clone(), &forced)?; + let (best_path, best_score) = lattice.viterbi().map_err(|_| { + "UNIGRAM nbest: no covering segmentation (fail-closed; no unk/byte_fallback)".to_string() + })?; + + if nbest_size == 1 { + let ids = expand_path_ids(model, &chars, &lattice, &best_path, opts)?; + return Ok(vec![(ids, best_score)]); + } + + // For the non-sample call site, any Rng type works; use StdRng as a dummy type param. + let limits = NbestLimits::from_opts(&opts); + let outcome = + nbest_astar::(&lattice, nbest_size, false, 0.0, None, None, limits)?; + let raw = match outcome { + NbestAstarOutcome::ViterbiFallback => { + let ids = expand_path_ids(model, &chars, &lattice, &best_path, opts)?; + return Ok(vec![(ids, best_score)]); + } + NbestAstarOutcome::Paths(paths) => paths, + }; + let mut out = Vec::with_capacity(raw.len()); + for (path, score) in raw { + let ids = expand_path_ids(model, &chars, &lattice, &path, opts.clone())?; + out.push((ids, score)); + } + if out.is_empty() { + // Fallback: at least return Viterbi best. + let ids = expand_path_ids(model, &chars, &lattice, &best_path, opts)?; + out.push((ids, best_score)); + } + Ok(out) +} + +/// Sample one segmentation: lattice forward-filter / backward sample (`nbest_size < 0` +/// path in official SP when SampleEncodeAvailable). +pub fn sample_encode_unigram_lattice( + model: &SentencePieceModel, + normalized: &str, + alpha: f32, + opts: EncodeOptions, + rng: &mut impl Rng, +) -> Result, String> { + if !(alpha.is_finite() && alpha >= 0.0) { + return Err("sample_encode: alpha must be finite and >= 0".into()); + } + let chars: Vec = normalized.chars().collect(); + if chars.is_empty() { + return Ok(Vec::new()); + } + let forced = resolve_forced(model, &opts)?; + let lattice = populate_lattice(model, &chars, opts.clone(), &forced)?; + // Ensure the lattice connects; run a coverage check via attempting Viterbi on a clone. + { + let mut check = populate_lattice(model, &chars, opts.clone(), &forced)?; + let _ = check.viterbi().map_err(|_| { + "UNIGRAM sample: no covering segmentation (fail-closed; no unk/byte_fallback)" + .to_string() + })?; + } + + let inv_theta = alpha; + let alpha_fwd = lattice.forward(inv_theta); + let mut results = Vec::new(); + let mut z = alpha_fwd[lattice.eos()]; + let mut node = lattice.eos(); + loop { + let ends = &lattice.end_nodes[lattice.nodes[node].pos]; + if ends.is_empty() { + return Err("UNIGRAM sample: empty end_nodes".into()); + } + let mut probs = Vec::with_capacity(ends.len()); + for &l_id in ends { + let p = (alpha_fwd[l_id] + inv_theta * lattice.nodes[l_id].score - z).exp(); + probs.push(p.max(0.0)); + } + let sum: f32 = probs.iter().sum(); + if sum <= 0.0 || !sum.is_finite() { + return Err("UNIGRAM sample: invalid probability mass".into()); + } + let mut r = rng.gen::() * sum; + let mut choice = ends[0]; + for (i, &l_id) in ends.iter().enumerate() { + r -= probs[i]; + if r <= 0.0 { + choice = l_id; + break; + } + } + node = choice; + if node == lattice.bos() { + break; + } + z = alpha_fwd[node]; + results.push(node); + } + results.reverse(); + expand_path_ids(model, &chars, &lattice, &results, opts) +} + +/// With-replacement SampleEncodeAndScore: draw `num_samples` lattice samples. +/// Score = `Σ α·node.score − marginal` (official SP `wor=false` path). +pub fn sample_encode_and_score_unigram( + model: &SentencePieceModel, + normalized: &str, + num_samples: usize, + alpha: f32, + opts: EncodeOptions, + rng: &mut impl Rng, +) -> Result, f32)>, String> { + if !(alpha.is_finite() && alpha >= 0.0) { + return Err("sample_encode_and_score: alpha must be finite and >= 0".into()); + } + if num_samples == 0 { + return Err("sample_encode_and_score: num_samples must be >= 1".into()); + } + let chars: Vec = normalized.chars().collect(); + if chars.is_empty() { + return Ok((0..num_samples).map(|_| (Vec::new(), 0.0)).collect()); + } + let forced = resolve_forced(model, &opts)?; + let lattice = populate_lattice(model, &chars, opts.clone(), &forced)?; + { + let mut check = populate_lattice(model, &chars, opts.clone(), &forced)?; + let _ = check.viterbi().map_err(|_| { + "UNIGRAM sample_encode_and_score: no covering segmentation (fail-closed)".to_string() + })?; + } + + let inv_theta = alpha; + let alpha_fwd = lattice.forward(inv_theta); + let marginal = alpha_fwd[lattice.eos()]; + + let mut out = Vec::with_capacity(num_samples); + for _ in 0..num_samples { + let (path, path_score) = sample_path_nodes(&lattice, inv_theta, &alpha_fwd, rng)?; + let ids = expand_path_ids(model, &chars, &lattice, &path, opts.clone())?; + out.push((ids, path_score - marginal)); + } + Ok(out) +} + +/// Without-replacement SampleEncodeAndScore (official SP `wor=true`). +/// +/// Draws Gumbel-perturbed A* n-best of size `num_samples + 1`, uses the +/// (k+1)-th perturbed score as κ for log inclusion probabilities +/// (`log Gumbel survival`). Optional `include_best` prepends the Viterbi path +/// with score `0` (inclusion probability 1) and removes it from the Gumbel draw. +pub fn sample_encode_and_score_unigram_wor( + model: &SentencePieceModel, + normalized: &str, + num_samples: usize, + alpha: f32, + include_best: bool, + opts: EncodeOptions, + rng: &mut impl Rng, +) -> Result, f32)>, String> { + if !(alpha.is_finite() && alpha >= 0.0) { + return Err("sample_encode_and_score: alpha must be finite and >= 0".into()); + } + if num_samples == 0 { + return Err("sample_encode_and_score: num_samples must be >= 1".into()); + } + let chars: Vec = normalized.chars().collect(); + if chars.is_empty() { + return Ok((0..num_samples).map(|_| (Vec::new(), 0.0)).collect()); + } + let forced = resolve_forced(model, &opts)?; + let mut lattice = populate_lattice(model, &chars, opts.clone(), &forced)?; + let (best_path, _) = lattice.viterbi().map_err(|_| { + "UNIGRAM sample_encode_and_score WOR: no covering segmentation (fail-closed)".to_string() + })?; + + let inv_theta = alpha; + let alpha_fwd = lattice.forward(inv_theta); + let marginal = alpha_fwd[lattice.eos()]; + + let mut results: Vec<(Vec, f32)> = Vec::new(); + if include_best { + let ids = expand_path_ids(model, &chars, &lattice, &best_path, opts.clone())?; + // Inclusion probability 1 → log-prob 0. + results.push((ids, 0.0)); + } + + // Need k+1 Gumbel-perturbed paths; κ is the (k+1)-th perturbed fx. + // WOR sampling keeps default refuse on pathological agenda (stock_compat + // still honored when explicitly requested via EncodeOptions). + let limits = NbestLimits::from_opts(&opts); + let outcome = nbest_astar( + &lattice, + num_samples + 1, + true, + inv_theta, + Some(&alpha_fwd), + Some(rng), + limits, + )?; + let mut nbest_samples = match outcome { + NbestAstarOutcome::ViterbiFallback => { + // Stock timeout under agenda pressure: return Viterbi-only score row. + let ids = expand_path_ids(model, &chars, &lattice, &best_path, opts)?; + return Ok(vec![(ids, 0.0)]); + } + NbestAstarOutcome::Paths(paths) => paths, + }; + + if include_best { + if let Some(idx) = nbest_samples.iter().position(|(p, _)| p == &best_path) { + nbest_samples.remove(idx); + } else if !nbest_samples.is_empty() { + nbest_samples.pop(); + } + } + + if nbest_samples.is_empty() { + // Single unique path (or nothing left after removing best): match SP + // post-#1218 — return whatever was already added (best if include_best). + return Ok(results); + } + + let kappa = f64::from(nbest_samples.last().unwrap().1); + nbest_samples.pop(); + + for (path, _perturbed_fx) in nbest_samples { + let mut path_score = 0.0f32; + for &nid in &path { + path_score += inv_theta * lattice.nodes[nid].score; + } + let ids = expand_path_ids(model, &chars, &lattice, &path, opts.clone())?; + results.push((ids, path_score - marginal)); + } + + // Convert path log-probs into log inclusion probabilities (skip exact 0.0 + // best entry, matching official `it.second != 0.0` guard). + for (_ids, score) in &mut results { + if *score != 0.0 { + let x = f64::from(*score) - kappa; + *score = log_gumbel_survival(x) as f32; + } + } + + Ok(results) +} + +fn sample_path_nodes( + lattice: &Lattice, + inv_theta: f32, + alpha_fwd: &[f32], + rng: &mut impl Rng, +) -> Result<(Vec, f32), String> { + let mut results = Vec::new(); + let mut path_score = 0.0f32; + let mut z = alpha_fwd[lattice.eos()]; + let mut node = lattice.eos(); + loop { + let ends = &lattice.end_nodes[lattice.nodes[node].pos]; + if ends.is_empty() { + return Err("UNIGRAM sample: empty end_nodes".into()); + } + let mut probs = Vec::with_capacity(ends.len()); + for &l_id in ends { + let p = (alpha_fwd[l_id] + inv_theta * lattice.nodes[l_id].score - z).exp(); + probs.push(p.max(0.0)); + } + let sum: f32 = probs.iter().sum(); + if sum <= 0.0 || !sum.is_finite() { + return Err("UNIGRAM sample: invalid probability mass".into()); + } + let mut r = rng.gen::() * sum; + let mut choice = ends[0]; + for (i, &l_id) in ends.iter().enumerate() { + r -= probs[i]; + if r <= 0.0 { + choice = l_id; + break; + } + } + node = choice; + if node == lattice.bos() { + break; + } + path_score += inv_theta * lattice.nodes[node].score; + z = alpha_fwd[node]; + results.push(node); + } + results.reverse(); + Ok((results, path_score)) +} + +/// Sample from the top-`nbest_size` lattice paths with temperature `alpha` +/// (official SP SampleEncode when `nbest_size > 1`). +pub fn sample_encode_from_nbest( + model: &SentencePieceModel, + normalized: &str, + nbest_size: usize, + alpha: f32, + opts: EncodeOptions, + rng: &mut impl Rng, +) -> Result, String> { + if !(alpha.is_finite() && alpha >= 0.0) { + return Err("sample_encode: alpha must be finite and >= 0".into()); + } + let nbests = nbest_encode_unigram(model, normalized, nbest_size, opts)?; + if nbests.is_empty() { + return Err("sample_encode: empty nbest".into()); + } + let log_probs: Vec = nbests.iter().map(|(_, s)| alpha * s).collect(); + let max_lp = log_probs.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let weights: Vec = log_probs.iter().map(|&lp| (lp - max_lp).exp()).collect(); + let sum: f32 = weights.iter().sum(); + let mut r = rng.gen::() * sum; + for (i, w) in weights.iter().enumerate() { + r -= w; + if r <= 0.0 { + return Ok(nbests[i].0.clone()); + } + } + Ok(nbests.last().unwrap().0.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sentencepiece::{load_model_proto, ModelProtoFixture}; + + #[test] + fn nbest_stock_compat_shrinks_under_tiny_agenda_cap() { + // Crafted UNIGRAM with multiple paths; force shrink with max_agenda=3. + let fixture = ModelProtoFixture { + pieces: vec![ + ("\u{2581}".into(), Some(-1.0), Some(1)), + ("a".into(), Some(-1.0), Some(1)), + ("b".into(), Some(-1.0), Some(1)), + ("ab".into(), Some(-0.5), Some(1)), + ("\u{2581}a".into(), Some(-0.5), Some(1)), + ("\u{2581}ab".into(), Some(-0.1), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let normalized = "\u{2581}ab"; + let chars: Vec = normalized.chars().collect(); + let opts = EncodeOptions { + nbest_stock_compat: true, + ..Default::default() + }; + let forced = resolve_forced(&model, &opts).unwrap(); + let mut lattice = populate_lattice(&model, &chars, opts.clone(), &forced).unwrap(); + let _ = lattice.viterbi().unwrap(); + let limits = NbestLimits { + max_agenda: 3, + min_agenda: 2, + timeout_ms: 30_000, + stock_compat: true, + }; + let outcome = + nbest_astar::(&lattice, 5, false, 0.0, None, None, limits).unwrap(); + match outcome { + NbestAstarOutcome::Paths(paths) => { + assert!(!paths.is_empty()); + assert!(paths.len() <= 5); + } + NbestAstarOutcome::ViterbiFallback => panic!("unexpected Viterbi fallback"), + } + } + + #[test] + fn nbest_default_refuses_tiny_agenda_overflow() { + let fixture = ModelProtoFixture { + pieces: vec![ + ("\u{2581}".into(), Some(-1.0), Some(1)), + ("a".into(), Some(-1.0), Some(1)), + ("b".into(), Some(-1.0), Some(1)), + ("ab".into(), Some(-0.5), Some(1)), + ("\u{2581}a".into(), Some(-0.5), Some(1)), + ("\u{2581}ab".into(), Some(-0.1), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + escape_whitespaces: Some(true), + remove_extra_whitespaces: Some(true), + ..Default::default() + }; + let model = load_model_proto(&fixture.encode()).unwrap(); + let normalized = "\u{2581}ab"; + let chars: Vec = normalized.chars().collect(); + let opts = EncodeOptions::default(); + let forced = resolve_forced(&model, &opts).unwrap(); + let mut lattice = populate_lattice(&model, &chars, opts.clone(), &forced).unwrap(); + let _ = lattice.viterbi().unwrap(); + let limits = NbestLimits { + max_agenda: 3, + min_agenda: 2, + timeout_ms: 30_000, + stock_compat: false, + }; + let err = nbest_astar::(&lattice, 5, false, 0.0, None, None, limits) + .unwrap_err(); + assert!(err.contains("refuse rather than shrink"), "{err}"); + } +} diff --git a/src/rust/guardd/src/sp_normalizer.rs b/src/rust/guardd/src/sp_normalizer.rs new file mode 100644 index 0000000..b5473ef --- /dev/null +++ b/src/rust/guardd/src/sp_normalizer.rs @@ -0,0 +1,570 @@ +//! SentencePiece `precompiled_charsmap` + whitespace normalization. +//! +//! Matches the official SP `Normalizer` algorithm in `normalizer.cc`: +//! - Decode blob: `u32 le trie_bytes_len` + Darts double-array units + `\0`-delimited +//! replacement string pool +//! - `NormalizePrefix`: leftmost-**longest** trie match over the UTF-8 byte stream +//! - Whitespace: ASCII-space collapse / dummy prefix / `▁` escape +//! +//! This is **not** a claim of grapheme-cluster parity with HuggingFace `tokenizers` +//! (which uses a different walk); cross-checks target the `sentencepiece` package. + +use std::borrow::Cow; +use std::collections::HashMap; + +const SP_SPACE_STR: &str = "\u{2581}"; +const REPLACEMENT_CHAR: &str = "\u{FFFD}"; +const K_MAX_TRIE_RESULTS: usize = 32; + +/// Leftmost-longest prefix matcher over USER_DEFINED / forced pieces. +/// +/// Matches official SP `normalizer::PrefixMatcher`: at each position, take the +/// longest dictionary entry that is a prefix; if none, consume one Unicode char. +#[derive(Debug, Clone, Default)] +pub struct PrefixMatcher { + /// Pieces sorted by descending byte length (stable among equal lengths). + entries: Vec, +} + +impl PrefixMatcher { + /// Build from dictionary pieces. Empty strings and NULs are refused. + pub fn from_pieces(pieces: I) -> Result + where + I: IntoIterator, + S: Into, + { + let mut entries: Vec = Vec::new(); + let mut seen = HashMap::new(); + for p in pieces { + let s = p.into(); + if s.is_empty() { + return Err("PrefixMatcher: empty piece not allowed".into()); + } + if s.as_bytes().contains(&0) { + return Err("PrefixMatcher: piece must not contain NUL".into()); + } + if seen.insert(s.clone(), ()).is_none() { + entries.push(s); + } + } + entries.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b))); + Ok(Self { entries }) + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn entries(&self) -> &[String] { + &self.entries + } + + /// Merge additional pieces into a new matcher (fail-closed on invalid). + pub fn with_extra(&self, extra: I) -> Result + where + I: IntoIterator, + S: Into, + { + let mut all: Vec = self.entries.clone(); + all.extend(extra.into_iter().map(Into::into)); + Self::from_pieces(all) + } + + /// Returns `(consume_bytes, found)`. + pub fn prefix_match(&self, input: &str) -> (usize, bool) { + if input.is_empty() { + return (0, false); + } + for e in &self.entries { + if input.starts_with(e.as_str()) { + return (e.len(), true); + } + } + let n = input + .chars() + .next() + .map(|c| c.len_utf8()) + .unwrap_or(0) + .min(input.len()); + (n, false) + } +} + +/// Lite knobs from `NormalizerSpec` (always applied after charsmap rewrite). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LiteNormalizer { + pub add_dummy_prefix: bool, + pub remove_extra_whitespaces: bool, + pub escape_whitespaces: bool, +} + +impl Default for LiteNormalizer { + fn default() -> Self { + Self { + add_dummy_prefix: true, + remove_extra_whitespaces: true, + escape_whitespaces: true, + } + } +} + +/// Parsed `precompiled_charsmap` (Darts double-array + replacement pool). +#[derive(Debug, Clone)] +pub struct PrecompiledCharsMap { + trie: Vec, + normalized: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct ArrayUnit(u32); + +impl ArrayUnit { + fn has_leaf(self) -> bool { + ((self.0 >> 8) & 1) == 1 + } + + fn value(self) -> u32 { + self.0 & ((1u32 << 31) - 1) + } + + fn label(self) -> u32 { + self.0 & ((1u32 << 31) | 0xFF) + } + + fn offset(self) -> usize { + let shifted = (self.0 >> 10) << ((self.0 & (1u32 << 9)) >> 6); + shifted as usize + } +} + +impl PrecompiledCharsMap { + /// Decode and validate an official SP charsmap blob. + pub fn parse(blob: &[u8]) -> Result { + if blob.len() < 4 { + return Err("precompiled_charsmap: truncated header".into()); + } + let trie_blob_size = u32::from_le_bytes(blob[0..4].try_into().unwrap()) as usize; + if trie_blob_size >= blob.len().saturating_sub(4) { + return Err("precompiled_charsmap: trie size exceeds blob".into()); + } + // Official SP: unit_size=4 and size must be a multiple of 1024. + if trie_blob_size < 1024 || (trie_blob_size & 0x3FF) != 0 { + return Err( + "precompiled_charsmap: trie size must be >= 1024 and divisible by 1024".into(), + ); + } + if trie_blob_size % 4 != 0 { + return Err("precompiled_charsmap: trie size not divisible by 4".into()); + } + let trie_bytes = &blob[4..4 + trie_blob_size]; + let normalized = blob[4 + trie_blob_size..].to_vec(); + if normalized.is_empty() || *normalized.last().unwrap() != 0 { + return Err("precompiled_charsmap: normalized pool must be NUL-terminated".into()); + } + let mut trie = Vec::with_capacity(trie_blob_size / 4); + for chunk in trie_bytes.chunks_exact(4) { + trie.push(u32::from_le_bytes(chunk.try_into().unwrap())); + } + let map = Self { trie, normalized }; + map.validate()?; + Ok(map) + } + + fn validate(&self) -> Result<(), String> { + if self.trie.is_empty() { + return Err("precompiled_charsmap: empty trie".into()); + } + // Soft bounds check: every leaf value must index into the pool. + for &unit in &self.trie { + let u = ArrayUnit(unit); + if u.has_leaf() { + // value is stored at the node reached after offset jump; we cannot + // cheaply enumerate leaves without walking. Skip deep validate — + // corrupt maps fail closed at normalize time via bounds checks. + let _ = u.value(); + } + } + Ok(()) + } + + /// Darts `commonPrefixSearch`: `(replacement_offset, match_byte_len)` pairs. + fn common_prefix_search(&self, key: &[u8]) -> Vec<(u32, usize)> { + let mut results = Vec::new(); + if self.trie.is_empty() || key.is_empty() { + return results; + } + let mut node_pos: usize = 0; + let mut unit = ArrayUnit(self.trie[node_pos]); + node_pos ^= unit.offset(); + for (i, &c) in key.iter().enumerate() { + if c == 0 { + break; + } + if node_pos >= self.trie.len() { + break; + } + node_pos ^= c as usize; + if node_pos >= self.trie.len() { + break; + } + unit = ArrayUnit(self.trie[node_pos]); + if unit.label() != u32::from(c) { + return results; + } + node_pos ^= unit.offset(); + if unit.has_leaf() { + if node_pos >= self.trie.len() { + break; + } + let value = ArrayUnit(self.trie[node_pos]).value(); + results.push((value, i + 1)); + if results.len() >= K_MAX_TRIE_RESULTS { + break; + } + } + } + results + } + + fn replacement_at(&self, offset: u32) -> Result<&str, String> { + let start = offset as usize; + if start >= self.normalized.len() { + return Err("precompiled_charsmap: replacement offset out of range".into()); + } + let rest = &self.normalized[start..]; + let end = rest + .iter() + .position(|&b| b == 0) + .ok_or_else(|| "precompiled_charsmap: replacement missing NUL".to_string())?; + std::str::from_utf8(&rest[..end]) + .map_err(|_| "precompiled_charsmap: replacement is not UTF-8".into()) + } +} + +/// One UTF-8 character length at the start of `input`, or `None` if empty/invalid. +fn utf8_char_len(input: &[u8]) -> Option { + if input.is_empty() { + return None; + } + let b0 = input[0]; + let want = if b0 < 0x80 { + 1 + } else if b0 & 0xE0 == 0xC0 { + 2 + } else if b0 & 0xF0 == 0xE0 { + 3 + } else if b0 & 0xF8 == 0xF0 { + 4 + } else { + return None; + }; + if input.len() < want { + return None; + } + if std::str::from_utf8(&input[..want]).is_err() { + return None; + } + Some(want) +} + +/// `NormalizePrefix`: `(rewritten, consume_bytes)`. +/// +/// Official order: PrefixMatcher (USER_DEFINED) wins over charsmap rewrite. +fn normalize_prefix<'a>( + charsmap: Option<&'a PrecompiledCharsMap>, + matcher: Option<&PrefixMatcher>, + input: &'a [u8], +) -> Result<(Cow<'a, str>, usize), String> { + if input.is_empty() { + return Ok((Cow::Borrowed(""), 0)); + } + + if let Some(m) = matcher { + if let Ok(s) = std::str::from_utf8(input) { + let (mblen, found) = m.prefix_match(s); + if found { + return Ok((Cow::Borrowed(&s[..mblen]), mblen)); + } + } + } + + let mut longest_length = 0usize; + let mut longest_value = 0u32; + if let Some(map) = charsmap { + let matches = map.common_prefix_search(input); + for (value, length) in matches { + if longest_length == 0 || length > longest_length { + longest_length = length; + longest_value = value; + } + } + } + + if longest_length == 0 || longest_length > input.len() { + match utf8_char_len(input) { + Some(n) => { + let s = std::str::from_utf8(&input[..n]) + .map_err(|_| "normalize: invalid UTF-8".to_string())?; + Ok((Cow::Borrowed(s), n)) + } + None => { + // Official SP: consume 1 byte, emit U+FFFD. + Ok((Cow::Borrowed(REPLACEMENT_CHAR), 1)) + } + } + } else { + let map = charsmap.expect("longest match implies charsmap"); + let repl = map.replacement_at(longest_value)?; + Ok((Cow::Borrowed(repl), longest_length)) + } +} + +/// Full SP normalize (charsmap + whitespace policy + optional PrefixMatcher). +/// +/// Encode path uses `escape_whitespaces=true`. Denormalizer specs typically set +/// `escape_whitespaces=false`, `add_dummy_prefix=false`, and +/// `remove_extra_whitespaces=false` (charsmap rewrite only). +pub fn sp_normalize( + text: &str, + lite: &LiteNormalizer, + charsmap: Option<&PrecompiledCharsMap>, + treat_whitespace_as_suffix: bool, +) -> Result { + sp_normalize_with_matcher(text, lite, charsmap, None, treat_whitespace_as_suffix) +} + +/// Like [`sp_normalize`], with USER_DEFINED / forced-piece PrefixMatcher. +pub fn sp_normalize_with_matcher( + text: &str, + lite: &LiteNormalizer, + charsmap: Option<&PrecompiledCharsMap>, + matcher: Option<&PrefixMatcher>, + treat_whitespace_as_suffix: bool, +) -> Result { + let mut input = text.as_bytes(); + if input.is_empty() { + return Ok(String::new()); + } + + // Ignore heading ASCII spaces (after charsmap rewrite of each prefix). + if lite.remove_extra_whitespaces { + while !input.is_empty() { + let (sp, n) = normalize_prefix(charsmap, matcher, input)?; + if sp.as_ref() != " " { + break; + } + input = &input[n..]; + } + } + if input.is_empty() { + return Ok(String::new()); + } + + let space_sym: &str = if lite.escape_whitespaces { + SP_SPACE_STR + } else { + " " + }; + + let mut out = String::with_capacity(input.len().saturating_mul(2)); + + // Official SP: dummy whitespace is a prefix unless treat_whitespace_as_suffix. + if !treat_whitespace_as_suffix && lite.add_dummy_prefix { + out.push_str(space_sym); + } + + let mut is_prev_space = lite.remove_extra_whitespaces; + while !input.is_empty() { + let (sp_cow, n) = normalize_prefix(charsmap, matcher, input)?; + let mut sp = sp_cow.as_ref(); + while is_prev_space && sp.starts_with(' ') { + sp = &sp[1..]; + } + if !sp.is_empty() { + for ch in sp.chars() { + if ch == ' ' { + out.push_str(space_sym); + } else { + out.push(ch); + } + } + is_prev_space = sp.ends_with(' '); + } + input = &input[n..]; + if !lite.remove_extra_whitespaces { + is_prev_space = false; + } + } + + if lite.remove_extra_whitespaces { + let trail = if lite.escape_whitespaces { + SP_SPACE_STR + } else { + " " + }; + while out.ends_with(trail) { + out.truncate(out.len() - trail.len()); + } + } + + // Official SP: with treat_whitespace_as_suffix, dummy whitespace is a suffix. + if treat_whitespace_as_suffix && lite.add_dummy_prefix { + out.push_str(space_sym); + } + + Ok(out) +} + +/// Apply denormalizer charsmap (official SP: `denormalizer_->Normalize` after piece decode). +pub fn sp_denormalize( + text: &str, + lite: &LiteNormalizer, + charsmap: Option<&PrecompiledCharsMap>, +) -> Result { + sp_normalize(text, lite, charsmap, false) +} + +/// Convert a byte to the SentencePiece byte-piece spelling `<0xXX>` (uppercase hex). +pub fn byte_to_piece(b: u8) -> String { + format!("<0x{b:02X}>") +} + +/// Parse `<0xXX>` → byte, or `None`. +pub fn piece_to_byte(piece: &str) -> Option { + let rest = piece.strip_prefix("<0x")?.strip_suffix('>')?; + if rest.len() != 2 { + return None; + } + u8::from_str_radix(rest, 16).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/fixtures") + .join(name) + } + + #[test] + fn parses_custom_charsmap_blob() { + let blob = std::fs::read(fixture("sp_custom_charsmap.bin")).expect("fixture"); + let map = PrecompiledCharsMap::parse(&blob).expect("parse"); + // Fullwidth A (U+FF21) → "A" + let ff21 = "\u{FF21}".as_bytes(); + let matches = map.common_prefix_search(ff21); + assert!(!matches.is_empty()); + let (off, len) = matches.iter().max_by_key(|(_, l)| *l).copied().unwrap(); + assert_eq!(len, ff21.len()); + assert_eq!(map.replacement_at(off).unwrap(), "A"); + } + + #[test] + fn normalize_applies_charsmap_and_dummy_prefix() { + let blob = std::fs::read(fixture("sp_custom_charsmap.bin")).expect("fixture"); + let map = PrecompiledCharsMap::parse(&blob).unwrap(); + let lite = LiteNormalizer::default(); + let out = sp_normalize("\u{FF21}bc", &lite, Some(&map), false).unwrap(); + assert_eq!(out, "\u{2581}Abc"); + let tm = sp_normalize("x\u{2122}y", &lite, Some(&map), false).unwrap(); + assert_eq!(tm, "\u{2581}xTMy"); + } + + #[test] + fn identity_normalize_matches_whitespace_policy() { + let lite = LiteNormalizer::default(); + let out = sp_normalize(" foo bar ", &lite, None, false).unwrap(); + assert_eq!(out, "\u{2581}foo\u{2581}bar"); + let hello = sp_normalize("hello", &lite, None, false).unwrap(); + assert_eq!(hello, "\u{2581}hello"); + } + + #[test] + fn treat_whitespace_as_suffix_moves_dummy_ws() { + let lite = LiteNormalizer::default(); + let out = sp_normalize("hello world", &lite, None, true).unwrap(); + assert_eq!(out, "hello\u{2581}world\u{2581}"); + let single = sp_normalize("hello", &lite, None, true).unwrap(); + assert_eq!(single, "hello\u{2581}"); + } + + #[test] + fn rejects_truncated_charsmap() { + let err = PrecompiledCharsMap::parse(&[1, 2, 3]).unwrap_err(); + assert!(err.contains("truncated") || err.contains("trie")); + } + + #[test] + fn byte_piece_roundtrip() { + for b in [0u8, 0x5A, 0xF0, 0xFF] { + let p = byte_to_piece(b); + assert_eq!(piece_to_byte(&p), Some(b)); + } + assert_eq!(piece_to_byte("<0xGG>"), None); + assert_eq!(piece_to_byte(""), None); + } + + #[test] + fn denormalize_applies_reverse_charsmap() { + let blob = std::fs::read(fixture("sp_denorm_charsmap.bin")).expect("fixture"); + let map = PrecompiledCharsMap::parse(&blob).unwrap(); + let lite = LiteNormalizer { + add_dummy_prefix: false, + remove_extra_whitespaces: false, + escape_whitespaces: false, + }; + assert_eq!( + sp_denormalize("xTMy", &lite, Some(&map)).unwrap(), + "x\u{2122}y" + ); + assert_eq!(sp_denormalize("TM", &lite, Some(&map)).unwrap(), "\u{2122}"); + // No match → identity. + assert_eq!(sp_denormalize("hello", &lite, Some(&map)).unwrap(), "hello"); + } + + /// Offline hygiene for Wave 27 curated goldens: identity rows use charsmap-only + /// knobs (no dummy-prefix / ▁ escape). Full nfkc / nfkc_cf blobs stay + /// integration-only — not an exhaustive Unicode NFKC claim. + #[test] + fn curated_nfkc_identity_goldens_charsmap_only() { + let raw = std::fs::read_to_string(fixture("sp_nfkc_normalize_goldens.json")) + .expect("sp_nfkc_normalize_goldens.json"); + let v: serde_json::Value = serde_json::from_str(&raw).expect("golden json"); + let identity = v["rule"]["identity"] + .as_array() + .expect("rule.identity array"); + assert!( + identity.len() > 100, + "Wave 27 expects curated identity goldens expanded beyond 100/rule, got {}", + identity.len() + ); + for rule in ["nfkc", "nfkc_cf"] { + let n = v["rule"][rule] + .as_array() + .unwrap_or_else(|| panic!("rule.{rule} array")) + .len(); + assert_eq!( + n, + identity.len(), + "golden rule lengths must stay aligned ({rule}={n}, identity={})", + identity.len() + ); + assert!(n > 100, "Wave 27: {rule} goldens beyond 100/rule, got {n}"); + } + + let lite = LiteNormalizer { + add_dummy_prefix: false, + remove_extra_whitespaces: false, + escape_whitespaces: false, + }; + for row in identity { + let inp = row["in"].as_str().expect("in"); + let expected = row["out"].as_str().expect("out"); + let out = sp_normalize(inp, &lite, None, false).expect("normalize"); + assert_eq!(out, expected, "identity golden mismatch for {inp:?}"); + } + } +} From ae151953794bcb84fda764617143aabc9f85d04c Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:13 -0700 Subject: [PATCH 16/35] Expand guardd FFI surface and deprecate legacy quant ABI Wire SentencePiece/CHD modules into the cdylib API and mark guardd_verify_quant deprecated in favor of the 128-vector verifier. --- src/rust/guardd/src/lib.rs | 1050 +++++++++++++++++++++++++++++++----- 1 file changed, 929 insertions(+), 121 deletions(-) diff --git a/src/rust/guardd/src/lib.rs b/src/rust/guardd/src/lib.rs index 0d8e81a..dd89aa8 100644 --- a/src/rust/guardd/src/lib.rs +++ b/src/rust/guardd/src/lib.rs @@ -1,25 +1,182 @@ -use sha2::{Sha256, Digest}; -use serde::{Serialize, Deserialize}; +use memmap2::Mmap; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::ffi::{CStr, CString}; +use std::fs::File; use std::os::raw::c_char; +use std::path::Path; use std::ptr; -use memmap2::Mmap; -use std::fs::File; -// Include the bit-flip corpus module -mod bitflip_corpus; +pub mod bitflip_corpus; use bitflip_corpus::BitFlipCorpusTester; -// Include the quantization verification module -mod quant_verification; +pub mod quant_verification; use quant_verification::{ - QuantizationConfig, - verify_layer_128_vectors, verify_model_128_vectors + verify_layer_128_vectors, verify_model_128_vectors, LayerSpec, QuantizationConfig, }; -mod perfect_hash; +pub mod perfect_hash; use perfect_hash::PerfectHashVocab; +pub mod bpe; +pub mod sentencepiece; +pub mod sp_lattice; +pub mod sp_normalizer; + +/// Safe Rust view of a checked-load result (not part of the C ABI). +#[derive(Debug, Clone)] +pub struct ModelInfo { + pub path: String, + pub size: u64, + pub digest: [u8; 32], + pub valid: bool, +} + +/// Safe digest verification. Returns `Ok(true)` on match, `Ok(false)` on mismatch. +pub fn verify_digest(path: &str, expected_digest: &[u8]) -> Result { + if expected_digest.len() != 32 { + return Err(GuarddError::InvalidDigest); + } + let digest = compute_file_digest(path)?; + Ok(digest.as_slice() == expected_digest) +} + +/// Safe checked load with optional expected digest. +pub fn checked_load(path: &str, expected_digest: Option<&[u8]>) -> Result { + if let Some(expected) = expected_digest { + if expected.len() != 32 { + return Err(GuarddError::InvalidDigest); + } + } + + let file = File::open(path).map_err(|_| GuarddError::FileNotFound)?; + let metadata = file.metadata().map_err(|_| GuarddError::FileNotFound)?; + let mmap = unsafe { Mmap::map(&file) }.map_err(|_| GuarddError::MemoryError)?; + + let mut hasher = Sha256::new(); + hasher.update(&mmap); + let computed: [u8; 32] = hasher.finalize().into(); + + let valid = match expected_digest { + Some(expected) => computed.as_slice() == expected, + None => true, + }; + + Ok(ModelInfo { + path: path.to_string(), + size: metadata.len(), + digest: computed, + valid, + }) +} + +/// Safe 128-vector quantization check for one layer. +pub fn verify_quantization_128_vectors( + layer_name: &str, + weights: &[f32], + fan_in: u32, + fan_out: u32, + quant_type: &str, +) -> quant_verification::LayerVerification128 { + let config = QuantizationConfig { + fan_in, + fan_out, + quant_type: quant_type.to_string(), + }; + verify_layer_128_vectors(layer_name, weights, &config) +} + +/// Safe multi-layer 128-vector check from object-form [`LayerSpec`] values. +pub fn verify_model_quantization_128_vectors( + layers: Vec<(String, Vec, QuantizationConfig)>, +) -> quant_verification::ModelVerification128 { + verify_model_128_vectors(layers) +} + +/// Safe bit-flip corpus check. `temp_dir = None` uses the platform temp dir +/// (see [`bitflip_corpus::default_bitflip_temp_dir`]). +pub fn run_bitflip_corpus_test( + file_size_gb: usize, + num_corruptions: usize, + temp_dir: Option, +) -> Result { + let tester = BitFlipCorpusTester::new(1024, temp_dir); + tester + .run_bitflip_test(file_size_gb, num_corruptions) + .map_err(|e| e.to_string()) +} + +/// Safe open-addressed hash-vocab encode (whitespace-split words → token ids). +pub fn perfect_hash_encode(vocab_json: &str, text: &str) -> Result, String> { + let vocab: PerfectHashVocab = + serde_json::from_str(vocab_json).map_err(|e| format!("invalid vocab JSON: {e}"))?; + Ok(text.split_whitespace().map(|w| vocab.encode(w)).collect()) +} + +/// Safe open-addressed hash-vocab decode for a single token id. +pub fn perfect_hash_decode(vocab_json: &str, token: u32) -> Result { + let vocab: PerfectHashVocab = + serde_json::from_str(vocab_json).map_err(|e| format!("invalid vocab JSON: {e}"))?; + Ok(vocab.decode(token).to_string()) +} + +/// Safe SentencePiece encode from a `.model` path. +pub fn sp_encode( + model_path: &str, + text: &str, + options: sentencepiece::EncodeOptions, +) -> Result, String> { + let model = sentencepiece::load_model_proto_file(model_path)?; + sentencepiece::encode_with_options(&model, text, options) +} + +/// Safe SentencePiece decode from a `.model` path. +pub fn sp_decode(model_path: &str, tokens: &[u32]) -> Result { + sp_decode_with_options(model_path, tokens, sentencepiece::DecodeOptions::default()) +} + +/// Safe SentencePiece decode with ExtraOptions-style knobs (`reverse`). +pub fn sp_decode_with_options( + model_path: &str, + tokens: &[u32], + options: sentencepiece::DecodeOptions, +) -> Result { + let model = sentencepiece::load_model_proto_file(model_path)?; + sentencepiece::decode_pieces_with_options(&model, tokens, options) +} + +/// Safe UNIGRAM n-best encode (other model types error). +pub fn sp_nbest_encode( + model_path: &str, + text: &str, + nbest_size: usize, + options: sentencepiece::EncodeOptions, +) -> Result, f32)>, String> { + let model = sentencepiece::load_model_proto_file(model_path)?; + sentencepiece::nbest_encode_with_options(&model, text, nbest_size, options) +} + +/// Safe sample encode (UNIGRAM lattice sample / BPE-dropout). +pub fn sp_sample_encode( + model_path: &str, + text: &str, + sample: sentencepiece::SampleEncodeOptions, +) -> Result, String> { + let model = sentencepiece::load_model_proto_file(model_path)?; + sentencepiece::sample_encode_with_options(&model, text, sample) +} + +fn compute_file_digest(path: &str) -> Result<[u8; 32], GuarddError> { + if !Path::new(path).exists() { + return Err(GuarddError::FileNotFound); + } + let file = File::open(path).map_err(|_| GuarddError::FileNotFound)?; + let mmap = unsafe { Mmap::map(&file) }.map_err(|_| GuarddError::MemoryError)?; + let mut hasher = Sha256::new(); + hasher.update(&mmap); + Ok(hasher.finalize().into()) +} + #[repr(C)] pub struct ModelHandle { path: *mut c_char, @@ -29,6 +186,7 @@ pub struct ModelHandle { } #[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GuarddError { Success = 0, FileNotFound = 1, @@ -60,7 +218,10 @@ pub struct LayerVerification { /// `path` must be a valid, null-terminated UTF-8 C string and `expected_digest` /// must point to at least 32 readable bytes. #[no_mangle] -pub unsafe extern "C" fn guardd_verify_digest(path: *const c_char, expected_digest: *const u8) -> GuarddError { +pub unsafe extern "C" fn guardd_verify_digest( + path: *const c_char, + expected_digest: *const u8, +) -> GuarddError { if path.is_null() || expected_digest.is_null() { return GuarddError::InvalidPath; } @@ -85,7 +246,7 @@ pub unsafe extern "C" fn guardd_verify_digest(path: *const c_char, expected_dige let computed_digest = hasher.finalize(); let expected_slice = unsafe { std::slice::from_raw_parts(expected_digest, 32) }; - + if computed_digest.as_slice() == expected_slice { GuarddError::Success } else { @@ -99,7 +260,10 @@ pub unsafe extern "C" fn guardd_verify_digest(path: *const c_char, expected_dige /// `path` must be a valid, null-terminated UTF-8 C string. If non-null, /// `expected_digest` must point to at least 32 readable bytes. #[no_mangle] -pub unsafe extern "C" fn guardd_checked_load(path: *const c_char, expected_digest: *const u8) -> *mut ModelHandle { +pub unsafe extern "C" fn guardd_checked_load( + path: *const c_char, + expected_digest: *const u8, +) -> *mut ModelHandle { if path.is_null() { return ptr::null_mut(); } @@ -165,56 +329,40 @@ pub unsafe extern "C" fn guardd_free_handle(handle: *mut ModelHandle) { } } -/// Verify quantization bounds for a layer (legacy function) -#[no_mangle] +/// Legacy quantization entry point — **unimplemented** (fail-closed). +/// +/// Always returns null; never allocates and never reports a pass/fail verdict. +/// Callers must treat null as “API unavailable”, not as success. +/// +/// **Deprecated:** use [`guardd_verify_quant_128_vectors`] (or +/// [`guardd_verify_model_128_vectors`]) for the supported 128-vector sampling path. /// /// # Safety -/// `weights` must point to `weights_len` readable `f32` values and `quant_type` -/// must be a valid, null-terminated UTF-8 C string. +/// Pointers are unused; null is returned without reading them. +#[deprecated( + note = "unimplemented; always returns null. Use guardd_verify_quant_128_vectors" +)] +#[no_mangle] pub unsafe extern "C" fn guardd_verify_quant( - weights: *const f32, - weights_len: usize, - fan_in: u32, - quant_type: *const c_char, - tolerance: f64, + _weights: *const f32, + _weights_len: usize, + _fan_in: u32, + _quant_type: *const c_char, + _tolerance: f64, ) -> *mut LayerVerification { - if weights.is_null() || quant_type.is_null() { - return ptr::null_mut(); - } - - let quant_type_str = match unsafe { CStr::from_ptr(quant_type).to_str() } { - Ok(s) => s, - Err(_) => return ptr::null_mut(), - }; - - let _weights_slice = unsafe { std::slice::from_raw_parts(weights, weights_len) }; - - // Compute epsilon bound based on quantization type - let epsilon_bound = match quant_type_str { - "int8" => 0.5 * (fan_in as f64).sqrt(), - "fp16" => 2.0 * (fan_in as f64).sqrt(), - _ => 1.0 * (fan_in as f64).sqrt(), - }; - - // Simplified quantization error computation - // In practice, this would implement the full quantization algorithm - let actual_error = 0.1; // Placeholder - - let verification = Box::new(LayerVerification { - layer_name: "layer".to_string(), - epsilon_bound, - actual_error, - passed: actual_error <= epsilon_bound && actual_error <= tolerance, - }); - - Box::into_raw(verification) + // Fail-closed: do not report false pass/fail results. + ptr::null_mut() } -/// Free a layer verification result +/// Free a layer verification result from the legacy [`guardd_verify_quant`] ABI. +/// +/// That entry point always returns null today, so this free is a no-op on the +/// supported path. Prefer JSON results from `guardd_verify_quant_128_vectors` +/// (freed with [`guardd_free_json_result`]). /// /// # Safety -/// `verification` must be a pointer previously returned by `guardd_verify_quant` -/// and must not be freed more than once. +/// `verification` must be null or a pointer previously returned by +/// `guardd_verify_quant`, and must not be freed more than once. #[no_mangle] pub unsafe extern "C" fn guardd_free_verification(verification: *mut LayerVerification) { if !verification.is_null() { @@ -261,12 +409,10 @@ pub unsafe extern "C" fn guardd_verify_quant_128_vectors( let verification = verify_layer_128_vectors(layer_name_str, weights_slice, &config); match serde_json::to_string(&verification) { - Ok(json_str) => { - match CString::new(json_str) { - Ok(c_string) => c_string.into_raw(), - Err(_) => ptr::null_mut(), - } - } + Ok(json_str) => match CString::new(json_str) { + Ok(c_string) => c_string.into_raw(), + Err(_) => ptr::null_mut(), + }, Err(_) => ptr::null_mut(), } } @@ -275,7 +421,8 @@ pub unsafe extern "C" fn guardd_verify_quant_128_vectors( /// /// # Safety /// `layers_json` must be a valid, null-terminated UTF-8 C string containing -/// JSON for `Vec<(String, Vec, QuantizationConfig)>`. +/// a JSON array of [`LayerSpec`] objects: +/// `[{"name","weights","fan_in","fan_out","quant_type"}, ...]`. #[no_mangle] pub unsafe extern "C" fn guardd_verify_model_128_vectors( layers_json: *const c_char, @@ -289,24 +436,20 @@ pub unsafe extern "C" fn guardd_verify_model_128_vectors( Err(_) => return ptr::null_mut(), }; - // Parse layers from JSON - let layers_data: Result, QuantizationConfig)>, _> = - serde_json::from_str(layers_json_str); + let layers_data: Result, _> = serde_json::from_str(layers_json_str); let layers = match layers_data { - Ok(layers) => layers, + Ok(specs) => specs.into_iter().map(LayerSpec::into_tuple).collect(), Err(_) => return ptr::null_mut(), }; let verification = verify_model_128_vectors(layers); match serde_json::to_string(&verification) { - Ok(json_str) => { - match CString::new(json_str) { - Ok(c_string) => c_string.into_raw(), - Err(_) => ptr::null_mut(), - } - } + Ok(json_str) => match CString::new(json_str) { + Ok(c_string) => c_string.into_raw(), + Err(_) => ptr::null_mut(), + }, Err(_) => ptr::null_mut(), } } @@ -342,28 +485,24 @@ pub unsafe extern "C" fn guardd_run_bitflip_corpus_test( temp_dir: *const c_char, ) -> *mut c_char { let temp_dir_str = if temp_dir.is_null() { - "/tmp/bitflip_corpus".to_string() + None } else { match unsafe { CStr::from_ptr(temp_dir).to_str() } { - Ok(s) => s.to_string(), - Err(_) => "/tmp/bitflip_corpus".to_string(), + Ok(s) => Some(s.to_string()), + Err(_) => None, } }; - let tester = BitFlipCorpusTester::new(1024, Some(temp_dir_str)); - + let tester = BitFlipCorpusTester::new(1024, temp_dir_str); + match tester.run_bitflip_test(file_size_gb, num_corruptions) { - Ok(result) => { - match serde_json::to_string(&result) { - Ok(json_str) => { - match CString::new(json_str) { - Ok(c_string) => c_string.into_raw(), - Err(_) => ptr::null_mut(), - } - } + Ok(result) => match serde_json::to_string(&result) { + Ok(json_str) => match CString::new(json_str) { + Ok(c_string) => c_string.into_raw(), Err(_) => ptr::null_mut(), - } - } + }, + Err(_) => ptr::null_mut(), + }, Err(_) => ptr::null_mut(), } } @@ -377,28 +516,24 @@ pub unsafe extern "C" fn guardd_run_comprehensive_bitflip_test( temp_dir: *const c_char, ) -> *mut c_char { let temp_dir_str = if temp_dir.is_null() { - "/tmp/bitflip_corpus".to_string() + None } else { match unsafe { CStr::from_ptr(temp_dir).to_str() } { - Ok(s) => s.to_string(), - Err(_) => "/tmp/bitflip_corpus".to_string(), + Ok(s) => Some(s.to_string()), + Err(_) => None, } }; - let tester = BitFlipCorpusTester::new(1024, Some(temp_dir_str)); - + let tester = BitFlipCorpusTester::new(1024, temp_dir_str); + match tester.run_comprehensive_test() { - Ok(summary) => { - match serde_json::to_string(&summary) { - Ok(json_str) => { - match CString::new(json_str) { - Ok(c_string) => c_string.into_raw(), - Err(_) => ptr::null_mut(), - } - } + Ok(summary) => match serde_json::to_string(&summary) { + Ok(json_str) => match CString::new(json_str) { + Ok(c_string) => c_string.into_raw(), Err(_) => ptr::null_mut(), - } - } + }, + Err(_) => ptr::null_mut(), + }, Err(_) => ptr::null_mut(), } } @@ -422,7 +557,10 @@ pub unsafe extern "C" fn guardd_free_json_result(result: *mut c_char) { /// # Safety /// `vocab_json` and `text` must be valid, null-terminated UTF-8 C strings. #[no_mangle] -pub unsafe extern "C" fn guardd_perfect_hash_encode(vocab_json: *const c_char, text: *const c_char) -> *mut c_char { +pub unsafe extern "C" fn guardd_perfect_hash_encode( + vocab_json: *const c_char, + text: *const c_char, +) -> *mut c_char { if vocab_json.is_null() || text.is_null() { return std::ptr::null_mut(); } @@ -438,7 +576,10 @@ pub unsafe extern "C" fn guardd_perfect_hash_encode(vocab_json: *const c_char, t Ok(v) => v, Err(_) => return std::ptr::null_mut(), }; - let tokens: Vec = text_str.split_whitespace().map(|w| vocab.encode(w)).collect(); + let tokens: Vec = text_str + .split_whitespace() + .map(|w| vocab.encode(w)) + .collect(); match serde_json::to_string(&tokens) { Ok(json_str) => match CString::new(json_str) { Ok(c_string) => c_string.into_raw(), @@ -453,7 +594,10 @@ pub unsafe extern "C" fn guardd_perfect_hash_encode(vocab_json: *const c_char, t /// # Safety /// `vocab_json` must be a valid, null-terminated UTF-8 C string. #[no_mangle] -pub unsafe extern "C" fn guardd_perfect_hash_decode(vocab_json: *const c_char, token: u32) -> *mut c_char { +pub unsafe extern "C" fn guardd_perfect_hash_decode( + vocab_json: *const c_char, + token: u32, +) -> *mut c_char { if vocab_json.is_null() { return std::ptr::null_mut(); } @@ -472,13 +616,547 @@ pub unsafe extern "C" fn guardd_perfect_hash_decode(vocab_json: *const c_char, t } } +fn sp_json_ok_tokens(tokens: &[u32], model_type: &str) -> *mut c_char { + let payload = serde_json::json!({ + "ok": true, + "tokens": tokens, + "model_type": model_type, + }); + match CString::new(payload.to_string()) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } +} + +fn sp_json_ok_text(text: &str) -> *mut c_char { + let payload = serde_json::json!({ + "ok": true, + "text": text, + }); + match CString::new(payload.to_string()) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } +} + +fn sp_json_err(msg: &str) -> *mut c_char { + let payload = serde_json::json!({ + "ok": false, + "error": msg, + }); + match CString::new(payload.to_string()) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } +} + +fn sp_json_ok_nbest(nbests: &[(Vec, f32)], model_type: &str) -> *mut c_char { + let rows: Vec = nbests + .iter() + .map(|(tokens, score)| { + serde_json::json!({ + "tokens": tokens, + "score": score, + }) + }) + .collect(); + let payload = serde_json::json!({ + "ok": true, + "nbests": rows, + "model_type": model_type, + }); + match CString::new(payload.to_string()) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } +} + +fn parse_sp_encode_opts(opt_str: &str) -> Result { + #[derive(Deserialize)] + struct SpEncodeOpts { + #[serde(default)] + allow_silent_unk: bool, + #[serde(default)] + forced_pieces: Vec, + #[serde(default)] + word_official_split: bool, + #[serde(default = "default_merge_consecutive_unk")] + merge_consecutive_unk: bool, + #[serde(default)] + reverse: bool, + #[serde(default)] + add_bos: bool, + #[serde(default)] + add_eos: bool, + #[serde(default)] + nbest_stock_compat: bool, + } + fn default_merge_consecutive_unk() -> bool { + true + } + let o: SpEncodeOpts = + serde_json::from_str(opt_str).map_err(|e| format!("invalid options JSON: {e}"))?; + Ok(sentencepiece::EncodeOptions { + allow_silent_unk: o.allow_silent_unk, + forced_pieces: o.forced_pieces, + word_official_split: o.word_official_split, + merge_consecutive_unk: o.merge_consecutive_unk, + reverse: o.reverse, + add_bos: o.add_bos, + add_eos: o.add_eos, + nbest_stock_compat: o.nbest_stock_compat, + }) +} + +fn parse_sp_decode_opts(opt_str: &str) -> Result { + #[derive(Deserialize)] + struct SpDecodeOpts { + #[serde(default)] + reverse: bool, + } + let o: SpDecodeOpts = + serde_json::from_str(opt_str).map_err(|e| format!("invalid decode options JSON: {e}"))?; + Ok(sentencepiece::DecodeOptions { reverse: o.reverse }) +} + +/// SentencePiece encode from a `.model` / `.proto` path (CHAR/BPE/UNIGRAM when gated). +/// +/// Returns owned JSON: `{"ok":true,"tokens":[...],"model_type":"..."}` or +/// `{"ok":false,"error":"..."}`. Caller must free with `guardd_free_json_result`. +/// +/// Fail-closed OOV default (no silent ``). +/// +/// # Safety +/// `model_path` and `text` must be valid, null-terminated UTF-8 C strings. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_encode( + model_path: *const c_char, + text: *const c_char, +) -> *mut c_char { + unsafe { guardd_sp_encode_with_options(model_path, text, ptr::null()) } +} + +/// SentencePiece encode with optional JSON options. +/// +/// `options_json` may be null or `{}` for defaults. Supported keys: +/// - `allow_silent_unk` (bool, default false): stock-SP OOV → UNKNOWN id +/// - `forced_pieces` (string[], default []): extra PrefixMatcher pieces; each +/// must already exist in the model vocab +/// - `word_official_split` (bool, default false): WORD flag-blind SplitIntoWords +/// - `merge_consecutive_unk` (bool, default true): stock SP processor merge of +/// consecutive UNKNOWN ids (ignored when `byte_fallback` is enabled) +/// - `reverse` (bool, default false): ExtraOptions reverse **before** bos/eos +/// (matches stock documented `reverse:bos:eos`; not free-form colon order) +/// - `add_bos` / `add_eos` (bool, default false): ExtraOptions CONTROL wrap after merge +/// - `nbest_stock_compat` (bool, default false): UNIGRAM n-best stock agenda +/// shrink + timeout→Viterbi (default refuses pathological cases) +/// +/// # Safety +/// Pointers must be valid null-terminated UTF-8 C strings when non-null. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_encode_with_options( + model_path: *const c_char, + text: *const c_char, + options_json: *const c_char, +) -> *mut c_char { + if model_path.is_null() || text.is_null() { + return sp_json_err("null argument"); + } + let path = match unsafe { CStr::from_ptr(model_path).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("model_path is not valid UTF-8"), + }; + let text_str = match unsafe { CStr::from_ptr(text).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("text is not valid UTF-8"), + }; + let mut opts = sentencepiece::EncodeOptions::default(); + if !options_json.is_null() { + let opt_str = match unsafe { CStr::from_ptr(options_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("options_json is not valid UTF-8"), + }; + if !opt_str.is_empty() { + match parse_sp_encode_opts(opt_str) { + Ok(o) => opts = o, + Err(e) => return sp_json_err(&e), + } + } + } + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => return sp_json_err(&e), + }; + match sentencepiece::encode_with_options(&model, text_str, opts) { + Ok(tokens) => sp_json_ok_tokens(&tokens, model.model_type.as_str()), + Err(e) => sp_json_err(&e), + } +} + +/// UNIGRAM lattice n-best encode. +/// +/// `options_json` may include encode opts plus `nbest_size` (default 1, max 1024). +/// +/// # Safety +/// Pointers must be valid null-terminated UTF-8 C strings when non-null. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_nbest_encode( + model_path: *const c_char, + text: *const c_char, + options_json: *const c_char, +) -> *mut c_char { + if model_path.is_null() || text.is_null() { + return sp_json_err("null argument"); + } + let path = match unsafe { CStr::from_ptr(model_path).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("model_path is not valid UTF-8"), + }; + let text_str = match unsafe { CStr::from_ptr(text).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("text is not valid UTF-8"), + }; + #[derive(Deserialize, Default)] + struct NbestOpts { + #[serde(default)] + allow_silent_unk: bool, + #[serde(default)] + forced_pieces: Vec, + #[serde(default)] + word_official_split: bool, + #[serde(default = "default_merge_unk_nbest")] + merge_consecutive_unk: bool, + #[serde(default)] + reverse: bool, + #[serde(default)] + add_bos: bool, + #[serde(default)] + add_eos: bool, + #[serde(default)] + nbest_stock_compat: bool, + #[serde(default = "default_nbest")] + nbest_size: usize, + } + fn default_nbest() -> usize { + 1 + } + fn default_merge_unk_nbest() -> bool { + true + } + let mut nbest_size = 1usize; + let mut opts = sentencepiece::EncodeOptions::default(); + if !options_json.is_null() { + let opt_str = match unsafe { CStr::from_ptr(options_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("options_json is not valid UTF-8"), + }; + if !opt_str.is_empty() { + match serde_json::from_str::(opt_str) { + Ok(o) => { + opts.allow_silent_unk = o.allow_silent_unk; + opts.forced_pieces = o.forced_pieces; + opts.word_official_split = o.word_official_split; + opts.merge_consecutive_unk = o.merge_consecutive_unk; + opts.reverse = o.reverse; + opts.add_bos = o.add_bos; + opts.add_eos = o.add_eos; + opts.nbest_stock_compat = o.nbest_stock_compat; + nbest_size = o.nbest_size; + } + Err(e) => return sp_json_err(&format!("invalid options JSON: {e}")), + } + } + } + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => return sp_json_err(&e), + }; + match sentencepiece::nbest_encode_with_options(&model, text_str, nbest_size, opts) { + Ok(nbests) => sp_json_ok_nbest(&nbests, model.model_type.as_str()), + Err(e) => sp_json_err(&e), + } +} + +/// UNIGRAM sample-encode. +/// +/// Options: encode keys plus `nbest_size` (i32), `alpha` (f32), optional `seed` (u64). +/// +/// # Safety +/// Pointers must be valid null-terminated UTF-8 C strings when non-null. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_sample_encode( + model_path: *const c_char, + text: *const c_char, + options_json: *const c_char, +) -> *mut c_char { + if model_path.is_null() || text.is_null() { + return sp_json_err("null argument"); + } + let path = match unsafe { CStr::from_ptr(model_path).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("model_path is not valid UTF-8"), + }; + let text_str = match unsafe { CStr::from_ptr(text).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("text is not valid UTF-8"), + }; + #[derive(Deserialize)] + struct SampleOpts { + #[serde(default)] + allow_silent_unk: bool, + #[serde(default)] + forced_pieces: Vec, + #[serde(default)] + word_official_split: bool, + #[serde(default = "default_merge_unk_sample")] + merge_consecutive_unk: bool, + #[serde(default)] + reverse: bool, + #[serde(default)] + add_bos: bool, + #[serde(default)] + add_eos: bool, + #[serde(default)] + nbest_stock_compat: bool, + #[serde(default = "default_sample_nbest")] + nbest_size: i32, + #[serde(default)] + alpha: f32, + #[serde(default)] + seed: Option, + } + fn default_sample_nbest() -> i32 { + -1 + } + fn default_merge_unk_sample() -> bool { + true + } + let mut sample = sentencepiece::SampleEncodeOptions::default(); + if !options_json.is_null() { + let opt_str = match unsafe { CStr::from_ptr(options_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("options_json is not valid UTF-8"), + }; + if !opt_str.is_empty() { + match serde_json::from_str::(opt_str) { + Ok(o) => { + sample.encode.allow_silent_unk = o.allow_silent_unk; + sample.encode.forced_pieces = o.forced_pieces; + sample.encode.word_official_split = o.word_official_split; + sample.encode.merge_consecutive_unk = o.merge_consecutive_unk; + sample.encode.reverse = o.reverse; + sample.encode.add_bos = o.add_bos; + sample.encode.add_eos = o.add_eos; + sample.encode.nbest_stock_compat = o.nbest_stock_compat; + sample.nbest_size = o.nbest_size; + sample.alpha = o.alpha; + sample.seed = o.seed; + } + Err(e) => return sp_json_err(&format!("invalid options JSON: {e}")), + } + } + } + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => return sp_json_err(&e), + }; + match sentencepiece::sample_encode_with_options(&model, text_str, sample) { + Ok(tokens) => sp_json_ok_tokens(&tokens, model.model_type.as_str()), + Err(e) => sp_json_err(&e), + } +} + +/// UNIGRAM SampleEncodeAndScore (with- or without-replacement). +/// +/// Options: encode keys plus `num_samples` (usize), `alpha` (f32), `wor` (bool), +/// `include_best` (bool), optional `seed` (u64). +/// +/// Returns `{"ok":true,"samples":[{"tokens":[...],"score":...},...],"model_type":"..."}`. +/// +/// # Safety +/// Pointers must be valid null-terminated UTF-8 C strings when non-null. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_sample_encode_and_score( + model_path: *const c_char, + text: *const c_char, + options_json: *const c_char, +) -> *mut c_char { + if model_path.is_null() || text.is_null() { + return sp_json_err("null argument"); + } + let path = match unsafe { CStr::from_ptr(model_path).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("model_path is not valid UTF-8"), + }; + let text_str = match unsafe { CStr::from_ptr(text).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("text is not valid UTF-8"), + }; + #[derive(Deserialize)] + struct ScoreOpts { + #[serde(default)] + allow_silent_unk: bool, + #[serde(default)] + forced_pieces: Vec, + #[serde(default)] + word_official_split: bool, + #[serde(default = "default_merge_unk_score")] + merge_consecutive_unk: bool, + #[serde(default)] + reverse: bool, + #[serde(default)] + add_bos: bool, + #[serde(default)] + add_eos: bool, + #[serde(default)] + nbest_stock_compat: bool, + #[serde(default = "default_num_samples")] + num_samples: usize, + #[serde(default)] + alpha: f32, + #[serde(default)] + wor: bool, + #[serde(default)] + include_best: bool, + #[serde(default)] + seed: Option, + } + fn default_num_samples() -> usize { + 1 + } + fn default_merge_unk_score() -> bool { + true + } + let mut sample = sentencepiece::SampleEncodeAndScoreOptions::default(); + if !options_json.is_null() { + let opt_str = match unsafe { CStr::from_ptr(options_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("options_json is not valid UTF-8"), + }; + if !opt_str.is_empty() { + match serde_json::from_str::(opt_str) { + Ok(o) => { + sample.encode.allow_silent_unk = o.allow_silent_unk; + sample.encode.forced_pieces = o.forced_pieces; + sample.encode.word_official_split = o.word_official_split; + sample.encode.merge_consecutive_unk = o.merge_consecutive_unk; + sample.encode.reverse = o.reverse; + sample.encode.add_bos = o.add_bos; + sample.encode.add_eos = o.add_eos; + sample.encode.nbest_stock_compat = o.nbest_stock_compat; + sample.num_samples = o.num_samples; + sample.alpha = o.alpha; + sample.wor = o.wor; + sample.include_best = o.include_best; + sample.seed = o.seed; + } + Err(e) => return sp_json_err(&format!("invalid options JSON: {e}")), + } + } + } + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => return sp_json_err(&e), + }; + match sentencepiece::sample_encode_and_score_with_options(&model, text_str, sample) { + Ok(rows) => { + let samples: Vec = rows + .iter() + .map(|(tokens, score)| { + serde_json::json!({ + "tokens": tokens, + "score": score, + }) + }) + .collect(); + let payload = serde_json::json!({ + "ok": true, + "samples": samples, + "model_type": model.model_type.as_str(), + }); + match CString::new(payload.to_string()) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } + } + Err(e) => sp_json_err(&e), + } +} + +/// SentencePiece decode: concatenate piece strings for token ids (JSON array). +/// +/// `tokens_json` is a JSON array of u32, e.g. `[1,2,3]`. +/// Returns owned JSON: `{"ok":true,"text":"..."}` or `{"ok":false,"error":"..."}`. +/// +/// # Safety +/// `model_path` and `tokens_json` must be valid, null-terminated UTF-8 C strings. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_decode( + model_path: *const c_char, + tokens_json: *const c_char, +) -> *mut c_char { + unsafe { guardd_sp_decode_with_options(model_path, tokens_json, ptr::null()) } +} + +/// SentencePiece decode with optional JSON options. +/// +/// `options_json` may be null or `{}` for defaults. Supported keys: +/// - `reverse` (bool, default false): reverse ids before detokenize (stock +/// decode ExtraOptions `reverse`) +/// +/// # Safety +/// Pointers must be valid null-terminated UTF-8 C strings when non-null. +#[no_mangle] +pub unsafe extern "C" fn guardd_sp_decode_with_options( + model_path: *const c_char, + tokens_json: *const c_char, + options_json: *const c_char, +) -> *mut c_char { + if model_path.is_null() || tokens_json.is_null() { + return sp_json_err("null argument"); + } + let path = match unsafe { CStr::from_ptr(model_path).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("model_path is not valid UTF-8"), + }; + let tokens_str = match unsafe { CStr::from_ptr(tokens_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("tokens_json is not valid UTF-8"), + }; + let tokens: Vec = match serde_json::from_str(tokens_str) { + Ok(t) => t, + Err(e) => return sp_json_err(&format!("invalid tokens JSON: {e}")), + }; + let mut opts = sentencepiece::DecodeOptions::default(); + if !options_json.is_null() { + let opt_str = match unsafe { CStr::from_ptr(options_json).to_str() } { + Ok(s) => s, + Err(_) => return sp_json_err("options_json is not valid UTF-8"), + }; + if !opt_str.is_empty() { + match parse_sp_decode_opts(opt_str) { + Ok(o) => opts = o, + Err(e) => return sp_json_err(&e), + } + } + } + let model = match sentencepiece::load_model_proto_file(path) { + Ok(m) => m, + Err(e) => return sp_json_err(&e), + }; + match sentencepiece::decode_pieces_with_options(&model, &tokens, opts) { + Ok(text) => sp_json_ok_text(&text), + Err(e) => sp_json_err(&e), + } +} + #[cfg(test)] mod tests { use super::*; - use std::fs; - use tempfile::NamedTempFile; use crate::bitflip_corpus::BitFlipTestResult; use crate::quant_verification::LayerVerification128; + use std::fs; + use tempfile::NamedTempFile; #[test] fn test_verify_digest() { @@ -490,9 +1168,8 @@ mod tests { let expected_digest = hasher.finalize(); let path_cstring = CString::new(temp_file.path().to_str().unwrap()).unwrap(); - let result = unsafe { - guardd_verify_digest(path_cstring.as_ptr(), expected_digest.as_ptr()) - }; + let result = + unsafe { guardd_verify_digest(path_cstring.as_ptr(), expected_digest.as_ptr()) }; assert_eq!(result as i32, GuarddError::Success as i32); } @@ -521,13 +1198,11 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap(); let temp_dir_str = temp_dir.path().to_str().unwrap(); let temp_dir_cstring = CString::new(temp_dir_str).unwrap(); - - let result = unsafe { - guardd_run_bitflip_corpus_test(0, 5, temp_dir_cstring.as_ptr()) - }; - + + let result = unsafe { guardd_run_bitflip_corpus_test(0, 5, temp_dir_cstring.as_ptr()) }; + assert!(!result.is_null()); - + unsafe { let result_str = CStr::from_ptr(result).to_str().unwrap(); let test_result: BitFlipTestResult = serde_json::from_str(result_str).unwrap(); @@ -541,7 +1216,7 @@ mod tests { let layer_name = CString::new("test_layer").unwrap(); let quant_type = CString::new("int8").unwrap(); let weights = [0.1f32; 50]; // 5x10 matrix - + let result = unsafe { guardd_verify_quant_128_vectors( layer_name.as_ptr(), @@ -552,9 +1227,9 @@ mod tests { quant_type.as_ptr(), ) }; - + assert!(!result.is_null()); - + unsafe { let result_str = CStr::from_ptr(result).to_str().unwrap(); let verification: LayerVerification128 = serde_json::from_str(result_str).unwrap(); @@ -564,4 +1239,137 @@ mod tests { guardd_free_json_result(result); } } -} \ No newline at end of file + + #[test] + fn test_verify_model_object_json_contract() { + let layers = r#"[{ + "name": "linear.weight", + "weights": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "fan_in": 3, + "fan_out": 2, + "quant_type": "int8" + }]"#; + let layers_c = CString::new(layers).unwrap(); + let result = unsafe { guardd_verify_model_128_vectors(layers_c.as_ptr()) }; + assert!(!result.is_null(), "object-form layers JSON must parse"); + + unsafe { + let result_str = CStr::from_ptr(result).to_str().unwrap(); + let verification: crate::quant_verification::ModelVerification128 = + serde_json::from_str(result_str).unwrap(); + assert_eq!(verification.total_layers, 1); + assert_eq!(verification.layers[0].layer_name, "linear.weight"); + guardd_free_json_result(result); + } + } + + #[test] + fn test_guardd_verify_quant_unimplemented() { + let quant_type = CString::new("int8").unwrap(); + let weights = [0.1f32; 4]; + #[allow(deprecated)] + let result = unsafe { + guardd_verify_quant(weights.as_ptr(), weights.len(), 2, quant_type.as_ptr(), 0.1) + }; + assert!(result.is_null()); + } + + #[test] + fn test_safe_verify_digest_mismatch() { + let temp_file = NamedTempFile::new().unwrap(); + fs::write(&temp_file, b"test data").unwrap(); + let wrong = [0u8; 32]; + let ok = verify_digest(temp_file.path().to_str().unwrap(), &wrong).unwrap(); + assert!(!ok); + } + + #[test] + fn test_safe_perfect_hash_and_bitflip() { + let vocab = PerfectHashVocab::from_words(vec!["hello".into(), "world".into()]); + let json = serde_json::to_string(&vocab).unwrap(); + let tokens = perfect_hash_encode(&json, "hello world").unwrap(); + assert_eq!(tokens, vec![1, 2]); + assert_eq!(perfect_hash_decode(&json, 1).unwrap(), "hello"); + + let dir = tempfile::tempdir().unwrap(); + let result = + run_bitflip_corpus_test(0, 3, Some(dir.path().to_str().unwrap().to_string())).unwrap(); + assert!(result.test_passed); + } + + #[test] + fn test_perfect_hash_ffi_roundtrip() { + let mph = PerfectHashVocab::from_words(vec!["hello".into(), "world".into(), "test".into()]); + let vocab_json = serde_json::to_string(&mph).unwrap(); + let vocab_c = CString::new(vocab_json).unwrap(); + let text_c = CString::new("hello world test").unwrap(); + + let enc = unsafe { guardd_perfect_hash_encode(vocab_c.as_ptr(), text_c.as_ptr()) }; + assert!(!enc.is_null()); + let tokens: Vec = unsafe { + let s = CStr::from_ptr(enc).to_str().unwrap(); + let parsed = serde_json::from_str(s).unwrap(); + guardd_free_json_result(enc); + parsed + }; + assert_eq!(tokens, vec![1, 2, 3]); + + for (token, word) in tokens.into_iter().zip(["hello", "world", "test"]) { + let dec = unsafe { guardd_perfect_hash_decode(vocab_c.as_ptr(), token) }; + assert!(!dec.is_null()); + unsafe { + assert_eq!(CStr::from_ptr(dec).to_str().unwrap(), word); + guardd_free_json_result(dec); + } + } + } + + #[test] + fn test_sp_encode_decode_ffi_unigram() { + use crate::sentencepiece::ModelProtoFixture; + use std::io::Write; + + let fixture = ModelProtoFixture { + pieces: vec![ + ("".into(), Some(0.0), Some(2)), + ("▁".into(), Some(-10.0), Some(1)), + ("a".into(), Some(-5.0), Some(1)), + ("b".into(), Some(-5.0), Some(1)), + ("▁ab".into(), Some(-0.5), Some(1)), + ], + model_type: Some(1), + add_dummy_prefix: Some(true), + remove_extra_whitespaces: Some(true), + escape_whitespaces: Some(true), + ..Default::default() + }; + let mut tmp = NamedTempFile::new().unwrap(); + tmp.write_all(&fixture.encode()).unwrap(); + let path_c = CString::new(tmp.path().to_str().unwrap()).unwrap(); + let text_c = CString::new("ab").unwrap(); + + let enc = unsafe { guardd_sp_encode(path_c.as_ptr(), text_c.as_ptr()) }; + assert!(!enc.is_null()); + let enc_json: serde_json::Value = unsafe { + let s = CStr::from_ptr(enc).to_str().unwrap(); + let v = serde_json::from_str(s).unwrap(); + guardd_free_json_result(enc); + v + }; + assert_eq!(enc_json["ok"], true); + assert_eq!(enc_json["model_type"], "UNIGRAM"); + assert_eq!(enc_json["tokens"], serde_json::json!([4])); + + let tokens_c = CString::new("[4]").unwrap(); + let dec = unsafe { guardd_sp_decode(path_c.as_ptr(), tokens_c.as_ptr()) }; + assert!(!dec.is_null()); + let dec_json: serde_json::Value = unsafe { + let s = CStr::from_ptr(dec).to_str().unwrap(); + let v = serde_json::from_str(s).unwrap(); + guardd_free_json_result(dec); + v + }; + assert_eq!(dec_json["ok"], true); + assert_eq!(dec_json["text"], "ab"); + } +} From 7667402f07082df44d0c21d91ac911db53284479 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:20 -0700 Subject: [PATCH 17/35] Harden 128-vector quantization verification path Strengthen runtime L2 checks that remain authoritative while Lean Float claims stay obligation markers. --- src/rust/guardd/src/quant_verification.rs | 290 +++++++++++++++++----- 1 file changed, 233 insertions(+), 57 deletions(-) diff --git a/src/rust/guardd/src/quant_verification.rs b/src/rust/guardd/src/quant_verification.rs index 7b6dcca..1860e0c 100644 --- a/src/rust/guardd/src/quant_verification.rs +++ b/src/rust/guardd/src/quant_verification.rs @@ -1,6 +1,6 @@ -use rand::{Rng, SeedableRng}; use rand::rngs::StdRng; -use serde::{Serialize, Deserialize}; +use rand::{Rng, SeedableRng}; +use serde::{Deserialize, Serialize}; /// Configuration for quantization verification #[derive(Debug, Clone, Serialize, Deserialize)] @@ -10,6 +10,37 @@ pub struct QuantizationConfig { pub quant_type: String, } +/// Object-form layer payload accepted by `guardd_verify_model_128_vectors`. +/// +/// JSON shape (matches Python bindings): +/// ```json +/// {"name": "layer", "weights": [0.1, ...], "fan_in": 10, "fan_out": 5, "quant_type": "int8"} +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LayerSpec { + pub name: String, + pub weights: Vec, + pub fan_in: u32, + pub fan_out: u32, + #[serde(default = "default_quant_type")] + pub quant_type: String, +} + +fn default_quant_type() -> String { + "int8".to_string() +} + +impl LayerSpec { + pub fn into_tuple(self) -> (String, Vec, QuantizationConfig) { + let config = QuantizationConfig { + fan_in: self.fan_in, + fan_out: self.fan_out, + quant_type: self.quant_type, + }; + (self.name, self.weights, config) + } +} + /// Detailed verification result with 128 vectors analysis #[derive(Debug, Serialize, Deserialize)] pub struct LayerVerification128 { @@ -40,25 +71,25 @@ pub struct ModelVerification128 { pub fn generate_random_vector(n: usize, seed: u64) -> Vec { let mut rng = StdRng::seed_from_u64(seed); let mut vector = Vec::with_capacity(n); - + for _ in 0..n { // Generate values in range [-1, 1] let value = rng.gen_range(-1.0..1.0); vector.push(value); } - + vector } /// Generate 128 random activation vectors pub fn generate_128_vectors(n: usize) -> Vec> { let mut vectors = Vec::with_capacity(128); - + for i in 0..128 { let vector = generate_random_vector(n, i as u64); vectors.push(vector); } - + vectors } @@ -70,7 +101,7 @@ pub fn l2_norm(vector: &[f32]) -> f32 { /// Matrix-vector multiplication: W * x pub fn matvec_mul(weights: &[f32], fan_in: usize, fan_out: usize, vector: &[f32]) -> Vec { let mut result = Vec::with_capacity(fan_out); - + for i in 0..fan_out { let mut sum = 0.0; for j in 0..fan_in { @@ -78,16 +109,19 @@ pub fn matvec_mul(weights: &[f32], fan_in: usize, fan_out: usize, vector: &[f32] } result.push(sum); } - + result } /// Quantize weights to int8 pub fn quantize_int8(weights: &[f32]) -> Vec { - weights.iter().map(|&w| { - let quantized = (w * 127.0).round().clamp(-127.0, 127.0); - quantized as i8 - }).collect() + weights + .iter() + .map(|&w| { + let quantized = (w * 127.0).round().clamp(-127.0, 127.0); + quantized as i8 + }) + .collect() } /// Dequantize int8 weights back to float @@ -104,60 +138,99 @@ pub fn compute_epsilon_bound(config: &QuantizationConfig) -> f64 { } } -/// Verify a single layer with 128 random activation vectors +/// Verify a single layer with 128 random activation vectors. +/// +/// Error metric (ε definition): for each activation `x`, +/// `e = ||W_q x - W x||_2 / ||x||_2`, then compare `max e` to `epsilon_bound`. +/// +/// Only `quant_type == "int8"` is implemented. Other types fail closed +/// (`passed_128_vectors = false`, infinite max error). pub fn verify_layer_128_vectors( layer_name: &str, weights: &[f32], config: &QuantizationConfig, ) -> LayerVerification128 { let start_time = std::time::Instant::now(); - + let epsilon_bound = compute_epsilon_bound(config); let fan_in = config.fan_in as usize; let fan_out = config.fan_out as usize; - - // Quantize and dequantize weights + let expected_len = fan_in.saturating_mul(fan_out); + + if config.quant_type != "int8" { + let computation_time = start_time.elapsed().as_millis() as u64; + return LayerVerification128 { + layer_name: layer_name.to_string(), + config: config.clone(), + epsilon_bound, + max_error_128_vectors: f64::INFINITY, + mean_error_128_vectors: f64::INFINITY, + error_std_deviation: 0.0, + passed_128_vectors: false, + error_distribution: vec![f64::INFINITY; 128], + computation_time_ms: computation_time, + }; + } + + if weights.len() != expected_len || fan_in == 0 || fan_out == 0 { + let computation_time = start_time.elapsed().as_millis() as u64; + return LayerVerification128 { + layer_name: layer_name.to_string(), + config: config.clone(), + epsilon_bound, + max_error_128_vectors: f64::INFINITY, + mean_error_128_vectors: f64::INFINITY, + error_std_deviation: 0.0, + passed_128_vectors: false, + error_distribution: vec![f64::INFINITY; 128], + computation_time_ms: computation_time, + }; + } + + // Quantize and dequantize weights (int8 only) let quantized = quantize_int8(weights); let dequantized = dequantize_int8(&quantized); - + // Generate 128 random activation vectors let vectors = generate_128_vectors(fan_in); - - // Compute errors for all 128 vectors + + // Compute errors for all 128 vectors: ||(Wq - W) x|| / ||x|| let mut error_distribution = Vec::with_capacity(128); let mut sum_errors = 0.0f64; let mut max_error = 0.0f64; - + for vector in &vectors { - // Compute original output let original_output = matvec_mul(weights, fan_in, fan_out, vector); - let original_norm = l2_norm(&original_output); - - // Compute quantized output let quantized_output = matvec_mul(&dequantized, fan_in, fan_out, vector); - let quantized_norm = l2_norm(&quantized_output); - - // Compute error - let error = if original_norm > 0.0 { - (quantized_norm - original_norm).abs() / original_norm + + let mut diff = Vec::with_capacity(fan_out); + for i in 0..fan_out { + diff.push(quantized_output[i] - original_output[i]); + } + + let x_norm = l2_norm(vector); + let error = if x_norm > 0.0 { + (l2_norm(&diff) / x_norm) as f64 } else { 0.0 }; - - error_distribution.push(error as f64); - sum_errors += error as f64; - max_error = max_error.max(error as f64); + + error_distribution.push(error); + sum_errors += error; + max_error = max_error.max(error); } - + // Compute statistics let mean_error = sum_errors / 128.0; - let variance = error_distribution.iter() + let variance = error_distribution + .iter() .map(|&err| (err - mean_error).powi(2)) - .sum::() / 128.0; + .sum::() + / 128.0; let std_deviation = variance.sqrt(); - + let computation_time = start_time.elapsed().as_millis() as u64; - + LayerVerification128 { layer_name: layer_name.to_string(), config: config.clone(), @@ -177,33 +250,44 @@ pub fn verify_model_128_vectors( ) -> ModelVerification128 { let mut layer_results = Vec::with_capacity(layers.len()); let mut total_computation_time = 0u64; - + for (layer_name, weights, config) in layers { let verification = verify_layer_128_vectors(&layer_name, &weights, &config); total_computation_time += verification.computation_time_ms; layer_results.push(verification); } - + // Compute overall statistics let total_layers = layer_results.len(); - let passed_layers = layer_results.iter().filter(|v| v.passed_128_vectors).count(); + let passed_layers = layer_results + .iter() + .filter(|v| v.passed_128_vectors) + .count(); let overall_pass_rate = if total_layers > 0 { (passed_layers as f64 / total_layers as f64) * 100.0 } else { 0.0 }; - - let max_errors: Vec = layer_results.iter().map(|v| v.max_error_128_vectors).collect(); + + let max_errors: Vec = layer_results + .iter() + .map(|v| v.max_error_128_vectors) + .collect(); let mean_max_error = if !max_errors.is_empty() { max_errors.iter().sum::() / max_errors.len() as f64 } else { 0.0 }; - - let worst_layer = layer_results.iter() - .max_by(|a, b| a.max_error_128_vectors.partial_cmp(&b.max_error_128_vectors).unwrap()) + + let worst_layer = layer_results + .iter() + .max_by(|a, b| { + a.max_error_128_vectors + .partial_cmp(&b.max_error_128_vectors) + .unwrap() + }) .map(|v| v.layer_name.clone()); - + ModelVerification128 { layers: layer_results, total_layers, @@ -223,7 +307,7 @@ mod tests { fn test_generate_random_vector() { let vector = generate_random_vector(10, 42); assert_eq!(vector.len(), 10); - + // Test determinism let vector2 = generate_random_vector(10, 42); assert_eq!(vector, vector2); @@ -248,7 +332,7 @@ mod tests { let weights = vec![1.0, 2.0, 3.0, 4.0]; // 2x2 matrix let vector = vec![1.0, 2.0]; let result = matvec_mul(&weights, 2, 2, &vector); - + assert_eq!(result.len(), 2); assert!((result[0] - 5.0).abs() < 1e-6); // 1*1 + 2*2 assert!((result[1] - 11.0).abs() < 1e-6); // 3*1 + 4*2 @@ -259,10 +343,10 @@ mod tests { let weights = vec![0.5, -0.25, 0.75]; let quantized = quantize_int8(&weights); let dequantized = dequantize_int8(&quantized); - + assert_eq!(quantized.len(), 3); assert_eq!(dequantized.len(), 3); - + // Check that dequantized values are close to original for (orig, deq) in weights.iter().zip(dequantized.iter()) { assert!((orig - deq).abs() < 0.01); @@ -276,13 +360,49 @@ mod tests { fan_out: 5, quant_type: "int8".to_string(), }; - + let weights = vec![0.1; 50]; // 5x10 matrix let verification = verify_layer_128_vectors("test_layer", &weights, &config); - + assert_eq!(verification.layer_name, "test_layer"); assert_eq!(verification.error_distribution.len(), 128); assert!(verification.computation_time_ms > 0); + assert!(verification.max_error_128_vectors.is_finite()); + assert!(verification.passed_128_vectors); + } + + #[test] + fn test_error_is_vector_delta_not_output_norm_delta() { + // Construct a case where ||Wq x|| ≈ ||W x|| but Wq x ≠ W x would still + // be caught by vector error. Here we just assert the metric matches + // ||(Wq-W)x||/||x|| on a single known vector path via the API shape. + let config = QuantizationConfig { + fan_in: 2, + fan_out: 2, + quant_type: "int8".to_string(), + }; + let weights = vec![0.5, -0.25, 0.75, 0.125]; + let verification = verify_layer_128_vectors("metric", &weights, &config); + let x = generate_random_vector(2, 0); + let q = dequantize_int8(&quantize_int8(&weights)); + let orig = matvec_mul(&weights, 2, 2, &x); + let quant = matvec_mul(&q, 2, 2, &x); + let diff: Vec = orig.iter().zip(quant.iter()).map(|(a, b)| b - a).collect(); + let expected = (l2_norm(&diff) / l2_norm(&x)) as f64; + assert!((verification.error_distribution[0] - expected).abs() < 1e-5); + } + + #[test] + fn test_non_int8_fails_closed() { + let config = QuantizationConfig { + fan_in: 2, + fan_out: 2, + quant_type: "fp16".to_string(), + }; + let weights = vec![0.1, 0.2, 0.3, 0.4]; + let verification = verify_layer_128_vectors("fp16_layer", &weights, &config); + assert!(!verification.passed_128_vectors); + assert!(verification.max_error_128_vectors.is_infinite()); } #[test] @@ -292,16 +412,72 @@ mod tests { fan_out: 3, quant_type: "int8".to_string(), }; - + let layers = vec![ ("layer1".to_string(), vec![0.1; 15], config.clone()), ("layer2".to_string(), vec![0.1; 15], config), ]; - + let verification = verify_model_128_vectors(layers); - + assert_eq!(verification.total_layers, 2); assert_eq!(verification.layers.len(), 2); assert!(verification.total_computation_time_ms > 0); } -} \ No newline at end of file + + #[test] + fn test_layer_spec_object_json_contract() { + let json = r#"[ + { + "name": "linear.weight", + "weights": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "fan_in": 3, + "fan_out": 2, + "quant_type": "int8" + } + ]"#; + let specs: Vec = serde_json::from_str(json).unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].name, "linear.weight"); + let layers: Vec<_> = specs.into_iter().map(LayerSpec::into_tuple).collect(); + let verification = verify_model_128_vectors(layers); + assert_eq!(verification.total_layers, 1); + assert_eq!(verification.layers[0].layer_name, "linear.weight"); + } + + #[test] + fn test_shared_epsilon_fixture_int8() { + // tests/fixtures/quant_epsilon_int8.json — ε = 0.5 * sqrt(fan_in) + let fixture_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../tests/fixtures/quant_epsilon_int8.json" + ); + let raw = std::fs::read_to_string(fixture_path).expect("shared epsilon fixture"); + let v: serde_json::Value = serde_json::from_str(&raw).unwrap(); + let fan_in = v["fan_in"].as_u64().unwrap() as u32; + let fan_out = v["fan_out"].as_u64().unwrap() as u32; + let expected_eps = v["epsilon_bound"].as_f64().unwrap(); + let weights: Vec = v["weights"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_f64().unwrap() as f32) + .collect(); + assert_eq!(weights.len(), (fan_in * fan_out) as usize); + + let config = QuantizationConfig { + fan_in, + fan_out, + quant_type: "int8".to_string(), + }; + let eps = compute_epsilon_bound(&config); + assert!((eps - expected_eps).abs() < 1e-9); + // Lean LayerBound uses the same closed form: 0.5 * sqrt(fan_in) + assert!((eps - 0.5 * (fan_in as f64).sqrt()).abs() < 1e-12); + + let verification = verify_layer_128_vectors("fixture.linear", &weights, &config); + assert_eq!(verification.epsilon_bound, eps); + assert!(verification.max_error_128_vectors.is_finite()); + assert!(verification.passed_128_vectors); + } +} From 81289e9c85c130346979575ce26af06fd0314690 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:20 -0700 Subject: [PATCH 18/35] Expand bitflip corpus generation and coverage helpers Improve deterministic corpus tooling used by the guardd_bitflip sidecar and integration tests. --- src/rust/guardd/src/bitflip_corpus.rs | 378 +++++++++++++++++--------- 1 file changed, 253 insertions(+), 125 deletions(-) diff --git a/src/rust/guardd/src/bitflip_corpus.rs b/src/rust/guardd/src/bitflip_corpus.rs index eabad22..c92e269 100644 --- a/src/rust/guardd/src/bitflip_corpus.rs +++ b/src/rust/guardd/src/bitflip_corpus.rs @@ -1,10 +1,19 @@ -use sha2::{Sha256, Digest}; -use std::fs::{File, OpenOptions}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; +use std::path::PathBuf; use std::time::Instant; -use rand::{Rng, SeedableRng}; -use rand::rngs::StdRng; -use serde::{Serialize, Deserialize}; + +/// Default maximum corpus size without explicit `--allow-large` / opt-in. +pub const MAX_DEFAULT_CORPUS_BYTES: u64 = 64 * 1024 * 1024; + +/// Platform temp dir + `bitflip_corpus` subdirectory (Windows-safe; not `/tmp`). +pub fn default_bitflip_temp_dir() -> PathBuf { + std::env::temp_dir().join("bitflip_corpus") +} #[derive(Debug, Serialize, Deserialize, Clone)] pub struct BitFlipTestResult { @@ -31,32 +40,49 @@ pub struct CorpusTestSummary { pub struct BitFlipCorpusTester { chunk_size_mb: usize, - temp_dir: String, + temp_dir: PathBuf, } impl BitFlipCorpusTester { pub fn new(chunk_size_mb: usize, temp_dir: Option) -> Self { - let temp_dir = temp_dir.unwrap_or_else(|| "/tmp/bitflip_corpus".to_string()); + let temp_dir = temp_dir + .map(PathBuf::from) + .unwrap_or_else(default_bitflip_temp_dir); Self { chunk_size_mb, temp_dir, } } + /// Ensure the corpus working directory exists (idempotent). + pub fn ensure_temp_dir(&self) -> Result<(), Box> { + fs::create_dir_all(&self.temp_dir)?; + Ok(()) + } + + fn path_in_temp(&self, name: &str) -> PathBuf { + self.temp_dir.join(name) + } + /// Generate deterministic random data for reproducible testing pub fn generate_deterministic_data(&self, size_bytes: usize, seed: u64) -> Vec { let mut rng = StdRng::seed_from_u64(seed); let mut data = Vec::with_capacity(size_bytes); - + for _ in 0..size_bytes { data.push(rng.gen::()); } - + data } /// Apply random bit flips to data - pub fn apply_bit_flips(&self, data: &[u8], flip_probability: f64, seed: u64) -> (Vec, Vec) { + pub fn apply_bit_flips( + &self, + data: &[u8], + flip_probability: f64, + seed: u64, + ) -> (Vec, Vec) { let mut rng = StdRng::seed_from_u64(seed); let mut corrupted_data = data.to_vec(); let mut flipped_positions = Vec::new(); @@ -74,9 +100,17 @@ impl BitFlipCorpusTester { } /// Create a large test file with specified size - pub fn create_test_file(&self, file_path: &str, size_bytes: usize) -> Result<(), Box> { - println!("Creating test file: {} ({:.2} GB)", file_path, size_bytes as f64 / (1024.0 * 1024.0 * 1024.0)); - + pub fn create_test_file( + &self, + file_path: &str, + size_bytes: usize, + ) -> Result<(), Box> { + println!( + "Creating test file: {} ({:.2} GB)", + file_path, + size_bytes as f64 / (1024.0 * 1024.0 * 1024.0) + ); + let chunk_size_bytes = self.chunk_size_mb * 1024 * 1024; let mut file = OpenOptions::new() .create(true) @@ -90,7 +124,7 @@ impl BitFlipCorpusTester { while remaining_bytes > 0 { let chunk_size = std::cmp::min(chunk_size_bytes, remaining_bytes); let chunk_data = self.generate_deterministic_data(chunk_size, chunk_num); - + file.write_all(&chunk_data)?; remaining_bytes -= chunk_size; chunk_num += 1; @@ -123,38 +157,68 @@ impl BitFlipCorpusTester { } /// Test file integrity using our Rust verification function - pub fn test_file_integrity(&self, file_path: &str, expected_hash: Option<&str>) -> Result> { + pub fn test_file_integrity( + &self, + file_path: &str, + expected_hash: Option<&str>, + ) -> Result> { let computed_hash = self.compute_file_hash(file_path)?; - + match expected_hash { Some(expected) => Ok(computed_hash == expected), None => Ok(true), // No expected hash provided, assume valid } } - /// Run bit-flip test on a file of specified size - pub fn run_bitflip_test(&self, file_size_gb: usize, num_corruptions: usize) -> Result> { - println!("\n=== Running {}GB Bit-Flip Test ===", file_size_gb); - + /// Run bit-flip test with an explicit byte size. + /// + /// Sizes above [`MAX_DEFAULT_CORPUS_BYTES`] require `allow_large = true`. + pub fn run_bitflip_test_bytes( + &self, + file_size_bytes: u64, + num_corruptions: usize, + allow_large: bool, + ) -> Result> { + if file_size_bytes == 0 { + return Err("file size must be > 0".into()); + } + if !allow_large && file_size_bytes > MAX_DEFAULT_CORPUS_BYTES { + return Err(format!( + "refusing corpus size {} bytes (cap {} without allow_large)", + file_size_bytes, MAX_DEFAULT_CORPUS_BYTES + ) + .into()); + } + if num_corruptions == 0 { + return Err("corruptions must be > 0".into()); + } + + println!( + "\n=== Running Bit-Flip Test ({:.2} MB) ===", + file_size_bytes as f64 / (1024.0 * 1024.0) + ); + let start_time = Instant::now(); - let file_size_bytes = if file_size_gb == 0 { - 1024 * 1024 - } else { - file_size_gb * 1024 * 1024 * 1024 - }; - - // Create test file - let test_file = format!("{}/test_{}gb.bin", self.temp_dir, file_size_gb); - self.create_test_file(&test_file, file_size_bytes)?; - + let size_usize = file_size_bytes as usize; + + self.ensure_temp_dir()?; + + // Create test file (PathBuf: works on Windows and Unix) + let test_file = self.path_in_temp(&format!("test_{file_size_bytes}b.bin")); + self.create_test_file(test_file.to_str().ok_or("non-UTF-8 temp path")?, size_usize)?; + // Compute original hash println!("Computing original hash..."); - let original_hash = self.compute_file_hash(&test_file)?; - + let original_hash = + self.compute_file_hash(test_file.to_str().ok_or("non-UTF-8 temp path")?)?; + // Test original file (should pass) println!("Testing original file integrity..."); - let original_valid = self.test_file_integrity(&test_file, Some(&original_hash))?; - + let original_valid = self.test_file_integrity( + test_file.to_str().ok_or("non-UTF-8 temp path")?, + Some(&original_hash), + )?; + if !original_valid { return Err("Original file failed integrity check".into()); } @@ -165,10 +229,11 @@ impl BitFlipCorpusTester { for i in 0..num_corruptions { println!("Applying corruption {}/{}...", i + 1, num_corruptions); - + // Create corrupted copy - let corrupted_file = format!("{}/corrupted_{}_{}gb.bin", self.temp_dir, i, file_size_gb); - + let corrupted_file = + self.path_in_temp(&format!("corrupted_{i}_{file_size_bytes}b.bin")); + // Copy and corrupt { let mut src = File::open(&test_file)?; @@ -177,42 +242,46 @@ impl BitFlipCorpusTester { .write(true) .truncate(true) .open(&corrupted_file)?; - + let mut buffer = Vec::new(); src.read_to_end(&mut buffer)?; - - let (corrupted_data, _flipped_positions) = self.apply_bit_flips(&buffer, 0.0001, i as u64); + + let (corrupted_data, _flipped_positions) = + self.apply_bit_flips(&buffer, 0.0001, i as u64); dst.write_all(&corrupted_data)?; dst.flush()?; } - + // Test corrupted file (should fail) - let corrupted_valid = self.test_file_integrity(&corrupted_file, Some(&original_hash))?; - + let corrupted_valid = self.test_file_integrity( + corrupted_file.to_str().ok_or("non-UTF-8 temp path")?, + Some(&original_hash), + )?; + if !corrupted_valid { rejection_count += 1; } - + corruption_results.push((i, !corrupted_valid)); - + // Clean up corrupted file - std::fs::remove_file(&corrupted_file)?; - - println!(" Corruption {}: rejected: {}", - i + 1, !corrupted_valid); + let _ = fs::remove_file(&corrupted_file); + + println!(" Corruption {}: rejected: {}", i + 1, !corrupted_valid); } - + // Clean up test file - std::fs::remove_file(&test_file)?; - + let _ = fs::remove_file(&test_file); + let duration = start_time.elapsed(); let duration_ms = duration.as_millis() as u64; - let throughput_mbps = (file_size_bytes as f64 / (1024.0 * 1024.0)) / (duration.as_secs_f64()); + let secs = duration.as_secs_f64().max(1e-9); + let throughput_mbps = (file_size_bytes as f64 / (1024.0 * 1024.0)) / secs; let rejection_rate = (rejection_count as f64 / num_corruptions as f64) * 100.0; let test_passed = rejection_rate == 100.0; - + let result = BitFlipTestResult { - file_size_bytes: file_size_bytes as u64, + file_size_bytes, original_hash, corruption_count: num_corruptions, rejection_count, @@ -221,33 +290,69 @@ impl BitFlipCorpusTester { duration_ms, throughput_mbps, }; - + println!("\nTest Results:"); - println!(" File size: {}GB", file_size_gb); + println!( + " File size: {:.2} MB", + file_size_bytes as f64 / (1024.0 * 1024.0) + ); println!(" Duration: {:.2}s", duration.as_secs_f64()); println!(" Corruptions tested: {}", num_corruptions); println!(" Rejections: {}/{}", rejection_count, num_corruptions); println!(" Rejection rate: {:.1}%", rejection_rate); println!(" Throughput: {:.2} MB/s", throughput_mbps); - println!(" Test passed: {}", if test_passed { "✓" } else { "✗" }); - + println!( + " Test passed: {}", + if test_passed { "PASS" } else { "FAIL" } + ); + Ok(result) } - /// Run scalability test with different file sizes + /// Run bit-flip test on a file of specified size in GB. + /// + /// `file_size_gb == 0` means 1 MiB (unit-test / CI default). + /// Sizes above the default cap require `allow_large`. + pub fn run_bitflip_test( + &self, + file_size_gb: usize, + num_corruptions: usize, + ) -> Result> { + let file_size_bytes = if file_size_gb == 0 { + 1024 * 1024 + } else { + (file_size_gb as u64).saturating_mul(1024 * 1024 * 1024) + }; + let allow_large = file_size_bytes <= MAX_DEFAULT_CORPUS_BYTES + || std::env::var("GUARDD_BITFLIP_ALLOW_LARGE").ok().as_deref() == Some("1"); + self.run_bitflip_test_bytes(file_size_bytes, num_corruptions, allow_large) + } + + /// Run scalability test with small default sizes (CI-safe). + /// + /// Large GB suites require `GUARDD_BITFLIP_ALLOW_LARGE=1`. pub fn run_scalability_test(&self) -> Result> { - println!("\n=== Running Scalability Test ==="); - - // Test different file sizes: 1GB, 10GB, 50GB, 100GB - let test_sizes = vec![1, 10, 50, 100]; + println!("\n=== Running Scalability Test (capped defaults) ==="); + + let allow_large = std::env::var("GUARDD_BITFLIP_ALLOW_LARGE").ok().as_deref() == Some("1"); + // Default: 1 MiB only. Opt-in: 1 GiB + 1 MiB. + let test_sizes_bytes: Vec = if allow_large { + vec![1024 * 1024, 1024 * 1024 * 1024] + } else { + vec![1024 * 1024] + }; + let mut test_results = Vec::new(); let mut total_duration_ms = 0u64; let mut total_throughput = 0.0; let mut test_count = 0; - - for &size_gb in &test_sizes { - println!("\nTesting {}GB file...", size_gb); - match self.run_bitflip_test(size_gb, 5) { + + for &size_bytes in &test_sizes_bytes { + println!( + "\nTesting {:.2} MB file...", + size_bytes as f64 / (1024.0 * 1024.0) + ); + match self.run_bitflip_test_bytes(size_bytes, 5, allow_large) { Ok(result) => { test_results.push(result.clone()); total_duration_ms += result.duration_ms; @@ -255,9 +360,9 @@ impl BitFlipCorpusTester { test_count += 1; } Err(e) => { - println!("Error testing {}GB: {}", size_gb, e); + println!("Error testing {} bytes: {}", size_bytes, e); test_results.push(BitFlipTestResult { - file_size_bytes: (size_gb * 1024 * 1024 * 1024) as u64, + file_size_bytes: size_bytes, original_hash: "".to_string(), corruption_count: 0, rejection_count: 0, @@ -269,12 +374,16 @@ impl BitFlipCorpusTester { } } } - + let passed_tests = test_results.iter().filter(|r| r.test_passed).count(); let failed_tests = test_results.len() - passed_tests; let overall_success_rate = (passed_tests as f64 / test_results.len() as f64) * 100.0; - let average_throughput = if test_count > 0 { total_throughput / test_count as f64 } else { 0.0 }; - + let average_throughput = if test_count > 0 { + total_throughput / test_count as f64 + } else { + 0.0 + }; + Ok(CorpusTestSummary { total_tests: test_results.len(), passed_tests, @@ -286,54 +395,30 @@ impl BitFlipCorpusTester { }) } - /// Run comprehensive 100GB bit-flip corpus test + /// Run comprehensive bit-flip corpus test (defaults stay under the size cap). + /// + /// Set `GUARDD_BITFLIP_ALLOW_LARGE=1` to include a 1 GiB case. There is no + /// automatic 100 GiB path anymore. pub fn run_comprehensive_test(&self) -> Result> { println!("{}", "=".repeat(80)); - println!("100GB Bit-Flip Corpus Test for Model Asset Guard (Rust)"); + println!("Bit-Flip Corpus Test for Model Asset Guard (Rust)"); println!("{}", "=".repeat(80)); - + let start_time = Instant::now(); - - // Run scalability test - let mut summary = self.run_scalability_test()?; - - // Run full 100GB test if system has enough resources - println!("\nAttempting full 100GB test..."); - match self.run_bitflip_test(100, 20) { - Ok(full_test_result) => { - summary.test_results.push(full_test_result.clone()); - summary.total_tests += 1; - if full_test_result.test_passed { - summary.passed_tests += 1; - } else { - summary.failed_tests += 1; - } - } - Err(e) => { - println!("Full 100GB test failed: {}", e); - summary.test_results.push(BitFlipTestResult { - file_size_bytes: 100 * 1024 * 1024 * 1024, - original_hash: "".to_string(), - corruption_count: 0, - rejection_count: 0, - rejection_rate: 0.0, - test_passed: false, - duration_ms: 0, - throughput_mbps: 0.0, - }); - summary.total_tests += 1; - summary.failed_tests += 1; - } - } - + let summary = self.run_scalability_test()?; let total_time = start_time.elapsed(); + + let mut summary = summary; summary.total_duration_ms = total_time.as_millis() as u64; - summary.overall_success_rate = (summary.passed_tests as f64 / summary.total_tests as f64) * 100.0; - - // Save results + summary.overall_success_rate = if summary.total_tests > 0 { + (summary.passed_tests as f64 / summary.total_tests as f64) * 100.0 + } else { + 0.0 + }; + let json_result = serde_json::to_string_pretty(&summary)?; std::fs::write("bitflip_corpus_rust_results.json", json_result)?; - + println!("\n{}", "=".repeat(80)); println!("Test Suite Summary (Rust)"); println!("{}", "=".repeat(80)); @@ -342,10 +427,20 @@ impl BitFlipCorpusTester { println!("Passed tests: {}", summary.passed_tests); println!("Failed tests: {}", summary.failed_tests); println!("Success rate: {:.1}%", summary.overall_success_rate); - println!("Average throughput: {:.2} MB/s", summary.average_throughput_mbps); - println!("All tests passed: {}", if summary.failed_tests == 0 { "✓" } else { "✗" }); + println!( + "Average throughput: {:.2} MB/s", + summary.average_throughput_mbps + ); + println!( + "All tests passed: {}", + if summary.failed_tests == 0 { + "PASS" + } else { + "FAIL" + } + ); println!("Results saved to: bitflip_corpus_rust_results.json"); - + Ok(summary) } } @@ -368,30 +463,63 @@ mod tests { let tester = BitFlipCorpusTester::new(1, None); let original_data = vec![0u8; 100]; let (corrupted_data, flipped_positions) = tester.apply_bit_flips(&original_data, 0.1, 42); - + assert_ne!(original_data, corrupted_data); assert!(!flipped_positions.is_empty()); } + #[test] + fn default_temp_dir_is_platform_temp() { + let tester = BitFlipCorpusTester::new(1, None); + let expected = default_bitflip_temp_dir(); + assert_eq!(tester.temp_dir, expected); + // Must track std::env::temp_dir() (Windows: %TEMP%; Unix: often /tmp). + assert!(tester.temp_dir.starts_with(std::env::temp_dir())); + assert_eq!( + tester.temp_dir.file_name().and_then(|s| s.to_str()), + Some("bitflip_corpus") + ); + tester.ensure_temp_dir().unwrap(); + assert!(tester.temp_dir.is_dir()); + } + #[test] fn test_file_creation_and_hash() { let temp_dir = tempdir().unwrap(); - let tester = BitFlipCorpusTester::new(1, Some(temp_dir.path().to_str().unwrap().to_string())); - + let tester = + BitFlipCorpusTester::new(1, Some(temp_dir.path().to_str().unwrap().to_string())); + let test_file = temp_dir.path().join("test.bin"); - tester.create_test_file(test_file.to_str().unwrap(), 1024).unwrap(); - - let hash = tester.compute_file_hash(test_file.to_str().unwrap()).unwrap(); + tester + .create_test_file(test_file.to_str().unwrap(), 1024) + .unwrap(); + + let hash = tester + .compute_file_hash(test_file.to_str().unwrap()) + .unwrap(); assert_eq!(hash.len(), 64); // SHA-256 hex string length } #[test] fn test_small_bitflip_test() { let temp_dir = tempdir().unwrap(); - let tester = BitFlipCorpusTester::new(1, Some(temp_dir.path().to_str().unwrap().to_string())); - + let tester = + BitFlipCorpusTester::new(1, Some(temp_dir.path().to_str().unwrap().to_string())); + // Test with 1MB file (much smaller for unit test) let result = tester.run_bitflip_test(0, 5).unwrap(); // 0GB = 1MB for testing assert!(result.test_passed); } -} \ No newline at end of file + + #[test] + fn test_size_cap_without_allow_large() { + let temp_dir = tempdir().unwrap(); + let tester = + BitFlipCorpusTester::new(1, Some(temp_dir.path().to_str().unwrap().to_string())); + let oversized = MAX_DEFAULT_CORPUS_BYTES + 1; + let err = tester + .run_bitflip_test_bytes(oversized, 1, false) + .unwrap_err(); + assert!(err.to_string().contains("allow_large")); + } +} From c498281fd98593fc2eb606abe4782375df484b88 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:27 -0700 Subject: [PATCH 19/35] Document zero-axiom inventory and CHD tokenizer status Record proven CHD/Fixed theorems and honest MPH runtime limits so formal claims stay auditable. --- docs/axioms.md | 125 +++++++++++++++++++++++++++++++++ docs/perfect-hash-tokenizer.md | 104 +++++++++++++++++++++++---- 2 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 docs/axioms.md diff --git a/docs/axioms.md b/docs/axioms.md new file mode 100644 index 0000000..7ab7afb --- /dev/null +++ b/docs/axioms.md @@ -0,0 +1,125 @@ +# Lean axiom inventory + +Model Asset Guard does **not** claim formal completeness. The Lean +sources type-check and build under `warningAsError`. There are currently +**no** `axiom` declarations under `src/lean/`; remaining Float-facing +properties are honest obligation markers (`Prop := True`), not axioms. + +## Allowed axioms (inventory) + +No axioms. Wave 6 reformulated the last Float bridge +(`round_int8_error_bound`) into `round_int8_error_bound_float_obligation` +(obligation marker). Wave 7 kept count at 0 and documented the Float +refinement path in `float-obligation.md`. Proven Int analogue remains +`Fixed.roundDivInt_error` / `fixed_round_half_ulp`. Wave 28 evaluated a +Mathlib / IEEE refinement gate and **aborted** (opaque `Float`, CI cost, +no non-vacuous 0-axiom morphism); markers unchanged — see +`float-obligation.md`. + +**Count:** 0 axioms. + +## Proven (Wave 4–18) — not axioms + +| Module | Theorem | Statement (informal) | +| --- | --- | --- | +| `ModelAssetGuard.Quant.Fixed` | `roundDivNat_error` / `roundDivInt_error` / `*_half_unit` | Half-ULP: `2 * \|n - round(n/d)*d\| ≤ d` | +| `ModelAssetGuard.Token.HashSeeded` | `slotMod_lt` | CHD slot `hash % n < n` | +| `ModelAssetGuard.Token.CHD` | `wellSized_of_displaceCandidatesOk` | Found displace candidates ⇒ wellSized bucket place | +| `ModelAssetGuard.Token.CHD` | `pairwiseNeBool_eq_true_iff_pairwiseDistinct` | Bool pairwise-≠ ↔ Prop pairwiseDistinct | +| `ModelAssetGuard.Token.CHD` | `wellFormed_of_wellFormedBool` / `wellSized_of_wellSizedBool` | Executable gates imply abstract well-formedness | +| `ModelAssetGuard.Token.CHDBuild` | `singleton_wellSized` | Singleton key→slot place is well-sized | +| `ModelAssetGuard.Token.CHDBuild` | `candidateSlots_length` / `candidateSlots_lt` | HashSeeded candidate slots: length + `< n` | +| `ModelAssetGuard.Token.CHDBuild` | `findDisplace_eq_some_imp` | Search return ⇒ `d ≤ maxD` and Bool gate true | +| `ModelAssetGuard.Token.CHDBuild` | `slotMod_one` / `candidateSlots_tableSize_one` | Table size 1 ⇒ every slot is `0` | +| `ModelAssetGuard.Token.CHDBuild` | `displaceCandidatesOk_of_candidatesOkBool` | Bool candidate gate ⇒ Prop `displaceCandidatesOk` | +| `ModelAssetGuard.Token.CHDBuild` | `findDisplace_singleton_freeOccupied` / `findDisplace_nil_keys` | Completeness: singleton or empty keys on free mask ⇒ `d = 0` | +| `ModelAssetGuard.Token.CHDBuild` | `candidatesOkBool_singleton_free` / `slotFreeBool_freeOccupied` | Free-mask Bool gate lemmas | +| `ModelAssetGuard.Token.CHDBuild` | `finalizePlacement_eq_some_imp_wellFormed` / `assembleBucketed_eq_some_imp_wellFormed` | Builder finalize ⇒ abstract `wellFormed` | +| `ModelAssetGuard.Token.CHDBuild` | `assembleBucketed_eq_some_imp` | Assemble success ⇒ `wellFormed` + `nBuckets` / `pilots` fields | +| `ModelAssetGuard.Token.CHDBuild` | `wellSized_of_findDisplace_success` | Compositional gate chain: findDisplace ⇒ wellSized (no mega-unfold) | +| `ModelAssetGuard.Token.CHDBuild` | `wellSized_merge_bucket_places` | Append step preserves wellSized under `BucketPlace.disjoint` | +| `ModelAssetGuard.Token.CHDBuild` | `findDisplace_returns_zero_of_candidatesOk_at_zero` | Restricted ∃d: gate OK at `d = 0` ⇒ search returns `some 0` | +| `ModelAssetGuard.Token.CHDBuild` | `placeBucket_singleton_freeOccupied` | Singleton placeBucket on free table succeeds at `d = 0` | +| `ModelAssetGuard.Quant.Fixed` | `dot_abs_bound_of_entrywise` | Entrywise Int ⇒ `2 * \|e·x\| ≤ scale * \|x\|₁` | +| `ModelAssetGuard.Quant.Fixed` | `rows_dot_abs_bound` | Same entrywise hypothesis on every row | +| `ModelAssetGuard.Quant.LayerBound` | `epsilonBoundSquaredUnits_nonneg` | Nat ε-scale is nonnegative (replaces old Float axiom) | +| `ModelAssetGuard.Quant.Core` | `layer_row_l1_bound` / `layer_rows_l1_bound` | Float L2 layer claim weakened to proven Int L1 forms | +| `ModelAssetGuard.Quant.Core` | `fixed_round_half_ulp` | Alias of Int half-ULP (replaces Float axiom role) | +| `ModelAssetGuard.Token.Tokenizer` | `bpe_string_roundtrip_of_faithful_vocab` | Char-level BPE round-trip under decode-faithful vocab (includes empty) | +| `ModelAssetGuard.Token.Tokenizer` | `sp_string_roundtrip_of_faithful_vocab` | Same for SentencePiece stub | +| `ModelAssetGuard.Token.Tokenizer` | `vocab_decode_faithful_of_forall_single_char` | Nonempty single-char identity vocabs are decode-faithful | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_once_length_le` | One BPE merge step never increases token-list length | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_once_length_lt_of_occurs` | Merge that fires strictly shortens the list | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_once_length_eq_pred_of_occurs` | Firing merge shortens by exactly one | +| `ModelAssetGuard.Token.Tokenizer` | `bpe_encode_with_merge_table_cons_length_of_occurs` | Firing first merge ⇒ encode length ≤ chars − 1 | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_once_eq_of_not_occurs` | No-pair merge is identity | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_once_length_eq_of_not_occurs` | Absent pair ⇒ length unchanged | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_table_length_le` | Multi-merge schedule never lengthens the token list | +| `ModelAssetGuard.Token.Tokenizer` | `apply_merge_table_append` | Merge schedules compose under `++` | +| `ModelAssetGuard.Token.Tokenizer` | `bpe_encode_with_merge_table_append` | Encode-with-merge-table schedules compose under `++` | +| `ModelAssetGuard.Token.Tokenizer` | `perfect_hash_injective_of_unique` | Unique key→token vocab ⇒ token injectivity | +| `ModelAssetGuard.Token.Tokenizer` | `perfect_hash_encode_eq_of_find` / `perfect_hash_key_roundtrip_of_unique` | Assoc-list encode/decode round-trip under uniqueness | +| `ModelAssetGuard.Token.CHD` | `slots_injective_of_wellFormed` / `keys_injective_of_wellFormed` | Abstract MPH placement: unique keys + unique slots ⇒ injective index maps | +| `ModelAssetGuard.Token.CHD` | `slot_eq_implies_index_eq` | Slot equality ↔ index equality under well-formed CHD placement | +| `ModelAssetGuard.Token.CHD` | `pairwiseDistinct_append` | Distinct lists with cross-disjointness concatenate | +| `ModelAssetGuard.Token.CHD` | `BucketPlace.wellSized_append` | Merging cross-disjoint well-sized bucket places stays well-sized | +| `ModelAssetGuard.Token.CHD` | `wellFormed_mk` | Constructor interface for well-formed placements | + +## Explicitly not axioms + +These used to look like formal claims; they are honest markers only: + +- `per_channel_error_bound` — trivial `theorem … : True` +- `verify_128_vectors_sound` — `def … : Prop := True` +- `layer_verification_sound` — `def … : Prop := True` +- `layer_error_bound_float_l2_obligation` — `def … : Prop := True` (former Float L2 axiom; runtime L2 is Rust) +- `round_int8_error_bound_float_obligation` — `def … : Prop := True` (former Float half-ULP axiom; Int form proven) + +## Removed / reformulated (no longer axioms) + +- `epsilon_bound_positive` — Float ≥ 0 not constructive; replaced by `epsilonBoundSquaredUnits_nonneg` +- `layer_error_bound` — Float L2 layer axiom removed (Wave 5); Int L1 forms proven; Float L2 is an obligation marker only +- `round_int8_error_bound` — Float half-ULP axiom removed (Wave 6); Int half-ULP proven; Float is obligation marker only +- `bpe_deterministic` — false/unprovable for the merge stub; replaced by faithful-vocab round-trip +- `sp_surjective_utf8` — replaced by empty/faithful-vocab forms +- `perfect_hash_injective` — replaced by `perfect_hash_injective_of_unique` (structural) + +BPE merge *tables as learned from data* are **not** formalized as correctness of +a trained tokenizer. Lean proves length / faithfulness / composition properties of +the structural merge operators. Runtime tokenization uses Rust `guardd_*` +helpers: CHD MPH hash vocab JSON by default (`kind: "chd_mph"`; legacy open-address +still loads), plus SentencePiece CHAR/BPE/UNIGRAM encode when +the support gate passes (see [`sentencepiece-bpe.md`](sentencepiece-bpe.md)). +Abstract CHD placement lemmas: `Token/CHD.lean` (not a proof of hash/displace search). +Lean `hash_seeded` mirror: `Token/HashSeeded.lean` (executable Nat-mod-2⁶⁴; ASCII goldens vs Rust). +Executable tiny CHD builder + displace search: `Token/CHDBuild.lean` (singleton theorems; +Wave 23 free-occupied completeness for singleton/empty keys; Wave 24 stable bucket order + +λ-plan goldens + compositional wellSized gates; multi-key hash-family ∃d still an obligation). +Lean↔Rust CHD goldens: `goldenBucketedNOk` / `goldenBucketedPlanOk` and Rust +`test_chd_lean_golden_n_buckets_eq_n` / `test_chd_lean_golden_lambda_plans`. +Search budgets: Rust production `MAX_DISPLACE = 4_000_000`; Lean smokes default +`leanSmokeMaxD = 10_000` (builders accept explicit `maxD`). +Float bridges: [`float-obligation.md`](float-obligation.md). + +## Policy + +1. New `axiom` declarations under `src/lean/` must be added to this table + in the same change. Prefer obligation markers or proven theorems. +2. CI runs `scripts/check_axioms.py` and fails if: + - the live axiom count differs from this inventory, or + - an axiom name appears in Lean but not in this file. +3. Prefer a real theorem over an axiom when the proof is tractable. +4. Prefer an honest obligation marker over a lying theorem when Float/opaque + ops block a proof. +5. Runtime guarantees come from Rust (`guardd_*`) + bindings; Lean markers + are not substitutes for those checks. + +## Runtime alignment (not formal) + +- Quantization error metric: `||W_q x - W x||_2 / ||x||_2` +- int8 ε bound: `0.5 * sqrt(fan_in)` (Lean `compute_epsilon_bound` and Rust `compute_epsilon_bound`) +- Hash vocab: CHD minimal perfect hash by default (`kind: "chd_mph"`); legacy + open-address JSON still loads (see [`perfect-hash-tokenizer.md`](perfect-hash-tokenizer.md)). + Lean `Token.CHD` models placement injectivity; not a formal CHD construction proof. +- SentencePiece `.model`: pieces + optional CHAR/BPE/UNIGRAM encode (support matrix in `docs/sentencepiece-bpe.md`) +- Float obligations: `docs/float-obligation.md` (Wave 28 aborted; markers retained; axiom count 0) diff --git a/docs/perfect-hash-tokenizer.md b/docs/perfect-hash-tokenizer.md index 84715f7..babdf22 100644 --- a/docs/perfect-hash-tokenizer.md +++ b/docs/perfect-hash-tokenizer.md @@ -1,36 +1,114 @@ # Perfect hash tokenizer -This document describes the perfect-hash tokenizer track (T-3) in Model Asset Guard: deterministic encoding and decoding aligned with the Lean specifications in `src/lean/ModelAssetGuard/Token/Tokenizer.lean`, exercised by CLI tools and integration tests. +This document describes the hash-vocab tokenizer track (T-3) in Model Asset Guard. -## Lean entry points +## What works today + +| Layer | Status | +| --- | --- | +| Rust `PerfectHashVocab` | **CHD minimal perfect hash** by default (`kind: "chd_mph"`); encode/decode round-trip with string verify | +| Legacy open-addressing | Still **loaded** when JSON has `hash_table` / `table_size` (or `kind: "open_address"`) | +| `gen_perfect_hash` CLI | Emits CHD MPH JSON schema | +| C ABI / Python FFI | `guardd_perfect_hash_encode` / `guardd_perfect_hash_decode` | +| Safe Rust API | `perfect_hash_encode` / `perfect_hash_decode` | +| Lean `lake exe perfecthash` | Delegates to Rust `guardd_perfect_hash` | +| Lean specs | `perfect_hash_injective_of_unique` + key round-trip; Waves 18–24 `Token/CHD.lean` / `CHDBuild.lean` placement injectivity, bucket merge, compositional displace gates — **not** full hash-family ∃d | +| SentencePiece `model_proto` | Pieces + CHAR/BPE/UNIGRAM/WORD encode when support gate passes; internal piece table also uses `PerfectHashVocab::build` (CHD) | +| C ABI SP encode/decode | `guardd_sp_encode` / `guardd_sp_decode` (+ nbest/sample); safe Rust: `sp_encode` / `sp_decode` / … | + +Wave 17 cut over the **runtime default** from open-addressing to a real CHD MPH. +Wave 18–19: Lean placement / bucket-place merge; fetch scripts; release native zips. +Wave 20: Lean `HashSeeded` mirrors Rust `hash_seeded` (ASCII goldens); displace +candidates theorem once a displace is found; Float half-unit aliases (no Mathlib). +Wave 21: `CHDBuild.findDisplace` / `buildAsciiMph` executable search + singleton +`wellSized` / `candidateSlots_*` theorems; completeness of ∃d still an obligation. +Wave 22: `findDisplace_eq_some_imp` (partial correctness); `n=1` slot lemmas + smoke; +`buildAsciiMphBucketed` (seed0 groups, largest-first) closer to Rust `try_chd`. +Wave 23: Bool↔Prop gates (`wellFormed_of_wellFormedBool`, `displaceCandidatesOk_of_candidatesOkBool`); +`findDisplace_singleton_freeOccupied` / `findDisplace_nil_keys`; Rust-aligned bucket plans + +pilots (`tryAsciiMphBucketed` / `buildAsciiMphBucketedN`); Lean↔Rust slot/pilot goldens. +Wave 24: stable bucket order (largest length, then ascending bucket id) shared by Rust +`bucket_process_order` and Lean `bucketProcessOrder`; first-successful λ-plan goldens +(`buildAsciiMphBucketedFull` / `test_chd_lean_golden_lambda_plans`); compositional +`wellSized_of_findDisplace_success` + `wellSized_merge_bucket_places`; restricted +`findDisplace_returns_zero_of_candidatesOk_at_zero` (not a full hash-family theorem). +Multi-key ∃d / general hash-family completeness remains `chd_displace_search_obligation`. +Search budgets: Rust `MAX_DISPLACE = 4e6`; Lean smoke/golden default `leanSmokeMaxD = 1e4`. +Legacy open-address JSON remains readable. Formal completeness is still **not** claimed. + +## Schema (CHD MPH — default) -- `lake exe perfecthash` — perfect-hash tokenizer CLI (vocabulary-driven encode/decode). -- `lake exe tokenizertest` — tokenizer tests and fuzzing against a tokenizer artifact. +```json +{ + "vocab": [["hello", 1], ["world", 2]], + "kind": "chd_mph", + "pilots": [0, 1], + "n_buckets": 1, + "indices": [0, 1], + "seed0": 11400714819323198485, + "seed1": 13787848793156543929 +} +``` + +- `vocab`: `(word, token_id)` pairs (tokens `1..N` when generated from a word list; `0` is unknown) +- `pilots`: per-bucket displace values (CHD) +- `indices`: `indices[slot] → vocab index` for the unique slot in `0..n-1` +- Lookup verifies the stored string (unknown keys → token `0`) -Build the project first: +## Schema (legacy open-addressing) + +```json +{ + "vocab": [["hello", 1], ["world", 2]], + "hash_table": [0, 18446744073709551615, 1], + "table_size": 3 +} +``` + +- `hash_table`: slot → vocab index; empty slots are `usize::MAX` +- Generator historically emitted this; new builds emit CHD instead + +CLI: ```bash -lake build +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin gen_perfect_hash -- vocab.txt out.json +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin gen_perfect_hash -- vocab.json --output out.json ``` -Lean builds treat warnings as errors (see `lakefile.lean`, package option `warningAsError`). +## Lean entry points + +- `lake exe perfecthash -- encode|decode|test|generate ...` — shells to `guardd_perfect_hash` +- `lake exe tokenizertest -- ` — runs Rust round-trip validation +- `lake exe tokenizertest` — separate tokenizer harness (see that CLI for its own honesty status). -## Runtime validation +Lean builds treat warnings as errors (`lakefile.lean`, `warningAsError`). -The Rust sidecar (`src/rust/guardd`) exposes C ABI helpers for perfect-hash encode/decode used by bindings. See `guardd_perfect_hash_encode` and `guardd_perfect_hash_decode` in `src/rust/guardd/src/lib.rs`. +## Tests -## Integration tests +Offline (default preflight): + +```bash +pytest tests/e2e/test_offline_guard.py tests/e2e/test_perfect_hash_integration.py --maxfail=1 +``` -Python coverage lives in `tests/e2e/test_perfect_hash_integration.py`. Run after installing test dependencies: +Optional integration (Node FFI / Lean stub honesty): ```bash -pytest tests/e2e/test_perfect_hash_integration.py --maxfail=1 +pytest tests/e2e -m integration --maxfail=1 ``` ## Related files | Area | Path | | ---- | ---- | +| Runtime vocab | `src/rust/guardd/src/perfect_hash.rs` | +| SP encode | `src/rust/guardd/src/sentencepiece.rs` + [`sentencepiece-bpe.md`](sentencepiece-bpe.md) | +| Structural BPE | `src/rust/guardd/src/bpe.rs` (Lean-aligned merges) | +| Generator | `src/rust/guardd/src/bin/gen_perfect_hash.rs` | +| FFI | `src/rust/guardd/src/lib.rs` | | Lean tokenizer spec | `src/lean/ModelAssetGuard/Token/Tokenizer.lean` | -| Perfect-hash CLI | `src/lean/cli/PerfectHash/Main.lean` | +| Lean CHD placement | `src/lean/ModelAssetGuard/Token/CHD.lean` | +| Lean CHD builder | `src/lean/ModelAssetGuard/Token/CHDBuild.lean` | +| Lean CLI stub | `src/lean/cli/PerfectHash/Main.lean` | | Python bindings | `bindings/python/pytorch_guard.py` | +| Native packaging | [`native-packaging.md`](native-packaging.md) | From 0ecbfabd6c63230dbd082145b8f7217b076aa968 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:27 -0700 Subject: [PATCH 20/35] Document aborted Float obligation discharge Explain why Mathlib/IEEE refinement was refused and keep Float markers as non-axiom obligations. --- docs/float-obligation.md | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docs/float-obligation.md diff --git a/docs/float-obligation.md b/docs/float-obligation.md new file mode 100644 index 0000000..f2bdcc4 --- /dev/null +++ b/docs/float-obligation.md @@ -0,0 +1,85 @@ +# Float obligation (executable refinement) + +Model Asset Guard keeps **0 Lean `axiom`s**. Float-facing quantization claims +are honest obligation markers (`Prop := True`), not proofs. + +## Wave 28 status: aborted (honest gate) + +Wave 28 evaluated replacing +`round_int8_error_bound_float_obligation` / +`layer_error_bound_float_l2_obligation` with a real refinement from +`Quant.Fixed` into an IEEE / Float model. **Aborted.** Markers stay markers. + +### Why discharge was refused + +1. **Lean 4 `Float` is opaque.** Arithmetic (`+`, `*`, `sqrt`, comparisons, + `toUInt64`) is external. There is no kernel-level IEEE model to rewrite + against without either Mathlib-scale infrastructure or new `axiom`s about + those ops. Fake lemmas over `Float` would be axiom theater. + +2. **Mathlib cost vs payoff.** Pulling Mathlib solely for a Float bridge would + dominate Lean CI (toolchain fetch + compile) while still leaving a gap: + Mathlib’s IEEE theories do not automatically refine Lean’s opaque `Float` + without a morphism that is itself axiomatic or incomplete. That fails the + “non-vacuous, 0 axioms” exit criterion. + +3. **Local Rational / Fixed already exists.** Half-ULP and entrywise L1 bounds + are proven on cleared-denominator `Nat`/`Int` in `Quant.Fixed`. Replacing + Float markers with those theorems (or with `True`-shaped wrappers) would + rename theater, not discharge the Float claim. + +4. **L2 layer claim needs more than rounding.** `||Ex||₂ ≤ (1/2)||x||₂` on + opaque `Float` additionally needs a sound sqrt / norm model. No incomplete + stub was landed. + +5. **Sampling soundness stays out of scope.** `verify_128_vectors_sound` and + `layer_verification_sound` remain markers. Proving Monte Carlo / 128-vector + coverage via `True` is forbidden. + +### What would reopen the gate + +A future change may discharge Float markers only if all hold: + +- A CI-buildable IEEE (or equivalent) model with **0 new axioms**, or an + inventory-documented temporary axiom count that `check_axioms.py` accepts +- A **stated** refinement morphism from `Fixed` rounding into that model that + transports `roundDivInt_error` / `rows_dot_abs_bound` non-vacuously +- Theorems that replace the obligation defs (not `Prop := True` aliases) +- Explicit non-claims for 128-vector / layer verification soundness unless a + real statistical theorem exists + +Until then: markers stay; runtime L2 authority remains Rust +`guardd_verify_quant_128_vectors`. + +## Why Float is not proven here + +Lean 4 `Float` is an opaque external type. Without a IEEE-754 model (typically +via Mathlib or a custom refinement), statements such as + +- `|round_int8(x) - x| ≤ 1/2` +- `||E x||₂ ≤ (1/2) ||x||₂` + +cannot be discharged as theorems over `Float`. Inventing fake proofs would be +worse than markers. + +## What is proven instead + +Cleared-denominator / fixed-point forms in `ModelAssetGuard.Quant.Fixed`: + +- `roundDivNat_error` / `roundDivInt_error` — half-ULP on `Nat`/`Int` +- `roundDivNat_error_half_unit` / `roundDivInt_error_half_unit` — aliases for + the bridge narrative (same theorems; no Mathlib) +- `dot_abs_bound_of_entrywise` / `rows_dot_abs_bound` — Int L1-style bounds + +`Core.fixed_round_half_ulp` and `layer_row_l1_bound` alias those results. +Runtime L2 sampling remains Rust (`guardd_verify_quant_128_vectors`). + +## Inventory pointers + +| Marker | Role | +| --- | --- | +| `round_int8_error_bound_float_obligation` | Former Float half-ULP axiom; Wave 28 left as marker | +| `layer_error_bound_float_l2_obligation` | Former Float L2 layer axiom; Wave 28 left as marker | +| `verify_128_vectors_sound` / `layer_verification_sound` | Runtime soundness not claimed in Lean | + +CI `scripts/check_axioms.py` enforces axiom count **0**. From 7808219647dcf765c0f8caf6c4563c99891dd36e Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:28 -0700 Subject: [PATCH 21/35] Document SentencePiece capability matrix honestly State MT19937/nbest and NFKC limits clearly so support is not over-claimed versus stock SentencePiece. --- docs/sentencepiece-bpe.md | 289 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 docs/sentencepiece-bpe.md diff --git a/docs/sentencepiece-bpe.md b/docs/sentencepiece-bpe.md new file mode 100644 index 0000000..4aea9e5 --- /dev/null +++ b/docs/sentencepiece-bpe.md @@ -0,0 +1,289 @@ +# SentencePiece / BPE support matrix + +Honest status for Model Asset Guard Wave 18. This is **not** a claim of full +SentencePiece compatibility or formal completeness. + +## Runtime (Rust `guardd`) + +| Capability | Status | +| --- | --- | +| Parse `ModelProto` pieces (UTF-8, scores, types) | **Yes** — minimal wire parser; skips unknown fields | +| Open-addressed hash vocab from pieces | **Yes** — token id = piece index; fail on duplicates / empty | +| `guardd_perfect_hash test` on `.model` | **Yes** — vocab round-trip + reports `model_type` / encode gate | +| CHAR encode (`TrainerSpec.model_type=CHAR`) | **Yes** — SP normalize → PrefixMatcher seeds (USER_DEFINED) | +| BPE encode (`model_type=BPE`) | **Yes** — PrefixMatcher freeze + priority-queue merges (official `Encode` ≡ `SampleEncode(α=0)`) | +| UNIGRAM encode (`model_type=UNIGRAM`, default if trainer absent) | **Yes** — lattice Viterbi; USER_DEFINED score bonus | +| WORD encode | **Yes** — `SplitIntoWords` + vocab lookup (fail-closed OOV) | +| `precompiled_charsmap` (Darts double-array) | **Yes** — official leftmost-longest rewrite + whitespace policy | +| Full Unicode NFKC corpus parity | **Not claimed** — curated goldens (221/rule, Wave 27) + integration cross-check; not exhaustive | +| `byte_fallback` | **Yes** — when vocab contains all 256 `<0xXX>` pieces; else refuse | +| `treat_whitespace_as_suffix` | **Yes** — dummy `▁` as suffix; WORD split attaches `▁` as suffix | +| `allow_whitespace_only_pieces` | **Yes** — parsed; affects WORD `SplitIntoWords` | +| USER_DEFINED / PrefixMatcher forced pieces | **Yes** — model type-4 + optional runtime `forced_pieces` (must be in vocab) | +| Unsupported forced-piece shapes | **Refuse** — empty / NUL / not-in-vocab | +| Reserved ids (`unk` / `bos` / `eos` / `pad`) | **Yes** — name + CONTROL/UNKNOWN type checks (SP `PieceToId` rules) | +| CONTROL pieces in segmentation | **Excluded** — not matchable edges (reserved); decode surface empty | +| ExtraOptions `bos` / `eos` / `reverse` | **Yes** — application order is always `reverse → bos → eos` (stock). CLI `--extra bos:eos:reverse` accepts those tokens in **any order** (flag set). Decode `reverse` reverses ids before detokenize | +| ExtraOptions `unk` / `unk_piece` | **N/A on id APIs** — stock only rewrites piece *strings* in `SentencePieceText`; our encode/decode return ids / detokenized text | +| Consecutive `` merge (processor) | **Yes (default)** — stock SP processor merge when not `byte_fallback`; CONTROL ids reset the run; `merge_consecutive_unk=false` / `--no-merge-unk` keeps piece-level ids | +| UNIGRAM lattice n-best | **Yes** — A* over lattice (cap 1024); pathological agenda/timeout **refuse** by default | +| UNIGRAM n-best stock agenda policy | **Opt-in** — `nbest_stock_compat` / `--nbest-stock-compat`: shrink to `min(512, n*10)` when agenda ≥ 10 000; timeout under overflow → Viterbi (matches `unigram_model.cc`) | +| UNIGRAM sample-encode | **Yes** — lattice sample (`nbest_size<0`) or sample-from-nbest (`nbest_size>1`) | +| UNIGRAM SampleEncodeAndScore (with-replacement) | **Yes** — scores = path log-prob − marginal | +| SampleEncodeAndScore WOR / `include_best` | **Yes** — Gumbel-perturbed A* n-best + log inclusion probs | +| BPE/CHAR n-best or lattice sample | **No** — clear error (no stub algorithms) | +| BPE-dropout sample-encode | **Yes** — priority-queue BPE; `alpha` = skip-merge probability; UNUSED reseg | +| Decode (ids → text) | **Yes** — ▁ dummy-prefix strip, BYTE→UTF-8, UNKNOWN→`unk_surface`, CONTROL→empty | +| Denormalizer / reverse charsmap | **Yes** — when `denormalizer_spec.precompiled_charsmap` non-empty | +| Reverse NFKC without denorm blob | **Permanent gap** — same as stock SP (most models leave denorm empty) | +| Silent `` without `byte_fallback` | **Opt-in** — `EncodeOptions.allow_silent_unk` / CLI `--silent-unk`; **default fail-closed** | +| WORD official flag-blind split | **Opt-in** — `word_official_split` matches stock `word_model.cc` (ignores trainer suffix flag) | +| Bit-identical sample RNG vs stock SP | **Permanent non-goal** — structural cross-check only; see MT19937 note below | +| Structural Lean-aligned merge table | **Separate** — `guardd::bpe` matches Lean `apply_merge_*`, not SP scores | +| Safe Rust API (`sp_encode` / `sp_decode` / …) | **Yes** — thin wrappers over the same paths as the C ABI | + +### What HF / SP `.model` files can encode now + +| Model characteristic | Encode? | +| --- | --- | +| CHAR / BPE / UNIGRAM / WORD, `identity` normalizer (empty charsmap) | **Yes** (if vocab covers the normalized string, or `byte_fallback` / silent-unk) | +| Same + non-empty `precompiled_charsmap` (custom / `nfkc` / `nfkc_cf` blobs) | **Yes** — charsmap applied; OOV still fail-closed unless BF / silent-unk | +| `treat_whitespace_as_suffix=true` | **Yes** — normalize + WORD split | +| `USER_DEFINED` pieces in the model | **Yes** — PrefixMatcher protect + CHAR/BPE freeze + UNIGRAM bonus | +| `byte_fallback=true` with full `<0xXX>` set (common modern LLM SP models) | **Yes** | +| Non-empty `denormalizer_spec` charsmap | **Decode applies** reverse rewrite after piece surfaces | +| Invalid charsmap / incomplete byte set | **Refuse** | +| Models relying on silent `` for OOV (no byte_fallback) | **Refuse** by default; pass `allow_silent_unk` / `--silent-unk` | + +Crafted / cached fixtures under `tests/fixtures/`: + +| File | Role | +| --- | --- | +| `sp_tiny_identity_unigram.model` | Empty charsmap, UNIGRAM | +| `sp_tiny_custom_norm_unigram.model` | Tiny custom charsmap (`FF21→A`, `2122→TM`) | +| `sp_custom_charsmap.bin` | Raw charsmap blob for unit tests | +| `sp_denorm_charsmap.bin` | Denorm charsmap (`TM→™`) for unit tests | +| `sp_tiny_identity_denorm_unigram.model` | Identity encode + denorm charsmap on decode | +| `sp_tiny_identity_bpe_bytefallback.model` | BPE + `byte_fallback` | +| `sp_nfkc_normalize_goldens.json` | Curated identity/nfkc/nfkc_cf normalize goldens (221 each; compat + NFD/NFC edges; not exhaustive) | + +### Normalization (supported subset) + +Implemented to match official SP `normalizer.cc` (not the HuggingFace grapheme walk): + +1. Optional PrefixMatcher (USER_DEFINED / forced) leftmost-longest — **wins over** charsmap +2. Optional `precompiled_charsmap` leftmost-longest rewrite over the UTF-8 stream +3. ASCII-space collapse (`remove_extra_whitespaces`) +4. Optional leading **or trailing** `▁` (`add_dummy_prefix` × `treat_whitespace_as_suffix`) +5. Spaces → `▁` (`U+2581`) when `escape_whitespaces` is true (encode path requires this) + +Malformed / undecodable charsmap blobs refuse encode. We do **not** claim exhaustive +NFKC / `nfkc_cf` equivalence for every Unicode edge case; see the curated golden file +and optional integration tests that load the live SP normalizer blob. + +### USER_DEFINED / PrefixMatcher notes + +- Model pieces with type `USER_DEFINED` (4) build the default PrefixMatcher at load. +- Runtime `forced_pieces` must already appear in the vocab; empty / NUL / missing → refuse. +- CHAR: seeds via `PrefixMatch` (longest forced piece, else one Unicode scalar). +- BPE: matched forced regions are **frozen** (no merge across / into them). +- UNIGRAM: forced / USER_DEFINED edges use score `0.1 * (char_len - 1)` (official bonus). + +### WORD notes + +- After normalize, split on `▁` meta-whitespace via official `SplitIntoWords` rules + (prefix vs suffix mode; optional whitespace-only pieces). +- Each word must exist in vocab (NORMAL / USER_DEFINED); else fail-closed unless + `byte_fallback` / `allow_silent_unk`. +- **Official divergence (default):** stock `word_model.cc` calls `SplitIntoWords(normalized)` + with default `treat_ws_as_suffix=false`, so WORD *inference* ignores + `TrainerSpec.treat_whitespace_as_suffix`. The trainer *does* pass the flag. + We honor the trainer flag at encode time by default (suffix vocab works). +- **Compat:** `EncodeOptions.word_official_split=true` / Python + `word_official_split=True` / Node `wordOfficialSplit: true` forces the flag-blind + stock split (use with `allow_silent_unk` when suffix vocab becomes OOV under + prefix splitting). + +### Consecutive `` merge (processor) + +Stock `SentencePieceProcessor::Encode` / `PopulateSentencePieceText` merges a +continuous run of UNKNOWN piece ids into a single id when `byte_fallback` is +**disabled**. CONTROL ids (`IsControl`) reset the run (`is_prev_unk = false`) so +unks on either side of BOS/EOS/PAD do not merge across the control. Segmentation +encoders do not emit CONTROL as matchable edges; ExtraOptions `add_bos` / +`add_eos` wrap CONTROL ids **after** this merge (stock `ApplyExtraOptions` +order). Injected CONTROL ids in an id list still reset the run. With +`byte_fallback`, UNK surfaces expand to `<0xXX>` bytes instead — no merge. + +| Knob | Default | Effect | +| --- | --- | --- | +| `merge_consecutive_unk` / `mergeConsecutiveUnk` | **true** | Match stock processor public encode output | +| `false` / CLI `--no-merge-unk` | opt-out | Keep model-level per-span UNKNOWN ids | +| `add_bos` / `addBos` / CLI `--bos` | **false** | Prepend CONTROL `bos_id` after optional reverse | +| `add_eos` / `addEos` / CLI `--eos` | **false** | Append CONTROL `eos_id` after optional reverse | +| `reverse` / CLI `--reverse` | **false** | Reverse ids after merge and **before** bos/eos (documented stock `reverse:bos:eos`) | +| Decode `reverse` | **false** | Reverse ids before detokenize | + +Silent-unk remains **opt-in** (`allow_silent_unk`); without it, OOV still fails closed +and merge never observes UNKNOWN ids. + +ExtraOption `unk`/`unk_piece` is intentionally **not** exposed on id APIs (stock +no-op for integer encode/decode). + +### UNIGRAM lattice notes + +- Operates on Unicode character positions of the normalized string. +- Piece types marked `UNUSED` / `CONTROL` / `UNKNOWN` / `BYTE` are not segmentation + candidates (reserved); BYTE used only via `byte_fallback` expansion. +- 1-best is lattice Viterbi; n-best is A* from EOS using Viterbi backtrace heuristics. +- Sample-encode: `nbest_size < 0` → forward-filter / backward sample; `>1` → discrete + sample over lattice n-best with temperature `alpha`; `0|1` → Viterbi. +- SampleEncodeAndScore (`wor=false`): repeated lattice samples; score = + `Σ α·node.score − marginal`. +- SampleEncodeAndScore (`wor=true`): Gumbel-perturbed A* n-best of size `k+1`; + κ = (k+1)-th perturbed fx; returned scores are log inclusion probabilities + (`log Gumbel survival`). `include_best` prepends Viterbi with score `0` and + removes it from the Gumbel draw (official constraint: requires `wor=true`). + **MT19937 permanent non-goal:** stock SP uses a process-global C++ MT19937 + stream (`random::GetRandomGenerator()`). We use an explicit per-call seedable + `StdRng`. Bit-identical draws would require a real C++-compatible MT19937 plus + an identical call-order model — **not planned** and must not be stubbed. + Cross-checks assert structural properties (uniqueness, n-best membership, + `include_best`), not byte-identical samples. +- Pathological n-best: agenda ≥ 10 000 or (default) wall-clock >30 s → **refuse**. + Stock `Lattice::NBest` instead shrinks to `min(512, nbest_size*10)` when the + agenda overflows, and on timeout **inside that overflow branch** falls back to + Viterbi. Opt-in `nbest_stock_compat` / `--nbest-stock-compat` / JSON + `nbest_stock_compat=true` matches that policy. Default stays refuse so + pathological lattices never silently return a truncated n-best. +- With `byte_fallback`, single-char OOV edges use `min_score - 10` (SP `kUnkPenalty`) + then expand to `<0xXX>` pieces. +- With `allow_silent_unk` (and no BF), the same edge emits `unk_id`. +- Default without BF / silent-unk: incomplete coverage fails closed. + +### BPE notes + +- Deterministic encode uses the same priority-queue algorithm as official + `SampleEncode(α=0)` (score-desc, lower-left tie-break). +- Sample-encode / BPE-dropout: `alpha` is the skip-merge probability + ([BPE-dropout](https://arxiv.org/abs/1910.13267)); `nbest_size` ignored. +- `UNUSED` merge targets are recorded in a reverse-merge map and resegmented at + emit time (official `rev_merge` path). Typical trained models have no UNUSED + pieces; the path is still exercised when present. +- Not a claim of bit-identical RNG vs stock SP (MT19937 permanent non-goal). + +### Decode notes + +| Step | Status | +| --- | --- | +| Concatenate `IdToPiece` | Yes | +| Strip one leading `▁` (dummy prefix / whitespace policy) | Yes | +| `▁` → space elsewhere | Yes | +| BYTE runs → UTF-8 (invalid → U+FFFD) | Yes | +| UNKNOWN id → `TrainerSpec.unk_surface` (default `" ⁇ "`) | Yes | +| Control pieces | Empty surface | +| Decode ExtraOptions `reverse` | Yes — reverses ids before detokenize | +| `denormalizer_spec` charsmap | Applied when present (official SP order) | +| Implicit reverse NFKC without denorm | **No** — permanent gap (documented + tested) | + +### CLI + +```bash +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-encode path/to/model.model "hello" + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-encode path/to/model.model "z" --silent-unk + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-encode path/to/model.model "xyz" --silent-unk --no-merge-unk + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-encode path/to/model.model "hello" --reverse --bos --eos + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-nbest path/to/model.model "hello" 5 --nbest-stock-compat + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-sample path/to/model.model "hello" --nbest 5 --alpha 0.5 --seed 1 + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-sample-score path/to/model.model "hello" --samples 3 --alpha 0.5 --seed 1 + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-sample-score path/to/model.model "hello" --samples 3 --alpha 0.5 --seed 1 --wor --include-best + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-decode path/to/model.model 1 2 3 + +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + sp-decode path/to/model.model --reverse 3 2 1 + +# Vocab integrity only (whitespace-split hash encode — not SP segmentation): +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_perfect_hash -- \ + test path/to/model.model +``` + +### FFI / bindings + +| Surface | Status | +| --- | --- | +| `guardd_sp_encode` / `guardd_sp_encode_with_options` / `guardd_sp_decode` / `guardd_sp_decode_with_options` | **Yes** | +| `guardd_sp_nbest_encode` / `guardd_sp_sample_encode` | **Yes** — UNIGRAM lattice; BPE-dropout via sample_encode | +| `guardd_sp_sample_encode_and_score` | **Yes** — UNIGRAM WR + WOR | +| Options: `allow_silent_unk`, `forced_pieces`, `word_official_split`, `merge_consecutive_unk`, `reverse`, `add_bos`, `add_eos`, `nbest_stock_compat` | **Yes** — silent-unk fail-closed; merge default on; reverse/bos/eos opt-in; n-best stock-compat opt-in | +| Sample opts: `nbest_size`, `alpha`, `seed` | **Yes** | +| Score opts: `num_samples`, `alpha`, `wor`, `include_best`, `seed` | **Yes** | +| Python `sp_encode` / `sp_nbest_encode` / `sp_sample_encode` / `sp_sample_encode_and_score` / `sp_decode` | **Yes** | +| Node `spEncode` / `spNbestEncode` / `spSampleEncode` / `spSampleEncodeAndScore` / `spDecode` | **Yes** | + +## Formal (Lean) + +| Claim | Status | +| --- | --- | +| Char-level stub encode/decode round-trip under faithful vocab | Proven | +| `apply_merge_once` / `apply_merge_table` length bounds | Proven | +| Merge identity when pair absent | Proven (`apply_merge_once_eq_of_not_occurs`) | +| Absent merge ⇒ length unchanged | Proven (`apply_merge_once_length_eq_of_not_occurs`) | +| Merge schedule composition under `++` | Proven (`apply_merge_table_append`) | +| Encode-with-merge-table append composition | Proven (`bpe_encode_with_merge_table_append`) | +| Encode length ≤ character count | Proven | +| Firing one-step merge shortens by exactly 1 | Proven (`apply_merge_once_length_eq_pred_of_occurs`) | +| Trained BPE merge tables / SP score BPE / UNIGRAM Viterbi | **Not** formalized | +| Full SP ModelProto semantics | **Not** formalized | +| Charsmap / denormalizer / PrefixMatcher | **Not** formalized (runtime + docs only; Darts not tractable at 0 axioms) | +| Minimal perfect hash cutover | **Yes (runtime)** — CHD MPH default via `PerfectHashVocab::build`; Lean proves injectivity under unique keys only, not CHD correctness | + +## Float obligations + +Unrelated to tokenization; see [`float-obligation.md`](float-obligation.md). +Axiom count remains **0**. + +## Cross-check + +- Offline golden vectors: crafted BPE/UNIGRAM/WORD ModelProto fixtures in Rust unit + tests and `tests/e2e/test_offline_guard.py`. +- Cached tiny trained models in `tests/fixtures/` (identity / custom charsmap / + denorm / byte_fallback). +- Curated NFKC normalize goldens in `sp_nfkc_normalize_goldens.json` (221 rows per + identity / nfkc / nfkc_cf rule; Wave 27 expanded with compat forms and + composed/decomposed pairs; size-aware; not a full Unicode audit). Offline Rust + hygiene covers identity rows with charsmap-only knobs; live full nfkc blobs stay + integration-only. +- Optional integration (`pytest -m integration`): compare against the + `sentencepiece` Python package when installed (covered strings / unique-optimum + crafted fixtures / WORD / `treat_whitespace_as_suffix` / denorm / silent-unk / + consecutive-unk merge / USER_DEFINED / n-best / SampleEncodeAndScore WR+WOR / + BPE-dropout / `word_official_split` / live nfkc blob + expanded goldens). + +## Related paths + +| Area | Path | +| --- | --- | +| SP parser + encode/decode | `src/rust/guardd/src/sentencepiece.rs` | +| UNIGRAM lattice n-best / sample / WOR | `src/rust/guardd/src/sp_lattice.rs` | +| Charsmap + PrefixMatcher + normalize / denormalize | `src/rust/guardd/src/sp_normalizer.rs` | +| Lean-aligned BPE merges | `src/rust/guardd/src/bpe.rs` | +| Lean specs | `src/lean/ModelAssetGuard/Token/Tokenizer.lean` | +| CLI | `src/rust/guardd/src/bin/guardd_perfect_hash.rs` | +| C ABI | `guardd_sp_encode*` / `guardd_sp_nbest_encode` / `guardd_sp_sample_encode` / `guardd_sp_sample_encode_and_score` / `guardd_sp_decode` | From 04a592c6949a555008590683b2b219537423eca4 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:28 -0700 Subject: [PATCH 22/35] Document native fetch and stage packaging paths Describe release-asset-first fetch, CI fallback, and that npm/PyPI do not ship embedded prebuilts yet. --- docs/native-packaging.md | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 docs/native-packaging.md diff --git a/docs/native-packaging.md b/docs/native-packaging.md new file mode 100644 index 0000000..33ee254 --- /dev/null +++ b/docs/native-packaging.md @@ -0,0 +1,110 @@ +# Native packaging (guardd) + +Honest status for Model Asset Guard. This is **not** a claim that npm / PyPI +ship platform binaries today. + +## What works today + +| Path | Status | +| --- | --- | +| Local `cargo build` of `libguardd` / `guardd.dll` | **Yes** | +| `scripts/stage_guardd_native.sh` / `.ps1` copies into `dist/native/` + binding `native/` dirs | **Yes** | +| Node / Python search order includes staged `native/` then cargo `target/` | **Yes** | +| CI upload of per-OS `guardd-native-*` artifacts (quality job) | **Yes** — ubuntu, macOS, Windows | +| Tag release attaches `guardd-native-.zip` assets | **Yes** (`bundle-push.yml`) | +| `scripts/fetch_guardd_native.sh` / `.ps1` | **Yes** — **release asset first**, CI artifact fallback | +| Node `npm run fetch-native` | **Yes** — invokes repo fetch helper (needs `gh`) | +| Python `model-asset-guard-fetch-native` / `python -m fetch_native` | **Yes** — same helper (editable / repo checkout) | +| npm package with embedded prebuilt binaries for all platforms | **No** — not published | +| PyPI wheel with embedded `libguardd` | **No** — sdist/wheel build exists; publish is manual / not automated | + +## Local install (recommended) + +```bash +cargo build --release --manifest-path src/rust/guardd/Cargo.toml +bash scripts/stage_guardd_native.sh +# Windows: .\scripts\stage_guardd_native.ps1 + +# Or fetch a built library (needs GitHub CLI): +# 1) latest GitHub Release asset guardd-native-.zip +# 2) if missing (unreleased commit), CI artifact guardd-native- +bash scripts/fetch_guardd_native.sh +# Windows: .\scripts\fetch_guardd_native.ps1 + +# Node (from bindings/nodejs) +npm run fetch-native + +# Python (repo / editable install) +model-asset-guard-fetch-native +# or: python -m fetch_native + +# Python load check +export PYTHONPATH="$(pwd)/bindings/python${PYTHONPATH:+:$PYTHONPATH}" +python -c "from pytorch_guard import ModelAssetGuard; print(ModelAssetGuard().lib._name)" + +# Node +cd bindings/nodejs && npm install && npm test +``` + +Or set `GUARDD_LIB` to an absolute path of the shared library. + +### Fetch options + +```bash +bash scripts/fetch_guardd_native.sh --tag v0.1.0 +bash scripts/fetch_guardd_native.sh --from-release # no CI fallback +bash scripts/fetch_guardd_native.sh --from-ci # Actions artifact only +bash scripts/fetch_guardd_native.sh --from-ci --run-id 123 # specific run +``` + +PowerShell mirrors the same flags as `-Tag`, `-FromRelease`, `-FromCi`, `-RunId`. + +## Release assets vs CI artifacts + +On version tags (`v*`), `bundle-push.yml` attaches +`guardd-native-.zip` to the GitHub Release. Prefer these for consumers: +they last with the release. + +On `push` / `pull_request`, the quality workflow builds a **release** `guardd` +library, stages it, and uploads ephemeral Actions artifacts: + +- Artifact name: `guardd-native-` (e.g. `guardd-native-ubuntu-latest`, + `guardd-native-windows-latest`) +- Contents: `dist/native/` (library + `STAGED.txt`) + +`fetch_guardd_native` uses **release zip first**, then CI download for +unreleased commits or missing assets. Neither path embeds binaries into +npm/PyPI packages. + +### Windows runners + +Quality CI uses `shell: bash` (Git Bash) on Windows so `scripts/ci_preflight.sh` +and staging stay the same scripts. Prefer `stage_guardd_native.ps1` / +`fetch_guardd_native.ps1` locally on Windows PowerShell if bash is unavailable. + +## Wheel / npm consumers without a git checkout + +Published packages still do **not** ship `libguardd`. Options: + +1. Download `guardd-native-.zip` from the matching GitHub Release, extract + `libguardd.so` / `libguardd.dylib` / `guardd.dll`, and set `GUARDD_LIB` to + that file (or copy next to the binding’s `native/` if you maintain a checkout). +2. Build from source and run `scripts/stage_guardd_native.*`. + +The `fetch-native` entry points require the repo `scripts/` helpers (and `gh`). + +## What we do not claim + +- No fake “published binaries” in the Node `package.json` / npm registry story +- No multi-arch wheel that silently ships empty `native/` directories +- Hash vocab default is **CHD MPH** (`kind: "chd_mph"`); legacy open-address JSON + still loads (not claimed as MPH when `kind: "open_address"`) +- Formal completeness is not claimed (`docs/axioms.md`); Lean proves abstract + CHD placement injectivity + bucket-place merge (`Token/CHD.lean`), not the + hash/displace search + +## Residual (Wave 25) + +- Optional one-platform wheel that embeds a CI-tested `libguardd` is deferred + until release automation builds and tests that platform end-to-end (Wave 25b). +- Until then: fetch or local `cargo build` + stage. From ea0ef115bc033c9370e680948f3e9ce5684ed71d Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:35 -0700 Subject: [PATCH 23/35] Add fetch and stage scripts for guardd natives Prefer GitHub release assets with CI artifact fallback and copy libraries into binding native/ dirs for local use. --- scripts/fetch_guardd_native.ps1 | 171 +++++++++++++++++++++++++ scripts/fetch_guardd_native.sh | 217 ++++++++++++++++++++++++++++++++ scripts/stage_guardd_native.ps1 | 45 +++++++ scripts/stage_guardd_native.sh | 59 +++++++++ 4 files changed, 492 insertions(+) create mode 100644 scripts/fetch_guardd_native.ps1 create mode 100644 scripts/fetch_guardd_native.sh create mode 100644 scripts/stage_guardd_native.ps1 create mode 100644 scripts/stage_guardd_native.sh diff --git a/scripts/fetch_guardd_native.ps1 b/scripts/fetch_guardd_native.ps1 new file mode 100644 index 0000000..e9aa267 --- /dev/null +++ b/scripts/fetch_guardd_native.ps1 @@ -0,0 +1,171 @@ +# Fetch libguardd into bindings/*/native/ and dist/native/. +# Prefer GitHub Release assets (guardd-native-.zip); fall back to CI artifacts. +# Requires: gh (GitHub CLI). Honesty: not npm/PyPI prebuilts. +param( + [ValidateSet("ubuntu-latest", "macos-latest", "windows-latest", "")] + [string]$Os = "", + + [string]$Tag = "", + + [string]$RunId = "", + + [switch]$FromCi, + + [switch]$FromRelease, + + [switch]$Help +) + +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent $PSScriptRoot + +if ($Help) { + @" +Usage: fetch_guardd_native.ps1 [-Os ubuntu-latest|macos-latest|windows-latest] + [-Tag ] [-RunId ] [-FromCi] [-FromRelease] + +Prefer GitHub Release asset guardd-native-.zip; if unavailable, fall back +to CI artifact guardd-native-. Requires gh (GitHub CLI). + +Examples: + .\scripts\fetch_guardd_native.ps1 + .\scripts\fetch_guardd_native.ps1 -Os windows-latest -Tag v0.1.0 + .\scripts\fetch_guardd_native.ps1 -FromCi + .\scripts\fetch_guardd_native.ps1 -FromRelease +"@ + exit 0 +} + +if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + Write-Error "gh (GitHub CLI) is required" +} + +if ([string]::IsNullOrEmpty($Os)) { + $Os = "windows-latest" +} + +if ($FromCi -and $FromRelease) { + Write-Error "Specify only one of -FromCi / -FromRelease" +} + +$Source = "auto" +if ($FromCi -or -not [string]::IsNullOrEmpty($RunId)) { + $Source = "ci" +} +elseif ($FromRelease) { + $Source = "release" +} + +$Art = "guardd-native-$Os" +$ZipName = "$Art.zip" +$Tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("guardd-native-" + [guid]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $Tmp | Out-Null + +function Stage-FromTree { + param([string]$Tree) + + $dests = @( + (Join-Path $Root "bindings\nodejs\native"), + (Join-Path $Root "bindings\python\native"), + (Join-Path $Root "dist\native") + ) + foreach ($d in $dests) { + New-Item -ItemType Directory -Force -Path $d | Out-Null + } + + $names = @("libguardd.so", "libguardd.dylib", "guardd.dll", "STAGED.txt") + $copied = $false + Get-ChildItem -Path $Tree -Recurse -File | ForEach-Object { + if ($names -contains $_.Name) { + foreach ($d in $dests) { + Copy-Item -Force $_.FullName (Join-Path $d $_.Name) + } + Write-Host " staged $($_.Name)" + $copied = $true + } + } + if (-not $copied) { + Write-Error "No guardd library found under downloaded tree" + } +} + +function Try-Release { + param([string]$ReleaseTag) + + if ([string]::IsNullOrEmpty($ReleaseTag)) { + Write-Host "Resolving latest non-draft GitHub Release..." + $ReleaseTag = gh release list --limit 30 --json tagName,isDraft --jq "[.[] | select(.isDraft==false)][0].tagName" + } + if ([string]::IsNullOrEmpty($ReleaseTag) -or $ReleaseTag -eq "null") { + Write-Host "No GitHub Release found." + return $false + } + + Write-Host "Downloading release asset $ZipName from tag $ReleaseTag..." + $relDir = Join-Path $Tmp "release" + New-Item -ItemType Directory -Force -Path $relDir | Out-Null + & gh release download $ReleaseTag -p $ZipName -D $relDir 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "Release $ReleaseTag has no asset $ZipName." + return $false + } + + $zipPath = Join-Path $relDir $ZipName + if (-not (Test-Path $zipPath)) { + $found = Get-ChildItem -Path $relDir -Recurse -Filter $ZipName -File | Select-Object -First 1 + if ($null -eq $found) { + Write-Host "Release $ReleaseTag has no asset $ZipName." + return $false + } + $zipPath = $found.FullName + } + + $extractDir = Join-Path $Tmp "extracted" + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + Expand-Archive -Force -Path $zipPath -DestinationPath $extractDir + Stage-FromTree -Tree $extractDir + Write-Host "Done (GitHub Release $ReleaseTag). Set GUARDD_LIB if needed." + return $true +} + +function Try-Ci { + param([string]$Id) + + if ([string]::IsNullOrEmpty($Id)) { + Write-Host "Resolving latest successful CI run with artifact $Art..." + $Id = gh run list --workflow=ci.yml --status=success --limit 20 --json databaseId --jq ".[0].databaseId" + } + if ([string]::IsNullOrEmpty($Id) -or $Id -eq "null") { + Write-Error "Could not find a successful ci.yml run. Pass -RunId explicitly." + } + + Write-Host "Downloading CI artifact $Art from run $Id..." + $ciDir = Join-Path $Tmp "ci" + New-Item -ItemType Directory -Force -Path $ciDir | Out-Null + gh run download $Id -n $Art -D $ciDir + Stage-FromTree -Tree $ciDir + Write-Host "Done (CI run $Id). Set GUARDD_LIB if needed." + return $true +} + +try { + switch ($Source) { + "release" { + if (-not (Try-Release -ReleaseTag $Tag)) { + Write-Error "Release asset unavailable (-FromRelease)." + } + } + "ci" { + [void](Try-Ci -Id $RunId) + } + default { + if (-not (Try-Release -ReleaseTag $Tag)) { + Write-Host "Falling back to CI artifact (unreleased commit or missing release asset)..." + [void](Try-Ci -Id $RunId) + } + } + } +} +finally { + Remove-Item -Recurse -Force $Tmp -ErrorAction SilentlyContinue +} diff --git a/scripts/fetch_guardd_native.sh b/scripts/fetch_guardd_native.sh new file mode 100644 index 0000000..ce01898 --- /dev/null +++ b/scripts/fetch_guardd_native.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +# Fetch libguardd into bindings/*/native/ and dist/native/. +# Prefer GitHub Release assets (guardd-native-.zip); fall back to CI artifacts. +# Requires: gh (GitHub CLI). Honesty: not npm/PyPI prebuilts. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OS_FILTER="" +TAG="" +RUN_ID="" +SOURCE="auto" # auto | release | ci + +usage() { + cat <.zip; if unavailable (unreleased +commits / missing asset), fall back to CI artifact guardd-native-. + +Options: + --tag TAG Release tag (default: latest non-draft release) + --run-id ID CI run id (implies --from-ci when SOURCE=auto) + --from-ci Skip release; download from Actions only + --from-release Release only; do not fall back to CI + -h, --help Show this help + +Examples: + $0 # auto OS; release then CI fallback + $0 ubuntu-latest + $0 --tag v0.1.0 windows-latest + $0 --from-ci + $0 --from-ci --run-id 123456789 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --tag) + TAG="${2:-}" + [[ -n "$TAG" ]] || { echo "Error: --tag requires a value" >&2; exit 2; } + shift 2 + ;; + --run-id) + RUN_ID="${2:-}" + [[ -n "$RUN_ID" ]] || { echo "Error: --run-id requires a value" >&2; exit 2; } + SOURCE="ci" + shift 2 + ;; + --from-ci) + SOURCE="ci" + shift + ;; + --from-release) + SOURCE="release" + shift + ;; + ubuntu-latest|macos-latest|windows-latest) + OS_FILTER="$1" + shift + ;; + *) + # Back-compat: bare numeric second positional was run_id + if [[ "$1" =~ ^[0-9]+$ ]]; then + RUN_ID="$1" + SOURCE="ci" + shift + else + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 2 + fi + ;; + esac +done + +if ! command -v gh >/dev/null 2>&1; then + echo "Error: gh (GitHub CLI) is required" >&2 + exit 2 +fi + +detect_os() { + case "$(uname -s)" in + Linux*) echo "ubuntu-latest" ;; + Darwin*) echo "macos-latest" ;; + MINGW*|MSYS*|CYGWIN*) echo "windows-latest" ;; + *) echo "ubuntu-latest" ;; + esac +} + +OS_NAME="${OS_FILTER:-$(detect_os)}" +ART="guardd-native-${OS_NAME}" +ZIP_NAME="${ART}.zip" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +stage_from_tree() { + local tree="$1" + mkdir -p "$ROOT/bindings/nodejs/native" "$ROOT/bindings/python/native" "$ROOT/dist/native" + local copied=0 + local f base + while IFS= read -r -d '' f; do + base="$(basename "$f")" + case "$base" in + libguardd.so|libguardd.dylib|guardd.dll|STAGED.txt) + cp -f "$f" "$ROOT/bindings/nodejs/native/$base" + cp -f "$f" "$ROOT/bindings/python/native/$base" + cp -f "$f" "$ROOT/dist/native/$base" + echo " staged $base" + copied=1 + ;; + esac + done < <(find "$tree" -type f \( \ + -name 'libguardd.so' -o -name 'libguardd.dylib' -o -name 'guardd.dll' -o -name 'STAGED.txt' \ + \) -print0) + + if [[ "$copied" -eq 0 ]]; then + echo "Error: no guardd library found under downloaded tree" >&2 + find "$tree" -type f | head -50 >&2 || true + return 1 + fi + return 0 +} + +try_release() { + local tag="$1" + if [[ -z "$tag" ]]; then + echo "Resolving latest non-draft GitHub Release..." + tag="$(gh release list --limit 30 --json tagName,isDraft \ + --jq '[.[] | select(.isDraft==false)][0].tagName' 2>/dev/null || true)" + fi + if [[ -z "$tag" || "$tag" == "null" ]]; then + echo "No GitHub Release found." + return 1 + fi + + echo "Downloading release asset ${ZIP_NAME} from tag ${tag}..." + rm -rf "$TMP/release" + mkdir -p "$TMP/release" + if ! gh release download "$tag" -p "$ZIP_NAME" -D "$TMP/release" 2>/dev/null; then + echo "Release ${tag} has no asset ${ZIP_NAME}." + return 1 + fi + + local zip_path="$TMP/release/$ZIP_NAME" + if [[ ! -f "$zip_path" ]]; then + # gh may nest or keep original name; locate the zip + zip_path="$(find "$TMP/release" -type f -name "$ZIP_NAME" | head -1 || true)" + fi + if [[ -z "$zip_path" || ! -f "$zip_path" ]]; then + echo "Error: downloaded zip missing: ${ZIP_NAME}" >&2 + return 1 + fi + + mkdir -p "$TMP/extracted" + if command -v unzip >/dev/null 2>&1; then + unzip -qo "$zip_path" -d "$TMP/extracted" + else + # Python is commonly available when bindings are used + python -c "import zipfile,sys; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])" \ + "$zip_path" "$TMP/extracted" + fi + stage_from_tree "$TMP/extracted" + echo "Done (GitHub Release ${tag}). Set GUARDD_LIB if your loader needs an absolute path." + return 0 +} + +try_ci() { + local run_id="$1" + if [[ -z "$run_id" ]]; then + echo "Resolving latest successful CI run with artifact ${ART}..." + run_id="$(gh run list --workflow=ci.yml --status=success --limit 20 --json databaseId \ + --jq '.[0].databaseId' 2>/dev/null || true)" + fi + if [[ -z "$run_id" || "$run_id" == "null" ]]; then + echo "Error: could not find a successful ci.yml run. Pass --run-id explicitly." >&2 + return 1 + fi + + echo "Downloading CI artifact ${ART} from run ${run_id}..." + rm -rf "$TMP/ci" + mkdir -p "$TMP/ci" + gh run download "$run_id" -n "$ART" -D "$TMP/ci" + stage_from_tree "$TMP/ci" + echo "Done (CI run ${run_id}). Set GUARDD_LIB if your loader needs an absolute path." + return 0 +} + +case "$SOURCE" in + release) + if ! try_release "$TAG"; then + echo "Error: release asset unavailable (--from-release)." >&2 + exit 1 + fi + ;; + ci) + if ! try_ci "$RUN_ID"; then + exit 1 + fi + ;; + auto) + if try_release "$TAG"; then + exit 0 + fi + echo "Falling back to CI artifact (unreleased commit or missing release asset)..." + if ! try_ci "$RUN_ID"; then + exit 1 + fi + ;; + *) + echo "Error: invalid SOURCE=${SOURCE}" >&2 + exit 2 + ;; +esac diff --git a/scripts/stage_guardd_native.ps1 b/scripts/stage_guardd_native.ps1 new file mode 100644 index 0000000..4d0cf48 --- /dev/null +++ b/scripts/stage_guardd_native.ps1 @@ -0,0 +1,45 @@ +# Stage built guardd shared libraries into binding-friendly native/ dirs. +# Windows companion to scripts/stage_guardd_native.sh (local cargo copy only). + +param( + [ValidateSet("release", "debug")] + [string]$Profile = "release" +) + +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path) +Set-Location $Root + +$TargetDir = Join-Path $Root "src/rust/guardd/target/$Profile" +$Candidates = @( + (Join-Path $TargetDir "libguardd.so"), + (Join-Path $TargetDir "libguardd.dylib"), + (Join-Path $TargetDir "guardd.dll") +) +$Found = @($Candidates | Where-Object { Test-Path $_ }) +if ($Found.Count -eq 0) { + Write-Error "No guardd shared library under $TargetDir. Build with: cargo build --$Profile --manifest-path src/rust/guardd/Cargo.toml" +} + +$Dests = @( + "dist/native", + "bindings/nodejs/native", + "bindings/python/native" +) +$Stamp = @" +source_profile=$Profile +staged_at=$([DateTime]::UtcNow.ToString("o")) +note=Local cargo build copy. Not an npm/PyPI prebuilt publish. +"@ + +foreach ($dest in $Dests) { + $full = Join-Path $Root $dest + New-Item -ItemType Directory -Force -Path $full | Out-Null + foreach ($lib in $Found) { + Copy-Item -Force $lib $full + Write-Host "Staged $(Split-Path -Leaf $lib) → $dest/" + } + Set-Content -Encoding utf8 (Join-Path $full "STAGED.txt") $Stamp +} + +Write-Host "Done. See docs/native-packaging.md" diff --git a/scripts/stage_guardd_native.sh b/scripts/stage_guardd_native.sh new file mode 100644 index 0000000..15cf22e --- /dev/null +++ b/scripts/stage_guardd_native.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Stage built guardd shared libraries into binding-friendly native/ dirs. +# Does not download or publish binaries — copies from a local cargo build only. +# +# Usage: +# cargo build --release --manifest-path src/rust/guardd/Cargo.toml +# bash scripts/stage_guardd_native.sh +# # optional: PROFILE=debug bash scripts/stage_guardd_native.sh + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +PROFILE="${PROFILE:-release}" +TARGET_DIR="src/rust/guardd/target/${PROFILE}" + +CANDIDATES=( + "${TARGET_DIR}/libguardd.so" + "${TARGET_DIR}/libguardd.dylib" + "${TARGET_DIR}/guardd.dll" +) + +FOUND=() +for c in "${CANDIDATES[@]}"; do + if [[ -f "$c" ]]; then + FOUND+=("$c") + fi +done + +if [[ ${#FOUND[@]} -eq 0 ]]; then + echo "Error: no guardd shared library under ${TARGET_DIR}" + echo "Build first:" + echo " cargo build --${PROFILE} --manifest-path src/rust/guardd/Cargo.toml" + exit 1 +fi + +DESTS=( + "dist/native" + "bindings/nodejs/native" + "bindings/python/native" +) + +for dest in "${DESTS[@]}"; do + mkdir -p "$dest" + for lib in "${FOUND[@]}"; do + cp -f "$lib" "$dest/" + echo "Staged $(basename "$lib") → ${dest}/" + done + # Sidecar metadata (honest: local build, not a published npm binary). + { + echo "source_profile=${PROFILE}" + echo "staged_at=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" + echo "note=Local cargo build copy. Not an npm/PyPI prebuilt publish." + } > "${dest}/STAGED.txt" +done + +echo "Done. Point GUARDD_LIB at dist/native/* or rely on binding search order." +echo "See docs/native-packaging.md" From 452b959c6958d046b6c7ff547e0d623ac1981362 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:35 -0700 Subject: [PATCH 24/35] Add axiom inventory CI checker script Fail closed if Lean sources introduce axiom declarations contrary to the documented zero-axiom policy. --- scripts/check_axioms.py | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 scripts/check_axioms.py diff --git a/scripts/check_axioms.py b/scripts/check_axioms.py new file mode 100644 index 0000000..61e9b66 --- /dev/null +++ b/scripts/check_axioms.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Fail if Lean axioms drift from docs/axioms.md inventory.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LEAN_ROOT = ROOT / "src" / "lean" +INVENTORY = ROOT / "docs" / "axioms.md" + +AXIOM_DECL = re.compile(r"(?m)^\s*axiom\s+([A-Za-z0-9_']+)") +INVENTORY_ROW = re.compile( + r"\|\s*`[^`]+`\s*\|\s*`([A-Za-z0-9_']+)`\s*\|" +) + + +def collect_lean_axioms() -> dict[str, list[Path]]: + found: dict[str, list[Path]] = {} + for path in LEAN_ROOT.rglob("*.lean"): + text = path.read_text(encoding="utf-8") + for match in AXIOM_DECL.finditer(text): + name = match.group(1) + found.setdefault(name, []).append(path.relative_to(ROOT)) + return found + + +def collect_inventory() -> set[str]: + text = INVENTORY.read_text(encoding="utf-8") + # Only the "Allowed axioms" table counts; later theorem tables must not match. + allowed = re.search( + r"## Allowed axioms \(inventory\)(.*?)(?:\n## |\Z)", + text, + flags=re.DOTALL, + ) + if not allowed: + raise SystemExit("docs/axioms.md missing '## Allowed axioms (inventory)' section") + section = allowed.group(1) + names = set(INVENTORY_ROW.findall(section)) + count_match = re.search(r"\*\*Count:\*\*\s*(\d+)\s*axioms", section) + if not count_match: + raise SystemExit( + "docs/axioms.md missing '**Count:** N axioms' in Allowed axioms section" + ) + declared_count = int(count_match.group(1)) + if declared_count != len(names): + raise SystemExit( + f"docs/axioms.md Count={declared_count} but table lists {len(names)} axioms: " + f"{sorted(names)}" + ) + return names + + +def main() -> int: + lean = collect_lean_axioms() + inventory = collect_inventory() + lean_names = set(lean) + + errors: list[str] = [] + missing_docs = sorted(lean_names - inventory) + extra_docs = sorted(inventory - lean_names) + if missing_docs: + errors.append(f"axioms in Lean but not in docs/axioms.md: {missing_docs}") + if extra_docs: + errors.append(f"axioms in docs/axioms.md but not in Lean: {extra_docs}") + if len(lean_names) != len(inventory): + errors.append( + f"axiom count mismatch: lean={len(lean_names)} inventory={len(inventory)}" + ) + + if errors: + print("Axiom inventory check FAILED:") + for err in errors: + print(f" - {err}") + print("\nLean axioms:") + for name in sorted(lean): + locs = ", ".join(str(p) for p in lean[name]) + print(f" {name}: {locs}") + return 1 + + print(f"Axiom inventory OK ({len(inventory)} axioms; no formal-completeness claim).") + for name in sorted(inventory): + print(f" - {name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 1e8f9048455b4dca11817ec58eeec0927826f20e Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:36 -0700 Subject: [PATCH 25/35] Update bundle and CI preflight for native artifacts Align packaging helpers with per-OS guardd-native uploads and fetch/stage workflows. --- scripts/bundle.sh | 229 +++++++++++++++++++--------------------- scripts/ci_preflight.sh | 34 +++++- 2 files changed, 138 insertions(+), 125 deletions(-) diff --git a/scripts/bundle.sh b/scripts/bundle.sh index 25a7327..d23b400 100644 --- a/scripts/bundle.sh +++ b/scripts/bundle.sh @@ -1,167 +1,157 @@ #!/bin/bash +# Create a distributable bundle of specs, bindings, and a built guardd library. +# Does not claim formal completeness. -set -e +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" echo "Creating Model Asset Guard bundle..." -# Create bundle directory BUNDLE_DIR="model-asset-guard-bundle" rm -rf "$BUNDLE_DIR" mkdir -p "$BUNDLE_DIR" -# Copy specification files echo "Copying specification files..." -mkdir -p "$BUNDLE_DIR/Spec" +mkdir -p "$BUNDLE_DIR/Spec/Quant" "$BUNDLE_DIR/Spec/Token" "$BUNDLE_DIR/docs" +cp src/lean/ModelAssetGuard.lean "$BUNDLE_DIR/Spec/" 2>/dev/null || true cp src/lean/ModelAssetGuard/Weights.lean "$BUNDLE_DIR/Spec/" -cp src/lean/ModelAssetGuard/Quant/Core.lean "$BUNDLE_DIR/Spec/" -cp src/lean/ModelAssetGuard/Quant/LayerBound.lean "$BUNDLE_DIR/Spec/" -cp src/lean/ModelAssetGuard/Token/Tokenizer.lean "$BUNDLE_DIR/Spec/" - -# Copy HuggingFace integration files (G-3 requirement) -echo "Copying HuggingFace integration files..." -mkdir -p "$BUNDLE_DIR/bindings" -cp bindings/python/pytorch_guard.py "$BUNDLE_DIR/bindings/" -cp bindings/nodejs/node_guard.js "$BUNDLE_DIR/bindings/" -cp tests/e2e/test_huggingface_integration.py "$BUNDLE_DIR/bindings/" - -# Copy perfect hash tokenizer files (T-3 requirement) -echo "Copying perfect hash tokenizer files..." -cp tests/e2e/test_perfect_hash_integration.py "$BUNDLE_DIR/bindings/" -mkdir -p "$BUNDLE_DIR/docs" +cp src/lean/ModelAssetGuard/RustSidecar.lean "$BUNDLE_DIR/Spec/" +cp src/lean/ModelAssetGuard/Quant/Core.lean "$BUNDLE_DIR/Spec/Quant/" +cp src/lean/ModelAssetGuard/Quant/LayerBound.lean "$BUNDLE_DIR/Spec/Quant/" +cp src/lean/ModelAssetGuard/Quant/Fixed.lean "$BUNDLE_DIR/Spec/Quant/" +cp src/lean/ModelAssetGuard/Token/Tokenizer.lean "$BUNDLE_DIR/Spec/Token/" +cp docs/axioms.md "$BUNDLE_DIR/docs/" cp docs/perfect-hash-tokenizer.md "$BUNDLE_DIR/docs/" +cp docs/float-obligation.md "$BUNDLE_DIR/docs/" +cp docs/sentencepiece-bpe.md "$BUNDLE_DIR/docs/" +cp docs/native-packaging.md "$BUNDLE_DIR/docs/" + +echo "Copying bindings..." +mkdir -p "$BUNDLE_DIR/bindings/python" "$BUNDLE_DIR/bindings/nodejs" +cp bindings/python/pytorch_guard.py "$BUNDLE_DIR/bindings/python/" +cp bindings/nodejs/node_guard.js "$BUNDLE_DIR/bindings/nodejs/" +cp bindings/nodejs/package.json "$BUNDLE_DIR/bindings/nodejs/" +cp bindings/nodejs/README.md "$BUNDLE_DIR/bindings/nodejs/" +cp pyproject.toml "$BUNDLE_DIR/" + +if [ -f examples/huggingface_example.py ]; then + mkdir -p "$BUNDLE_DIR/examples" + cp examples/huggingface_example.py "$BUNDLE_DIR/examples/" +fi -# Copy examples -echo "Copying examples..." -mkdir -p "$BUNDLE_DIR/examples" -cp examples/huggingface_example.py "$BUNDLE_DIR/examples/" - -# Copy extracted Rust library -echo "Copying Rust library..." +echo "Copying Rust library (platform-specific)..." mkdir -p "$BUNDLE_DIR/guardd" -if [ -f "src/rust/guardd/target/release/libguardd.so" ]; then - cp src/rust/guardd/target/release/libguardd.so "$BUNDLE_DIR/guardd/" -else - echo "Warning: libguardd.so not found, building..." - cargo build --release --manifest-path src/rust/guardd/Cargo.toml - cp src/rust/guardd/target/release/libguardd.so "$BUNDLE_DIR/guardd/" +LIB_COPIED=0 +for candidate in \ + src/rust/guardd/target/release/libguardd.so \ + src/rust/guardd/target/release/guardd.dll \ + src/rust/guardd/target/release/libguardd.dylib \ + src/rust/guardd/target/debug/libguardd.so \ + src/rust/guardd/target/debug/guardd.dll \ + src/rust/guardd/target/debug/libguardd.dylib +do + if [ -f "$candidate" ]; then + cp "$candidate" "$BUNDLE_DIR/guardd/" + LIB_COPIED=1 + break + fi +done + +if [ "$LIB_COPIED" -eq 0 ]; then + echo "guardd library not found; building release..." + cargo build --release --manifest-path src/rust/guardd/Cargo.toml --locked + for candidate in \ + src/rust/guardd/target/release/libguardd.so \ + src/rust/guardd/target/release/guardd.dll \ + src/rust/guardd/target/release/libguardd.dylib + do + if [ -f "$candidate" ]; then + cp "$candidate" "$BUNDLE_DIR/guardd/" + LIB_COPIED=1 + break + fi + done fi -# Generate Lean kernel hash -echo "Generating Lean kernel hash..." +if [ "$LIB_COPIED" -eq 0 ]; then + echo "Error: could not locate or build guardd shared library" + exit 1 +fi + +# Copy CLI helpers when present (best-effort) +mkdir -p "$BUNDLE_DIR/guardd/bin" +for bin in guardd_sha256 guardd_perfect_hash guardd_bitflip guardd_quant128 gen_perfect_hash; do + for ext in "" .exe; do + for profile in release debug; do + candidate="src/rust/guardd/target/${profile}/${bin}${ext}" + if [ -f "$candidate" ]; then + cp "$candidate" "$BUNDLE_DIR/guardd/bin/" + fi + done + done +done + +echo "Generating Lean toolchain stamp..." lake build -KERNEL_HASH=$(lake exe lean --version 2>/dev/null | head -1 | sha256sum | cut -d' ' -f1) +KERNEL_HASH=$( (lake --version 2>/dev/null || true; cat lean-toolchain) | sha256sum | cut -d' ' -f1 ) echo "$KERNEL_HASH" > "$BUNDLE_DIR/lean-hash.txt" +cp lean-toolchain "$BUNDLE_DIR/" +cp lakefile.lean "$BUNDLE_DIR/" +cp LICENSE "$BUNDLE_DIR/" 2>/dev/null || true -# Create README for bundle cat > "$BUNDLE_DIR/README.md" << 'EOF' # Model Asset Guard Bundle -This bundle contains the verified components of the Model Asset Guard system. +Runtime-checked components plus Lean specifications. Formal completeness is +**not** claimed; see `docs/axioms.md`. ## Contents -- `Spec/` - Lean specification files with formal proofs -- `guardd/` - Rust sidecar library for high-performance validation -- `bindings/` - Python and Node.js bindings for HuggingFace integration (G-3) -- `docs/` - Documentation including perfect hash tokenizer (T-3) -- `lean-hash.txt` - Lean kernel hash for verification +- `Spec/` — Lean specifications (0 axioms; Float markers; Int/Fixed + BPE lemmas) +- `guardd/` — Rust sidecar library (+ `bin/` helpers when built) +- `bindings/` — Python and Node.js bindings +- `docs/axioms.md` — axiom inventory (must stay at 0) +- `docs/float-obligation.md` — Float bridge path (markers, not proofs) +- `docs/sentencepiece-bpe.md` — SP/BPE support matrix (honest gaps) +- `docs/perfect-hash-tokenizer.md` — open-addressed hash vocab (not MPH) +- `docs/native-packaging.md` — local stage + CI artifact path (no fake npm binaries) +- `lean-hash.txt` — toolchain/content stamp (not a proof certificate) ## Verification -To verify this bundle: - -1. Check the Lean kernel hash: - ```bash - lake exe lean --version | head -1 | sha256sum - ``` - Compare with the contents of `lean-hash.txt` - -2. Build and test the components: - ```bash - lake build - lake exe tests - cargo test --manifest-path src/rust/guardd/Cargo.toml - ``` - -3. Run the benchmark suite: - ```bash - ./bench/run.sh - ``` - -4. Test HuggingFace integration: - ```bash - pip install transformers torch numpy - python3 bindings/test_huggingface_integration.py - ``` - -5. Test perfect hash tokenizer: - ```bash - python3 bindings/test_perfect_hash_integration.py - lake exe perfecthash test vocab.json - ``` - -## Integration - -### Python/HuggingFace Integration - -The `pytorch_guard.py` module provides seamless integration with HuggingFace transformers: - -```python -from bindings.pytorch_guard import checked_load_pretrained - -# Load a model with full verification -model, tokenizer, verification_results = checked_load_pretrained( - "gpt2", - verify_quantization=True -) +```bash +lake build && lake exe tests +python scripts/check_axioms.py +cargo test --manifest-path src/rust/guardd/Cargo.toml --locked +pytest tests/e2e -m "not integration" ``` -### Node.js Integration - -The `node_guard.js` module provides JavaScript/TypeScript bindings: - -```javascript -const { createGuard } = require('./bindings/node_guard.js'); - -const guard = createGuard(); -const verified = guard.verifyDigest('model.bin', expectedDigest); - -// Perfect hash tokenizer -const tokens = guard.perfectHashEncode(vocabJson, "hello world"); -const decoded = guard.perfectHashDecodeSequence(vocabJson, tokens); -``` +## Integration notes -### Direct FFI Integration - -The `libguardd.so` library can be integrated directly: - -```python -import ctypes -lib = ctypes.CDLL("./guardd/libguardd.so") -# Use the FFI functions for model validation -``` - -## License - -MIT License - see the main repository for details. +- Python: `pip install .` from the bundle root (module `pytorch_guard`); set + `GUARDD_LIB` or place the library under `bindings/python/native/` +- Node: `cd bindings/nodejs && npm install` — requires staged/built `guardd` + (see `docs/native-packaging.md`); npm does not ship prebuilts +- Hash vocab is a deterministic open-addressed table, not a proven MPH +- SentencePiece: see `docs/sentencepiece-bpe.md` (not full SP parity) EOF -# Create bundle archive BUNDLE_NAME="model-asset-guard-$(date +%Y%m%d).zip" echo "Creating bundle archive: $BUNDLE_NAME" zip -r "$BUNDLE_NAME" "$BUNDLE_DIR" -# Generate SHA-256 of bundle BUNDLE_SHA256=$(sha256sum "$BUNDLE_NAME" | cut -d' ' -f1) echo "Bundle SHA-256: $BUNDLE_SHA256" -# Create release info cat > "release-info.txt" << EOF Model Asset Guard Bundle Date: $(date) Bundle: $BUNDLE_NAME SHA-256: $BUNDLE_SHA256 Kernel Hash: $KERNEL_HASH +Formal completeness: NOT claimed (see docs/axioms.md) EOF echo "Bundle created successfully!" @@ -169,5 +159,4 @@ echo "Bundle: $BUNDLE_NAME" echo "SHA-256: $BUNDLE_SHA256" echo "Release info saved to: release-info.txt" -# Cleanup -rm -rf "$BUNDLE_DIR" \ No newline at end of file +rm -rf "$BUNDLE_DIR" diff --git a/scripts/ci_preflight.sh b/scripts/ci_preflight.sh index aba0834..2db8960 100644 --- a/scripts/ci_preflight.sh +++ b/scripts/ci_preflight.sh @@ -2,21 +2,45 @@ set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + echo "[ci] Lean build (warnings are errors via lakefile.lean leanOptions)" lake build +# CLI tools used by docs and smoke steps (not all are default_target) +lake build verifyweights perfecthash benchmarks quantbound quantverify128 bitflipcorpus tokenizertest echo "[ci] Lean tests" lake exe tests +echo "[ci] Axiom inventory gate (no formal-completeness claim)" +python scripts/check_axioms.py + echo "[ci] Rust fmt + clippy + tests" cargo fmt --manifest-path src/rust/guardd/Cargo.toml --all -- --check cargo clippy --manifest-path src/rust/guardd/Cargo.toml --all-targets --locked -- -D warnings cargo test --manifest-path src/rust/guardd/Cargo.toml --locked -echo "[ci] Python lint + tests" -ruff check bindings/python tests/e2e -python -m py_compile bindings/python/pytorch_guard.py tests/e2e/test_huggingface_integration.py tests/e2e/test_perfect_hash_integration.py -pytest tests/e2e --maxfail=1 --disable-warnings --cov=bindings/python --cov-report=xml || true +echo "[ci] Build libguardd + Lean/CLI helper bins" +cargo build --manifest-path src/rust/guardd/Cargo.toml --locked +cargo build --manifest-path src/rust/guardd/Cargo.toml --locked \ + --bin guardd_sha256 --bin guardd_perfect_hash --bin guardd_bitflip --bin guardd_quant128 + +# Optional local convenience: stage debug lib into binding native/ dirs when present. +if [[ "${STAGE_GUARDD_NATIVE:-0}" == "1" ]]; then + echo "[ci] Staging guardd native (STAGE_GUARDD_NATIVE=1)" + PROFILE=debug bash scripts/stage_guardd_native.sh || true +fi -echo "[ci] Smoke benchmark" +echo "[ci] Python lint + offline e2e tests (integration markers excluded)" +ruff check bindings/python tests/e2e scripts/check_axioms.py +python -m py_compile bindings/python/pytorch_guard.py tests/e2e/test_offline_guard.py \ + tests/e2e/test_huggingface_integration.py tests/e2e/test_perfect_hash_integration.py \ + scripts/check_axioms.py +# Default addopts in pyproject.toml exclude @pytest.mark.integration +pytest tests/e2e --maxfail=1 --disable-warnings \ + --cov=bindings/python --cov-report=xml + +echo "[ci] Smoke benchmark (measured via guardd_sha256)" lake exe benchmarks > benchmark_results.txt + From ea5993b3f6e939e170d4a96fc54432a63889dc59 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:42 -0700 Subject: [PATCH 26/35] Add Node native fetch helper and package metadata Provide npm fetch-native/smoke scripts and document that binaries are staged locally, not published in the package. --- bindings/nodejs/README.md | 65 ++++++++++++++++++++++ bindings/nodejs/fetch_native.js | 91 +++++++++++++++++++++++++++++++ bindings/nodejs/native/README.md | 19 +++++++ bindings/nodejs/package.json | 34 ++++++++++++ bindings/nodejs/smoke_test.js | 94 ++++++++++++++++++++++++++++++++ 5 files changed, 303 insertions(+) create mode 100644 bindings/nodejs/README.md create mode 100644 bindings/nodejs/fetch_native.js create mode 100644 bindings/nodejs/native/README.md create mode 100644 bindings/nodejs/package.json create mode 100644 bindings/nodejs/smoke_test.js diff --git a/bindings/nodejs/README.md b/bindings/nodejs/README.md new file mode 100644 index 0000000..0d2f539 --- /dev/null +++ b/bindings/nodejs/README.md @@ -0,0 +1,65 @@ +# `@model-asset-guard/node` + +Node bindings over the Rust `guardd` C ABI using [koffi](https://koffi.dev/) +(prebuilt N-API; no `node-gyp` / `ffi-napi` toolchain required). + +## Install (from repo) + +```bash +cargo build --release --manifest-path ../../src/rust/guardd/Cargo.toml +bash ../../scripts/stage_guardd_native.sh +cd bindings/nodejs +npm install +npm test +``` + +Or fetch a CI/release-built library (needs `gh` CLI; does **not** embed prebuilts): + +```bash +cd bindings/nodejs +npm run fetch-native +# optional: npm run fetch-native -- --tag v0.1.0 +``` + +### Pointing at `libguardd` + +Search order: + +1. Constructor argument / `createGuard(libPath)` +2. Environment variable `GUARDD_LIB` (absolute path to `.dll` / `.so` / `.dylib`) +3. `bindings/nodejs/native/` (staged by `scripts/stage_guardd_native.sh`) +4. `dist/native/` at the repo root +5. `src/rust/guardd/target/{release,debug}/` relative to the repo root +6. This package directory + +On Windows the artifact is typically `guardd.dll`; on Linux/macOS, +`libguardd.so` / `libguardd.dylib`. + +Prefer GitHub Release assets `guardd-native-.zip` (via `npm run fetch-native`); +CI artifacts remain a fallback for unreleased commits. Or set `GUARDD_LIB`. +See [`docs/native-packaging.md`](../../docs/native-packaging.md). + +**Honesty:** this npm package does **not** ship platform prebuilts. A local +`cargo build` or `npm run fetch-native` (release/CI download) is required. + +## Scripts + +| Script | Purpose | +| --- | --- | +| `npm test` / `npm run smoke` | Digest + `checkedLoad` smoke (`exit 2` = skip if lib/koffi missing) | +| `npm run selfcheck` | Inline self-check via `node_guard.js` | +| `npm run fetch-native` | Download `libguardd` via repo helper (release-first; needs `gh`) | + +## Honesty notes + +- `checkedLoad` sets `valid` from a real digest comparison when an expected + digest is supplied; it does not hardcode success. +- Bit-flip helpers default to small corpora; large sizes require opt-in on the + Rust side (`GUARDD_BITFLIP_ALLOW_LARGE=1` or CLI `--allow-large`). +- Hash-vocab encode/decode uses CHD MPH by default (`kind: "chd_mph"`) with + string verify; legacy open-address JSON still loads. Lean proves structural + placement / assoc-list round-trip properties, not the Rust displace search. +- `spEncode` / `spDecode` mirror the Rust SentencePiece gate (CHAR/BPE/UNIGRAM/WORD + + charsmap / byte_fallback / n-best / sample). Options include `reverse`, + `addBos` / `addEos`, and n-best `nbestStockCompat`. Smoke tests remain + skippable when the shared library or `koffi` is missing (`exit 2`). diff --git a/bindings/nodejs/fetch_native.js b/bindings/nodejs/fetch_native.js new file mode 100644 index 0000000..baf0439 --- /dev/null +++ b/bindings/nodejs/fetch_native.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node +"use strict"; + +/** + * Thin wrapper: invoke repo fetch helper into bindings native dirs. + * Honesty: does not embed or publish prebuilts; requires gh CLI. + */ + +const { spawnSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const root = path.resolve(__dirname, "..", ".."); +const isWin = process.platform === "win32"; +const script = isWin + ? path.join(root, "scripts", "fetch_guardd_native.ps1") + : path.join(root, "scripts", "fetch_guardd_native.sh"); + +if (!fs.existsSync(script)) { + console.error( + "Error: fetch helper not found at", + script, + "\nRun from a full git checkout of model-asset-guard (npm does not ship prebuilts)." + ); + process.exit(2); +} + +/** Map GNU-style flags to PowerShell param names for the .ps1 helper. */ +function toPowershellArgs(argv) { + const out = []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + switch (a) { + case "--tag": + out.push("-Tag", argv[++i] ?? ""); + break; + case "--run-id": + out.push("-RunId", argv[++i] ?? ""); + break; + case "--from-ci": + out.push("-FromCi"); + break; + case "--from-release": + out.push("-FromRelease"); + break; + case "-h": + case "--help": + out.push("-Help"); + break; + case "ubuntu-latest": + case "macos-latest": + case "windows-latest": + out.push("-Os", a); + break; + default: + // Bare run id (back-compat) or already-PowerShell-shaped args + if (/^\d+$/.test(a)) { + out.push("-RunId", a); + } else { + out.push(a); + } + break; + } + } + return out; +} + +const passthrough = process.argv.slice(2); +let cmd; +let args; +if (isWin) { + cmd = "powershell"; + args = [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + script, + ...toPowershellArgs(passthrough), + ]; +} else { + cmd = "bash"; + args = [script, ...passthrough]; +} + +const result = spawnSync(cmd, args, { stdio: "inherit", cwd: root }); +if (result.error) { + console.error("Error: failed to spawn", cmd, result.error.message); + process.exit(2); +} +process.exit(result.status === null ? 1 : result.status); diff --git a/bindings/nodejs/native/README.md b/bindings/nodejs/native/README.md new file mode 100644 index 0000000..1e6c93d --- /dev/null +++ b/bindings/nodejs/native/README.md @@ -0,0 +1,19 @@ +# Node native library staging + +Place `libguardd.so` / `libguardd.dylib` / `guardd.dll` here after: + +```bash +cargo build --release --manifest-path ../../../src/rust/guardd/Cargo.toml +bash ../../../scripts/stage_guardd_native.sh +``` + +Or fetch a release/CI library (see `docs/native-packaging.md`): + +```bash +npm run fetch-native +# or: bash ../../../scripts/fetch_guardd_native.sh +``` + +This directory is empty in git by design. The npm package does **not** ship +prebuilt binaries. `fetch-native` prefers GitHub Release +`guardd-native-.zip`, then CI artifacts for unreleased commits. diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json new file mode 100644 index 0000000..02f2536 --- /dev/null +++ b/bindings/nodejs/package.json @@ -0,0 +1,34 @@ +{ + "name": "@model-asset-guard/node", + "version": "0.1.0", + "description": "Node.js bindings for Model Asset Guard (Rust guardd sidecar via koffi)", + "main": "node_guard.js", + "type": "commonjs", + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node smoke_test.js", + "smoke": "node smoke_test.js", + "selfcheck": "node node_guard.js", + "fetch-native": "node fetch_native.js" + }, + "dependencies": { + "koffi": "^2.9.0" + }, + "files": [ + "node_guard.js", + "smoke_test.js", + "fetch_native.js", + "README.md", + "native/README.md" + ], + "keywords": [ + "model-asset-guard", + "sha256", + "quantization", + "koffi", + "ffi" + ], + "license": "MIT" +} diff --git a/bindings/nodejs/smoke_test.js b/bindings/nodejs/smoke_test.js new file mode 100644 index 0000000..e9da725 --- /dev/null +++ b/bindings/nodejs/smoke_test.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node +/** + * Smoke test: load guardd, verify digest, checkedLoad valid flag. + * Exit 0 on success, 2 if native deps/lib unavailable (CI skip), 1 on failure. + */ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +function skip(msg) { + console.log(`SKIP: ${msg}`); + process.exitCode = 2; +} + +function fail(msg) { + console.error(`FAIL: ${msg}`); + process.exitCode = 1; +} + +function pass(msg) { + console.log(`PASS: ${msg}`); + process.exitCode = 0; +} + +let testFile = null; + +function cleanup() { + if (!testFile) return; + try { + if (fs.existsSync(testFile)) fs.unlinkSync(testFile); + } catch (_) { + /* ignore */ + } +} + +try { + require('koffi'); +} catch (err) { + skip(`koffi not installable/available (${err.message})`); + process.exit(process.exitCode); +} + +let createGuard; +try { + ({ createGuard } = require('./node_guard')); +} catch (err) { + skip(`node_guard load failed (${err.message})`); + process.exit(process.exitCode); +} + +let guard; +try { + guard = createGuard(); +} catch (err) { + skip(`libguardd not found or failed to load (${err.message})`); + process.exit(process.exitCode); +} + +testFile = path.join(os.tmpdir(), `mag-smoke-${process.pid}-${Date.now()}.bin`); +try { + const payload = Buffer.from('model-asset-guard-node-smoke'); + fs.writeFileSync(testFile, payload); + const digest = guard.computeDigest(testFile); + if (!Buffer.isBuffer(digest) || digest.length !== 32) { + fail('computeDigest must return a 32-byte Buffer'); + } else if (!guard.verifyDigest(testFile, digest)) { + fail('verifyDigest returned false for matching digest'); + } else { + const bad = Buffer.alloc(32, 0); + if (guard.verifyDigest(testFile, bad) !== false) { + fail('verifyDigest must return false on mismatch'); + } else { + const ok = guard.checkedLoad(testFile, digest); + if (!ok || !ok.valid) { + fail('checkedLoad.valid expected true'); + } else { + const badLoad = guard.checkedLoad(testFile, bad); + if (!badLoad || badLoad.valid) { + fail('checkedLoad.valid must be false on digest mismatch'); + } else { + pass(`node smoke (lib=${guard.libPath})`); + } + } + } + } +} catch (err) { + fail(err.message || String(err)); +} finally { + cleanup(); +} + +process.exit(process.exitCode ?? 0); From 4f0656b0a856cdaf7874c06c439b6ef1fa872ba1 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:43 -0700 Subject: [PATCH 27/35] Add Python fetch-native entry point and packaging metadata Expose model-asset-guard-fetch-native via setuptools and mark integration tests so default pytest stays offline. --- bindings/python/fetch_native.py | 88 ++++++++++++++++++++++++++++++++ bindings/python/native/README.md | 20 ++++++++ pyproject.toml | 36 ++++++++++++- 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 bindings/python/fetch_native.py create mode 100644 bindings/python/native/README.md diff --git a/bindings/python/fetch_native.py b/bindings/python/fetch_native.py new file mode 100644 index 0000000..f27dd72 --- /dev/null +++ b/bindings/python/fetch_native.py @@ -0,0 +1,88 @@ +"""Fetch staged libguardd into binding native/ dirs (release-first). + +Honesty: downloads GitHub Release / CI assets via the repo helper scripts. +Does not embed or publish prebuilts inside the wheel. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def _repo_root() -> Path: + # bindings/python/fetch_native.py -> repo root + return Path(__file__).resolve().parents[2] + + +def _to_powershell_args(argv: list[str]) -> list[str]: + """Map GNU-style flags to PowerShell param names for the .ps1 helper.""" + out: list[str] = [] + i = 0 + while i < len(argv): + a = argv[i] + if a == "--tag": + i += 1 + out.extend(["-Tag", argv[i] if i < len(argv) else ""]) + elif a == "--run-id": + i += 1 + out.extend(["-RunId", argv[i] if i < len(argv) else ""]) + elif a == "--from-ci": + out.append("-FromCi") + elif a == "--from-release": + out.append("-FromRelease") + elif a in ("-h", "--help"): + out.append("-Help") + elif a in ("ubuntu-latest", "macos-latest", "windows-latest"): + out.extend(["-Os", a]) + elif a.isdigit(): + out.extend(["-RunId", a]) + else: + out.append(a) + i += 1 + return out + + +def main(argv: list[str] | None = None) -> None: + """Console entry: `model-asset-guard-fetch-native` / `python -m fetch_native`.""" + args = list(sys.argv[1:] if argv is None else argv) + root = _repo_root() + if sys.platform == "win32": + script = root / "scripts" / "fetch_guardd_native.ps1" + if not script.is_file(): + print( + "Error: fetch helper not found at", + script, + "\nUse a full git checkout, or download guardd-native-.zip " + "from a GitHub Release (see docs/native-packaging.md).", + file=sys.stderr, + ) + raise SystemExit(2) + cmd = [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script), + *_to_powershell_args(args), + ] + else: + script = root / "scripts" / "fetch_guardd_native.sh" + if not script.is_file(): + print( + "Error: fetch helper not found at", + script, + "\nUse a full git checkout, or download guardd-native-.zip " + "from a GitHub Release (see docs/native-packaging.md).", + file=sys.stderr, + ) + raise SystemExit(2) + cmd = ["bash", str(script), *args] + + raise SystemExit(int(subprocess.call(cmd, cwd=str(root)))) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/native/README.md b/bindings/python/native/README.md new file mode 100644 index 0000000..8b05175 --- /dev/null +++ b/bindings/python/native/README.md @@ -0,0 +1,20 @@ +# Python native library staging + +Place `libguardd.so` / `libguardd.dylib` / `guardd.dll` here after: + +```bash +cargo build --release --manifest-path ../../../src/rust/guardd/Cargo.toml +bash ../../../scripts/stage_guardd_native.sh +``` + +Or fetch a release/CI library (see `docs/native-packaging.md`): + +```bash +model-asset-guard-fetch-native +# or: python -m fetch_native +# or: bash ../../../scripts/fetch_guardd_native.sh +``` + +This directory is empty in git by design. PyPI wheels do **not** currently +embed `libguardd`. Fetch prefers GitHub Release `guardd-native-.zip`, then +CI artifacts for unreleased commits. Or set `GUARDD_LIB`. diff --git a/pyproject.toml b/pyproject.toml index b5e2ce8..42f2a21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,34 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "model-asset-guard" +version = "0.1.0" +description = "Python bindings for Model Asset Guard (Rust guardd sidecar)" +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +authors = [{ name = "Model Asset Guard contributors" }] +dependencies = [] + +[project.optional-dependencies] +quant = ["numpy"] +hf = ["numpy", "transformers", "torch"] +dev = ["pytest", "pytest-cov", "ruff"] + +[project.urls] +Repository = "https://github.com/SentinelOps-CI/model-asset-guard" + +[project.scripts] +model-asset-guard-fetch-native = "fetch_native:main" + +[tool.setuptools] +py-modules = ["pytorch_guard", "fetch_native"] + +[tool.setuptools.package-dir] +"" = "bindings/python" + [tool.ruff] line-length = 100 target-version = "py311" @@ -8,4 +39,7 @@ select = ["E", "F", "I", "B", "UP"] [tool.pytest.ini_options] testpaths = ["tests/e2e"] pythonpath = ["bindings/python"] -addopts = "-ra" +addopts = "-ra -m 'not integration'" +markers = [ + "integration: requires network, HuggingFace downloads, Node, Lean CLI, or optional packages", +] From 517f3af2da839c73d1422cc8146a2982938d3279 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:49 -0700 Subject: [PATCH 28/35] Improve Node guardd loader search and FFI wrappers Prefer staged native/ then cargo target paths so local fetch/stage workflows work without embedding binaries. --- bindings/nodejs/node_guard.js | 938 ++++++++++++++++------------------ 1 file changed, 428 insertions(+), 510 deletions(-) diff --git a/bindings/nodejs/node_guard.js b/bindings/nodejs/node_guard.js index 64de08c..48df631 100644 --- a/bindings/nodejs/node_guard.js +++ b/bindings/nodejs/node_guard.js @@ -1,557 +1,475 @@ #!/usr/bin/env node /** - * Model Asset Guard Node.js Bindings - * - * This module provides Node.js bindings for the Model Asset Guard Rust sidecar, - * enabling integration with JavaScript/TypeScript applications. - * - * Requirements: - * - Node.js >= 16.0.0 - * - ffi-napi (for FFI bindings) - * - ref-napi (for memory management) + * Model Asset Guard Node.js bindings (koffi over guardd C ABI). + * + * Build the sidecar first: + * cargo build --release --manifest-path ../../src/rust/guardd/Cargo.toml + * + * Optional override: + * set GUARDD_LIB=/absolute/path/to/guardd.dll|libguardd.so|libguardd.dylib */ -const ffi = require('ffi-napi'); -const ref = require('ref-napi'); +'use strict'; + const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); +const os = require('os'); + +let koffi; +try { + koffi = require('koffi'); +} catch (err) { + throw new Error( + 'Missing dependency `koffi`. From bindings/nodejs run: npm install\n' + + `Underlying error: ${err.message}` + ); +} -/** - * Model Asset Guard error codes - */ const GuarddError = { - Success: 0, - FileNotFound: 1, - InvalidDigest: 2, - QuantizationError: 3, - MemoryError: 4, - InvalidPath: 5 + Success: 0, + FileNotFound: 1, + InvalidDigest: 2, + QuantizationError: 3, + MemoryError: 4, + InvalidPath: 5, }; -/** - * Model Asset Guard Node.js bindings - */ -class ModelAssetGuard { - constructor(libPath = null) { - this.lib = this._loadLibrary(libPath); - this._setupFunctionSignatures(); +function candidateLibPaths(explicit) { + if (explicit) return [explicit]; + if (process.env.GUARDD_LIB) return [process.env.GUARDD_LIB]; + + const repoRoot = path.resolve(__dirname, '..', '..'); + const names = [ + 'libguardd.so', + 'guardd.dll', + 'libguardd.dylib', + // Windows often emits guardd.dll; MSVC may also emit guardd.dll only. + ]; + const roots = [ + // Staged native packaging path (scripts/stage_guardd_native.sh) + path.join(__dirname, 'native'), + path.join(repoRoot, 'dist', 'native'), + path.join(repoRoot, 'bindings', 'python', 'native'), + path.join(repoRoot, 'src/rust/guardd/target/release'), + path.join(repoRoot, 'src/rust/guardd/target/debug'), + path.join(repoRoot, 'src/rust/guardd/target/x86_64-unknown-linux-gnu/release'), + path.join(repoRoot, 'src/rust/guardd/target/x86_64-unknown-linux-gnu/debug'), + path.join(repoRoot, 'src/rust/guardd/target/aarch64-apple-darwin/release'), + path.join(repoRoot, 'src/rust/guardd/target/aarch64-apple-darwin/debug'), + __dirname, + process.cwd(), + ]; + const out = []; + for (const root of roots) { + for (const name of names) { + out.push(path.join(root, name)); } + } + return out; +} - /** - * Load the Rust sidecar library - */ - _loadLibrary(libPath) { - if (!libPath) { - // Search for library in common locations - const searchPaths = [ - './src/rust/guardd/target/release/libguardd.so', - './src/rust/guardd/target/debug/libguardd.so', - './target/release/libguardd.so', - './target/debug/libguardd.so', - './src/rust/guardd/target/release/guardd.dll', - './src/rust/guardd/target/debug/guardd.dll', - './target/release/guardd.dll', - './target/debug/guardd.dll' - ]; - - for (const searchPath of searchPaths) { - if (fs.existsSync(searchPath)) { - libPath = searchPath; - break; - } - } - - if (!libPath) { - throw new Error( - 'Could not find libguardd library. Please build the Rust sidecar first:\n' + - 'cargo build --release --manifest-path src/rust/guardd/Cargo.toml' - ); - } - } +function resolveLibPath(libPath) { + for (const candidate of candidateLibPaths(libPath)) { + if (candidate && fs.existsSync(candidate)) return path.resolve(candidate); + } + throw new Error( + 'Could not find libguardd / guardd shared library.\n' + + 'Build + stage it with:\n' + + ' cargo build --release --manifest-path src/rust/guardd/Cargo.toml\n' + + ' bash scripts/stage_guardd_native.sh\n' + + 'Or set GUARDD_LIB to the absolute library path.\n' + + 'See docs/native-packaging.md (CI uploads platform artifacts; npm does not ship binaries).' + ); +} - try { - return ffi.Library(libPath, { - 'guardd_error_message': ['string', ['int']], - 'guardd_verify_digest': ['int', ['string', 'pointer']], - 'guardd_checked_load': ['pointer', ['string', 'pointer']], - 'guardd_free_handle': ['void', ['pointer']], - 'guardd_verify_quant_128_vectors': ['pointer', ['string', 'pointer', 'size_t', 'uint32', 'uint32', 'string']], - 'guardd_verify_model_128_vectors': ['pointer', ['string']], - 'guardd_run_bitflip_corpus_test': ['pointer', ['size_t', 'size_t', 'string']], - 'guardd_free_json_result': ['void', ['pointer']], - 'guardd_perfect_hash_encode': ['pointer', ['string', 'string']], - 'guardd_perfect_hash_decode': ['pointer', ['string', 'uint32']] - }); - } catch (error) { - throw new Error(`Failed to load library ${libPath}: ${error.message}`); - } +class ModelAssetGuard { + constructor(libPath = null) { + this.libPath = resolveLibPath(libPath); + this.lib = koffi.load(this.libPath); + this._bind(); + } + + _bind() { + this._error_message = this.lib.func('guardd_error_message', 'str', ['int']); + this._verify_digest = this.lib.func('guardd_verify_digest', 'int', [ + 'str', + 'void *', + ]); + this._checked_load = this.lib.func('guardd_checked_load', 'void *', [ + 'str', + 'void *', + ]); + this._free_handle = this.lib.func('guardd_free_handle', 'void', ['void *']); + this._verify_quant_128 = this.lib.func( + 'guardd_verify_quant_128_vectors', + 'void *', + ['str', 'void *', 'size_t', 'uint32', 'uint32', 'str'] + ); + this._verify_model_128 = this.lib.func( + 'guardd_verify_model_128_vectors', + 'void *', + ['str'] + ); + this._bitflip = this.lib.func('guardd_run_bitflip_corpus_test', 'void *', [ + 'size_t', + 'size_t', + 'str', + ]); + this._free_json = this.lib.func('guardd_free_json_result', 'void', ['void *']); + this._ph_encode = this.lib.func('guardd_perfect_hash_encode', 'void *', [ + 'str', + 'str', + ]); + this._ph_decode = this.lib.func('guardd_perfect_hash_decode', 'void *', [ + 'str', + 'uint32', + ]); + this._sp_encode = this.lib.func('guardd_sp_encode', 'void *', ['str', 'str']); + this._sp_encode_opts = this.lib.func('guardd_sp_encode_with_options', 'void *', [ + 'str', + 'str', + 'str', + ]); + this._sp_nbest = this.lib.func('guardd_sp_nbest_encode', 'void *', ['str', 'str', 'str']); + this._sp_sample = this.lib.func('guardd_sp_sample_encode', 'void *', ['str', 'str', 'str']); + this._sp_sample_score = this.lib.func('guardd_sp_sample_encode_and_score', 'void *', [ + 'str', + 'str', + 'str', + ]); + this._sp_decode = this.lib.func('guardd_sp_decode', 'void *', ['str', 'str']); + this._sp_decode_opts = this.lib.func('guardd_sp_decode_with_options', 'void *', [ + 'str', + 'str', + 'str', + ]); + } + + _readCString(ptr) { + if (!ptr) throw new Error('Native call returned null pointer'); + return koffi.decode(ptr, 'str'); + } + + _getErrorMessage(errorCode) { + try { + return this._error_message(errorCode); + } catch (_) { + return `Unknown error code: ${errorCode}`; } + } - /** - * Setup function signatures for FFI calls - */ - _setupFunctionSignatures() { - // Additional setup if needed + _digestPtr(expectedDigest) { + if (expectedDigest == null) return null; + if (!Buffer.isBuffer(expectedDigest) || expectedDigest.length !== 32) { + throw new Error('Expected digest must be 32 bytes'); } - - _ptrToString(ptr) { - if (!ptr || ref.isNull(ptr)) { - throw new Error('Native call returned null pointer'); - } - return ref.readCString(ptr, 0); + return expectedDigest; + } + + verifyDigest(filePath, expectedDigest) { + const dig = this._digestPtr(expectedDigest); + const result = this._verify_digest(filePath, dig); + if (result === GuarddError.Success) return true; + if (result === GuarddError.InvalidDigest) return false; + throw new Error(`Verification failed: ${this._getErrorMessage(result)}`); + } + + /** + * Load with integrity check. `valid` reflects a real digest comparison when + * `expectedDigest` is supplied (never hard-coded success). + */ + checkedLoad(filePath, expectedDigest = null) { + const dig = this._digestPtr(expectedDigest); + const handlePtr = this._checked_load(filePath, dig); + if (!handlePtr) { + throw new Error('Failed to load model file (guardd_checked_load returned null)'); } - - /** - * Get error message for error code - */ - _getErrorMessage(errorCode) { - try { - return this.lib.guardd_error_message(errorCode); - } catch (error) { - return `Unknown error code: ${errorCode}`; - } + try { + const digestBuf = this.computeDigest(filePath); + const valid = expectedDigest ? digestBuf.equals(expectedDigest) : true; + return { + path: filePath, + size: fs.statSync(filePath).size, + digest: digestBuf.toString('hex'), + valid, + libPath: this.libPath, + }; + } finally { + this._free_handle(handlePtr); } - - /** - * Verify SHA-256 digest of a file - * - * @param {string} filePath - Path to the file to verify - * @param {Buffer} expectedDigest - Expected SHA-256 digest (32 bytes) - * @returns {boolean} - True if digest matches, false otherwise - * @throws {Error} - If verification fails due to system error - */ - verifyDigest(filePath, expectedDigest) { - if (!Buffer.isBuffer(expectedDigest) || expectedDigest.length !== 32) { - throw new Error('Expected digest must be 32 bytes'); - } - - const result = this.lib.guardd_verify_digest(filePath, expectedDigest); - - if (result === GuarddError.Success) { - return true; - } else if (result === GuarddError.InvalidDigest) { - return false; - } else { - const errorMsg = this._getErrorMessage(result); - throw new Error(`Verification failed: ${errorMsg}`); - } + } + + verifyQuantization128Vectors(layerName, weights, fanIn, fanOut, quantType = 'int8') { + const arr = + weights instanceof Float32Array ? weights : new Float32Array(weights); + const resultPtr = this._verify_quant_128( + layerName, + arr, + arr.length, + fanIn, + fanOut, + quantType + ); + if (!resultPtr) throw new Error('Quantization verification failed'); + try { + return JSON.parse(this._readCString(resultPtr)); + } finally { + this._free_json(resultPtr); } + } - /** - * Load a model file with integrity checks - * - * @param {string} filePath - Path to the model file - * @param {Buffer} expectedDigest - Expected SHA-256 digest (32 bytes). If null, only computes digest. - * @returns {Object} - Model information - * @throws {Error} - If loading fails - */ - checkedLoad(filePath, expectedDigest = null) { - if (expectedDigest && (!Buffer.isBuffer(expectedDigest) || expectedDigest.length !== 32)) { - throw new Error('Expected digest must be 32 bytes'); - } - const digestPtr = expectedDigest ? expectedDigest : null; - const handlePtr = this.lib.guardd_checked_load(filePath, digestPtr); - - if (!handlePtr) { - throw new Error('Failed to load model file'); - } - - try { - const result = { - path: filePath, - size: fs.statSync(filePath).size, - digest: this.computeDigest(filePath).toString('hex'), - valid: true - }; - - return result; - } finally { - this.lib.guardd_free_handle(handlePtr); - } + verifyModelQuantization(layers) { + const resultPtr = this._verify_model_128(JSON.stringify(layers)); + if (!resultPtr) throw new Error('Model quantization verification failed'); + try { + return JSON.parse(this._readCString(resultPtr)); + } finally { + this._free_json(resultPtr); } + } - /** - * Verify quantization bounds for a layer using 128 random activation vectors - * - * @param {string} layerName - Name of the layer - * @param {Float32Array} weights - Weight matrix - * @param {number} fanIn - Number of input features - * @param {number} fanOut - Number of output features - * @param {string} quantType - Quantization type ("int8", "fp16", etc.) - * @returns {Object} - Verification results - * @throws {Error} - If verification fails - */ - verifyQuantization128Vectors(layerName, weights, fanIn, fanOut, quantType = 'int8') { - if (!(weights instanceof Float32Array)) { - weights = new Float32Array(weights); - } - - const weightsPtr = ref.ref(weights); - const resultPtr = this.lib.guardd_verify_quant_128_vectors( - layerName, - weightsPtr, - weights.length, - fanIn, - fanOut, - quantType - ); - - if (!resultPtr) { - throw new Error('Quantization verification failed'); - } - - try { - const resultJson = this._ptrToString(resultPtr); - return JSON.parse(resultJson); - } finally { - this.lib.guardd_free_json_result(resultPtr); - } + runBitflipCorpusTest(fileSizeGb = 0, numCorruptions = 10, tempDir = null) { + const dir = tempDir || os.tmpdir(); + const resultPtr = this._bitflip(fileSizeGb, numCorruptions, dir); + if (!resultPtr) throw new Error('Bit-flip corpus test failed'); + try { + return JSON.parse(this._readCString(resultPtr)); + } finally { + this._free_json(resultPtr); } + } - /** - * Verify quantization bounds for an entire model - * - * @param {Array} layers - List of layer objects with weights and metadata - * @returns {Object} - Model verification results - * @throws {Error} - If verification fails - */ - verifyModelQuantization(layers) { - const layersJson = JSON.stringify(layers); - const resultPtr = this.lib.guardd_verify_model_128_vectors(layersJson); - - if (!resultPtr) { - throw new Error('Model quantization verification failed'); - } - - try { - const resultJson = this._ptrToString(resultPtr); - return JSON.parse(resultJson); - } finally { - this.lib.guardd_free_json_result(resultPtr); - } + perfectHashEncode(vocabJson, text) { + const resultPtr = this._ph_encode(vocabJson, text); + if (!resultPtr) throw new Error('Perfect hash encoding failed'); + try { + return JSON.parse(this._readCString(resultPtr)); + } finally { + this._free_json(resultPtr); } + } - /** - * Run bit-flip corpus test for weight integrity validation - * - * @param {number} fileSizeGb - Size of test files in GB - * @param {number} numCorruptions - Number of corruption attempts - * @param {string} tempDir - Temporary directory for test files - * @returns {Object} - Test results - * @throws {Error} - If test fails - */ - runBitflipCorpusTest(fileSizeGb = 1, numCorruptions = 10, tempDir = null) { - if (!tempDir) { - tempDir = require('os').tmpdir(); - } - - const resultPtr = this.lib.guardd_run_bitflip_corpus_test( - fileSizeGb, - numCorruptions, - tempDir - ); - - if (!resultPtr) { - throw new Error('Bit-flip corpus test failed'); - } - - try { - const resultJson = this._ptrToString(resultPtr); - return JSON.parse(resultJson); - } finally { - this.lib.guardd_free_json_result(resultPtr); - } + perfectHashDecode(vocabJson, token) { + const resultPtr = this._ph_decode(vocabJson, token >>> 0); + if (!resultPtr) throw new Error('Perfect hash decoding failed'); + try { + return this._readCString(resultPtr); + } finally { + this._free_json(resultPtr); } - - /** - * Encode text using perfect hash tokenizer - * - * @param {string} vocabJson - JSON string containing the perfect hash vocabulary - * @param {string} text - Text to encode - * @returns {Array} - Array of token IDs - * @throws {Error} - If encoding fails - */ - perfectHashEncode(vocabJson, text) { - const resultPtr = this.lib.guardd_perfect_hash_encode(vocabJson, text); - - if (!resultPtr) { - throw new Error('Perfect hash encoding failed'); - } - - try { - const resultJson = this._ptrToString(resultPtr); - return JSON.parse(resultJson); - } finally { - this.lib.guardd_free_json_result(resultPtr); - } + } + + perfectHashDecodeSequence(vocabJson, tokens) { + return tokens.map((t) => this.perfectHashDecode(vocabJson, t)).join(' '); + } + + /** + * SentencePiece encode (CHAR / BPE / UNIGRAM when the Rust support gate passes). + * See docs/sentencepiece-bpe.md — not full SP parity. + * @param {string} modelPath + * @param {string} text + * @param {{allowSilentUnk?: boolean, forcedPieces?: string[], wordOfficialSplit?: boolean, mergeConsecutiveUnk?: boolean, reverse?: boolean, addBos?: boolean, addEos?: boolean}} [options] + */ + spEncode(modelPath, text, options = {}) { + const optsJson = JSON.stringify({ + allow_silent_unk: Boolean(options.allowSilentUnk), + forced_pieces: Array.isArray(options.forcedPieces) ? options.forcedPieces : [], + word_official_split: Boolean(options.wordOfficialSplit), + merge_consecutive_unk: + options.mergeConsecutiveUnk === undefined ? true : Boolean(options.mergeConsecutiveUnk), + reverse: Boolean(options.reverse), + add_bos: Boolean(options.addBos), + add_eos: Boolean(options.addEos), + }); + const resultPtr = this._sp_encode_opts(modelPath, text, optsJson); + if (!resultPtr) throw new Error('spEncode returned null'); + try { + const payload = JSON.parse(this._readCString(resultPtr)); + if (!payload.ok) { + throw new Error(payload.error || 'spEncode failed'); + } + return payload.tokens; + } finally { + this._free_json(resultPtr); } - - /** - * Decode a single token using perfect hash tokenizer - * - * @param {string} vocabJson - JSON string containing the perfect hash vocabulary - * @param {number} token - Token ID to decode - * @returns {string} - Decoded word - * @throws {Error} - If decoding fails - */ - perfectHashDecode(vocabJson, token) { - const resultPtr = this.lib.guardd_perfect_hash_decode(vocabJson, token); - - if (!resultPtr) { - throw new Error('Perfect hash decoding failed'); - } - - try { - return this._ptrToString(resultPtr); - } finally { - this.lib.guardd_free_json_result(resultPtr); - } + } + + /** + * UNIGRAM lattice n-best. Returns `[{tokens, score}, ...]`. + * @param {{allowSilentUnk?: boolean, forcedPieces?: string[], mergeConsecutiveUnk?: boolean, reverse?: boolean, addBos?: boolean, addEos?: boolean, nbestStockCompat?: boolean}} [options] + */ + spNbestEncode(modelPath, text, nbestSize = 1, options = {}) { + const optsJson = JSON.stringify({ + allow_silent_unk: Boolean(options.allowSilentUnk), + forced_pieces: Array.isArray(options.forcedPieces) ? options.forcedPieces : [], + merge_consecutive_unk: + options.mergeConsecutiveUnk === undefined ? true : Boolean(options.mergeConsecutiveUnk), + reverse: Boolean(options.reverse), + add_bos: Boolean(options.addBos), + add_eos: Boolean(options.addEos), + nbest_stock_compat: Boolean(options.nbestStockCompat), + nbest_size: Number(nbestSize) || 1, + }); + const resultPtr = this._sp_nbest(modelPath, text, optsJson); + if (!resultPtr) throw new Error('spNbestEncode returned null'); + try { + const payload = JSON.parse(this._readCString(resultPtr)); + if (!payload.ok) { + throw new Error(payload.error || 'spNbestEncode failed'); + } + return payload.nbests; + } finally { + this._free_json(resultPtr); } - - /** - * Decode a sequence of tokens using perfect hash tokenizer - * - * @param {string} vocabJson - JSON string containing the perfect hash vocabulary - * @param {Array} tokens - Array of token IDs to decode - * @returns {string} - Decoded text - * @throws {Error} - If decoding fails - */ - perfectHashDecodeSequence(vocabJson, tokens) { - const decodedWords = []; - for (const token of tokens) { - const word = this.perfectHashDecode(vocabJson, token); - decodedWords.push(word); - } - return decodedWords.join(' '); + } + + /** + * UNIGRAM sample-encode or BPE-dropout (alpha = dropout probability). + * @param {{nbestSize?: number, alpha?: number, seed?: number, allowSilentUnk?: boolean, forcedPieces?: string[], mergeConsecutiveUnk?: boolean, reverse?: boolean, addBos?: boolean, addEos?: boolean}} [options] + */ + spSampleEncode(modelPath, text, options = {}) { + const body = { + allow_silent_unk: Boolean(options.allowSilentUnk), + forced_pieces: Array.isArray(options.forcedPieces) ? options.forcedPieces : [], + merge_consecutive_unk: + options.mergeConsecutiveUnk === undefined ? true : Boolean(options.mergeConsecutiveUnk), + reverse: Boolean(options.reverse), + add_bos: Boolean(options.addBos), + add_eos: Boolean(options.addEos), + nbest_size: options.nbestSize === undefined ? -1 : Number(options.nbestSize), + alpha: options.alpha === undefined ? 0.0 : Number(options.alpha), + }; + if (options.seed !== undefined && options.seed !== null) { + body.seed = Number(options.seed); } - - /** - * Compute SHA-256 digest of a file using Node.js crypto - * - * @param {string} filePath - Path to the file - * @returns {Buffer} - SHA-256 digest - */ - computeDigest(filePath) { - const hash = crypto.createHash('sha256'); - const data = fs.readFileSync(filePath); - hash.update(data); - return hash.digest(); + const resultPtr = this._sp_sample(modelPath, text, JSON.stringify(body)); + if (!resultPtr) throw new Error('spSampleEncode returned null'); + try { + const payload = JSON.parse(this._readCString(resultPtr)); + if (!payload.ok) { + throw new Error(payload.error || 'spSampleEncode failed'); + } + return payload.tokens; + } finally { + this._free_json(resultPtr); } - - /** - * Verify file integrity using both Rust and Node.js implementations - * - * @param {string} filePath - Path to the file - * @param {Buffer} expectedDigest - Expected digest - * @returns {Object} - Verification results - */ - verifyFileIntegrity(filePath, expectedDigest) { - const results = { - filePath, - expectedDigest: expectedDigest.toString('hex'), - rustVerification: null, - nodeVerification: null, - match: false - }; - - try { - // Rust verification - results.rustVerification = this.verifyDigest(filePath, expectedDigest); - } catch (error) { - results.rustVerification = { error: error.message }; - } - - try { - // Node.js verification - const computedDigest = this.computeDigest(filePath); - results.nodeVerification = Buffer.compare(computedDigest, expectedDigest) === 0; - } catch (error) { - results.nodeVerification = { error: error.message }; - } - - // Check if both verifications match - if (results.rustVerification === true && results.nodeVerification === true) { - results.match = true; - } - - return results; + } + + /** + * UNIGRAM SampleEncodeAndScore (WR or WOR / Gumbel inclusion probs). + * @param {{numSamples?: number, alpha?: number, wor?: boolean, includeBest?: boolean, seed?: number, allowSilentUnk?: boolean, forcedPieces?: string[], mergeConsecutiveUnk?: boolean, reverse?: boolean, addBos?: boolean, addEos?: boolean}} [options] + */ + spSampleEncodeAndScore(modelPath, text, options = {}) { + const body = { + allow_silent_unk: Boolean(options.allowSilentUnk), + forced_pieces: Array.isArray(options.forcedPieces) ? options.forcedPieces : [], + merge_consecutive_unk: + options.mergeConsecutiveUnk === undefined ? true : Boolean(options.mergeConsecutiveUnk), + reverse: Boolean(options.reverse), + add_bos: Boolean(options.addBos), + add_eos: Boolean(options.addEos), + num_samples: options.numSamples === undefined ? 1 : Number(options.numSamples), + alpha: options.alpha === undefined ? 0.0 : Number(options.alpha), + wor: Boolean(options.wor), + include_best: Boolean(options.includeBest), + }; + if (options.seed !== undefined && options.seed !== null) { + body.seed = Number(options.seed); } -} - -/** - * HuggingFace-style integration for JavaScript - */ -class JavaScriptGuard { - constructor(guard) { - this.guard = guard; + const resultPtr = this._sp_sample_score(modelPath, text, JSON.stringify(body)); + if (!resultPtr) throw new Error('spSampleEncodeAndScore returned null'); + try { + const payload = JSON.parse(this._readCString(resultPtr)); + if (!payload.ok) { + throw new Error(payload.error || 'spSampleEncodeAndScore failed'); + } + return payload.samples; + } finally { + this._free_json(resultPtr); } - - /** - * Load a model with verification (JavaScript equivalent of HuggingFace integration) - * - * @param {string} modelPath - Path to the model file - * @param {Buffer} expectedDigest - Expected digest - * @param {boolean} verifyQuantization - Whether to verify quantization - * @returns {Object} - Model and verification results - */ - checkedLoadModel(modelPath, expectedDigest = null, verifyQuantization = true) { - console.log(`Loading model ${modelPath} with integrity verification...`); - - const results = { - modelPath, - verified: false, - fileIntegrity: null, - quantization: null, - errors: [] - }; - - try { - // Verify file integrity - if (expectedDigest) { - results.fileIntegrity = this.guard.verifyFileIntegrity(modelPath, expectedDigest); - console.log(`✓ File integrity verified: ${results.fileIntegrity.expectedDigest}`); - } else { - const computedDigest = this.guard.computeDigest(modelPath); - results.fileIntegrity = { - computedDigest: computedDigest.toString('hex'), - verified: true - }; - console.log(`✓ File digest computed: ${results.fileIntegrity.computedDigest}`); - } - } catch (error) { - console.log(`✗ File integrity verification failed: ${error.message}`); - results.fileIntegrity = { error: error.message }; - results.errors.push(error.message); - } - - // Note: Quantization verification would require model loading - // which is beyond the scope of this basic Node.js implementation - if (verifyQuantization) { - results.quantization = { note: 'Quantization verification requires model loading implementation' }; - } - - results.verified = results.errors.length === 0; - return results; + } + + /** + * SentencePiece decode: ▁/BYTE/unk_surface + optional denormalizer charsmap. + * @param {string} modelPath + * @param {number[]} tokens + * @param {{reverse?: boolean}} [options] + */ + spDecode(modelPath, tokens, options = {}) { + const optsJson = JSON.stringify({ reverse: Boolean(options.reverse) }); + const resultPtr = this._sp_decode_opts(modelPath, JSON.stringify(tokens), optsJson); + if (!resultPtr) throw new Error('spDecode returned null'); + try { + const payload = JSON.parse(this._readCString(resultPtr)); + if (!payload.ok) { + throw new Error(payload.error || 'spDecode failed'); + } + return payload.text; + } finally { + this._free_json(resultPtr); } - - /** - * Verify tokenizer determinism (JavaScript equivalent) - * - * @param {Function} encode - Encoding function - * @param {Function} decode - Decoding function - * @param {Array} testStrings - Test strings - * @returns {Object} - Test results - */ - verifyTokenizerDeterminism(encode, decode, testStrings = null) { - if (!testStrings) { - testStrings = this._generateTestStrings(1000); - } - - const results = { - totalTests: testStrings.length, - passed: 0, - failed: 0, - errors: [] - }; - - for (let i = 0; i < testStrings.length; i++) { - try { - const testStr = testStrings[i]; - const tokens = encode(testStr); - const decoded = decode(tokens); - - if (decoded === testStr) { - results.passed++; - } else { - results.failed++; - results.errors.push({ - index: i, - original: testStr, - decoded: decoded, - tokens: tokens - }); - } - } catch (error) { - results.failed++; - results.errors.push({ - index: i, - original: testStrings[i], - error: error.message - }); - } - } - - return results; + } + + computeDigest(filePath) { + const hash = crypto.createHash('sha256'); + hash.update(fs.readFileSync(filePath)); + return hash.digest(); + } + + verifyFileIntegrity(filePath, expectedDigest) { + const results = { + filePath, + expectedDigest: expectedDigest.toString('hex'), + rustVerification: null, + nodeVerification: null, + match: false, + }; + try { + results.rustVerification = this.verifyDigest(filePath, expectedDigest); + } catch (error) { + results.rustVerification = { error: error.message }; } - - /** - * Generate random test strings - */ - _generateTestStrings(numTests) { - const strings = []; - const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 '; - - for (let i = 0; i < numTests; i++) { - const length = Math.floor(Math.random() * 100) + 1; - let testStr = ''; - for (let j = 0; j < length; j++) { - testStr += chars.charAt(Math.floor(Math.random() * chars.length)); - } - strings.push(testStr); - } - - return strings; + try { + results.nodeVerification = + Buffer.compare(this.computeDigest(filePath), expectedDigest) === 0; + } catch (error) { + results.nodeVerification = { error: error.message }; } + results.match = + results.rustVerification === true && results.nodeVerification === true; + return results; + } } -/** - * Convenience functions - */ function createGuard(libPath = null) { - return new ModelAssetGuard(libPath); -} - -function createJSGuard(libPath = null) { - const guard = new ModelAssetGuard(libPath); - return new JavaScriptGuard(guard); + return new ModelAssetGuard(libPath); } -/** - * Example usage - */ if (require.main === module) { - // Example: Verify a test file - try { - const guard = createGuard(); - - // Create a test file - const testContent = 'Hello, Model Asset Guard!'; - const testFile = path.join(require('os').tmpdir(), 'test_model.bin'); - fs.writeFileSync(testFile, testContent); - - // Compute digest - const digest = guard.computeDigest(testFile); - console.log(`Test file digest: ${digest.toString('hex')}`); - - // Verify integrity - const verified = guard.verifyDigest(testFile, digest); - console.log(`Verification result: ${verified}`); - - // Clean up - fs.unlinkSync(testFile); - - console.log('✓ Node.js bindings test completed successfully!'); - } catch (error) { - console.error(`✗ Node.js bindings test failed: ${error.message}`); - process.exit(1); + try { + const guard = createGuard(); + const testFile = path.join(os.tmpdir(), `mag-node-${process.pid}.bin`); + fs.writeFileSync(testFile, 'Hello, Model Asset Guard!'); + const digest = guard.computeDigest(testFile); + const verified = guard.verifyDigest(testFile, digest); + const loaded = guard.checkedLoad(testFile, digest); + fs.unlinkSync(testFile); + if (!verified || !loaded.valid) { + console.error('Node bindings self-check failed: digest mismatch'); + process.exit(1); } + console.log(`Node bindings OK (lib=${guard.libPath})`); + } catch (error) { + console.error(`Node bindings test failed: ${error.message}`); + process.exit(1); + } } module.exports = { - ModelAssetGuard, - JavaScriptGuard, - GuarddError, - createGuard, - createJSGuard -}; \ No newline at end of file + ModelAssetGuard, + GuarddError, + createGuard, + resolveLibPath, + candidateLibPaths, +}; From 46ecab0d65123e5eb49fd839e8492b6940a05426 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:49 -0700 Subject: [PATCH 29/35] Improve Python guardd loader and quant helpers Match Node search order for staged natives and keep optional HF/torch paths behind extras. --- bindings/python/pytorch_guard.py | 672 ++++++++++++++++++++++++------- 1 file changed, 524 insertions(+), 148 deletions(-) diff --git a/bindings/python/pytorch_guard.py b/bindings/python/pytorch_guard.py index 0355dbe..a9badb2 100644 --- a/bindings/python/pytorch_guard.py +++ b/bindings/python/pytorch_guard.py @@ -6,35 +6,46 @@ enabling integration with HuggingFace transformers and PyTorch models. Requirements: -- transformers >= 4.20.0 -- torch >= 1.12.0 -- numpy >= 1.21.0 - ctypes (built-in) +- Optional: numpy (single-layer quant helper), transformers/torch (HF helpers) """ -import os -import json -import tempfile +from __future__ import annotations + import ctypes import ctypes.util -from typing import Dict, List, Optional, Union, Any, Tuple +import json +import os +import tempfile import warnings +from typing import Any -# Try to import optional dependencies try: import numpy as np + + NUMPY_AVAILABLE = True +except ImportError: + np = None # type: ignore[assignment] + NUMPY_AVAILABLE = False + +try: from transformers import ( - PreTrainedModel, - PreTrainedTokenizer, AutoModel, AutoTokenizer, + PreTrainedModel, + PreTrainedTokenizer, ) TRANSFORMERS_AVAILABLE = True except ImportError: TRANSFORMERS_AVAILABLE = False + AutoModel = Any # type: ignore[misc,assignment] + AutoTokenizer = Any # type: ignore[misc,assignment] + PreTrainedModel = Any # type: ignore[misc,assignment] + PreTrainedTokenizer = Any # type: ignore[misc,assignment] warnings.warn( - "transformers or torch not available. HuggingFace integration disabled." + "transformers or torch not available. HuggingFace integration disabled.", + stacklevel=2, ) @@ -52,7 +63,7 @@ class ModelAssetGuard: verification functions, including integration with HuggingFace transformers. """ - def __init__(self, lib_path: Optional[str] = None): + def __init__(self, lib_path: str | None = None): """ Initialize Model Asset Guard bindings. @@ -62,15 +73,33 @@ def __init__(self, lib_path: Optional[str] = None): self.lib = self._load_library(lib_path) self._setup_function_signatures() - def _load_library(self, lib_path: Optional[str]) -> ctypes.CDLL: + def _load_library(self, lib_path: str | None) -> ctypes.CDLL: """Load the Rust sidecar library""" + _pkg_dir = os.path.dirname(os.path.abspath(__file__)) + _repo_root = os.path.abspath(os.path.join(_pkg_dir, "..", "..")) if lib_path is None: - # Search for library in common locations + # Search for library in common locations; prefer newest mtime when + # both debug and release artifacts exist (avoids stale release DLL). search_paths = [ + os.environ.get("GUARDD_LIB"), + # Staged native dir (see scripts/stage_guardd_native.sh / docs/native-packaging.md) + os.path.join(_pkg_dir, "native", "libguardd.so"), + os.path.join(_pkg_dir, "native", "libguardd.dylib"), + os.path.join(_pkg_dir, "native", "guardd.dll"), + os.path.join(_repo_root, "dist", "native", "libguardd.so"), + os.path.join(_repo_root, "dist", "native", "libguardd.dylib"), + os.path.join(_repo_root, "dist", "native", "guardd.dll"), + os.path.join(_repo_root, "bindings", "nodejs", "native", "libguardd.so"), + os.path.join(_repo_root, "bindings", "nodejs", "native", "libguardd.dylib"), + os.path.join(_repo_root, "bindings", "nodejs", "native", "guardd.dll"), "./src/rust/guardd/target/release/libguardd.so", "./src/rust/guardd/target/debug/libguardd.so", + "./src/rust/guardd/target/release/libguardd.dylib", + "./src/rust/guardd/target/debug/libguardd.dylib", "./target/release/libguardd.so", "./target/debug/libguardd.so", + "./target/release/libguardd.dylib", + "./target/debug/libguardd.dylib", "./src/rust/guardd/target/release/guardd.dll", "./src/rust/guardd/target/debug/guardd.dll", "./target/release/guardd.dll", @@ -78,10 +107,13 @@ def _load_library(self, lib_path: Optional[str]) -> ctypes.CDLL: ctypes.util.find_library("guardd"), ] - for path in search_paths: - if path and os.path.exists(path): - lib_path = path - break + existing = [p for p in search_paths if p and os.path.exists(p)] + if existing: + # GUARDD_LIB (first entry) wins if set and present. + if search_paths[0] and os.path.exists(search_paths[0]): + lib_path = search_paths[0] + else: + lib_path = max(existing, key=lambda p: os.path.getmtime(p)) else: raise FileNotFoundError( "Could not find libguardd library. Please build the Rust sidecar first:\n" @@ -92,7 +124,7 @@ def _load_library(self, lib_path: Optional[str]) -> ctypes.CDLL: lib = ctypes.CDLL(lib_path) return lib except Exception as e: - raise RuntimeError(f"Failed to load library {lib_path}: {e}") + raise RuntimeError(f"Failed to load library {lib_path}: {e}") from e def _setup_function_signatures(self): """Setup function signatures for FFI calls""" @@ -129,7 +161,8 @@ class ModelHandle(ctypes.Structure): self.lib.guardd_free_handle.argtypes = [ctypes.c_void_p] self.lib.guardd_free_handle.restype = None - self.lib.guardd_free_json_result.argtypes = [ctypes.c_char_p] + # Owned C strings must use c_void_p so ctypes does not steal/free the pointer. + self.lib.guardd_free_json_result.argtypes = [ctypes.c_void_p] self.lib.guardd_free_json_result.restype = None # Quantization verification @@ -141,13 +174,13 @@ class ModelHandle(ctypes.Structure): ctypes.c_uint32, # fan_out ctypes.c_char_p, # quant_type ] - self.lib.guardd_verify_quant_128_vectors.restype = ctypes.c_char_p + self.lib.guardd_verify_quant_128_vectors.restype = ctypes.c_void_p # Model verification self.lib.guardd_verify_model_128_vectors.argtypes = [ ctypes.c_char_p, # layers_json ] - self.lib.guardd_verify_model_128_vectors.restype = ctypes.c_char_p + self.lib.guardd_verify_model_128_vectors.restype = ctypes.c_void_p # Bit-flip corpus testing self.lib.guardd_run_bitflip_corpus_test.argtypes = [ @@ -155,20 +188,80 @@ class ModelHandle(ctypes.Structure): ctypes.c_size_t, # num_corruptions ctypes.c_char_p, # temp_dir ] - self.lib.guardd_run_bitflip_corpus_test.restype = ctypes.c_char_p + self.lib.guardd_run_bitflip_corpus_test.restype = ctypes.c_void_p # Perfect hash tokenizer functions self.lib.guardd_perfect_hash_encode.argtypes = [ ctypes.c_char_p, # vocab_json ctypes.c_char_p, # text ] - self.lib.guardd_perfect_hash_encode.restype = ctypes.c_char_p + self.lib.guardd_perfect_hash_encode.restype = ctypes.c_void_p self.lib.guardd_perfect_hash_decode.argtypes = [ ctypes.c_char_p, # vocab_json ctypes.c_uint32, # token ] - self.lib.guardd_perfect_hash_decode.restype = ctypes.c_char_p + self.lib.guardd_perfect_hash_decode.restype = ctypes.c_void_p + + # SentencePiece encode/decode (CHAR / BPE / UNIGRAM when support gate passes) + self.lib.guardd_sp_encode.argtypes = [ + ctypes.c_char_p, # model_path + ctypes.c_char_p, # text + ] + self.lib.guardd_sp_encode.restype = ctypes.c_void_p + + self.lib.guardd_sp_encode_with_options.argtypes = [ + ctypes.c_char_p, # model_path + ctypes.c_char_p, # text + ctypes.c_char_p, # options_json + ] + self.lib.guardd_sp_encode_with_options.restype = ctypes.c_void_p + + self.lib.guardd_sp_nbest_encode.argtypes = [ + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ] + self.lib.guardd_sp_nbest_encode.restype = ctypes.c_void_p + + self.lib.guardd_sp_sample_encode.argtypes = [ + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ] + self.lib.guardd_sp_sample_encode.restype = ctypes.c_void_p + + self.lib.guardd_sp_sample_encode_and_score.argtypes = [ + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ] + self.lib.guardd_sp_sample_encode_and_score.restype = ctypes.c_void_p + + self.lib.guardd_sp_decode.argtypes = [ + ctypes.c_char_p, # model_path + ctypes.c_char_p, # tokens_json + ] + self.lib.guardd_sp_decode.restype = ctypes.c_void_p + + self.lib.guardd_sp_decode_with_options.argtypes = [ + ctypes.c_char_p, # model_path + ctypes.c_char_p, # tokens_json + ctypes.c_char_p, # options_json + ] + self.lib.guardd_sp_decode_with_options.restype = ctypes.c_void_p + + def _consume_owned_cstring(self, result_ptr: int | None) -> str: + """Read an owned NUL-terminated C string and free it via guardd.""" + if not result_ptr: + raise GuarddError("Native call returned null pointer") + try: + raw = ctypes.cast(result_ptr, ctypes.c_char_p).value + if raw is None: + raise GuarddError("Native call returned empty string pointer") + return raw.decode("utf-8") + finally: + self.lib.guardd_free_json_result(result_ptr) def _get_error_message(self, error_code: int) -> str: """Get error message for error code""" @@ -208,8 +301,8 @@ def verify_digest(self, file_path: str, expected_digest: bytes) -> bool: raise GuarddError(f"Verification failed: {error_msg}") def checked_load( - self, file_path: str, expected_digest: Optional[bytes] = None - ) -> Dict[str, Any]: + self, file_path: str, expected_digest: bytes | None = None + ) -> dict[str, Any]: """ Load a model file with integrity checks. @@ -259,11 +352,11 @@ def checked_load( def verify_quantization_128_vectors( self, layer_name: str, - weights: Union[List[float], np.ndarray], + weights: list[float] | np.ndarray, fan_in: int, fan_out: int, quant_type: str = "int8", - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Verify quantization bounds for a layer using 128 random activation vectors. @@ -280,6 +373,12 @@ def verify_quantization_128_vectors( Raises: GuarddError: If verification fails """ + if not NUMPY_AVAILABLE: + raise GuarddError( + "numpy is required for verify_quantization_128_vectors; " + "use verify_model_quantization with plain lists instead" + ) + if not isinstance(weights, np.ndarray): weights = np.array(weights, dtype=np.float32) @@ -296,20 +395,9 @@ def verify_quantization_128_vectors( fan_out, quant_type.encode("utf-8"), ) + return json.loads(self._consume_owned_cstring(result_ptr)) - if not result_ptr: - raise GuarddError("Quantization verification failed") - - try: - result_casted = ctypes.cast(result_ptr, ctypes.c_char_p) - if result_casted.value is None: - raise GuarddError("Quantization verification returned null result") - result_json = result_casted.value.decode("utf-8") - return json.loads(result_json) - finally: - self.lib.guardd_free_json_result(result_ptr) - - def verify_model_quantization(self, layers: List[Dict[str, Any]]) -> Dict[str, Any]: + def verify_model_quantization(self, layers: list[dict[str, Any]]) -> dict[str, Any]: """ Verify quantization bounds for an entire model. @@ -323,36 +411,24 @@ def verify_model_quantization(self, layers: List[Dict[str, Any]]) -> Dict[str, A GuarddError: If verification fails """ layers_json = json.dumps(layers) - result_ptr = self.lib.guardd_verify_model_128_vectors( layers_json.encode("utf-8") ) - - if not result_ptr: - raise GuarddError("Model quantization verification failed") - - try: - result_casted = ctypes.cast(result_ptr, ctypes.c_char_p) - if result_casted.value is None: - raise GuarddError( - "Model quantization verification returned null result" - ) - result_json = result_casted.value.decode("utf-8") - return json.loads(result_json) - finally: - self.lib.guardd_free_json_result(result_ptr) + return json.loads(self._consume_owned_cstring(result_ptr)) def run_bitflip_corpus_test( self, - file_size_gb: int = 1, + file_size_gb: int = 0, num_corruptions: int = 10, - temp_dir: Optional[str] = None, - ) -> Dict[str, Any]: + temp_dir: str | None = None, + ) -> dict[str, Any]: """ Run bit-flip corpus test for weight integrity validation. Args: - file_size_gb: Size of test files in GB + file_size_gb: Size in GB. ``0`` means 1 MiB (CI/default). Values + above the Rust default cap (64 MiB) require + ``GUARDD_BITFLIP_ALLOW_LARGE=1`` in the environment. num_corruptions: Number of corruption attempts temp_dir: Temporary directory for test files @@ -368,20 +444,9 @@ def run_bitflip_corpus_test( result_ptr = self.lib.guardd_run_bitflip_corpus_test( file_size_gb, num_corruptions, temp_dir.encode("utf-8") ) + return json.loads(self._consume_owned_cstring(result_ptr)) - if not result_ptr: - raise GuarddError("Bit-flip corpus test failed") - - try: - result_casted = ctypes.cast(result_ptr, ctypes.c_char_p) - if result_casted.value is None: - raise GuarddError("Bit-flip corpus test returned null result") - result_json = result_casted.value.decode("utf-8") - return json.loads(result_json) - finally: - self.lib.guardd_free_json_result(result_ptr) - - def perfect_hash_encode(self, vocab_json: str, text: str) -> List[int]: + def perfect_hash_encode(self, vocab_json: str, text: str) -> list[int]: """ Encode text using perfect hash tokenizer. @@ -398,18 +463,7 @@ def perfect_hash_encode(self, vocab_json: str, text: str) -> List[int]: result_ptr = self.lib.guardd_perfect_hash_encode( vocab_json.encode("utf-8"), text.encode("utf-8") ) - - if not result_ptr: - raise GuarddError("Perfect hash encoding failed") - - try: - result_casted = ctypes.cast(result_ptr, ctypes.c_char_p) - if result_casted.value is None: - raise GuarddError("Perfect hash encoding returned null result") - result_json = result_casted.value.decode("utf-8") - return json.loads(result_json) - finally: - self.lib.guardd_free_json_result(result_ptr) + return json.loads(self._consume_owned_cstring(result_ptr)) def perfect_hash_decode(self, vocab_json: str, token: int) -> str: """ @@ -428,20 +482,272 @@ def perfect_hash_decode(self, vocab_json: str, token: int) -> str: result_ptr = self.lib.guardd_perfect_hash_decode( vocab_json.encode("utf-8"), token ) + return self._consume_owned_cstring(result_ptr) - if not result_ptr: - raise GuarddError("Perfect hash decoding failed") + def sp_encode( + self, + model_path: str, + text: str, + *, + allow_silent_unk: bool = False, + forced_pieces: list[str] | None = None, + word_official_split: bool = False, + merge_consecutive_unk: bool = True, + reverse: bool = False, + add_bos: bool = False, + add_eos: bool = False, + ) -> list[int]: + """ + Encode text with a SentencePiece ``.model`` (CHAR / BPE / UNIGRAM / WORD). - try: - result_casted = ctypes.cast(result_ptr, ctypes.c_char_p) - if result_casted.value is None: - raise GuarddError("Perfect hash decoding returned null result") - result_str = result_casted.value.decode("utf-8") - return result_str - finally: - self.lib.guardd_free_json_result(result_ptr) + Uses the Rust support gate (SP normalize + optional charsmap / + byte_fallback / USER_DEFINED PrefixMatcher / treat_whitespace_as_suffix; + incomplete OOV refuse unless ``allow_silent_unk``). See + ``docs/sentencepiece-bpe.md``. - def perfect_hash_decode_sequence(self, vocab_json: str, tokens: List[int]) -> str: + Args: + model_path: Path to a SentencePiece ModelProto ``.model`` file + text: Text to encode + allow_silent_unk: When True, emit UNKNOWN for OOV without + ``byte_fallback`` (stock SP). Default False (fail-closed). + forced_pieces: Extra PrefixMatcher pieces; each must already be in + the model vocab (fail-closed otherwise). + word_official_split: When True, WORD ``SplitIntoWords`` ignores the + trainer ``treat_whitespace_as_suffix`` flag (stock + ``word_model.cc``). Default False honors the trainer flag. + merge_consecutive_unk: When True (default), merge consecutive + UNKNOWN ids like stock ``SentencePieceProcessor`` (no-op when + ``byte_fallback`` is enabled). Set False for piece-level ids. + reverse: Reverse piece ids after merge and before bos/eos (stock + documented ExtraOptions order ``reverse:bos:eos``). + add_bos: Prepend CONTROL bos id after merge (stock ExtraOptions + ``bos``). Refuses if the model has no CONTROL bos piece. + add_eos: Append CONTROL eos id after merge (stock ExtraOptions + ``eos``). Refuses if the model has no CONTROL eos piece. + + Returns: + List of token IDs (piece indices) + + Raises: + GuarddError: If loading or encoding fails + """ + options = json.dumps( + { + "allow_silent_unk": bool(allow_silent_unk), + "forced_pieces": list(forced_pieces or []), + "word_official_split": bool(word_official_split), + "merge_consecutive_unk": bool(merge_consecutive_unk), + "reverse": bool(reverse), + "add_bos": bool(add_bos), + "add_eos": bool(add_eos), + } + ) + result_ptr = self.lib.guardd_sp_encode_with_options( + model_path.encode("utf-8"), + text.encode("utf-8"), + options.encode("utf-8"), + ) + payload = json.loads(self._consume_owned_cstring(result_ptr)) + if not payload.get("ok"): + raise GuarddError(payload.get("error", "sp_encode failed")) + tokens = payload.get("tokens") + if not isinstance(tokens, list): + raise GuarddError("sp_encode response missing tokens list") + return [int(t) for t in tokens] + + def sp_nbest_encode( + self, + model_path: str, + text: str, + nbest_size: int = 1, + *, + allow_silent_unk: bool = False, + forced_pieces: list[str] | None = None, + merge_consecutive_unk: bool = True, + reverse: bool = False, + add_bos: bool = False, + add_eos: bool = False, + nbest_stock_compat: bool = False, + ) -> list[dict]: + """UNIGRAM lattice n-best. Returns ``[{tokens, score}, ...]`` best-first. + + ``nbest_stock_compat`` enables stock agenda shrink / timeout→Viterbi; + default refuses pathological lattices. + """ + options = json.dumps( + { + "allow_silent_unk": bool(allow_silent_unk), + "forced_pieces": list(forced_pieces or []), + "merge_consecutive_unk": bool(merge_consecutive_unk), + "reverse": bool(reverse), + "add_bos": bool(add_bos), + "add_eos": bool(add_eos), + "nbest_stock_compat": bool(nbest_stock_compat), + "nbest_size": int(nbest_size), + } + ) + result_ptr = self.lib.guardd_sp_nbest_encode( + model_path.encode("utf-8"), + text.encode("utf-8"), + options.encode("utf-8"), + ) + payload = json.loads(self._consume_owned_cstring(result_ptr)) + if not payload.get("ok"): + raise GuarddError(payload.get("error", "sp_nbest_encode failed")) + rows = payload.get("nbests") + if not isinstance(rows, list): + raise GuarddError("sp_nbest_encode response missing nbests") + out = [] + for row in rows: + out.append( + { + "tokens": [int(t) for t in row["tokens"]], + "score": float(row["score"]), + } + ) + return out + + def sp_sample_encode( + self, + model_path: str, + text: str, + *, + nbest_size: int = -1, + alpha: float = 0.0, + seed: int | None = None, + allow_silent_unk: bool = False, + forced_pieces: list[str] | None = None, + merge_consecutive_unk: bool = True, + reverse: bool = False, + add_bos: bool = False, + add_eos: bool = False, + ) -> list[int]: + """UNIGRAM lattice sample / sample-from-nbest, or BPE-dropout. + + For BPE, ``alpha`` is the dropout probability (``nbest_size`` ignored). + """ + options: dict = { + "allow_silent_unk": bool(allow_silent_unk), + "forced_pieces": list(forced_pieces or []), + "merge_consecutive_unk": bool(merge_consecutive_unk), + "reverse": bool(reverse), + "add_bos": bool(add_bos), + "add_eos": bool(add_eos), + "nbest_size": int(nbest_size), + "alpha": float(alpha), + } + if seed is not None: + options["seed"] = int(seed) + result_ptr = self.lib.guardd_sp_sample_encode( + model_path.encode("utf-8"), + text.encode("utf-8"), + json.dumps(options).encode("utf-8"), + ) + payload = json.loads(self._consume_owned_cstring(result_ptr)) + if not payload.get("ok"): + raise GuarddError(payload.get("error", "sp_sample_encode failed")) + tokens = payload.get("tokens") + if not isinstance(tokens, list): + raise GuarddError("sp_sample_encode response missing tokens list") + return [int(t) for t in tokens] + + def sp_sample_encode_and_score( + self, + model_path: str, + text: str, + *, + num_samples: int = 1, + alpha: float = 0.0, + wor: bool = False, + include_best: bool = False, + seed: int | None = None, + allow_silent_unk: bool = False, + forced_pieces: list[str] | None = None, + merge_consecutive_unk: bool = True, + reverse: bool = False, + add_bos: bool = False, + add_eos: bool = False, + ) -> list[dict]: + """UNIGRAM SampleEncodeAndScore. + + With-replacement (``wor=False``): score = path log-prob − marginal. + Without-replacement (``wor=True``): Gumbel-Top-k draws; score = log + inclusion probability. ``include_best`` requires ``wor=True``. + """ + options: dict = { + "allow_silent_unk": bool(allow_silent_unk), + "forced_pieces": list(forced_pieces or []), + "merge_consecutive_unk": bool(merge_consecutive_unk), + "reverse": bool(reverse), + "add_bos": bool(add_bos), + "add_eos": bool(add_eos), + "num_samples": int(num_samples), + "alpha": float(alpha), + "wor": bool(wor), + "include_best": bool(include_best), + } + if seed is not None: + options["seed"] = int(seed) + result_ptr = self.lib.guardd_sp_sample_encode_and_score( + model_path.encode("utf-8"), + text.encode("utf-8"), + json.dumps(options).encode("utf-8"), + ) + payload = json.loads(self._consume_owned_cstring(result_ptr)) + if not payload.get("ok"): + raise GuarddError(payload.get("error", "sp_sample_encode_and_score failed")) + rows = payload.get("samples") + if not isinstance(rows, list): + raise GuarddError("sp_sample_encode_and_score response missing samples") + out = [] + for row in rows: + out.append( + { + "tokens": [int(t) for t in row["tokens"]], + "score": float(row["score"]), + } + ) + return out + + def sp_decode( + self, model_path: str, tokens: list[int], *, reverse: bool = False + ) -> str: + """ + Decode token IDs toward official SP (▁ dummy-prefix strip, BYTE→UTF-8, + UNKNOWN→unk_surface, then optional ``denormalizer_spec`` charsmap). + + Reverse NFKC only applies when the model embeds a denormalizer charsmap. + ``reverse=True`` reverses ids before detokenize (stock decode + ExtraOptions ``reverse``). ExtraOption ``unk``/``unk_piece`` is a no-op + on id APIs (stock only rewrites piece strings). + + Args: + model_path: Path to a SentencePiece ModelProto ``.model`` file + tokens: Token IDs to decode + reverse: Reverse ids before detokenize + + Returns: + Decoded string + + Raises: + GuarddError: If loading or decoding fails + """ + tokens_json = json.dumps([int(t) for t in tokens]) + options = json.dumps({"reverse": bool(reverse)}) + result_ptr = self.lib.guardd_sp_decode_with_options( + model_path.encode("utf-8"), + tokens_json.encode("utf-8"), + options.encode("utf-8"), + ) + payload = json.loads(self._consume_owned_cstring(result_ptr)) + if not payload.get("ok"): + raise GuarddError(payload.get("error", "sp_decode failed")) + text = payload.get("text") + if not isinstance(text, str): + raise GuarddError("sp_decode response missing text") + return text + + def perfect_hash_decode_sequence(self, vocab_json: str, tokens: list[int]) -> str: """ Decode a sequence of tokens using perfect hash tokenizer. @@ -484,19 +790,87 @@ def __init__(self, guard: ModelAssetGuard): self.guard = guard + def _resolve_local_weight_file(self, model_path: str) -> str | None: + """Return a local pytorch weight file path, or None if not found.""" + candidates = [ + os.path.join(model_path, "pytorch_model.bin"), + os.path.join(model_path, "model.safetensors"), + ] + for path in candidates: + if os.path.exists(path): + return path + try: + for file in os.listdir(model_path): + if file.endswith(".bin") and "pytorch" in file: + return os.path.join(model_path, file) + except OSError: + return None + return None + + def _materialize_model_dir(self, model_name: str) -> tuple[str, str]: + """ + Return ``(local_dir, mode)`` for integrity-checked loading. + + - Local directories: verify-before-load. + - Hub IDs: snapshot download, then verify-after-download-before-use + (HuggingFace does not expose a stable verify-before-download API for + arbitrary weight blobs). Fail closed if download is impossible. + """ + if os.path.isdir(model_name): + return model_name, "verify_before_load" + + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise GuarddError( + "Remote Hub load requires huggingface_hub for " + "verify-after-download-before-use; install huggingface_hub or " + "pass a local model directory" + ) from e + + try: + local_dir = snapshot_download( + repo_id=model_name, + allow_patterns=[ + "*.bin", + "*.safetensors", + "*.json", + "tokenizer*", + "vocab*", + "merges.txt", + "config.json", + ], + ) + except Exception as e: + raise GuarddError( + f"Failed to download Hub snapshot for '{model_name}' " + f"(fail-closed): {e}" + ) from e + + if not os.path.isdir(local_dir): + raise GuarddError( + f"Hub snapshot for '{model_name}' did not yield a local directory" + ) + return local_dir, "verify_after_download_before_use" + def checked_load_pretrained( self, model_name: str, - expected_digest: Optional[bytes] = None, + expected_digest: bytes | None = None, verify_quantization: bool = True, **kwargs, - ) -> Tuple[PreTrainedModel, PreTrainedTokenizer, Dict[str, Any]]: + ) -> tuple[PreTrainedModel, PreTrainedTokenizer, dict[str, Any]]: """ Load a pretrained model with Model Asset Guard verification. + Local directories: verify weight file **before** ``from_pretrained``. + Remote Hub IDs: download a snapshot, verify the weight-file digest, + then load from the local snapshot (verify-after-download-before-use). + There is no reliable verify-before-download for Hub blobs. + Args: model_name: HuggingFace model name or path - expected_digest: Expected SHA-256 digest of the model file + expected_digest: Expected SHA-256 digest of the primary weight file verify_quantization: Whether to verify quantization bounds **kwargs: Additional arguments for from_pretrained @@ -504,60 +878,62 @@ def checked_load_pretrained( Tuple of (model, tokenizer, verification_results) Raises: - GuarddError: If verification fails + GuarddError: If verification or download fails (fail-closed) """ print(f"Loading model {model_name} with integrity verification...") + verification_results: dict[str, Any] = {} - # Load model and tokenizer normally first - model = AutoModel.from_pretrained(model_name, **kwargs) - tokenizer = AutoTokenizer.from_pretrained(model_name, **kwargs) - - # Get model file path - model_path = model.config._name_or_path - if os.path.isdir(model_path): - # Local directory - pytorch_model_path = os.path.join(model_path, "pytorch_model.bin") - if not os.path.exists(pytorch_model_path): - # Try to find the actual model file - for file in os.listdir(model_path): - if file.endswith(".bin") and "pytorch" in file: - pytorch_model_path = os.path.join(model_path, file) - break - else: - # Remote model, we can't verify the file directly - print("Warning: Remote model, skipping file integrity verification") - return model, tokenizer, {"verified": False, "reason": "remote_model"} + local_dir, mode = self._materialize_model_dir(model_name) + verification_results["mode"] = mode + verification_results["local_dir"] = local_dir - # Verify model file integrity - verification_results = {} + pytorch_model_path = self._resolve_local_weight_file(local_dir) + if pytorch_model_path is None: + raise GuarddError( + f"Could not find weight file under '{local_dir}' " + "(expected pytorch_model.bin or model.safetensors)" + ) - if os.path.exists(pytorch_model_path): - try: - file_info = self.guard.checked_load(pytorch_model_path, expected_digest) - verification_results["file_integrity"] = file_info - print(f"✓ File integrity verified: {file_info['digest']}") - except Exception as e: - print(f"✗ File integrity verification failed: {e}") - verification_results["file_integrity"] = {"error": str(e)} - else: - print("Warning: Could not find pytorch_model.bin file") - verification_results["file_integrity"] = {"error": "file_not_found"} + try: + file_info = self.guard.checked_load(pytorch_model_path, expected_digest) + verification_results["file_integrity"] = file_info + if expected_digest is not None and not file_info.get("valid", False): + raise GuarddError( + f"File integrity verification failed for '{pytorch_model_path}'" + ) + print(f"File integrity digest ({mode}): {file_info['digest']}") + except GuarddError: + raise + except Exception as e: + raise GuarddError(f"File integrity verification failed: {e}") from e + + # Load only from the verified local tree (never from an unverified remote id). + model = AutoModel.from_pretrained(local_dir, **kwargs) + tokenizer = AutoTokenizer.from_pretrained(local_dir, **kwargs) - # Verify quantization bounds if requested if verify_quantization and hasattr(model, "state_dict"): + print("Verifying quantization bounds...") try: - print("Verifying quantization bounds...") quant_results = self._verify_model_quantization(model) - verification_results["quantization"] = quant_results - print("✓ Quantization verification completed") except Exception as e: - print(f"✗ Quantization verification failed: {e}") - verification_results["quantization"] = {"error": str(e)} + raise GuarddError(f"Quantization verification failed: {e}") from e + verification_results["quantization"] = quant_results + if isinstance(quant_results, dict) and "error" in quant_results: + raise GuarddError( + f"Quantization verification failed: {quant_results['error']}" + ) + if isinstance(quant_results, dict) and quant_results.get( + "overall_pass_rate", 100.0 + ) < 100.0: + raise GuarddError( + "Quantization verification failed: not all layers passed" + ) + print("Quantization verification completed") verification_results["verified"] = True return model, tokenizer, verification_results - def _verify_model_quantization(self, model: PreTrainedModel) -> Dict[str, Any]: + def _verify_model_quantization(self, model: PreTrainedModel) -> dict[str, Any]: """ Verify quantization bounds for all layers in a model. @@ -593,9 +969,9 @@ def _verify_model_quantization(self, model: PreTrainedModel) -> Dict[str, Any]: def verify_tokenizer_determinism( self, tokenizer: PreTrainedTokenizer, - test_strings: Optional[List[str]] = None, + test_strings: list[str] | None = None, num_tests: int = 1000, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Verify tokenizer determinism property. @@ -644,7 +1020,7 @@ def verify_tokenizer_determinism( return results - def _generate_test_strings(self, num_tests: int) -> List[str]: + def _generate_test_strings(self, num_tests: int) -> list[str]: """Generate random test strings for tokenizer testing""" import random import string @@ -662,12 +1038,12 @@ def _generate_test_strings(self, num_tests: int) -> List[str]: # Convenience functions for easy usage -def create_guard(lib_path: Optional[str] = None) -> ModelAssetGuard: +def create_guard(lib_path: str | None = None) -> ModelAssetGuard: """Create a ModelAssetGuard instance""" return ModelAssetGuard(lib_path) -def create_hf_guard(lib_path: Optional[str] = None) -> HuggingFaceGuard: +def create_hf_guard(lib_path: str | None = None) -> HuggingFaceGuard: """Create a HuggingFaceGuard instance""" guard = ModelAssetGuard(lib_path) return HuggingFaceGuard(guard) @@ -675,11 +1051,11 @@ def create_hf_guard(lib_path: Optional[str] = None) -> HuggingFaceGuard: def checked_load_pretrained( model_name: str, - expected_digest: Optional[bytes] = None, + expected_digest: bytes | None = None, verify_quantization: bool = True, - lib_path: Optional[str] = None, + lib_path: str | None = None, **kwargs, -) -> Tuple[PreTrainedModel, PreTrainedTokenizer, Dict[str, Any]]: +) -> tuple[PreTrainedModel, PreTrainedTokenizer, dict[str, Any]]: """ Convenience function to load a pretrained model with verification. From 7e46aecbcd59114d7c0ba0ee71099df0a4b613ed Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:56 -0700 Subject: [PATCH 30/35] Update CI workflows for natives, axioms, and formal verify Upload per-OS guardd-native artifacts, run axiom checks, and keep formal/bundle jobs aligned with the new sidecars. --- .github/dependabot.yml | 6 +- .github/workflows/bundle-push.yml | 255 +++++++++++----------------- .github/workflows/ci.yml | 78 ++++++++- .github/workflows/formal-verify.yml | 96 +++++------ 4 files changed, 221 insertions(+), 214 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3ecb010..ae6ef98 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,10 @@ updates: schedule: interval: "weekly" - package-ecosystem: "pip" - directory: "/bindings/python" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "npm" + directory: "/bindings/nodejs" schedule: interval: "weekly" diff --git a/.github/workflows/bundle-push.yml b/.github/workflows/bundle-push.yml index 97be874..368fb63 100644 --- a/.github/workflows/bundle-push.yml +++ b/.github/workflows/bundle-push.yml @@ -49,120 +49,35 @@ jobs: - name: Create bundle run: | - echo "Creating Model Asset Guard bundle..." - - # Determine version + set -euo pipefail if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then VERSION="${{ github.event.inputs.version }}" else VERSION="${GITHUB_REF#refs/tags/}" fi - echo "Creating bundle for version: $VERSION" - - # Create bundle directory - BUNDLE_DIR="model-asset-guard-$VERSION" - mkdir -p "$BUNDLE_DIR" - - # Copy specification files - echo "Copying specification files..." - mkdir -p "$BUNDLE_DIR/Spec" - cp src/lean/ModelAssetGuard/Weights.lean "$BUNDLE_DIR/Spec/" 2>/dev/null || echo "Weights.lean not found" - cp src/lean/ModelAssetGuard/Quant/Core.lean "$BUNDLE_DIR/Spec/" 2>/dev/null || echo "Core.lean not found" - cp src/lean/ModelAssetGuard/Quant/LayerBound.lean "$BUNDLE_DIR/Spec/" 2>/dev/null || echo "LayerBound.lean not found" - cp src/lean/ModelAssetGuard/Token/Tokenizer.lean "$BUNDLE_DIR/Spec/" 2>/dev/null || echo "Tokenizer.lean not found" - - # Copy bindings - echo "Copying language bindings..." - mkdir -p "$BUNDLE_DIR/bindings" - cp bindings/python/pytorch_guard.py "$BUNDLE_DIR/bindings/" 2>/dev/null || echo "pytorch_guard.py not found" - cp bindings/nodejs/node_guard.js "$BUNDLE_DIR/bindings/" 2>/dev/null || echo "node_guard.js not found" - - # Copy Rust library - echo "Copying Rust library..." - mkdir -p "$BUNDLE_DIR/guardd" - if [ -f "src/rust/guardd/target/release/libguardd.so" ]; then - cp src/rust/guardd/target/release/libguardd.so "$BUNDLE_DIR/guardd/" - elif [ -f "src/rust/guardd/target/release/guardd.dll" ]; then - cp src/rust/guardd/target/release/guardd.dll "$BUNDLE_DIR/guardd/" - elif [ -f "src/rust/guardd/target/release/libguardd.dylib" ]; then - cp src/rust/guardd/target/release/libguardd.dylib "$BUNDLE_DIR/guardd/" - else - echo "Warning: No Rust library found" + # Canonical packager (honest README + axioms inventory + guardd bin helpers). + bash scripts/bundle.sh + # Rename dated artifact to versioned name for releases. + SRC="$(ls -1 model-asset-guard-*.zip | head -1)" + DST="model-asset-guard-${VERSION}.zip" + if [ "$SRC" != "$DST" ]; then + mv "$SRC" "$DST" fi - - # Copy configuration files - echo "Copying configuration files..." - cp lakefile.lean "$BUNDLE_DIR/" - cp lean-toolchain "$BUNDLE_DIR/" - cp LICENSE "$BUNDLE_DIR/" - - # Generate Lean kernel hash - echo "Generating Lean kernel hash..." - KERNEL_HASH=$(lake exe lean --version 2>/dev/null | head -1 | sha256sum | cut -d' ' -f1) - echo "$KERNEL_HASH" > "$BUNDLE_DIR/lean-hash.txt" - - # Create bundle README - cat > "$BUNDLE_DIR/README.md" << EOF - # Model Asset Guard Bundle v$VERSION - - This bundle contains the verified components of the Model Asset Guard system. - - ## Contents - - - \`Spec/\` - Lean specification files with formal proofs - - \`guardd/\` - Rust sidecar library for high-performance validation - - \`bindings/\` - Python and Node.js bindings - - \`lean-hash.txt\` - Lean kernel hash for verification - - ## Verification - - To verify this bundle: - - 1. Check the Lean kernel hash: - \`\`\`bash - lake exe lean --version | head -1 | sha256sum - \`\`\` - Compare with the contents of \`lean-hash.txt\` - - 2. Build and test the components: - \`\`\`bash - lake build - lake exe tests - cargo test --manifest-path src/rust/guardd/Cargo.toml - \`\`\` - - ## License - - MIT License - see LICENSE file for details. - EOF - - # Create bundle archive - BUNDLE_NAME="model-asset-guard-$VERSION.zip" - echo "Creating bundle archive: $BUNDLE_NAME" - zip -r "$BUNDLE_NAME" "$BUNDLE_DIR" - - # Generate SHA-256 of bundle - BUNDLE_SHA256=$(sha256sum "$BUNDLE_NAME" | cut -d' ' -f1) - echo "Bundle SHA-256: $BUNDLE_SHA256" - - # Create release info - cat > "release-info.txt" << EOF + # Refresh release-info with version tag. + BUNDLE_SHA256="$(sha256sum "$DST" | cut -d' ' -f1)" + cat > release-info.txt << EOF Model Asset Guard Bundle Version: $VERSION - Date: $(date) - Bundle: $BUNDLE_NAME + Date: $(date -u) + Bundle: $DST SHA-256: $BUNDLE_SHA256 - Kernel Hash: $KERNEL_HASH + Formal completeness: NOT claimed (see docs/axioms.md) + Hash vocab: CHD MPH by default (kind=chd_mph); legacy open-address still loads EOF - - echo "Bundle created successfully!" - echo "Bundle: $BUNDLE_NAME" + echo "Bundle: $DST" echo "SHA-256: $BUNDLE_SHA256" - # Cleanup - rm -rf "$BUNDLE_DIR" - - name: Upload bundle artifact uses: actions/upload-artifact@v4 with: @@ -184,10 +99,38 @@ jobs: name: sbom-${{ github.sha }} path: sbom.spdx.json + # Wave 19: multi-OS native libs attached to the GitHub Release (not npm/PyPI). + native-libs: + name: Release native (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + if: startsWith(github.ref, 'refs/tags/') + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Build + stage release guardd + run: | + set -euo pipefail + cargo build --release --manifest-path src/rust/guardd/Cargo.toml --locked + bash scripts/stage_guardd_native.sh + - uses: actions/upload-artifact@v4 + with: + name: release-native-${{ matrix.os }} + path: dist/native/ + if-no-files-found: error + publish-release: - needs: create-bundle + needs: [create-bundle, native-libs] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: write steps: - name: Checkout code @@ -199,48 +142,68 @@ jobs: name: model-asset-guard-bundle-${{ github.sha }} path: ./bundles + - name: Download native artifacts + uses: actions/download-artifact@v4 + with: + pattern: release-native-* + path: ./natives + merge-multiple: false + + - name: Pack per-OS native zips + run: | + set -euo pipefail + mkdir -p ./native-zips + for d in ./natives/release-native-*; do + [[ -d "$d" ]] || continue + os="$(basename "$d" | sed 's/^release-native-//')" + (cd "$d" && zip -r "$GITHUB_WORKSPACE/native-zips/guardd-native-${os}.zip" .) + done + ls -la ./native-zips + - name: Create GitHub Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: | bundles/model-asset-guard-*.zip bundles/release-info.txt + native-zips/guardd-native-*.zip body: | - ## Model Asset Guard v${{ github.ref_name }} + ## Model Asset Guard ${{ github.ref_name }} - This release includes: + Runtime bundle with Lean specs + Rust `guardd` + Python/Node bindings. - - ✅ Formal verification of all specifications - - ✅ High-performance Rust sidecar library - - ✅ Python and Node.js bindings - - ✅ Comprehensive test suite - - ✅ Bit-flip corpus validation - - ✅ Quantization error bounds with 128-vector verification - - ✅ Perfect hash tokenizer implementation + **Formal completeness is not claimed.** See `docs/axioms.md`. + Hash vocab defaults to **CHD MPH** (`kind: "chd_mph"`); legacy open-address JSON still loads. - ### Verification + ### Native libraries (per OS) - To verify this release: + Release assets `guardd-native-.zip` contain the staged `libguardd` / + `guardd.dll` for that runner. These are **GitHub Release binaries**, not + npm/PyPI embedded prebuilts. Locally you can also: - 1. Check the Lean kernel hash in `lean-hash.txt` - 2. Run `lake build && lake exe tests` - 3. Run `cargo test --manifest-path guardd/Cargo.toml` + ```bash + # Prefers these release zips; CI artifacts are fallback for unreleased commits + bash scripts/fetch_guardd_native.sh # needs gh CLI + # or + cargo build --release --manifest-path src/rust/guardd/Cargo.toml + bash scripts/stage_guardd_native.sh + ``` - ### Installation + ### Verification - Extract the bundle and follow the README for integration instructions. + 1. Inspect `docs/axioms.md` / `docs/perfect-hash-tokenizer.md` + 2. `lake build && lake exe tests && python scripts/check_axioms.py` + 3. `cargo test --manifest-path src/rust/guardd/Cargo.toml --locked` - ### SHA-256 + ### Installation - ``` - $(cat bundles/release-info.txt | grep SHA-256 | cut -d' ' -f2) - ``` + Extract the bundle and follow its README. draft: false prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - publish-to-registry: + build-python-package: needs: create-bundle runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') @@ -249,40 +212,24 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Download bundle artifact - uses: actions/download-artifact@v4 - with: - name: model-asset-guard-bundle-${{ github.sha }} - path: ./bundles - - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" - - name: Install Python dependencies - run: | - pip install build twine + - name: Install build tooling + run: python -m pip install --upgrade pip build - - name: Build Python package - run: | - cd bindings/python - python -m build + - name: Build Python package (sdist/wheel; no auto-publish) + run: python -m build - - name: Publish to PyPI - if: github.ref == 'refs/tags/main' - run: | - cd bindings/python - python -m twine upload dist/* --username __token__ --password ${{ secrets.PYPI_TOKEN }} - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + - name: Upload Python dist artifacts + uses: actions/upload-artifact@v4 + with: + name: python-dist-${{ github.sha }} + path: dist/* + + # PyPI upload is intentionally not automated here: packaging is real via + # root pyproject.toml, but publishing requires an explicit human/release step + # with configured trusted publishing or tokens. - - name: Publish to Test PyPI - if: github.ref != 'refs/tags/main' - run: | - cd bindings/python - python -m twine upload --repository testpypi dist/* --username __token__ --password ${{ secrets.TEST_PYPI_TOKEN }} - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22d9fb4..022c542 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,10 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - + os: [ubuntu-latest, macos-latest, windows-latest] + defaults: + run: + shell: bash steps: - name: Checkout code uses: actions/checkout@v4 @@ -55,8 +57,80 @@ jobs: pip install ruff pytest pytest-cov - name: Run canonical preflight + shell: bash run: bash scripts/ci_preflight.sh + - name: Node smoke (skip if koffi/lib unavailable) + shell: bash + working-directory: bindings/nodejs + run: | + set -euo pipefail + ROOT="${GITHUB_WORKSPACE}" + GUARDD_LIB="" + for cand in \ + "$ROOT/bindings/nodejs/native/libguardd.so" \ + "$ROOT/bindings/nodejs/native/libguardd.dylib" \ + "$ROOT/bindings/nodejs/native/guardd.dll" \ + "$ROOT/dist/native/libguardd.so" \ + "$ROOT/dist/native/libguardd.dylib" \ + "$ROOT/dist/native/guardd.dll" \ + "$ROOT/src/rust/guardd/target/debug/libguardd.so" \ + "$ROOT/src/rust/guardd/target/release/libguardd.so" \ + "$ROOT/src/rust/guardd/target/debug/libguardd.dylib" \ + "$ROOT/src/rust/guardd/target/release/libguardd.dylib" \ + "$ROOT/src/rust/guardd/target/debug/guardd.dll" \ + "$ROOT/src/rust/guardd/target/release/guardd.dll"; do + if [ -f "$cand" ]; then + GUARDD_LIB="$cand" + break + fi + done + # Fallback: first matching cdylib under target (ubuntu/mac layout variance). + if [ -z "$GUARDD_LIB" ]; then + GUARDD_LIB="$(find "$ROOT/src/rust/guardd/target" \ + \( -name 'libguardd.so' -o -name 'libguardd.dylib' -o -name 'guardd.dll' \) \ + 2>/dev/null | head -n 1 || true)" + fi + if [ -n "$GUARDD_LIB" ]; then + export GUARDD_LIB + echo "GUARDD_LIB=$GUARDD_LIB" + else + echo "WARN: libguardd not found under target/; Node smoke will skip if unresolved" + ls -la "$ROOT/src/rust/guardd/target/debug" 2>/dev/null || true + ls -la "$ROOT/src/rust/guardd/target/release" 2>/dev/null || true + fi + npm install --no-fund --no-audit + set +e + npm test + code=$? + set -e + if [ "$code" -eq 0 ]; then + echo "Node smoke passed" + elif [ "$code" -eq 2 ]; then + echo "Node smoke skipped (native dep or libguardd unavailable)" + else + echo "Node smoke failed with exit $code" + exit "$code" + fi + + - name: Stage release guardd native library + shell: bash + run: | + set -euo pipefail + cargo build --release --manifest-path src/rust/guardd/Cargo.toml --locked + if [[ "${{ runner.os }}" == "Windows" ]]; then + pwsh -File scripts/stage_guardd_native.ps1 -Profile release + else + bash scripts/stage_guardd_native.sh + fi + + - name: Upload guardd native artifact + uses: actions/upload-artifact@v4 + with: + name: guardd-native-${{ matrix.os }} + path: dist/native/ + if-no-files-found: error + - name: Upload quality artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/formal-verify.yml b/.github/workflows/formal-verify.yml index 0487543..ce7d914 100644 --- a/.github/workflows/formal-verify.yml +++ b/.github/workflows/formal-verify.yml @@ -6,7 +6,6 @@ on: pull_request: branches: [main] schedule: - # Run formal verification daily at 2 AM UTC - cron: "0 2 * * *" jobs: @@ -22,6 +21,11 @@ jobs: - name: Setup Lean 4 uses: leanprover/lean-action@v1 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Cache Lean dependencies uses: actions/cache@v4 with: @@ -30,41 +34,24 @@ jobs: restore-keys: | ${{ runner.os }}-lean- - # Lean warnings fail the build: `lakefile.lean` sets `-DwarningAsError=true` on the package. + # Lean warnings fail the build: `lakefile.lean` sets `-DwarningAsError=true`. - name: Build Lean project run: lake build - - name: Run formal verification + - name: Build Lean CLI targets run: | set -euo pipefail - echo "Running formal verification of Model Asset Guard specifications..." - - # Verify core specifications lake build ModelAssetGuard - - # Verify quantization bounds - lake build quantbound - - # Verify tokenizer specifications - lake build tokenizertest - - # Verify weight integrity specifications - lake build verifyweights - - # Verify bit-flip corpus specifications - lake build bitflipcorpus - - # Verify 128-vector quantization verification - lake build quantverify128 - - # Verify perfect hash tokenizer - lake build perfecthash - - echo "✅ All formal specifications verified successfully" + lake build quantbound tokenizertest verifyweights bitflipcorpus quantverify128 perfecthash benchmarks tests - name: Run Lean tests run: lake exe tests + - name: Axiom inventory gate + run: | + python scripts/check_axioms.py + echo "Formal completeness is NOT claimed; see docs/axioms.md" + - name: Check for sorry statements run: | echo "Checking for incomplete proofs (sorry statements)" @@ -78,45 +65,41 @@ PY ) echo "Found $SORRY_COUNT sorry statements" test "$SORRY_COUNT" -eq 0 - - - name: Verify type safety - run: | - echo "Verifying type safety of all Lean specifications..." - lake build --no-deps - echo "✅ All specifications are type-safe" + echo "SORRY_COUNT=$SORRY_COUNT" >> "$GITHUB_ENV" - name: Generate verification report run: | - echo "Generating formal verification report..." - + AXIOM_COUNT=$(python - <<'PY' +import re +from pathlib import Path +text = Path("docs/axioms.md").read_text(encoding="utf-8") +m = re.search(r"\*\*Count:\*\*\s*(\d+)\s*axioms", text) +print(m.group(1) if m else "?") +PY +) cat > verification_report.md << EOF - # Formal Verification Report +# Formal Verification Report - **Date:** $(date) - **Commit:** ${{ github.sha }} - **Branch:** ${{ github.ref_name }} +**Date:** $(date -u) +**Commit:** ${{ github.sha }} +**Branch:** ${{ github.ref_name }} - ## Verification Results +## What this job checks - ### Core Specifications - - ✅ ModelAssetGuard: Verified - - ✅ QuantBound: Verified - - ✅ TokenizerTest: Verified - - ✅ VerifyWeights: Verified - - ✅ BitFlipCorpus: Verified - - ✅ QuantVerify128: Verified - - ✅ PerfectHash: Verified +- Lean package builds with \`warningAsError\` +- Lean CLI targets build +- \`lake exe tests\` passes +- \`sorry\` count is zero +- Axiom inventory matches \`docs/axioms.md\` (count: ${AXIOM_COUNT}) - ### Type Safety - - ✅ All specifications are type-safe +## What this job does NOT claim - ### Incomplete Proofs - - Found $SORRY_COUNT sorry statements - - ## Summary - All formal specifications have been verified successfully. - EOF +- Formal completeness +- That Float / opaque obligation markers are proven theorems +- That runtime CLIs are proven correct in Lean (they delegate to Rust) +See \`docs/axioms.md\` for the explicit axiom list. +EOF echo "Report generated: verification_report.md" - name: Upload verification report @@ -132,10 +115,9 @@ PY script: | const fs = require('fs'); const report = fs.readFileSync('verification_report.md', 'utf8'); - github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: `## Formal Verification Results ✅\n\n${report}` + body: `## Formal verification job\n\n${report}` }); From fb0ee5c97542994ad4092f8bb156513a37a60af7 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:09:56 -0700 Subject: [PATCH 31/35] Refresh README for packaging honesty and build paths Document fetch/stage natives, formal modules, and repository URLs without claiming published prebuilt packages. --- README.md | 234 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 163 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 9ae5401..647dee9 100644 --- a/README.md +++ b/README.md @@ -18,44 +18,81 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Lean 4](https://img.shields.io/badge/Lean-4%20stable-blue.svg)](https://leanprover.github.io/) [![Rust](https://img.shields.io/badge/Rust-stable-orange.svg)](https://www.rust-lang.org/) -[![Python](https://img.shields.io/badge/Python-3.11+-green.svg)](https://www.python.org/) +[![Python](https://img.shields.io/badge/Python-3.10+-green.svg)](https://www.python.org/) -> **Machine-verified integrity for fixed model artifacts** -> Weights, quantization constraints, and tokenizer behavior validated from formal spec to runtime checks. +> Integrity checks for fixed model artifacts — Lean specs where they exist, Rust runtime where it matters. +> This README describes **what works today**, not aspirational completeness. -Model Asset Guard combines Lean 4 formal specifications with a Rust runtime sidecar and language bindings so teams can verify artifact integrity as part of normal model delivery. +Model Asset Guard combines Lean 4 specifications (partial; Float claims are obligation markers, not axioms) with a Rust runtime sidecar (`guardd`) and language bindings for digest, quantization sampling, and deterministic hash-vocab tokenization. --- +## Honesty status (read this first) + +| Claim area | Reality today | +| --- | --- | +| Weight SHA-256 | **Rust** computes real SHA-256 (`guardd_verify_digest`, `guardd_sha256` CLI). Lean `simpleDigest` is a **placeholder** and is not used for CLI verify. | +| `lake exe verifyweights --digest` | Calls Rust `guardd_sha256`. Exits non-zero if the binary is missing — does **not** fake success. | +| Quantization bounds | **Rust** runs 128-vector sampling checks (`guardd_verify_quant_128_vectors`). Legacy `guardd_verify_quant` is **unimplemented** (always null; fail-closed). Lean has Int half-ULP / L1 theorems; Float claims are obligation markers. | +| Tokenizer / hash vocab | **Rust** CHD MPH (default) + legacy open-address load + FFI. Lean `perfecthash` / `tokenizertest` shell to Rust. SP `.model`: CHAR/BPE/UNIGRAM/WORD + charsmap/`byte_fallback`/denormalizer/PrefixMatcher; ExtraOptions `bos`/`eos`/`reverse`; UNIGRAM n-best/sample/WOR SampleEncodeAndScore; BPE-dropout; silent-`` opt-in ([`docs/sentencepiece-bpe.md`](docs/sentencepiece-bpe.md)). Lean `bpe_encode` / `sp_encode` in `Tokenizer.lean` remain **char-level stubs** — not the Rust ModelProto path. | +| Formal completeness | **Not** claimed. **0** Lean `axiom`s; Float bridges are obligation markers — see below. | +| Safe Rust API | Thin safe wrappers exist (`verify_digest`, `checked_load`, …) in addition to the C ABI. | +| Node `checkedLoad` | Reports `valid` from digest comparison; does not hardcode `true`. | +| Native packaging | Stage locally via `scripts/stage_guardd_native.sh`, or `fetch_guardd_native` (**release zip first**, CI fallback). **No** fake npm/PyPI prebuilt publish — see [`docs/native-packaging.md`](docs/native-packaging.md). | +| CI pytest | Offline e2e tests **must pass**. `@pytest.mark.integration` (HF/network/Node/Lean) is excluded from default preflight. | + +### Lean axioms vs runtime + +Canonical inventory: [`docs/axioms.md`](docs/axioms.md) (CI-gated by `scripts/check_axioms.py`). + +Current axiom count: **0**. Former Float half-ULP axiom is now +`round_int8_error_bound_float_obligation` (marker, not a proof). + +Proven (not axioms): fixed-point half-ULP (`Fixed.roundDivInt_error`), +entrywise / row-wise Int L1 bounds (`layer_row_l1_bound`, `layer_rows_l1_bound`), +faithful-vocab / single-char nonempty tokenizer lemmas, BPE merge length / +identity / composition lemmas, hash-vocab injectivity under unique keys. + +Honest non-axioms: `per_channel_error_bound` (trivial theorem), +`verify_128_vectors_sound` / `layer_verification_sound` / +`layer_error_bound_float_l2_obligation` / +`round_int8_error_bound_float_obligation` (`Prop := True` markers; L2 runtime is Rust). +Float bridge (Wave 28 aborted): [`docs/float-obligation.md`](docs/float-obligation.md). +SP/BPE support matrix: [`docs/sentencepiece-bpe.md`](docs/sentencepiece-bpe.md). + +Runtime CLIs shell out to Rust `guardd_*` helpers; missing binaries fail closed. +Formal completeness is **not** claimed. + ## Why Model Asset Guard -ML artifact failures are often silent: corrupted checkpoints, drifted quantization assumptions, and tokenizer mismatches can pass standard unit tests and still break production behavior. -This project closes that gap with a verification-first pipeline: +ML artifact failures are often silent: corrupted checkpoints, drifted quantization assumptions, and tokenizer mismatches can pass standard unit tests and still break production behavior. This repo provides: -- **Formal guarantees** for key invariants in Lean 4 -- **Runtime enforcement** in Rust for fast, practical checks -- **Workflow integration** through CI gates and Python/Node entry points +- Lean 4 specifications and proof obligations (no `axiom`s; Float markers remain unproven) +- Rust runtime enforcement for fast SHA-256 and quantization sampling +- Python / Node bindings over the C ABI +- CI gates that fail on real test failures (no `|| true` on pytest) ## What You Get -| Capability | What it verifies | Implementation | +| Capability | What it verifies | Implementation status | | --- | --- | --- | -| Weight integrity | SHA-256 consistency and checkpoint validity | Lean + Rust | -| Quantization bounds | Layer error bounds for quantized weights | Lean | -| Tokenizer determinism | Stable encode/decode behavior | Lean + tests | -| Runtime guardrails | Fast validation in production paths | Rust FFI | -| Integration surface | Python and Node usage paths | Bindings | +| Weight integrity | SHA-256 of file bytes | Rust (real). Lean CLI delegates to Rust. | +| Quantization bounds | Layer error stats over 128 random activations | Rust runtime. Lean Int theorems + Float markers. | +| Perfect-hash vocab | CHD minimal perfect hash (default); legacy open-address still loads | Rust `guardd_perfect_hash` / FFI. Lean CLI delegates. | +| Bit-flip corpus | Corruption detection (default ≤64 MiB) | Rust `guardd_bitflip` (large sizes need opt-in) | +| HuggingFace helpers | Local verify-before-load; remote fail-closed | Python (network tests marked `integration`) | ## At a Glance | Area | Current policy | | --- | --- | -| Lean compiler warnings | **Fail build** via `warningAsError` in `lakefile.lean` | -| Formal completeness | CI fails on `sorry` in `src/lean` | +| Lean compiler warnings | Fail build via `warningAsError` in `lakefile.lean` | +| `sorry` in `src/lean` | Enforced separately where CI checks for it | | Rust quality gate | `cargo clippy ... -D warnings` | | Canonical local gate | `bash scripts/ci_preflight.sh` | +| Default pytest | Offline only (`-m 'not integration'`) | ## Repository Map @@ -63,15 +100,15 @@ This project closes that gap with a verification-first pipeline: model-asset-guard/ ├── src/ │ ├── lean/ -│ │ ├── ModelAssetGuard/ # Core specs -│ │ └── cli/ # Lean executables +│ │ ├── ModelAssetGuard/ # Specs (0 axioms; Float markers) +│ │ └── cli/ # Lean executables (some stubs) │ └── rust/ -│ └── guardd/ # Runtime sidecar (FFI boundary) +│ └── guardd/ # Runtime sidecar (C ABI + safe Rust API) ├── bindings/ │ ├── python/ │ └── nodejs/ ├── docs/ -├── tests/ +├── tests/e2e/ # Offline + integration pytest ├── scripts/ ├── lakefile.lean └── lean-toolchain @@ -83,77 +120,84 @@ model-asset-guard/ - Lean 4 (`leanprover/lean4:stable`, managed with [`elan`](https://github.com/leanprover/elan)) - Rust stable + Cargo -- Python 3.11+ -- Node.js 20+ +- Python 3.10+ (`requires-python` in `pyproject.toml`) +- Node.js 20+ (optional; Node tests are `integration`) ### 2) Build and verify ```bash -git clone https://github.com/fraware/model-asset-guard.git +git clone https://github.com/SentinelOps-CI/model-asset-guard.git cd model-asset-guard # Lean (warnings are errors) lake build lake exe tests -# Rust runtime +# Rust runtime (library + tools) cargo build --release --manifest-path src/rust/guardd/Cargo.toml cargo test --manifest-path src/rust/guardd/Cargo.toml --locked -# Optional: run full local CI gate +# Offline Python e2e (requires built libguardd) +export PYTHONPATH="$(pwd)/bindings/python${PYTHONPATH:+:$PYTHONPATH}" +pytest tests/e2e --maxfail=1 + +# Optional: full local CI gate (Lean + Rust + offline pytest) bash scripts/ci_preflight.sh ``` ### 3) Python path setup (local development) -The Python bindings are source-based in this repo; add them to `PYTHONPATH`: - ```bash export PYTHONPATH="$(pwd)/bindings/python${PYTHONPATH:+:$PYTHONPATH}" ``` ## CLI Usage -### Verify checkpoint integrity +### Verify checkpoint integrity (SHA-256 via Rust) ```bash +# Build the helper used by the Lean CLI +cargo build --release --manifest-path src/rust/guardd/Cargo.toml --bin guardd_sha256 + +# Print digest lake exe verifyweights /path/to/checkpoint.bin -lake exe verifyweights /path/to/checkpoint.bin --digest -``` -### Check quantization bounds +# Compare digest (hex) +lake exe verifyweights /path/to/checkpoint.bin --digest -```bash -lake exe quantbound /path/to/model.onnx --output bounds.json -lake exe quantbound /path/to/model.onnx --verify --tolerance 1e-6 +# Or call Rust directly +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin guardd_sha256 -- /path/to/checkpoint.bin ``` -### Test tokenizer behavior +### Quantization / tokenizer Lean CLIs + +`quantbound`, `quantverify128`, `bitflipcorpus`, `tokenizertest`, and `perfecthash` +delegate to Rust `guardd_*` helpers (fail closed if binaries are missing). +`lake exe benchmarks` is an honest smoke stub (no fake MB/s). + +Working perfect-hash path: ```bash -lake exe tokenizertest /path/to/tokenizer.json --type bpe -lake exe tokenizertest /path/to/tokenizer.json --fuzz 1000000 -lake exe perfecthash --help +cargo run --manifest-path src/rust/guardd/Cargo.toml --bin gen_perfect_hash -- vocab.txt out.json +# then encode/decode via Python ModelAssetGuard.perfect_hash_* or C ABI ``` +`lake exe perfecthash ...` shells to Rust `guardd_perfect_hash` (fail closed if missing). + ## Language Integrations ### Python ```python -from bindings.python.pytorch_guard import ModelAssetGuard, HuggingFaceGuard +from pytorch_guard import ModelAssetGuard -guard = ModelAssetGuard() -ok = guard.verify_digest("model.bin", expected_digest) +guard = ModelAssetGuard() # loads libguardd / guardd.dll +ok = guard.verify_digest("model.bin", expected_digest) # False on mismatch (no raise) model_info = guard.checked_load("model.bin", expected_digest) - -hf_guard = HuggingFaceGuard(guard) -model, tokenizer, info = hf_guard.checked_load_pretrained( - "gpt2", - verify_quantization=True, -) ``` +HuggingFace helpers require `transformers` / `torch` and are covered by `@pytest.mark.integration` tests. + ### Node.js ```javascript @@ -162,10 +206,25 @@ const { ModelAssetGuard } = require("./bindings/nodejs/node_guard.js"); const guard = new ModelAssetGuard(); const ok = guard.verifyDigest("model.bin", expectedDigest); const modelInfo = guard.checkedLoad("model.bin", expectedDigest); +// modelInfo.valid is true only when expectedDigest matches ``` +Requires `koffi` (`cd bindings/nodejs && npm install`) and a built `libguardd` +(or `GUARDD_LIB` pointing at it). Preferred local path: + +```bash +cargo build --release --manifest-path src/rust/guardd/Cargo.toml +bash scripts/stage_guardd_native.sh +cd bindings/nodejs && npm install && npm test +``` + +See `bindings/nodejs/README.md` and [`docs/native-packaging.md`](docs/native-packaging.md) +(CI uploads per-OS artifacts; npm does **not** ship prebuilts). + ### Rust +Public surface is primarily a **C ABI** (`extern "C" fn guardd_*`). Thin safe wrappers are also available when linking the `rlib`: + ```rust use guardd::{checked_load, verify_digest}; @@ -173,44 +232,77 @@ let ok = verify_digest("model.bin", &expected_digest)?; let info = checked_load("model.bin", Some(&expected_digest))?; ``` -## API Surface (Core) +C ABI equivalents: `guardd_verify_digest`, `guardd_checked_load`, `guardd_verify_quant_128_vectors`, `guardd_verify_model_128_vectors`, `guardd_perfect_hash_encode`, `guardd_perfect_hash_decode`, `guardd_sp_*`, bitflip helpers. -```rust -verify_digest(path: &str, expected_digest: &[u8]) -> Result -checked_load(path: &str, expected_digest: Option<&[u8]>) -> Result -verify_quantization_128_vectors( - layer_name: &str, - weights: &[f32], - fan_in: u32, - fan_out: u32, - quant_type: &str -) -> Result +Safe Rust also exposes `perfect_hash_encode` / `sp_encode` / `run_bitflip_corpus_test` (platform temp dir by default — not a hardcoded `/tmp`). + +`guardd_verify_quant` is **deprecated and unimplemented**: it always returns null +(fail-closed; null means unavailable, not success). Use +`guardd_verify_quant_128_vectors` / `guardd_verify_model_128_vectors`. + +### Model quantization JSON contract + +Python and Rust agree on object-form layers: + +```json +[{ + "name": "linear.weight", + "weights": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "fan_in": 3, + "fan_out": 2, + "quant_type": "int8" +}] ``` -`GuarddError` covers I/O and validation failures (`FileNotFound`, `InvalidDigest`, `QuantizationError`, `IoError`). +### Perfect-hash vocab JSON contract + +Default builds emit a CHD minimal perfect hash: + +```json +{ + "vocab": [["hello", 1], ["world", 2]], + "kind": "chd_mph", + "pilots": [0], + "n_buckets": 1, + "indices": [0, 1], + "seed0": 11400714819323198485, + "seed1": 13787848793156543929 +} +``` + +Produced by `gen_perfect_hash`; consumed by FFI encode/decode. Legacy +`hash_table` / `table_size` open-address JSON still loads. See +[`docs/perfect-hash-tokenizer.md`](docs/perfect-hash-tokenizer.md). ## Quality and CI Main workflows in `.github/workflows/`: -- `ci.yml`: cross-platform quality checks, security checks, and performance bench step +- `ci.yml`: runs `scripts/ci_preflight.sh` (Lean + Rust + offline pytest) and + uploads `guardd-native-` release library artifacts - `formal-verify.yml`: Lean build + formal targets + `sorry` enforcement - `bundle-push.yml`: release bundle and SBOM flow -Lean warnings are enforced as errors by project configuration in `lakefile.lean`, so formal and standard builds share the same warning-clean requirement. +Pytest failures fail the gate. Integration tests (HF downloads, Node FFI, Lean stub checks) are opt-in: + +```bash +pytest tests/e2e -m integration +``` ## Documentation - [Perfect hash tokenizer](docs/perfect-hash-tokenizer.md) +- [SentencePiece / BPE support matrix](docs/sentencepiece-bpe.md) +- [Native packaging](docs/native-packaging.md) +- [Lean axiom inventory](docs/axioms.md) +- [Float obligations](docs/float-obligation.md) ## Contributing -Until a dedicated contribution guide is added, use `scripts/ci_preflight.sh` and existing CI workflows as the acceptance baseline. - -Typical flow: +Use `scripts/ci_preflight.sh` and existing CI workflows as the acceptance baseline. 1. Fork and branch from `main` -2. Implement changes with tests +2. Implement changes with tests that assert real behavior 3. Run local gates 4. Open a PR with a clear test plan @@ -222,15 +314,15 @@ MIT. See [LICENSE](LICENSE). ```bibtex @software{model_asset_guard, - title={Model Asset Guard: Machine-verified integrity for ML artifacts}, + title={Model Asset Guard: integrity checks for ML artifacts}, author={Model Asset Guard Contributors}, year={2026}, - url={https://github.com/fraware/model-asset-guard} + url={https://github.com/SentinelOps-CI/model-asset-guard} } ``` ## Support -- Issues: [GitHub Issues](https://github.com/fraware/model-asset-guard/issues) -- Discussions: [GitHub Discussions](https://github.com/fraware/model-asset-guard/discussions) -- Security reports: [GitHub Security Advisories](https://github.com/fraware/model-asset-guard/security/advisories) +- Issues: [GitHub Issues](https://github.com/SentinelOps-CI/model-asset-guard/issues) +- Discussions: [GitHub Discussions](https://github.com/SentinelOps-CI/model-asset-guard/discussions) +- Security reports: [GitHub Security Advisories](https://github.com/SentinelOps-CI/model-asset-guard/security/advisories) From 83016299574fe39ce8bbe5b334a5be54eada5ac7 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:10:08 -0700 Subject: [PATCH 32/35] Add SentencePiece NFKC and tiny model fixtures Expand curated normalize goldens and miniature models for offline encode/normalize cross-checks. --- tests/fixtures/sp_nfkc_normalize_goldens.json | 2663 +++++++++++++++++ .../sp_tiny_custom_norm_unigram.model | Bin 0 -> 1421 bytes .../sp_tiny_identity_bpe_bytefallback.model | Bin 0 -> 5320 bytes .../sp_tiny_identity_denorm_unigram.model | Bin 0 -> 1437 bytes tests/fixtures/sp_tiny_identity_unigram.model | Bin 0 -> 321 bytes 5 files changed, 2663 insertions(+) create mode 100644 tests/fixtures/sp_nfkc_normalize_goldens.json create mode 100644 tests/fixtures/sp_tiny_custom_norm_unigram.model create mode 100644 tests/fixtures/sp_tiny_identity_bpe_bytefallback.model create mode 100644 tests/fixtures/sp_tiny_identity_denorm_unigram.model create mode 100644 tests/fixtures/sp_tiny_identity_unigram.model diff --git a/tests/fixtures/sp_nfkc_normalize_goldens.json b/tests/fixtures/sp_nfkc_normalize_goldens.json new file mode 100644 index 0000000..8bc0efd --- /dev/null +++ b/tests/fixtures/sp_nfkc_normalize_goldens.json @@ -0,0 +1,2663 @@ +{ + "rule": { + "identity": [ + { + "in": "hello", + "out": "hello" + }, + { + "in": " foo bar ", + "out": " foo bar " + }, + { + "in": "Ａ", + "out": "Ａ" + }, + { + "in": "ａ", + "out": "ａ" + }, + { + "in": "™", + "out": "™" + }, + { + "in": "café", + "out": "café" + }, + { + "in": "Café", + "out": "Café" + }, + { + "in": "fi", + "out": "fi" + }, + { + "in": " ", + "out": " " + }, + { + "in": "ＡＢＣ", + "out": "ＡＢＣ" + }, + { + "in": "１２３", + "out": "１２３" + }, + { + "in": "Ⅰ", + "out": "Ⅰ" + }, + { + "in": "file", + "out": "file" + }, + { + "in": "Straße", + "out": "Straße" + }, + { + "in": "½", + "out": "½" + }, + { + "in": "①", + "out": "①" + }, + { + "in": "Ⅷ", + "out": "Ⅷ" + }, + { + "in": "№", + "out": "№" + }, + { + "in": "℃", + "out": "℃" + }, + { + "in": "ffi", + "out": "ffi" + }, + { + "in": "①②③", + "out": "①②③" + }, + { + "in": "ｰ", + "out": "ｰ" + }, + { + "in": "　", + "out": "　" + }, + { + "in": "ij", + "out": "ij" + }, + { + "in": "Ǻ", + "out": "Ǻ" + }, + { + "in": "K", + "out": "K" + }, + { + "in": "Å", + "out": "Å" + }, + { + "in": "Ω", + "out": "Ω" + }, + { + "in": "ℌ", + "out": "ℌ" + }, + { + "in": "²", + "out": "²" + }, + { + "in": "³", + "out": "³" + }, + { + "in": "¼", + "out": "¼" + }, + { + "in": "¾", + "out": "¾" + }, + { + "in": "ｶ", + "out": "ｶ" + }, + { + "in": "⑩", + "out": "⑩" + }, + { + "in": "㈱", + "out": "㈱" + }, + { + "in": "fl", + "out": "fl" + }, + { + "in": "ffl", + "out": "ffl" + }, + { + "in": "Á", + "out": "Á" + }, + { + "in": "É", + "out": "É" + }, + { + "in": "Ñ", + "out": "Ñ" + }, + { + "in": "０", + "out": "０" + }, + { + "in": "ß", + "out": "ß" + }, + { + "in": "ẞ", + "out": "ẞ" + }, + { + "in": "İ", + "out": "İ" + }, + { + "in": "I", + "out": "I" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "ø", + "out": "ø" + }, + { + "in": "Ø", + "out": "Ø" + }, + { + "in": "œ", + "out": "œ" + }, + { + "in": "Œ", + "out": "Œ" + }, + { + "in": "æ", + "out": "æ" + }, + { + "in": "Æ", + "out": "Æ" + }, + { + "in": "µ", + "out": "µ" + }, + { + "in": "μ", + "out": "μ" + }, + { + "in": "℡", + "out": "℡" + }, + { + "in": "℠", + "out": "℠" + }, + { + "in": "㍉", + "out": "㍉" + }, + { + "in": "㌔", + "out": "㌔" + }, + { + "in": "ｱ", + "out": "ｱ" + }, + { + "in": "ア", + "out": "ア" + }, + { + "in": "㏄", + "out": "㏄" + }, + { + "in": "㎞", + "out": "㎞" + }, + { + "in": "ⅹ", + "out": "ⅹ" + }, + { + "in": "⑴", + "out": "⑴" + }, + { + "in": "⒜", + "out": "⒜" + }, + { + "in": "｡", + "out": "｡" + }, + { + "in": "｢", + "out": "｢" + }, + { + "in": "｣", + "out": "｣" + }, + { + "in": "ﾞ", + "out": "ﾞ" + }, + { + "in": "ﾟ", + "out": "ﾟ" + }, + { + "in": "₩", + "out": "₩" + }, + { + "in": "€", + "out": "€" + }, + { + "in": "℉", + "out": "℉" + }, + { + "in": "㏇", + "out": "㏇" + }, + { + "in": "⅕", + "out": "⅕" + }, + { + "in": "⅖", + "out": "⅖" + }, + { + "in": "⅗", + "out": "⅗" + }, + { + "in": "LJ", + "out": "LJ" + }, + { + "in": "nj", + "out": "nj" + }, + { + "in": "DZ", + "out": "DZ" + }, + { + "in": "Dz", + "out": "Dz" + }, + { + "in": "dz", + "out": "dz" + }, + { + "in": "ǰ", + "out": "ǰ" + }, + { + "in": "ȟ", + "out": "ȟ" + }, + { + "in": "ℏ", + "out": "ℏ" + }, + { + "in": "ℓ", + "out": "ℓ" + }, + { + "in": "℣", + "out": "℣" + }, + { + "in": "ℤ", + "out": "ℤ" + }, + { + "in": "ℰ", + "out": "ℰ" + }, + { + "in": "ℱ", + "out": "ℱ" + }, + { + "in": "ℳ", + "out": "ℳ" + }, + { + "in": "ℹ", + "out": "ℹ" + }, + { + "in": "ㄱ", + "out": "ㄱ" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "㊀", + "out": "㊀" + }, + { + "in": "㊥", + "out": "㊥" + }, + { + "in": "­", + "out": "­" + }, + { + "in": "​", + "out": "​" + }, + { + "in": "！", + "out": "！" + }, + { + "in": "？", + "out": "？" + }, + { + "in": "．", + "out": "．" + }, + { + "in": "，", + "out": "，" + }, + { + "in": "：", + "out": "：" + }, + { + "in": "；", + "out": "；" + }, + { + "in": "（", + "out": "（" + }, + { + "in": "）", + "out": "）" + }, + { + "in": "［", + "out": "［" + }, + { + "in": "］", + "out": "］" + }, + { + "in": "｛", + "out": "｛" + }, + { + "in": "｝", + "out": "｝" + }, + { + "in": "￠", + "out": "￠" + }, + { + "in": "￡", + "out": "￡" + }, + { + "in": "￥", + "out": "￥" + }, + { + "in": "ｶﾀｶﾅ", + "out": "ｶﾀｶﾅ" + }, + { + "in": "ﾊﾟ", + "out": "ﾊﾟ" + }, + { + "in": "ｳﾞ", + "out": "ｳﾞ" + }, + { + "in": "㊰", + "out": "㊰" + }, + { + "in": "㊠", + "out": "㊠" + }, + { + "in": "⑪", + "out": "⑪" + }, + { + "in": "⑫", + "out": "⑫" + }, + { + "in": "⑵", + "out": "⑵" + }, + { + "in": "Ⓐ", + "out": "Ⓐ" + }, + { + "in": "ⓐ", + "out": "ⓐ" + }, + { + "in": "㉑", + "out": "㉑" + }, + { + "in": "㉒", + "out": "㉒" + }, + { + "in": "⅓", + "out": "⅓" + }, + { + "in": "⅔", + "out": "⅔" + }, + { + "in": "⅛", + "out": "⅛" + }, + { + "in": "⅜", + "out": "⅜" + }, + { + "in": "⅝", + "out": "⅝" + }, + { + "in": "⅞", + "out": "⅞" + }, + { + "in": "¹", + "out": "¹" + }, + { + "in": "⁰", + "out": "⁰" + }, + { + "in": "⁴", + "out": "⁴" + }, + { + "in": "₀", + "out": "₀" + }, + { + "in": "₁", + "out": "₁" + }, + { + "in": "₂", + "out": "₂" + }, + { + "in": "ff", + "out": "ff" + }, + { + "in": "st", + "out": "st" + }, + { + "in": "¢", + "out": "¢" + }, + { + "in": "£", + "out": "£" + }, + { + "in": "¥", + "out": "¥" + }, + { + "in": "©", + "out": "©" + }, + { + "in": "®", + "out": "®" + }, + { + "in": "naive", + "out": "naive" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "á", + "out": "á" + }, + { + "in": "í", + "out": "í" + }, + { + "in": "ó", + "out": "ó" + }, + { + "in": "ú", + "out": "ú" + }, + { + "in": "Á", + "out": "Á" + }, + { + "in": "É", + "out": "É" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "SS", + "out": "SS" + }, + { + "in": "ss", + "out": "ss" + }, + { + "in": "Å", + "out": "Å" + }, + { + "in": "å", + "out": "å" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "Ά", + "out": "Ά" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "ㅏ", + "out": "ㅏ" + }, + { + "in": "각", + "out": "각" + }, + { + "in": "﨎", + "out": "﨎" + }, + { + "in": "塚", + "out": "塚" + }, + { + "in": "﨑", + "out": "﨑" + }, + { + "in": "⼀", + "out": "⼀" + }, + { + "in": "⼁", + "out": "⼁" + }, + { + "in": "㌀", + "out": "㌀" + }, + { + "in": "㌃", + "out": "㌃" + }, + { + "in": "㌶", + "out": "㌶" + }, + { + "in": "㍑", + "out": "㍑" + }, + { + "in": "㍱", + "out": "㍱" + }, + { + "in": "㍴", + "out": "㍴" + }, + { + "in": "㎀", + "out": "㎀" + }, + { + "in": "㎏", + "out": "㎏" + }, + { + "in": "㎜", + "out": "㎜" + }, + { + "in": "㎡", + "out": "㎡" + }, + { + "in": "㏑", + "out": "㏑" + }, + { + "in": "㏿", + "out": "㏿" + }, + { + "in": "ＡＢＣ file ½", + "out": "ＡＢＣ file ½" + }, + { + "in": "Café ™", + "out": "Café ™" + }, + { + "in": "１２３²", + "out": "１２３²" + }, + { + "in": "Ⅰ+Ⅱ=Ⅲ", + "out": "Ⅰ+Ⅱ=Ⅲ" + }, + { + "in": "file™", + "out": "file™" + }, + { + "in": "¼ + ½ = ¾", + "out": "¼ + ½ = ¾" + }, + { + "in": "a b", + "out": "a b" + }, + { + "in": "a​b", + "out": "a​b" + }, + { + "in": "a­b", + "out": "a­b" + }, + { + "in": " 　 ", + "out": " 　 " + }, + { + "in": "…", + "out": "…" + }, + { + "in": "–", + "out": "–" + }, + { + "in": "—", + "out": "—" + }, + { + "in": "‘", + "out": "‘" + }, + { + "in": "’", + "out": "’" + }, + { + "in": "“", + "out": "“" + }, + { + "in": "”", + "out": "”" + }, + { + "in": "«", + "out": "«" + }, + { + "in": "»", + "out": "»" + }, + { + "in": "℅", + "out": "℅" + }, + { + "in": "︐", + "out": "︐" + }, + { + "in": "︰", + "out": "︰" + }, + { + "in": "︱", + "out": "︱" + }, + { + "in": "Ａａ", + "out": "Ａａ" + }, + { + "in": "０９", + "out": "０９" + }, + { + "in": "㈠", + "out": "㈠" + }, + { + "in": "㈯", + "out": "㈯" + }, + { + "in": "ↀ", + "out": "ↀ" + }, + { + "in": "Ↄ", + "out": "Ↄ" + }, + { + "in": "́", + "out": "́" + }, + { + "in": "ㅤ", + "out": "ㅤ" + }, + { + "in": "㎡㎡", + "out": "㎡㎡" + }, + { + "in": "xfiyflz", + "out": "xfiyflz" + } + ], + "nfkc": [ + { + "in": "hello", + "out": "hello" + }, + { + "in": " foo bar ", + "out": " foo bar " + }, + { + "in": "Ａ", + "out": "A" + }, + { + "in": "ａ", + "out": "a" + }, + { + "in": "™", + "out": "TM" + }, + { + "in": "café", + "out": "café" + }, + { + "in": "Café", + "out": "Café" + }, + { + "in": "fi", + "out": "fi" + }, + { + "in": " ", + "out": " " + }, + { + "in": "ＡＢＣ", + "out": "ABC" + }, + { + "in": "１２３", + "out": "123" + }, + { + "in": "Ⅰ", + "out": "I" + }, + { + "in": "file", + "out": "file" + }, + { + "in": "Straße", + "out": "Straße" + }, + { + "in": "½", + "out": "1⁄2" + }, + { + "in": "①", + "out": "1" + }, + { + "in": "Ⅷ", + "out": "VIII" + }, + { + "in": "№", + "out": "No" + }, + { + "in": "℃", + "out": "°C" + }, + { + "in": "ffi", + "out": "ffi" + }, + { + "in": "①②③", + "out": "123" + }, + { + "in": "ｰ", + "out": "ー" + }, + { + "in": "　", + "out": " " + }, + { + "in": "ij", + "out": "ij" + }, + { + "in": "Ǻ", + "out": "Ǻ" + }, + { + "in": "K", + "out": "K" + }, + { + "in": "Å", + "out": "Å" + }, + { + "in": "Ω", + "out": "Ω" + }, + { + "in": "ℌ", + "out": "H" + }, + { + "in": "²", + "out": "2" + }, + { + "in": "³", + "out": "3" + }, + { + "in": "¼", + "out": "1⁄4" + }, + { + "in": "¾", + "out": "3⁄4" + }, + { + "in": "ｶ", + "out": "カ" + }, + { + "in": "⑩", + "out": "10" + }, + { + "in": "㈱", + "out": "(株)" + }, + { + "in": "fl", + "out": "fl" + }, + { + "in": "ffl", + "out": "ffl" + }, + { + "in": "Á", + "out": "Á" + }, + { + "in": "É", + "out": "É" + }, + { + "in": "Ñ", + "out": "Ñ" + }, + { + "in": "０", + "out": "0" + }, + { + "in": "ß", + "out": "ß" + }, + { + "in": "ẞ", + "out": "ẞ" + }, + { + "in": "İ", + "out": "İ" + }, + { + "in": "I", + "out": "I" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "ø", + "out": "ø" + }, + { + "in": "Ø", + "out": "Ø" + }, + { + "in": "œ", + "out": "œ" + }, + { + "in": "Œ", + "out": "Œ" + }, + { + "in": "æ", + "out": "æ" + }, + { + "in": "Æ", + "out": "Æ" + }, + { + "in": "µ", + "out": "μ" + }, + { + "in": "μ", + "out": "μ" + }, + { + "in": "℡", + "out": "TEL" + }, + { + "in": "℠", + "out": "SM" + }, + { + "in": "㍉", + "out": "ミリ" + }, + { + "in": "㌔", + "out": "キロ" + }, + { + "in": "ｱ", + "out": "ア" + }, + { + "in": "ア", + "out": "ア" + }, + { + "in": "㏄", + "out": "cc" + }, + { + "in": "㎞", + "out": "km" + }, + { + "in": "ⅹ", + "out": "x" + }, + { + "in": "⑴", + "out": "(1)" + }, + { + "in": "⒜", + "out": "(a)" + }, + { + "in": "｡", + "out": "。" + }, + { + "in": "｢", + "out": "「" + }, + { + "in": "｣", + "out": "」" + }, + { + "in": "ﾞ", + "out": "゙" + }, + { + "in": "ﾟ", + "out": "゚" + }, + { + "in": "₩", + "out": "₩" + }, + { + "in": "€", + "out": "€" + }, + { + "in": "℉", + "out": "°F" + }, + { + "in": "㏇", + "out": "Co." + }, + { + "in": "⅕", + "out": "1⁄5" + }, + { + "in": "⅖", + "out": "2⁄5" + }, + { + "in": "⅗", + "out": "3⁄5" + }, + { + "in": "LJ", + "out": "LJ" + }, + { + "in": "nj", + "out": "nj" + }, + { + "in": "DZ", + "out": "DZ" + }, + { + "in": "Dz", + "out": "Dz" + }, + { + "in": "dz", + "out": "dz" + }, + { + "in": "ǰ", + "out": "ǰ" + }, + { + "in": "ȟ", + "out": "ȟ" + }, + { + "in": "ℏ", + "out": "ħ" + }, + { + "in": "ℓ", + "out": "l" + }, + { + "in": "℣", + "out": "℣" + }, + { + "in": "ℤ", + "out": "Z" + }, + { + "in": "ℰ", + "out": "E" + }, + { + "in": "ℱ", + "out": "F" + }, + { + "in": "ℳ", + "out": "M" + }, + { + "in": "ℹ", + "out": "i" + }, + { + "in": "ㄱ", + "out": "ᄀ" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "㊀", + "out": "一" + }, + { + "in": "㊥", + "out": "中" + }, + { + "in": "­", + "out": "­" + }, + { + "in": "​", + "out": "​" + }, + { + "in": "！", + "out": "!" + }, + { + "in": "？", + "out": "?" + }, + { + "in": "．", + "out": "." + }, + { + "in": "，", + "out": "," + }, + { + "in": "：", + "out": ":" + }, + { + "in": "；", + "out": ";" + }, + { + "in": "（", + "out": "(" + }, + { + "in": "）", + "out": ")" + }, + { + "in": "［", + "out": "[" + }, + { + "in": "］", + "out": "]" + }, + { + "in": "｛", + "out": "{" + }, + { + "in": "｝", + "out": "}" + }, + { + "in": "￠", + "out": "¢" + }, + { + "in": "￡", + "out": "£" + }, + { + "in": "￥", + "out": "¥" + }, + { + "in": "ｶﾀｶﾅ", + "out": "カタカナ" + }, + { + "in": "ﾊﾟ", + "out": "パ" + }, + { + "in": "ｳﾞ", + "out": "ヴ" + }, + { + "in": "㊰", + "out": "夜" + }, + { + "in": "㊠", + "out": "項" + }, + { + "in": "⑪", + "out": "11" + }, + { + "in": "⑫", + "out": "12" + }, + { + "in": "⑵", + "out": "(2)" + }, + { + "in": "Ⓐ", + "out": "A" + }, + { + "in": "ⓐ", + "out": "a" + }, + { + "in": "㉑", + "out": "21" + }, + { + "in": "㉒", + "out": "22" + }, + { + "in": "⅓", + "out": "1⁄3" + }, + { + "in": "⅔", + "out": "2⁄3" + }, + { + "in": "⅛", + "out": "1⁄8" + }, + { + "in": "⅜", + "out": "3⁄8" + }, + { + "in": "⅝", + "out": "5⁄8" + }, + { + "in": "⅞", + "out": "7⁄8" + }, + { + "in": "¹", + "out": "1" + }, + { + "in": "⁰", + "out": "0" + }, + { + "in": "⁴", + "out": "4" + }, + { + "in": "₀", + "out": "0" + }, + { + "in": "₁", + "out": "1" + }, + { + "in": "₂", + "out": "2" + }, + { + "in": "ff", + "out": "ff" + }, + { + "in": "st", + "out": "st" + }, + { + "in": "¢", + "out": "¢" + }, + { + "in": "£", + "out": "£" + }, + { + "in": "¥", + "out": "¥" + }, + { + "in": "©", + "out": "©" + }, + { + "in": "®", + "out": "®" + }, + { + "in": "naive", + "out": "naive" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "á", + "out": "á" + }, + { + "in": "í", + "out": "í" + }, + { + "in": "ó", + "out": "ó" + }, + { + "in": "ú", + "out": "ú" + }, + { + "in": "Á", + "out": "Á" + }, + { + "in": "É", + "out": "É" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "SS", + "out": "SS" + }, + { + "in": "ss", + "out": "ss" + }, + { + "in": "Å", + "out": "Å" + }, + { + "in": "å", + "out": "å" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "Ά", + "out": "Ά" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "ㅏ", + "out": "ᅡ" + }, + { + "in": "각", + "out": "각" + }, + { + "in": "﨎", + "out": "﨎" + }, + { + "in": "塚", + "out": "塚" + }, + { + "in": "﨑", + "out": "﨑" + }, + { + "in": "⼀", + "out": "一" + }, + { + "in": "⼁", + "out": "丨" + }, + { + "in": "㌀", + "out": "アパート" + }, + { + "in": "㌃", + "out": "アール" + }, + { + "in": "㌶", + "out": "ヘクタール" + }, + { + "in": "㍑", + "out": "リットル" + }, + { + "in": "㍱", + "out": "hPa" + }, + { + "in": "㍴", + "out": "bar" + }, + { + "in": "㎀", + "out": "pA" + }, + { + "in": "㎏", + "out": "kg" + }, + { + "in": "㎜", + "out": "mm" + }, + { + "in": "㎡", + "out": "m2" + }, + { + "in": "㏑", + "out": "ln" + }, + { + "in": "㏿", + "out": "gal" + }, + { + "in": "ＡＢＣ file ½", + "out": "ABC file 1⁄2" + }, + { + "in": "Café ™", + "out": "Café TM" + }, + { + "in": "１２３²", + "out": "1232" + }, + { + "in": "Ⅰ+Ⅱ=Ⅲ", + "out": "I+II=III" + }, + { + "in": "file™", + "out": "fileTM" + }, + { + "in": "¼ + ½ = ¾", + "out": "1⁄4 + 1⁄2 = 3⁄4" + }, + { + "in": "a b", + "out": "a b" + }, + { + "in": "a​b", + "out": "a​b" + }, + { + "in": "a­b", + "out": "a­b" + }, + { + "in": " 　 ", + "out": " " + }, + { + "in": "…", + "out": "..." + }, + { + "in": "–", + "out": "–" + }, + { + "in": "—", + "out": "—" + }, + { + "in": "‘", + "out": "‘" + }, + { + "in": "’", + "out": "’" + }, + { + "in": "“", + "out": "“" + }, + { + "in": "”", + "out": "”" + }, + { + "in": "«", + "out": "«" + }, + { + "in": "»", + "out": "»" + }, + { + "in": "℅", + "out": "c/o" + }, + { + "in": "︐", + "out": "," + }, + { + "in": "︰", + "out": ".." + }, + { + "in": "︱", + "out": "—" + }, + { + "in": "Ａａ", + "out": "Aa" + }, + { + "in": "０９", + "out": "09" + }, + { + "in": "㈠", + "out": "(一)" + }, + { + "in": "㈯", + "out": "(土)" + }, + { + "in": "ↀ", + "out": "ↀ" + }, + { + "in": "Ↄ", + "out": "Ↄ" + }, + { + "in": "́", + "out": "́" + }, + { + "in": "ㅤ", + "out": "ᅠ" + }, + { + "in": "㎡㎡", + "out": "m2m2" + }, + { + "in": "xfiyflz", + "out": "xfiyflz" + } + ], + "nfkc_cf": [ + { + "in": "hello", + "out": "hello" + }, + { + "in": " foo bar ", + "out": " foo bar " + }, + { + "in": "Ａ", + "out": "a" + }, + { + "in": "ａ", + "out": "a" + }, + { + "in": "™", + "out": "tm" + }, + { + "in": "café", + "out": "café" + }, + { + "in": "Café", + "out": "café" + }, + { + "in": "fi", + "out": "fi" + }, + { + "in": " ", + "out": " " + }, + { + "in": "ＡＢＣ", + "out": "abc" + }, + { + "in": "１２３", + "out": "123" + }, + { + "in": "Ⅰ", + "out": "i" + }, + { + "in": "file", + "out": "file" + }, + { + "in": "Straße", + "out": "straße" + }, + { + "in": "½", + "out": "1⁄2" + }, + { + "in": "①", + "out": "1" + }, + { + "in": "Ⅷ", + "out": "viii" + }, + { + "in": "№", + "out": "no" + }, + { + "in": "℃", + "out": "°c" + }, + { + "in": "ffi", + "out": "ffi" + }, + { + "in": "①②③", + "out": "123" + }, + { + "in": "ｰ", + "out": "ー" + }, + { + "in": "　", + "out": " " + }, + { + "in": "ij", + "out": "ij" + }, + { + "in": "Ǻ", + "out": "ǻ" + }, + { + "in": "K", + "out": "k" + }, + { + "in": "Å", + "out": "å" + }, + { + "in": "Ω", + "out": "ω" + }, + { + "in": "ℌ", + "out": "h" + }, + { + "in": "²", + "out": "2" + }, + { + "in": "³", + "out": "3" + }, + { + "in": "¼", + "out": "1⁄4" + }, + { + "in": "¾", + "out": "3⁄4" + }, + { + "in": "ｶ", + "out": "カ" + }, + { + "in": "⑩", + "out": "10" + }, + { + "in": "㈱", + "out": "(株)" + }, + { + "in": "fl", + "out": "fl" + }, + { + "in": "ffl", + "out": "ffl" + }, + { + "in": "Á", + "out": "á" + }, + { + "in": "É", + "out": "é" + }, + { + "in": "Ñ", + "out": "ñ" + }, + { + "in": "０", + "out": "0" + }, + { + "in": "ß", + "out": "ß" + }, + { + "in": "ẞ", + "out": "ß" + }, + { + "in": "İ", + "out": "İ" + }, + { + "in": "I", + "out": "i" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "é", + "out": "é" + }, + { + "in": "ø", + "out": "ø" + }, + { + "in": "Ø", + "out": "ø" + }, + { + "in": "œ", + "out": "œ" + }, + { + "in": "Œ", + "out": "œ" + }, + { + "in": "æ", + "out": "æ" + }, + { + "in": "Æ", + "out": "æ" + }, + { + "in": "µ", + "out": "μ" + }, + { + "in": "μ", + "out": "μ" + }, + { + "in": "℡", + "out": "tel" + }, + { + "in": "℠", + "out": "sm" + }, + { + "in": "㍉", + "out": "ミリ" + }, + { + "in": "㌔", + "out": "キロ" + }, + { + "in": "ｱ", + "out": "ア" + }, + { + "in": "ア", + "out": "ア" + }, + { + "in": "㏄", + "out": "cc" + }, + { + "in": "㎞", + "out": "km" + }, + { + "in": "ⅹ", + "out": "x" + }, + { + "in": "⑴", + "out": "(1)" + }, + { + "in": "⒜", + "out": "(a)" + }, + { + "in": "｡", + "out": "。" + }, + { + "in": "｢", + "out": "「" + }, + { + "in": "｣", + "out": "」" + }, + { + "in": "ﾞ", + "out": "゙" + }, + { + "in": "ﾟ", + "out": "゚" + }, + { + "in": "₩", + "out": "₩" + }, + { + "in": "€", + "out": "€" + }, + { + "in": "℉", + "out": "°f" + }, + { + "in": "㏇", + "out": "co." + }, + { + "in": "⅕", + "out": "1⁄5" + }, + { + "in": "⅖", + "out": "2⁄5" + }, + { + "in": "⅗", + "out": "3⁄5" + }, + { + "in": "LJ", + "out": "lj" + }, + { + "in": "nj", + "out": "nj" + }, + { + "in": "DZ", + "out": "dz" + }, + { + "in": "Dz", + "out": "dz" + }, + { + "in": "dz", + "out": "dz" + }, + { + "in": "ǰ", + "out": "ǰ" + }, + { + "in": "ȟ", + "out": "ȟ" + }, + { + "in": "ℏ", + "out": "ħ" + }, + { + "in": "ℓ", + "out": "l" + }, + { + "in": "℣", + "out": "℣" + }, + { + "in": "ℤ", + "out": "z" + }, + { + "in": "ℰ", + "out": "e" + }, + { + "in": "ℱ", + "out": "f" + }, + { + "in": "ℳ", + "out": "m" + }, + { + "in": "ℹ", + "out": "i" + }, + { + "in": "ㄱ", + "out": "ᄀ" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "㊀", + "out": "一" + }, + { + "in": "㊥", + "out": "中" + }, + { + "in": "­", + "out": "­" + }, + { + "in": "​", + "out": "​" + }, + { + "in": "！", + "out": "!" + }, + { + "in": "？", + "out": "?" + }, + { + "in": "．", + "out": "." + }, + { + "in": "，", + "out": "," + }, + { + "in": "：", + "out": ":" + }, + { + "in": "；", + "out": ";" + }, + { + "in": "（", + "out": "(" + }, + { + "in": "）", + "out": ")" + }, + { + "in": "［", + "out": "[" + }, + { + "in": "］", + "out": "]" + }, + { + "in": "｛", + "out": "{" + }, + { + "in": "｝", + "out": "}" + }, + { + "in": "￠", + "out": "¢" + }, + { + "in": "￡", + "out": "£" + }, + { + "in": "￥", + "out": "¥" + }, + { + "in": "ｶﾀｶﾅ", + "out": "カタカナ" + }, + { + "in": "ﾊﾟ", + "out": "パ" + }, + { + "in": "ｳﾞ", + "out": "ヴ" + }, + { + "in": "㊰", + "out": "夜" + }, + { + "in": "㊠", + "out": "項" + }, + { + "in": "⑪", + "out": "11" + }, + { + "in": "⑫", + "out": "12" + }, + { + "in": "⑵", + "out": "(2)" + }, + { + "in": "Ⓐ", + "out": "a" + }, + { + "in": "ⓐ", + "out": "a" + }, + { + "in": "㉑", + "out": "21" + }, + { + "in": "㉒", + "out": "22" + }, + { + "in": "⅓", + "out": "1⁄3" + }, + { + "in": "⅔", + "out": "2⁄3" + }, + { + "in": "⅛", + "out": "1⁄8" + }, + { + "in": "⅜", + "out": "3⁄8" + }, + { + "in": "⅝", + "out": "5⁄8" + }, + { + "in": "⅞", + "out": "7⁄8" + }, + { + "in": "¹", + "out": "1" + }, + { + "in": "⁰", + "out": "0" + }, + { + "in": "⁴", + "out": "4" + }, + { + "in": "₀", + "out": "0" + }, + { + "in": "₁", + "out": "1" + }, + { + "in": "₂", + "out": "2" + }, + { + "in": "ff", + "out": "ff" + }, + { + "in": "st", + "out": "st" + }, + { + "in": "¢", + "out": "¢" + }, + { + "in": "£", + "out": "£" + }, + { + "in": "¥", + "out": "¥" + }, + { + "in": "©", + "out": "©" + }, + { + "in": "®", + "out": "®" + }, + { + "in": "naive", + "out": "naive" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "naïve", + "out": "naïve" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "não", + "out": "não" + }, + { + "in": "á", + "out": "á" + }, + { + "in": "í", + "out": "í" + }, + { + "in": "ó", + "out": "ó" + }, + { + "in": "ú", + "out": "ú" + }, + { + "in": "Á", + "out": "á" + }, + { + "in": "É", + "out": "é" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ç", + "out": "ç" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "ö", + "out": "ö" + }, + { + "in": "SS", + "out": "ss" + }, + { + "in": "ss", + "out": "ss" + }, + { + "in": "Å", + "out": "å" + }, + { + "in": "å", + "out": "å" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "ά", + "out": "ά" + }, + { + "in": "Ά", + "out": "ά" + }, + { + "in": "가", + "out": "가" + }, + { + "in": "ㅏ", + "out": "ᅡ" + }, + { + "in": "각", + "out": "각" + }, + { + "in": "﨎", + "out": "﨎" + }, + { + "in": "塚", + "out": "塚" + }, + { + "in": "﨑", + "out": "﨑" + }, + { + "in": "⼀", + "out": "一" + }, + { + "in": "⼁", + "out": "丨" + }, + { + "in": "㌀", + "out": "アパート" + }, + { + "in": "㌃", + "out": "アール" + }, + { + "in": "㌶", + "out": "ヘクタール" + }, + { + "in": "㍑", + "out": "リットル" + }, + { + "in": "㍱", + "out": "hpa" + }, + { + "in": "㍴", + "out": "bar" + }, + { + "in": "㎀", + "out": "pa" + }, + { + "in": "㎏", + "out": "kg" + }, + { + "in": "㎜", + "out": "mm" + }, + { + "in": "㎡", + "out": "m2" + }, + { + "in": "㏑", + "out": "ln" + }, + { + "in": "㏿", + "out": "gal" + }, + { + "in": "ＡＢＣ file ½", + "out": "abc file 1⁄2" + }, + { + "in": "Café ™", + "out": "café tm" + }, + { + "in": "１２３²", + "out": "1232" + }, + { + "in": "Ⅰ+Ⅱ=Ⅲ", + "out": "i+ii=iii" + }, + { + "in": "file™", + "out": "filetm" + }, + { + "in": "¼ + ½ = ¾", + "out": "1⁄4 + 1⁄2 = 3⁄4" + }, + { + "in": "a b", + "out": "a b" + }, + { + "in": "a​b", + "out": "a​b" + }, + { + "in": "a­b", + "out": "a­b" + }, + { + "in": " 　 ", + "out": " " + }, + { + "in": "…", + "out": "..." + }, + { + "in": "–", + "out": "–" + }, + { + "in": "—", + "out": "—" + }, + { + "in": "‘", + "out": "‘" + }, + { + "in": "’", + "out": "’" + }, + { + "in": "“", + "out": "“" + }, + { + "in": "”", + "out": "”" + }, + { + "in": "«", + "out": "«" + }, + { + "in": "»", + "out": "»" + }, + { + "in": "℅", + "out": "c/o" + }, + { + "in": "︐", + "out": "," + }, + { + "in": "︰", + "out": ".." + }, + { + "in": "︱", + "out": "—" + }, + { + "in": "Ａａ", + "out": "aa" + }, + { + "in": "０９", + "out": "09" + }, + { + "in": "㈠", + "out": "(一)" + }, + { + "in": "㈯", + "out": "(土)" + }, + { + "in": "ↀ", + "out": "ↀ" + }, + { + "in": "Ↄ", + "out": "ↄ" + }, + { + "in": "́", + "out": "́" + }, + { + "in": "ㅤ", + "out": "ᅠ" + }, + { + "in": "㎡㎡", + "out": "m2m2" + }, + { + "in": "xfiyflz", + "out": "xfiyflz" + } + ] + }, + "note": "Golden Normalizer.normalize outputs from official sentencepiece; curated subset (Wave 27 expanded beyond 100/rule with compat forms and composed/decomposed pairs). Not exhaustive Unicode NFKC parity. Live full nfkc blobs remain integration-only. Bit-identical C++ MT19937 sample streams are a permanent non-goal." +} diff --git a/tests/fixtures/sp_tiny_custom_norm_unigram.model b/tests/fixtures/sp_tiny_custom_norm_unigram.model new file mode 100644 index 0000000000000000000000000000000000000000..b87ef5a892ce93dbc29ab9e92c917198af5735df GIT binary patch literal 1421 zcmb8vcTCh!0D$qwDG64Bik_mP;@&8tfQl%Bdn@i!IF5n{$DLeHzzI%N+pgO9dCTgHIYN0ObpdOszg8Fbp12jTIxWf&N z;en=Tf@bhUb9liUE#QNeXa!%iL2I-_TXaBsbV5gTL1%PBSM)%4@L7Z(dZQQmq7V9` z9|k}Je++~c0ni~3GJ+6-V1!~2!Y~+mgu{Rc41p0QM8XUOQ5cFCL}NIHVI)RiG)5r~ zu^5Xn7>{w7hzXd4cuc`$Ohp2wV;W{+24-Ux=3)-!V;&Y_0TyEsmSPE(V;Pc=h?Q7@ z)mVkKScCOghmF{P&Deyk*n;iYhMm}f-Pnb_*n|Dphl4nP!#IRw9Klhf;5d%qBu?No zPT>qvaSmsZhV!_H3%HC+NXHdi!&ThCb=*V-ZsQj2;tn!#4-ar350Qn(c!Z~Tg6DXK zmw18Kc!g}d!8^Rg2fW88e8d-g#y5P$4}8Zj{6r3Z!&~@u5)It{?>`i24wp^NifQN{ DipLgB literal 0 HcmV?d00001 diff --git a/tests/fixtures/sp_tiny_identity_bpe_bytefallback.model b/tests/fixtures/sp_tiny_identity_bpe_bytefallback.model new file mode 100644 index 0000000000000000000000000000000000000000..44dfc9c0d353468014badd8fc610eb6ccfe376cd GIT binary patch literal 5320 zcmb8zKWrRD6vy%Nkr*?W5LqBuh{6R01wyy~caa6zo!J#diWG@NaT3|{?a2{m-^pHN zN1`xYN-C7nB_$;tzN=EEOc_y8QqsZ9+glNRJFfb@=e?PEpB*=+?x@>`-=BW><%a|N z*&nL+)%M}&ax~gBw)=+1hR;U8M#x6QM$AUSM#@IUM$SgThGxUCQL<5?;nBmRher>O z9v(eBdU*8k=;6`BqlZTij~*U9JbHNa@aW;w!>5N&51$@BJ$!oj^ziB7)5E8SPY<6S zK0SPT`1J7U5zr%`M?jB&9sxZ9dIaE(phrNDfF1!o0(u1W2gBcw-2kB}ZA zJwkef^a$w@(j%ltNRN;nAw5ERg!BmM5z!-}M?{Z^9uYkvdPMYy=n>H)qDMrJh#nC= zB6>vhi0Bd1Bc?}8kC+}YJz{#q^oZ#Z(<7!wOpllzF+E~>#Po>ikF z^vLOv(<7%xPLG@(IX!ZEZITVdfAySq~Fs#|~j{G;3Cpf8SXG3tv8 zTijbIrQWJ1M>ao@JZ)w+KbAaOxK)=cwX1I1rR3~a^7UpucYk^&#o|m3^3zJar?%~L z-Ezb7kDcLP66a3#Z;3MEVC~?}T=$m53pd@)dSj?(Re=4~H{7kYP(Dp~`4Z7Ttu&?BHq-CwI6wdEFgAjysOP?B5c8%b`Uu_SlY$CBKAno4rB zA4_rvo=b94OG)k(w36JVyOiW^+z;!$>n#=NFUF6~o3r!rN!>P!alTxdx~<3e7f1Dc z{IEG$j@y%Ea=JKbo?f1h+t1s}cF+HRUpo1AGTR>>tcH*5f6-UZhJ)wBt()QC#qgj0 Zv-R?g{OXPT(XQHgGHp)VC+*X{!N0Aq_+|hA literal 0 HcmV?d00001 diff --git a/tests/fixtures/sp_tiny_identity_denorm_unigram.model b/tests/fixtures/sp_tiny_identity_denorm_unigram.model new file mode 100644 index 0000000000000000000000000000000000000000..0c082a76e2fd8baad58632943ebe40f0f297184c GIT binary patch literal 1437 zcmb8uWmFVF0KoA@K+h2bPgf8OY(&LC1@R2*!aU4VJmC=JfD=~21VzOb1G~GiTd=#k z6}!9hf8aYmyYKzp{%3Y}KJ2^|`Gu{fe`I()S>kfeCG3Q?Cg$(RTG$I4O_hH`A*{2~ zlH^$TJsCnJtfFKmy=$hqdQ)|>D>|nfgo>4+2BXouh(~t9mIT}AE&2ma>DZQnFEEQ}iqov|36>q78>{2Vj ztC6oRAjS|Kql?fhhA5q%$<$b{=yk25g7ik6)(~ORDG{cS0p4L=K3v(vz41ufFvc>Pst^@*a=k_b4r+!pq4yRRq7O#WWy=0J`yR>d^1ny zL2l%Q3i%*l4?8#@KMKGR1)+xdWDCIwg;4~~aD@wsp(skAI7-3|rBMoHQ3mBv4i(^z zits>XR6-Saq8h5g3)N8*HQKKV-?n74c22FHXt3Fun}9Z8QZWG zJFp$QuoHW*8yVP(eaOTC?8hM-#1S0EF&xDS9LFh~#2K8%Ih@4>oW~_x#1&k|HC)9F zT*ock#2wtmJ>128WZ@wm;4vQIDW2dtp5Y~4;5A<1E#BZg-r*xY;4?nqE56`6zTqc+ Q;5U9DD|r$BryKv+A6z!?asU7T literal 0 HcmV?d00001 diff --git a/tests/fixtures/sp_tiny_identity_unigram.model b/tests/fixtures/sp_tiny_identity_unigram.model new file mode 100644 index 0000000000000000000000000000000000000000..ee180187b34fc0b905f65ed8f789c11b6e07281c GIT binary patch literal 321 zcmd<$<6^Zb&C9kEWdH&RCN3^6=10>SMYSHE-Ot6r#h5S3pUHOs#7Y!R*ryC;rHUp& zS!r-q4xE()WI2P?rNCK5qUlgEXHmoZjbPbiI4c9rDi^f?$_mY4)5 zO)N>xk8vz0a7ipljPc1&PRxl3NzE;YDakD`&&y9vt*j`HDbXvbC=oIuO;c`sW{L!( r0$(TtLxX(>6T<-}h7(NxVSw=h68i=cTZ)H^BQqs6uOzdiQiuTn-9>Xe literal 0 HcmV?d00001 From 20701e3fa6ce26b07bf21935ae07c0794f2546f9 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:10:09 -0700 Subject: [PATCH 33/35] Add int8 quant epsilon fixture for verifier tests Provide a stable epsilon corpus for guardd 128-vector quantization checks. --- tests/fixtures/quant_epsilon_int8.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/fixtures/quant_epsilon_int8.json diff --git a/tests/fixtures/quant_epsilon_int8.json b/tests/fixtures/quant_epsilon_int8.json new file mode 100644 index 0000000..916ece3 --- /dev/null +++ b/tests/fixtures/quant_epsilon_int8.json @@ -0,0 +1,18 @@ +{ + "description": "Shared Lean/Rust int8 epsilon fixture. ε = 0.5 * sqrt(fan_in). Error metric: ||Wq x - W x||_2 / ||x||_2.", + "quant_type": "int8", + "layer_name": "fixture.linear", + "fan_in": 16, + "fan_out": 4, + "epsilon_bound": 2.0, + "weights": [ + 0.10, -0.20, 0.30, -0.40, 0.15, -0.25, 0.35, -0.45, + 0.11, -0.21, 0.31, -0.41, 0.16, -0.26, 0.36, -0.46, + 0.12, -0.22, 0.32, -0.42, 0.17, -0.27, 0.37, -0.47, + 0.13, -0.23, 0.33, -0.43, 0.18, -0.28, 0.38, -0.48, + 0.14, -0.24, 0.34, -0.44, 0.19, -0.29, 0.39, -0.49, + 0.105, -0.205, 0.305, -0.405, 0.155, -0.255, 0.355, -0.455, + 0.115, -0.215, 0.315, -0.415, 0.165, -0.265, 0.365, -0.465, + 0.125, -0.225, 0.325, -0.425, 0.175, -0.275, 0.375, -0.475 + ] +} From ca1fbbe93966ba390324ce302efcbc62a2ac1bc4 Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:10:09 -0700 Subject: [PATCH 34/35] Add offline and SentencePiece e2e test suites Keep default pytest offline via markers while covering staged natives and SP normalize/encode paths. --- tests/e2e/conftest.py | 37 ++ tests/e2e/test_offline_guard.py | 454 +++++++++++++++++++ tests/e2e/test_sentencepiece_crosscheck.py | 498 +++++++++++++++++++++ 3 files changed, 989 insertions(+) create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/test_offline_guard.py create mode 100644 tests/e2e/test_sentencepiece_crosscheck.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..d01a2d7 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,37 @@ +"""Shared fixtures and markers for Model Asset Guard e2e tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +PYTHON_BINDINGS = REPO_ROOT / "bindings" / "python" + +if str(PYTHON_BINDINGS) not in sys.path: + sys.path.insert(0, str(PYTHON_BINDINGS)) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + ( + "integration: requires network, HuggingFace downloads, " + "Node, Lean CLI, or optional packages" + ), + ) + + +@pytest.fixture(scope="session") +def repo_root() -> Path: + return REPO_ROOT + + +@pytest.fixture(scope="session") +def guard(): + """Load ModelAssetGuard against a built libguardd (debug or release).""" + from pytorch_guard import ModelAssetGuard + + return ModelAssetGuard() diff --git a/tests/e2e/test_offline_guard.py b/tests/e2e/test_offline_guard.py new file mode 100644 index 0000000..68e0dce --- /dev/null +++ b/tests/e2e/test_offline_guard.py @@ -0,0 +1,454 @@ +""" +Offline e2e tests: require only a built libguardd (+ optional cargo for gen_perfect_hash). + +No HuggingFace downloads, Node, or Lean toolchain. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import tempfile +from pathlib import Path + +import pytest +from pytorch_guard import GuarddError, ModelAssetGuard + + +def test_verify_digest_match(guard: ModelAssetGuard) -> None: + content = b"Hello, Model Asset Guard!" + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp.write(content) + path = tmp.name + try: + expected = hashlib.sha256(content).digest() + assert guard.verify_digest(path, expected) is True + finally: + os.remove(path) + + +def test_verify_digest_mismatch_returns_false(guard: ModelAssetGuard) -> None: + """F026: mismatch must return False, not raise.""" + content = b"test content" + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp.write(content) + path = tmp.name + try: + wrong = b"\x00" * 32 + assert guard.verify_digest(path, wrong) is False + finally: + os.remove(path) + + +def test_verify_digest_missing_file_raises(guard: ModelAssetGuard) -> None: + with pytest.raises(GuarddError): + guard.verify_digest("/non/existent/file/model-asset-guard", b"\x00" * 32) + + +def test_verify_digest_short_digest_raises(guard: ModelAssetGuard) -> None: + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp.write(b"x") + path = tmp.name + try: + with pytest.raises(ValueError): + guard.verify_digest(path, b"short") + finally: + os.remove(path) + + +def test_checked_load_valid_flag(guard: ModelAssetGuard) -> None: + content = b"checked-load-payload" + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp.write(content) + path = tmp.name + try: + good = hashlib.sha256(content).digest() + info = guard.checked_load(path, good) + assert info["valid"] is True + assert info["digest"] == good.hex() + + bad = guard.checked_load(path, b"\xff" * 32) + assert bad["valid"] is False + finally: + os.remove(path) + + +def test_model_quantization_object_json_contract(guard: ModelAssetGuard) -> None: + """F004: Python object-form layers must be accepted by Rust.""" + layers = [ + { + "name": "linear.weight", + "weights": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "fan_in": 3, + "fan_out": 2, + "quant_type": "int8", + } + ] + result = guard.verify_model_quantization(layers) + assert isinstance(result, dict) + assert result.get("total_layers") == 1 + assert result["layers"][0]["layer_name"] == "linear.weight" + assert len(result["layers"][0]["error_distribution"]) == 128 + + +def _generate_mph_vocab(repo_root: Path, words: list[str]) -> dict: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".txt", delete=False, encoding="utf-8" + ) as vocab_in: + vocab_in.write("\n".join(words) + "\n") + vocab_path = vocab_in.name + out_fd, out_path = tempfile.mkstemp(suffix=".json") + os.close(out_fd) + try: + result = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), + "--bin", + "gen_perfect_hash", + "--", + vocab_path, + out_path, + ], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert result.returncode == 0, result.stderr + return json.loads(Path(out_path).read_text(encoding="utf-8")) + finally: + if os.path.exists(vocab_path): + os.remove(vocab_path) + if os.path.exists(out_path): + os.remove(out_path) + + +def test_perfect_hash_encode_decode_roundtrip( + guard: ModelAssetGuard, repo_root: Path +) -> None: + """F005: runtime schema round-trips via FFI.""" + vocab = _generate_mph_vocab( + repo_root, ["hello", "world", "test", "perfect", "hash"] + ) + vocab_json = json.dumps(vocab) + text = "hello world test" + tokens = guard.perfect_hash_encode(vocab_json, text) + assert tokens == [1, 2, 3] + assert guard.perfect_hash_decode(vocab_json, tokens[0]) == "hello" + assert guard.perfect_hash_decode_sequence(vocab_json, tokens) == text + + +def test_gen_perfect_hash_cli_positional_and_output_flag(repo_root: Path) -> None: + words_path = repo_root / "tmp_vocab_words.txt" + out_a = repo_root / "tmp_mph_a.json" + out_b = repo_root / "tmp_mph_b.json" + words_path.write_text("alpha\nbeta\ngamma\n", encoding="utf-8") + try: + base = [ + "cargo", + "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), + "--bin", + "gen_perfect_hash", + "--", + str(words_path), + ] + r1 = subprocess.run( + [*base, str(out_a)], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert r1.returncode == 0, r1.stderr + assert out_a.is_file() + + r2 = subprocess.run( + [*base, "--output", str(out_b)], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert r2.returncode == 0, r2.stderr + assert out_b.is_file() + + a = json.loads(out_a.read_text(encoding="utf-8")) + b = json.loads(out_b.read_text(encoding="utf-8")) + assert a["vocab"] == b["vocab"] + assert a.get("kind") == "chd_mph" + assert b.get("kind") == "chd_mph" + assert a["pilots"] == b["pilots"] + assert a["indices"] == b["indices"] + assert a["n_buckets"] == b["n_buckets"] + assert len(a["indices"]) == len(a["vocab"]) + finally: + for p in (words_path, out_a, out_b): + if p.exists(): + p.unlink() + + +def test_bitflip_corpus_small(guard: ModelAssetGuard) -> None: + result = guard.run_bitflip_corpus_test( + file_size_gb=0, + num_corruptions=3, + temp_dir=tempfile.gettempdir(), + ) + assert isinstance(result, dict) + assert "test_passed" in result + + +def test_hf_local_materialize_verify_before_load(guard: ModelAssetGuard, tmp_path: Path) -> None: + """Local dirs use verify-before-load; no Hub download.""" + pytest.importorskip("transformers") + from pytorch_guard import HuggingFaceGuard + + weight = tmp_path / "pytorch_model.bin" + weight.write_bytes(b"fake-weights-for-materialize") + (tmp_path / "config.json").write_text("{}", encoding="utf-8") + + hf = HuggingFaceGuard(guard) + local_dir, mode = hf._materialize_model_dir(str(tmp_path)) + assert mode == "verify_before_load" + assert local_dir == str(tmp_path) + resolved = hf._resolve_local_weight_file(local_dir) + assert resolved == str(weight) + + +def test_shared_epsilon_fixture_int8(guard: ModelAssetGuard) -> None: + """Align Python/Rust with tests/fixtures/quant_epsilon_int8.json (Lean uses same formula).""" + fixture = ( + Path(__file__).resolve().parents[2] + / "tests" + / "fixtures" + / "quant_epsilon_int8.json" + ) + data = json.loads(fixture.read_text(encoding="utf-8")) + model = guard.verify_model_quantization( + [ + { + "name": data["layer_name"], + "weights": data["weights"], + "fan_in": data["fan_in"], + "fan_out": data["fan_out"], + "quant_type": data["quant_type"], + } + ] + ) + assert model["total_layers"] == 1 + layer = model["layers"][0] + assert layer["epsilon_bound"] == pytest.approx(data["epsilon_bound"], abs=1e-9) + assert layer["epsilon_bound"] == pytest.approx( + 0.5 * (data["fan_in"] ** 0.5), abs=1e-12 + ) + assert layer["passed_128_vectors"] is True + assert len(layer["error_distribution"]) == 128 + + +def _pb_varint(v: int) -> bytes: + out = bytearray() + while True: + b = v & 0x7F + v >>= 7 + if v: + b |= 0x80 + out.append(b) + if not v: + return bytes(out) + + +def _pb_len_delim(field: int, payload: bytes) -> bytes: + return _pb_varint((field << 3) | 2) + _pb_varint(len(payload)) + payload + + +def _craft_sp_bpe_model() -> bytes: + """Minimal BPE ModelProto: ▁/a/b/ab/▁a/▁ab with scores (matches Rust unit fixture).""" + import struct + + def piece(text: str, score: float, typ: int = 1) -> bytes: + body = _pb_varint((1 << 3) | 2) + _pb_varint(len(text.encode())) + text.encode() + body += _pb_varint((2 << 3) | 5) + struct.pack("", 0.0, 2) + out += piece("\u2581", -5.0) + out += piece("a", -4.0) + out += piece("b", -4.0) + out += piece("ab", -1.0) + out += piece("\u2581a", -0.5) + out += piece("\u2581ab", 0.0) + trainer = _pb_varint((3 << 3) | 0) + _pb_varint(2) # model_type=BPE + out += _pb_len_delim(2, trainer) + norm = ( + _pb_varint((3 << 3) | 0) + + _pb_varint(1) # add_dummy_prefix + + _pb_varint((4 << 3) | 0) + + _pb_varint(1) # remove_extra_whitespaces + + _pb_varint((5 << 3) | 0) + + _pb_varint(1) # escape_whitespaces + ) + out += _pb_len_delim(3, norm) + return bytes(out) + + +def _craft_sp_unigram_model() -> bytes: + """Minimal UNIGRAM ModelProto with unique Viterbi optimum ▁ab.""" + import struct + + def piece(text: str, score: float, typ: int = 1) -> bytes: + body = _pb_varint((1 << 3) | 2) + _pb_varint(len(text.encode())) + text.encode() + body += _pb_varint((2 << 3) | 5) + struct.pack("", 0.0, 2) + out += piece("\u2581", -10.0) + out += piece("a", -5.0) + out += piece("b", -5.0) + out += piece("ab", -3.0) + out += piece("\u2581a", -2.0) + out += piece("\u2581ab", -0.5) + trainer = _pb_varint((3 << 3) | 0) + _pb_varint(1) # model_type=UNIGRAM + out += _pb_len_delim(2, trainer) + norm = ( + _pb_varint((3 << 3) | 0) + + _pb_varint(1) + + _pb_varint((4 << 3) | 0) + + _pb_varint(1) + + _pb_varint((5 << 3) | 0) + + _pb_varint(1) + ) + out += _pb_len_delim(3, norm) + return bytes(out) + + +def test_sentencepiece_bpe_sp_encode_cli(repo_root: Path) -> None: + """Wave 7/8: crafted BPE .model encodes via guardd_perfect_hash sp-encode.""" + model_path = repo_root / "tmp_sp_bpe_wave8.model" + model_path.write_bytes(_craft_sp_bpe_model()) + try: + result = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), + "--bin", + "guardd_perfect_hash", + "--", + "sp-encode", + str(model_path), + "ab", + ], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert result.returncode == 0, result.stderr + tokens = json.loads(result.stdout.strip()) + assert tokens == [6], tokens + + test = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), + "--bin", + "guardd_perfect_hash", + "--", + "test", + str(model_path), + ], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert test.returncode == 0, test.stderr + meta = json.loads(test.stdout.strip()) + assert meta["ok"] is True + assert meta["kind"] == "sentencepiece_model_proto" + assert meta["model_type"] == "BPE" + assert meta["encode"] == "supported" + finally: + if model_path.exists(): + model_path.unlink() + + +def test_sentencepiece_unigram_sp_encode_ffi(guard: ModelAssetGuard, repo_root: Path) -> None: + """Wave 8: UNIGRAM Viterbi via Python FFI + CLI golden vector.""" + model_path = repo_root / "tmp_sp_unigram_wave8.model" + model_path.write_bytes(_craft_sp_unigram_model()) + try: + tokens = guard.sp_encode(str(model_path), "ab") + assert tokens == [6], tokens + assert guard.sp_decode(str(model_path), tokens) == "ab" + + cli = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), + "--bin", + "guardd_perfect_hash", + "--", + "sp-encode", + str(model_path), + "ab", + ], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + assert cli.returncode == 0, cli.stderr + assert json.loads(cli.stdout.strip()) == [6] + + with pytest.raises(GuarddError, match="no covering segmentation|not in vocab|fail"): + # Missing character coverage must fail closed (no silent unk). + guard.sp_encode(str(model_path), "xyz") + finally: + if model_path.exists(): + model_path.unlink() + + +def test_sp_nfkc_normalize_goldens_offline_hygiene(repo_root: Path) -> None: + """Wave 27: curated NFKC golden shape without live SP / nfkc blobs. + + Identity rows must be pure passthrough under charsmap-only knobs (in == out). + Full nfkc / nfkc_cf charsmap cross-check remains integration-only. + """ + gold_path = repo_root / "tests" / "fixtures" / "sp_nfkc_normalize_goldens.json" + gold = json.loads(gold_path.read_text(encoding="utf-8")) + rules = gold["rule"] + assert set(rules) == {"identity", "nfkc", "nfkc_cf"} + counts = {name: len(rows) for name, rows in rules.items()} + assert len(set(counts.values())) == 1 + assert counts["identity"] > 100 + assert "permanent non-goal" in gold.get("note", "").lower() or "MT19937" in gold.get( + "note", "" + ) + for row in rules["identity"]: + assert row["in"] == row["out"], row + # Spot-check a few known NFKC transforms stay present (fixture integrity). + nfkc = {row["in"]: row["out"] for row in rules["nfkc"]} + assert nfkc["\uff21"] == "A" + assert nfkc["\ufb01"] == "fi" + assert nfkc["\u00b2"] == "2" diff --git a/tests/e2e/test_sentencepiece_crosscheck.py b/tests/e2e/test_sentencepiece_crosscheck.py new file mode 100644 index 0000000..fbfa2dc --- /dev/null +++ b/tests/e2e/test_sentencepiece_crosscheck.py @@ -0,0 +1,498 @@ +"""Optional cross-check against the official `sentencepiece` Python package. + +Marked `integration` — skipped from default offline preflight. +Requires: `pip install sentencepiece` (no network needed for crafted / fixture models). +""" + +from __future__ import annotations + +import json +import struct +from pathlib import Path + +import pytest +from pytorch_guard import GuarddError + +pytestmark = pytest.mark.integration + +sentencepiece = pytest.importorskip("sentencepiece") + +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" + + +def _pb_varint(v: int) -> bytes: + out = bytearray() + while True: + b = v & 0x7F + v >>= 7 + if v: + b |= 0x80 + out.append(b) + if not v: + return bytes(out) + + +def _pb_len_delim(field: int, payload: bytes) -> bytes: + return _pb_varint((field << 3) | 2) + _pb_varint(len(payload)) + payload + + +def _piece(text: str, score: float, typ: int = 1) -> bytes: + body = _pb_varint((1 << 3) | 2) + _pb_varint(len(text.encode())) + text.encode() + body += _pb_varint((2 << 3) | 5) + struct.pack(" bytes: + return ( + _pb_varint((3 << 3) | 0) + + _pb_varint(1) + + _pb_varint((4 << 3) | 0) + + _pb_varint(1) + + _pb_varint((5 << 3) | 0) + + _pb_varint(1) + ) + + +def craft_unigram_unique_optimum() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("\u2581", -10.0) + out += _piece("a", -5.0) + out += _piece("b", -5.0) + out += _piece("ab", -3.0) + out += _piece("\u2581a", -2.0) + out += _piece("\u2581ab", -0.5) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(1)) # UNIGRAM + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def craft_bpe_unique_optimum() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("\u2581", -5.0) + out += _piece("a", -4.0) + out += _piece("b", -4.0) + out += _piece("ab", -1.0) + out += _piece("\u2581a", -0.5) + out += _piece("\u2581ab", 0.0) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(2)) # BPE + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def craft_char_limited_vocab() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("\u2581", 0.0) + out += _piece("a", 0.0) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(4)) # CHAR + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def craft_word_model() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("\u2581hello", 0.0) + out += _piece("\u2581world", 0.0) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(3)) # WORD + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def craft_word_suffix_ws() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("hello\u2581", 0.0) + out += _piece("world\u2581", 0.0) + trainer = ( + _pb_varint((3 << 3) | 0) + + _pb_varint(3) # WORD + + _pb_varint((24 << 3) | 0) + + _pb_varint(1) # treat_whitespace_as_suffix + ) + out += _pb_len_delim(2, trainer) + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def craft_control_symbols() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("", 0.0, 3) + out += _piece("", 0.0, 3) + out += _piece("\u2581a", -1.0) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(1)) + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def test_unigram_matches_sentencepiece_package(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ug.model" + model_path.write_bytes(craft_unigram_unique_optimum()) + + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = list(sp.encode("ab", out_type=int)) + ours = guard.sp_encode(str(model_path), "ab") + assert ours == official == [6], (ours, official) + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) == "ab" + + +def test_bpe_matches_sentencepiece_package(guard, tmp_path: Path) -> None: + model_path = tmp_path / "bpe.model" + model_path.write_bytes(craft_bpe_unique_optimum()) + + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = list(sp.encode("ab", out_type=int)) + ours = guard.sp_encode(str(model_path), "ab") + assert ours == official == [6], (ours, official) + + +def test_identity_fixture_matches_sentencepiece(guard) -> None: + model_path = FIXTURES / "sp_tiny_identity_unigram.model" + assert model_path.is_file() + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("hello", " foo bar "): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) + + +def test_custom_charsmap_fixture_matches_on_covered_text(guard) -> None: + """Charsmap is applied; covered ASCII still matches official SP.""" + model_path = FIXTURES / "sp_tiny_custom_norm_unigram.model" + assert model_path.is_file() + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("hello", " foo bar "): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + + # Fullwidth A is rewritten by charsmap to ASCII A (OOV). Stock SP emits unk; + # we remain fail-closed without byte_fallback / silent_unk. + with pytest.raises(GuarddError): + guard.sp_encode(str(model_path), "\uff21bc") + + +def test_byte_fallback_fixture_matches_sentencepiece(guard) -> None: + model_path = FIXTURES / "sp_tiny_identity_bpe_bytefallback.model" + assert model_path.is_file() + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("hello", "Z", "ABC", " foo bar ", "\U0001f600"): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) + + +def test_denorm_fixture_roundtrip_matches_sentencepiece(guard) -> None: + model_path = FIXTURES / "sp_tiny_identity_denorm_unigram.model" + assert model_path.is_file() + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("hello", "TM", "xTMy", "ATM"): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) + + +def test_silent_unk_opt_in_matches_sentencepiece(guard, tmp_path: Path) -> None: + model_path = tmp_path / "char_lim.model" + model_path.write_bytes(craft_char_limited_vocab()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = list(sp.encode("z", out_type=int)) + with pytest.raises(GuarddError): + guard.sp_encode(str(model_path), "z") + ours = guard.sp_encode(str(model_path), "z", allow_silent_unk=True) + assert ours == official == [1, 0], (ours, official) + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) + # Consecutive OOV chars merge to one unk (stock processor). + official_xyz = list(sp.encode("xyz", out_type=int)) + ours_xyz = guard.sp_encode(str(model_path), "xyz", allow_silent_unk=True) + assert ours_xyz == official_xyz == [1, 0], (ours_xyz, official_xyz) + piece = guard.sp_encode( + str(model_path), "xyz", allow_silent_unk=True, merge_consecutive_unk=False + ) + assert piece == [1, 0, 0, 0], piece + + +def craft_unigram_with_user_defined() -> bytes: + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("\u2581", -10.0) + out += _piece("a", -5.0) + out += _piece("b", -5.0) + out += _piece("ab", -0.1) + out += _piece("\u2581a", -2.0) + out += _piece("", 0.0, 4) # USER_DEFINED + out += _piece("f", -5.0) + out += _piece("o", -5.0) + out += _piece("<", -5.0) + out += _piece(">", -5.0) + out += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(1)) # UNIGRAM + out += _pb_len_delim(3, _norm_spec()) + return bytes(out) + + +def test_user_defined_matches_sentencepiece(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ud.model" + model_path.write_bytes(craft_unigram_with_user_defined()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("", "ab", "ab"): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + + +def test_nbest_matches_sentencepiece_order(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ug.model" + model_path.write_bytes(craft_unigram_unique_optimum()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = [list(row) for row in sp.nbest_encode("ab", nbest_size=5, out_type=int)] + ours = [row["tokens"] for row in guard.sp_nbest_encode(str(model_path), "ab", 5)] + assert ours == official + # 1-best equals encode + assert ours[0] == guard.sp_encode(str(model_path), "ab") + + +def test_sample_encode_from_nbest_is_member(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ug.model" + model_path.write_bytes(craft_unigram_unique_optimum()) + nbest = {tuple(r["tokens"]) for r in guard.sp_nbest_encode(str(model_path), "ab", 5)} + sampled = tuple( + guard.sp_sample_encode(str(model_path), "ab", nbest_size=5, alpha=1.0, seed=99) + ) + assert sampled in nbest + + +def test_word_matches_sentencepiece(guard, tmp_path: Path) -> None: + model_path = tmp_path / "word.model" + model_path.write_bytes(craft_word_model()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = list(sp.encode("hello world", out_type=int)) + ours = guard.sp_encode(str(model_path), "hello world") + assert ours == official + assert guard.sp_decode(str(model_path), ours) == sp.decode(official) + + +def test_treat_whitespace_as_suffix_matches_sentencepiece(guard, tmp_path: Path) -> None: + """Normalize + UNIGRAM encode with suffix ▁ matches official SP. + + Note: official ``word_model.cc`` calls ``SplitIntoWords(normalized)`` with + default ``treat_ws_as_suffix=false``, so WORD *inference* ignores the trainer + flag. We apply the flag in WORD split (trainer-compatible). Cross-check the + flag via UNIGRAM, where segmentation does not depend on SplitIntoWords. + """ + out = bytearray() + out += _piece("", 0.0, 2) + out += _piece("h", -2.0) + out += _piece("e", -2.0) + out += _piece("l", -2.0) + out += _piece("o", -2.0) + out += _piece("hello\u2581", -0.1) + out += _piece("w", -2.0) + out += _piece("r", -2.0) + out += _piece("d", -2.0) + out += _piece("world\u2581", -0.1) + trainer = ( + _pb_varint((3 << 3) | 0) + + _pb_varint(1) # UNIGRAM + + _pb_varint((24 << 3) | 0) + + _pb_varint(1) # treat_whitespace_as_suffix + ) + out += _pb_len_delim(2, trainer) + out += _pb_len_delim(3, _norm_spec()) + model_path = tmp_path / "ug_suffix.model" + model_path.write_bytes(bytes(out)) + + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("hello", "hello world"): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text) + assert ours == official, (text, ours, official) + + # WORD + suffix: our SplitIntoWords honors the trainer flag (official WORD + # inference does not). Sanity-check encode succeeds for suffix vocab. + word_path = tmp_path / "word_suffix.model" + word_path.write_bytes(craft_word_suffix_ws()) + ours_word = guard.sp_encode(str(word_path), "hello world") + assert ours_word == [1, 2], ours_word + + +def test_control_decode_matches_sentencepiece(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ctrl.model" + model_path.write_bytes(craft_control_symbols()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + assert sp.bos_id() == 1 + assert sp.eos_id() == 2 + ids = [1] + list(sp.encode("a", out_type=int)) + [2] + official = sp.decode(ids) + ours = guard.sp_decode(str(model_path), ids) + assert ours == official == "a" + + +def test_sample_encode_and_score_with_replacement(guard, tmp_path: Path) -> None: + model_path = tmp_path / "ug.model" + model_path.write_bytes(craft_unigram_unique_optimum()) + rows = guard.sp_sample_encode_and_score( + str(model_path), "ab", num_samples=4, alpha=0.5, seed=3 + ) + assert len(rows) == 4 + for row in rows: + assert row["tokens"] + assert isinstance(row["score"], float) + + +def test_sample_encode_and_score_wor(guard, tmp_path: Path) -> None: + """WOR: unique samples; include_best puts Viterbi first with score 0.""" + model_path = tmp_path / "ug.model" + model_path.write_bytes(craft_unigram_unique_optimum()) + best = guard.sp_encode(str(model_path), "ab") + rows = guard.sp_sample_encode_and_score( + str(model_path), + "ab", + num_samples=3, + alpha=0.5, + wor=True, + include_best=True, + seed=11, + ) + assert rows[0]["tokens"] == best + assert rows[0]["score"] == 0.0 + seen = set() + for row in rows: + key = tuple(row["tokens"]) + assert key not in seen + seen.add(key) + assert isinstance(row["score"], float) + + # Cross-check vs official SP: same token sets appear in n-best; scores finite. + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = sp.sample_encode_and_score( + "ab", num_samples=3, alpha=0.5, wor=True, include_best=True, out_type=int + ) + assert list(official[0][0]) == best + assert float(official[0][1]) == 0.0 + # Our seeded draw need not match SP's global MT; membership in n-best holds. + nbest = {tuple(r) for r in sp.nbest_encode("ab", nbest_size=8, out_type=int)} + for row in rows: + assert tuple(row["tokens"]) in nbest + + +def test_bpe_dropout_variants(guard, tmp_path: Path) -> None: + model_path = tmp_path / "bpe.model" + model_path.write_bytes(craft_bpe_unique_optimum()) + det = guard.sp_encode(str(model_path), "ab") + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + assert det == list(sp.encode("ab", out_type=int)) + seen: set[tuple[int, ...]] = set() + for seed in range(40): + ids = tuple( + guard.sp_sample_encode( + str(model_path), "ab", alpha=0.5, seed=seed, nbest_size=-1 + ) + ) + seen.add(ids) + assert len(seen) >= 2, f"expected dropout diversity, got {seen}" + off_seen = set() + for _ in range(50): + off_seen.add(tuple(sp.encode("ab", out_type=int, enable_sampling=True, alpha=0.5))) + assert seen & off_seen, f"no overlap with official dropout variants: {seen} vs {off_seen}" + + +def test_word_official_split_compat(guard, tmp_path: Path) -> None: + """word_official_split uses flag-blind SplitIntoWords (stock word_model.cc). + + Default encode also merges consecutive ```` like stock + ``SentencePieceProcessor`` (disable with ``merge_consecutive_unk=False``). + """ + model_path = tmp_path / "word_suffix.model" + model_path.write_bytes(craft_word_suffix_ws()) + ours = guard.sp_encode(str(model_path), "hello world") + assert ours == [1, 2] + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + official = list(sp.encode("hello world", out_type=int)) + compat = guard.sp_encode( + str(model_path), "hello world", word_official_split=True, allow_silent_unk=True + ) + assert compat == official + piece = guard.sp_encode( + str(model_path), + "hello world", + word_official_split=True, + allow_silent_unk=True, + merge_consecutive_unk=False, + ) + assert piece == [0, 0, 0] + + +def test_consecutive_unk_merge_crosscheck(guard, tmp_path: Path) -> None: + """Processor-level consecutive-unk merge vs official SentencePiece.""" + model_path = tmp_path / "char_lim.model" + model_path.write_bytes(craft_char_limited_vocab()) + sp = sentencepiece.SentencePieceProcessor(model_file=str(model_path)) + for text in ("xyz", "xay", "zzazz", "a", "hello"): + official = list(sp.encode(text, out_type=int)) + ours = guard.sp_encode(str(model_path), text, allow_silent_unk=True) + assert ours == official, (text, ours, official) + + +def test_nfkc_normalize_goldens_vs_live_and_guardd(tmp_path: Path, repo_root: Path) -> None: + """Curated corpus vs live SP Normalizer; nfkc/nfkc_cf charsmap through guardd. + + Not an exhaustive Unicode NFKC audit. Full nfkc blobs (~240KB) are loaded from + the installed sentencepiece package at test time (size-aware: not checked in). + Wave 27 expanded the curated set beyond 100/rule (compat forms, NFD/NFC pairs). + + Note: ``SentencePieceNormalizer`` goldens use charsmap-only flags + (no dummy-prefix / ▁ escape). They are not compared to encode-path models. + """ + gold_path = FIXTURES / "sp_nfkc_normalize_goldens.json" + assert gold_path.is_file() + gold = json.loads(gold_path.read_text(encoding="utf-8")) + counts = {rule: len(rows) for rule, rows in gold["rule"].items()} + assert set(counts) == {"identity", "nfkc", "nfkc_cf"} + assert len(set(counts.values())) == 1 + assert next(iter(counts.values())) > 100 + + for rule, rows in gold["rule"].items(): + live = sentencepiece.SentencePieceNormalizer(rule_name=rule) + for row in rows: + assert live.normalize(row["in"]) == row["out"], (rule, row) + + import subprocess + + cli = repo_root / "src" / "rust" / "guardd" / "target" / "debug" / "guardd_perfect_hash.exe" + if not cli.is_file(): + cli = repo_root / "src" / "rust" / "guardd" / "target" / "debug" / "guardd_perfect_hash" + if not cli.is_file(): + pytest.skip("guardd_perfect_hash binary not built") + + # Embed live nfkc / nfkc_cf NormalizerSpec (charsmap-only flags) into a tiny model. + for rule in ("nfkc", "nfkc_cf"): + live = sentencepiece.SentencePieceNormalizer(rule_name=rule) + spec = live.serialized_normalizer_spec() + pieces = [("", 0.0, 2), ("\u2581", 0.0, 1)] + for ch in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789": + pieces.append((ch, 0.0, 1)) + body = bytearray() + for text, score, typ in pieces: + body += _piece(text, score, typ) + body += _pb_len_delim(2, _pb_varint((3 << 3) | 0) + _pb_varint(4)) # CHAR + body += _pb_len_delim(3, spec) + model_path = tmp_path / f"{rule}_norm.model" + model_path.write_bytes(bytes(body)) + + for row in gold["rule"][rule]: + proc = subprocess.run( + [str(cli), "sp-normalize", str(model_path), row["in"]], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + assert proc.returncode == 0, (rule, row, proc.stderr) + assert proc.stdout.rstrip("\n") == row["out"], (rule, row, proc.stdout) From 740569df9f35c52a02733acdfcc06c0c20ebb85a Mon Sep 17 00:00:00 2001 From: fraware Date: Mon, 20 Jul 2026 23:10:09 -0700 Subject: [PATCH 35/35] Mark heavy e2e integrations and tighten perfect-hash tests Gate network/HF cases behind the integration marker and focus perfect-hash coverage on CHD round-trips. --- tests/e2e/test_huggingface_integration.py | 570 +++++--------------- tests/e2e/test_perfect_hash_integration.py | 572 ++++++--------------- 2 files changed, 265 insertions(+), 877 deletions(-) diff --git a/tests/e2e/test_huggingface_integration.py b/tests/e2e/test_huggingface_integration.py index ec06158..07452b1 100644 --- a/tests/e2e/test_huggingface_integration.py +++ b/tests/e2e/test_huggingface_integration.py @@ -1,458 +1,126 @@ -#!/usr/bin/env python3 """ -Test HuggingFace Transformers Integration (G-3) +HuggingFace / network-facing integration tests. -This script tests the complete HuggingFace transformers integration -for Model Asset Guard, validating the G-3 requirement implementation. - -Requirements: -- transformers >= 4.20.0 -- torch >= 1.12.0 -- numpy >= 1.21.0 -- pytest (for testing) +Marked `integration` — excluded from default preflight (no HF downloads in CI gate). """ +from __future__ import annotations + +import hashlib import os -import sys -import json import tempfile -import hashlib -from pathlib import Path -from typing import Dict, Any - -# Add bindings directory to Python path -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(REPO_ROOT / "bindings" / "python")) - -try: - import numpy as np - from transformers import AutoModel, AutoTokenizer - from pytorch_guard import ModelAssetGuard, HuggingFaceGuard, checked_load_pretrained - - TRANSFORMERS_AVAILABLE = True -except ImportError as e: - print(f"Warning: Some dependencies not available: {e}") - TRANSFORMERS_AVAILABLE = False - - -class HuggingFaceIntegrationTester: - """Comprehensive tester for HuggingFace transformers integration""" - - def __init__(self): - self.test_results = [] - self.temp_files = [] - self.guard = None - self.hf_guard = None - - def setup(self): - """Setup test environment""" - print("Setting up HuggingFace integration test environment...") - - try: - # Initialize Model Asset Guard - self.guard = ModelAssetGuard() - self.hf_guard = HuggingFaceGuard(self.guard) - print("✓ Model Asset Guard initialized successfully") - except Exception as e: - print(f"✗ Failed to initialize Model Asset Guard: {e}") - return False - - return True - - def cleanup(self): - """Clean up test resources""" - for temp_file in self.temp_files: - try: - os.remove(temp_file) - except Exception: - pass - - def create_test_file(self, content: str) -> str: - """Create a temporary test file""" - fd, path = tempfile.mkstemp() - os.write(fd, content.encode()) +import time + +import pytest + +pytest.importorskip("numpy") +np = pytest.importorskip("numpy") +pytest.importorskip("transformers") + +from pytorch_guard import ( # noqa: E402 + HuggingFaceGuard, + ModelAssetGuard, + checked_load_pretrained, +) + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def hf_guard(guard: ModelAssetGuard) -> HuggingFaceGuard: + return HuggingFaceGuard(guard) + + +def test_quantization_single_layer(guard: ModelAssetGuard) -> None: + fan_in, fan_out = 64, 32 + weights = np.random.randn(fan_in, fan_out).astype(np.float32) + layer_result = guard.verify_quantization_128_vectors( + layer_name="test_layer", + weights=weights, + fan_in=fan_in, + fan_out=fan_out, + quant_type="int8", + ) + assert isinstance(layer_result, dict) + assert layer_result["layer_name"] == "test_layer" + assert len(layer_result["error_distribution"]) == 128 + + +# Hub LFS oid for sshleifer/tiny-gpt2 `pytorch_model.bin` (commit 0c95bc9). +# Integration-only: skip/fail if Hub replaces the blob. +TINY_GPT2_PYTORCH_BIN_SHA256 = bytes.fromhex( + "b706b24034032bdfe765ded5ab6403d201d295a995b790cb24c74becca5c04e6" +) + + +def test_huggingface_checked_load_pretrained(hf_guard: HuggingFaceGuard) -> None: + model, tokenizer, verification_results = hf_guard.checked_load_pretrained( + "sshleifer/tiny-gpt2", + verify_quantization=False, + ) + assert model is not None + assert tokenizer is not None + assert isinstance(verification_results, dict) + # Remote Hub: verify-after-download-before-use (no verify-before-download API). + assert verification_results.get("verified") is True + assert ( + verification_results.get("mode") == "verify_after_download_before_use" + ) + assert "file_integrity" in verification_results + assert verification_results["file_integrity"].get("digest") + + +def test_huggingface_pinned_weight_digest(hf_guard: HuggingFaceGuard) -> None: + """Pin Hub weight digest: verify-after-download must match the known LFS oid.""" + model, tokenizer, verification_results = hf_guard.checked_load_pretrained( + "sshleifer/tiny-gpt2", + expected_digest=TINY_GPT2_PYTORCH_BIN_SHA256, + verify_quantization=False, + ) + assert model is not None + assert tokenizer is not None + info = verification_results["file_integrity"] + assert info.get("valid") is True + assert info.get("digest") == TINY_GPT2_PYTORCH_BIN_SHA256.hex() + + +def test_convenience_checked_load_pretrained() -> None: + model, tokenizer, results = checked_load_pretrained( + "sshleifer/tiny-gpt2", + verify_quantization=False, + ) + assert model is not None + assert tokenizer is not None + assert results.get("verified") is True + assert results.get("mode") == "verify_after_download_before_use" + + +def test_tokenizer_determinism_sample(hf_guard: HuggingFaceGuard) -> None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("sshleifer/tiny-gpt2") + # HF decode(encode(x)) is not always identity; we only assert the API returns structure. + results = hf_guard.verify_tokenizer_determinism( + tokenizer, + test_strings=["hello", "world"], + num_tests=2, + ) + assert results["total_tests"] == 2 + assert "passed" in results and "failed" in results + + +def test_digest_throughput_smoke(guard: ModelAssetGuard) -> None: + content = b"x" * (256 * 1024) + fd, path = tempfile.mkstemp() + try: + os.write(fd, content) os.close(fd) - self.temp_files.append(path) - return path - - def test_basic_guard_functionality(self) -> Dict[str, Any]: - """Test basic Model Asset Guard functionality""" - print("\n=== Testing Basic Guard Functionality ===") - - results = { - "test_name": "basic_guard_functionality", - "passed": False, - "errors": [], - } - - try: - # Create test file - test_content = "Hello, Model Asset Guard!" - test_file = self.create_test_file(test_content) - - # Compute expected digest - expected_digest = hashlib.sha256(test_content.encode()).digest() - - # Test digest verification - verified = self.guard.verify_digest(test_file, expected_digest) - if not verified: - results["errors"].append("Digest verification failed") - - # Test checked_load - file_info = self.guard.checked_load(test_file, expected_digest) - if not file_info["valid"]: - results["errors"].append("Checked load validation failed") - - # Test bit-flip corpus - corpus_result = self.guard.run_bitflip_corpus_test( - file_size_gb=0.001, # Small test - num_corruptions=5, - temp_dir=tempfile.gettempdir(), - ) - - if not isinstance(corpus_result, dict): - results["errors"].append("Bit-flip corpus test failed") - - results["passed"] = len(results["errors"]) == 0 - print( - f"✓ Basic functionality test: {'PASSED' if results['passed'] else 'FAILED'}" - ) - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ Basic functionality test failed: {e}") - - return results - - def test_quantization_verification(self) -> Dict[str, Any]: - """Test quantization verification with 128 vectors""" - print("\n=== Testing Quantization Verification ===") - - results = { - "test_name": "quantization_verification", - "passed": False, - "errors": [], - } - - try: - # Create test weights - fan_in, fan_out = 256, 512 - weights = np.random.randn(fan_in, fan_out).astype(np.float32) - - # Test single layer verification - layer_result = self.guard.verify_quantization_128_vectors( - layer_name="test_layer", - weights=weights, - fan_in=fan_in, - fan_out=fan_out, - quant_type="int8", - ) - - if not isinstance(layer_result, dict): - results["errors"].append("Single layer verification failed") - - # Test model verification - layers = [ - { - "name": "test_layer", - "weights": weights.flatten().tolist(), - "fan_in": fan_in, - "fan_out": fan_out, - "quant_type": "int8", - } - ] - - model_result = self.guard.verify_model_quantization(layers) - - if not isinstance(model_result, dict): - results["errors"].append("Model verification failed") - - results["passed"] = len(results["errors"]) == 0 - print( - f"✓ Quantization verification test: {'PASSED' if results['passed'] else 'FAILED'}" - ) - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ Quantization verification test failed: {e}") - - return results - - def test_huggingface_integration(self) -> Dict[str, Any]: - """Test HuggingFace transformers integration""" - print("\n=== Testing HuggingFace Integration ===") - - results = { - "test_name": "huggingface_integration", - "passed": False, - "errors": [], - } - - if not TRANSFORMERS_AVAILABLE: - results["errors"].append("Transformers not available") - print("✗ HuggingFace integration test skipped (transformers not available)") - return results - - try: - # Test with a small model (GPT-2) - print("Loading GPT-2 model with verification...") - - model, tokenizer, verification_results = ( - self.hf_guard.checked_load_pretrained("gpt2", verify_quantization=True) - ) - - # Check verification results - if not verification_results.get("verified", False): - results["errors"].append("Model verification failed") - - # Test tokenizer determinism - tokenizer_results = self.hf_guard.verify_tokenizer_determinism( - tokenizer, num_tests=100 # Small test for speed - ) - - if tokenizer_results["failed"] > 0: - results["errors"].append( - f"Tokenizer determinism test failed: {tokenizer_results['failed']} failures" - ) - - results["passed"] = len(results["errors"]) == 0 - print( - f"✓ HuggingFace integration test: {'PASSED' if results['passed'] else 'FAILED'}" - ) - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ HuggingFace integration test failed: {e}") - - return results - - def test_convenience_functions(self) -> Dict[str, Any]: - """Test convenience functions""" - print("\n=== Testing Convenience Functions ===") - - results = {"test_name": "convenience_functions", "passed": False, "errors": []} - - if not TRANSFORMERS_AVAILABLE: - results["errors"].append("Transformers not available") - print("✗ Convenience functions test skipped (transformers not available)") - return results - - try: - # Test create_guard function - guard = ModelAssetGuard() - if not guard: - results["errors"].append("create_guard failed") - - # Test create_hf_guard function - hf_guard = HuggingFaceGuard(guard) - if not hf_guard: - results["errors"].append("create_hf_guard failed") - - # Test checked_load_pretrained convenience function - model, tokenizer, verification_results = checked_load_pretrained( - "gpt2", verify_quantization=False # Skip for speed - ) - - if not model or not tokenizer: - results["errors"].append("checked_load_pretrained failed") - - results["passed"] = len(results["errors"]) == 0 - print( - f"✓ Convenience functions test: {'PASSED' if results['passed'] else 'FAILED'}" - ) - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ Convenience functions test failed: {e}") - - return results - - def test_error_handling(self) -> Dict[str, Any]: - """Test error handling""" - print("\n=== Testing Error Handling ===") - - results = {"test_name": "error_handling", "passed": False, "errors": []} - - try: - # Test invalid digest - test_file = self.create_test_file("test content") - invalid_digest = b"invalid" * 4 # 32 bytes but wrong - - try: - self.guard.verify_digest(test_file, invalid_digest) - results["errors"].append("Should have failed with invalid digest") - except Exception: - pass # Expected to fail - - # Test non-existent file - try: - self.guard.verify_digest("/non/existent/file", b"0" * 32) - results["errors"].append("Should have failed with non-existent file") - except Exception: - pass # Expected to fail - - # Test invalid digest length - try: - self.guard.verify_digest(test_file, b"short") - results["errors"].append("Should have failed with short digest") - except ValueError: - pass # Expected to fail - - results["passed"] = len(results["errors"]) == 0 - print( - f"✓ Error handling test: {'PASSED' if results['passed'] else 'FAILED'}" - ) - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ Error handling test failed: {e}") - - return results - - def test_performance(self) -> Dict[str, Any]: - """Test performance characteristics""" - print("\n=== Testing Performance ===") - - results = { - "test_name": "performance_test", - "passed": False, - "errors": [], - "metrics": {}, - } - - try: - import time - - # Test digest verification performance - test_content = "x" * (1024 * 1024) # 1MB - test_file = self.create_test_file(test_content) - expected_digest = hashlib.sha256(test_content.encode()).digest() - - start_time = time.time() - for _ in range(10): - self.guard.verify_digest(test_file, expected_digest) - end_time = time.time() - - avg_time = (end_time - start_time) / 10 - throughput = (1024 * 1024) / avg_time # MB/s - - results["metrics"]["digest_throughput_mbps"] = throughput - - if throughput < 100: # Target: > 100 MB/s - results["errors"].append( - f"Digest throughput too low: {throughput:.2f} MB/s" - ) - - # Test quantization verification performance - fan_in, fan_out = 512, 512 - weights = np.random.randn(fan_in, fan_out).astype(np.float32) - - start_time = time.time() - self.guard.verify_quantization_128_vectors( - "test_layer", weights, fan_in, fan_out, "int8" - ) - end_time = time.time() - - layer_time = end_time - start_time - results["metrics"]["quantization_time_seconds"] = layer_time - - if layer_time > 1.0: # Target: < 1 second - results["errors"].append( - f"Quantization verification too slow: {layer_time:.2f}s" - ) - - results["passed"] = len(results["errors"]) == 0 - print(f"✓ Performance test: {'PASSED' if results['passed'] else 'FAILED'}") - print(f" Digest throughput: {throughput:.2f} MB/s") - print(f" Quantization time: {layer_time:.2f}s") - - except Exception as e: - results["errors"].append(f"Exception: {str(e)}") - print(f"✗ Performance test failed: {e}") - - return results - - def run_all_tests(self) -> Dict[str, Any]: - """Run all tests and return comprehensive results""" - print("Starting HuggingFace Transformers Integration Tests (G-3)") - print("=" * 60) - - if not self.setup(): - return {"error": "Failed to setup test environment"} - - try: - # Run all tests - tests = [ - self.test_basic_guard_functionality, - self.test_quantization_verification, - self.test_huggingface_integration, - self.test_convenience_functions, - self.test_error_handling, - self.test_performance, - ] - - for test_func in tests: - result = test_func() - self.test_results.append(result) - - # Compile results - total_tests = len(self.test_results) - passed_tests = sum(1 for r in self.test_results if r.get("passed", False)) - - summary = { - "total_tests": total_tests, - "passed_tests": passed_tests, - "failed_tests": total_tests - passed_tests, - "success_rate": passed_tests / total_tests if total_tests > 0 else 0, - "test_results": self.test_results, - "overall_status": "PASSED" if passed_tests == total_tests else "FAILED", - } - - print("\n" + "=" * 60) - print("TEST SUMMARY") - print("=" * 60) - print(f"Total tests: {total_tests}") - print(f"Passed: {passed_tests}") - print(f"Failed: {total_tests - passed_tests}") - print(f"Success rate: {summary['success_rate']:.1%}") - print(f"Overall status: {summary['overall_status']}") - - # Print detailed results - for result in self.test_results: - status = "PASSED" if result.get("passed", False) else "FAILED" - print(f"\n{result['test_name']}: {status}") - if result.get("errors"): - for error in result["errors"]: - print(f" - {error}") - if result.get("metrics"): - for key, value in result["metrics"].items(): - print(f" - {key}: {value}") - - return summary - - finally: - self.cleanup() - - -def main(): - """Main test runner""" - tester = HuggingFaceIntegrationTester() - results = tester.run_all_tests() - - # Save results to file - with open("huggingface_integration_test_results.json", "w") as f: - json.dump(results, f, indent=2) - - print("\nTest results saved to: huggingface_integration_test_results.json") - - # Exit with appropriate code - if results.get("overall_status") == "PASSED": - print("\n🎉 All HuggingFace integration tests passed!") - return 0 - else: - print("\n❌ Some HuggingFace integration tests failed!") - return 1 - - -if __name__ == "__main__": - exit(main()) + expected = hashlib.sha256(content).digest() + start = time.time() + for _ in range(5): + assert guard.verify_digest(path, expected) + elapsed = time.time() - start + assert elapsed < 5.0 + finally: + if os.path.exists(path): + os.remove(path) diff --git a/tests/e2e/test_perfect_hash_integration.py b/tests/e2e/test_perfect_hash_integration.py index 26e7d32..853cbb9 100644 --- a/tests/e2e/test_perfect_hash_integration.py +++ b/tests/e2e/test_perfect_hash_integration.py @@ -1,465 +1,185 @@ -#!/usr/bin/env python3 """ -Perfect Hash Tokenizer Integration Test +Perfect-hash integration tests. -This script tests the complete perfect hash tokenizer integration including: -- Python bindings -- Node.js bindings -- Rust CLI tools -- Lean CLI tools -- End-to-end functionality +Default (offline) coverage lives primarily in `test_offline_guard.py`. +This module covers optional Node / Lean surfaces, marked `integration`. """ -import os +from __future__ import annotations + import json -import tempfile +import os import subprocess -import sys +import tempfile from pathlib import Path -from typing import Dict, Any - - -def create_test_vocab() -> Dict[str, Any]: - """Create a test vocabulary for perfect hash tokenizer""" - return { - "words": [ - "hello", - "world", - "test", - "perfect", - "hash", - "tokenizer", - "model", - "asset", - "guard", - "rust", - "lean", - "python", - ], - "hash_table": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], - } +import pytest +from pytorch_guard import ModelAssetGuard -def test_rust_cli() -> Dict[str, Any]: - """Test Rust CLI tool for perfect hash generation""" - print("Testing Rust CLI tool...") - - # Create test vocab file - test_vocab = create_test_vocab() - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(test_vocab, f) - vocab_file = f.name +def test_generator_words_json_input(repo_root: Path, guard: ModelAssetGuard) -> None: + """Words-object input must produce a runtime-loadable vocab.""" + words_obj = { + "words": ["hello", "world", "test", "perfect", "hash", "tokenizer"], + } + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, encoding="utf-8" + ) as f: + json.dump(words_obj, f) + vocab_in = f.name + out_fd, out_path = tempfile.mkstemp(suffix=".json") + os.close(out_fd) try: - # Test perfect hash generation result = subprocess.run( [ "cargo", "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), "--bin", "gen_perfect_hash", - "--manifest-path", - "src/rust/guardd/Cargo.toml", - vocab_file, + "--", + vocab_in, "--output", - "test_perfect_hash.json", + out_path, ], capture_output=True, text=True, - timeout=30, - ) - - success = result.returncode == 0 - output_file_exists = os.path.exists("test_perfect_hash.json") - - if output_file_exists: - with open("test_perfect_hash.json", "r") as f: - generated_vocab = json.load(f) - os.remove("test_perfect_hash.json") - else: - generated_vocab = None - - print(f" Rust CLI test: {'✓' if success and output_file_exists else '✗'}") - - return { - "test": "rust_cli", - "success": success and output_file_exists, - "output": result.stdout, - "error": result.stderr, - "generated_vocab": generated_vocab, - } - except subprocess.TimeoutExpired: - return {"test": "rust_cli", "success": False, "error": "Timeout"} - finally: - if os.path.exists(vocab_file): - os.remove(vocab_file) - - -def test_python_bindings() -> Dict[str, Any]: - """Test Python bindings for perfect hash tokenizer""" - print("Testing Python bindings...") - - try: - # Import the Python bindings - repo_root = Path(__file__).resolve().parents[2] - sys.path.insert(0, str(repo_root / "bindings" / "python")) - from pytorch_guard import ModelAssetGuard - - # Create guard instance - guard = ModelAssetGuard() - - # Create test vocabulary - test_vocab = create_test_vocab() - vocab_json = json.dumps(test_vocab) - - # Test encoding - test_text = "hello world test" - try: - tokens = guard.perfect_hash_encode(vocab_json, test_text) - encode_success = isinstance(tokens, list) and len(tokens) > 0 - print(f" Python encode test: {'✓' if encode_success else '✗'}") - except Exception as e: - encode_success = False - print(f" Python encode test: ✗ ({e})") - - # Test decoding - try: - if encode_success and tokens: - decoded_word = guard.perfect_hash_decode(vocab_json, tokens[0]) - decode_success = isinstance(decoded_word, str) and len(decoded_word) > 0 - print(f" Python decode test: {'✓' if decode_success else '✗'}") - else: - decode_success = False - print(" Python decode test: ✗ (no tokens to decode)") - except Exception as e: - decode_success = False - print(f" Python decode test: ✗ ({e})") - - # Test sequence decoding - try: - if encode_success and tokens: - decoded_text = guard.perfect_hash_decode_sequence(vocab_json, tokens) - sequence_success = ( - isinstance(decoded_text, str) and len(decoded_text) > 0 - ) - print( - f" Python sequence decode test: {'✓' if sequence_success else '✗'}" - ) - else: - sequence_success = False - print(" Python sequence decode test: ✗ (no tokens to decode)") - except Exception as e: - sequence_success = False - print(f" Python sequence decode test: ✗ ({e})") - - overall_success = encode_success and decode_success and sequence_success - - return { - "test": "python_bindings", - "success": overall_success, - "encode_success": encode_success, - "decode_success": decode_success, - "sequence_success": sequence_success, - "tokens": tokens if encode_success else None, - "decoded_word": decoded_word if decode_success else None, - "decoded_text": decoded_text if sequence_success else None, - } - except ImportError as e: - print(f" Python bindings test: ✗ (Import error: {e})") - return { - "test": "python_bindings", - "success": False, - "error": f"Import error: {e}", - } - except Exception as e: - print(f" Python bindings test: ✗ ({e})") - return {"test": "python_bindings", "success": False, "error": str(e)} - - -def test_nodejs_bindings() -> Dict[str, Any]: - """Test Node.js bindings for perfect hash tokenizer""" - print("Testing Node.js bindings...") - - # Create test script - repo_root = Path(__file__).resolve().parents[2] - node_guard_path = str((repo_root / "bindings" / "nodejs" / "node_guard.js").resolve()).replace("\\", "/") - test_script = ( - """ -const { ModelAssetGuard } = require('""" - + node_guard_path - + """'); - -async function testPerfectHash() { - try { - const guard = new ModelAssetGuard(); - - const testVocab = { - words: ["hello", "world", "test", "perfect", "hash", "tokenizer"], - hash_table: [0, 1, 2, 3, 4, 5] - }; - - const vocabJson = JSON.stringify(testVocab); - const testText = "hello world test"; - - // Test encoding - const tokens = guard.perfectHashEncode(vocabJson, testText); - console.log('ENCODE_SUCCESS:', Array.isArray(tokens) && tokens.length > 0); - - // Test decoding - if (tokens && tokens.length > 0) { - const decodedWord = guard.perfectHashDecode(vocabJson, tokens[0]); - console.log('DECODE_SUCCESS:', typeof decodedWord === 'string' && decodedWord.length > 0); - - // Test sequence decoding - const decodedText = guard.perfectHashDecodeSequence(vocabJson, tokens); - console.log('SEQUENCE_SUCCESS:', typeof decodedText === 'string' && decodedText.length > 0); - } else { - console.log('DECODE_SUCCESS: false'); - console.log('SEQUENCE_SUCCESS: false'); - } - - } catch (error) { - console.log('ERROR:', error.message); - } -} - -testPerfectHash(); -""" - ) - - with tempfile.NamedTemporaryFile(mode="w", suffix=".js", delete=False) as f: - f.write(test_script) - test_file = f.name - - try: - result = subprocess.run( - ["node", test_file], capture_output=True, text=True, timeout=30 - ) - - output = result.stdout.strip() - encode_success = "ENCODE_SUCCESS: true" in output - decode_success = "DECODE_SUCCESS: true" in output - sequence_success = "SEQUENCE_SUCCESS: true" in output - has_error = "ERROR:" in output - - overall_success = ( - encode_success and decode_success and sequence_success and not has_error - ) - - print(f" Node.js encode test: {'✓' if encode_success else '✗'}") - print(f" Node.js decode test: {'✓' if decode_success else '✗'}") - print(f" Node.js sequence decode test: {'✓' if sequence_success else '✗'}") - - return { - "test": "nodejs_bindings", - "success": overall_success, - "encode_success": encode_success, - "decode_success": decode_success, - "sequence_success": sequence_success, - "output": output, - "error": result.stderr, - } - except subprocess.TimeoutExpired: - return {"test": "nodejs_bindings", "success": False, "error": "Timeout"} - finally: - if os.path.exists(test_file): - os.remove(test_file) - - -def test_lean_cli() -> Dict[str, Any]: - """Test Lean CLI tool for perfect hash tokenizer""" - print("Testing Lean CLI tool...") - - # Create test vocabulary file - test_vocab = create_test_vocab() - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(test_vocab, f) - vocab_file = f.name - - try: - # Test encode command - result = subprocess.run( - ["lake", "exe", "perfecthash", "encode", vocab_file, "hello world"], - capture_output=True, - text=True, - timeout=30, - ) - - encode_success = result.returncode == 0 - print(f" Lean CLI encode test: {'✓' if encode_success else '✗'}") - - # Test decode command - result = subprocess.run( - ["lake", "exe", "perfecthash", "decode", vocab_file, "0"], - capture_output=True, - text=True, - timeout=30, - ) - - decode_success = result.returncode == 0 - print(f" Lean CLI decode test: {'✓' if decode_success else '✗'}") - - # Test test command - result = subprocess.run( - ["lake", "exe", "perfecthash", "test", vocab_file], - capture_output=True, - text=True, - timeout=30, + check=False, + cwd=repo_root, ) - - test_success = result.returncode == 0 - print(f" Lean CLI test command: {'✓' if test_success else '✗'}") - - overall_success = encode_success and decode_success and test_success - - return { - "test": "lean_cli", - "success": overall_success, - "encode_success": encode_success, - "decode_success": decode_success, - "test_success": test_success, - "output": result.stdout, - "error": result.stderr, - } - except subprocess.TimeoutExpired: - return {"test": "lean_cli", "success": False, "error": "Timeout"} + assert result.returncode == 0, result.stderr + vocab = json.loads(Path(out_path).read_text(encoding="utf-8")) + assert "vocab" in vocab + assert vocab.get("kind") == "chd_mph" + assert "pilots" in vocab and "indices" in vocab and "n_buckets" in vocab + assert len(vocab["indices"]) == len(vocab["vocab"]) + vocab_json = json.dumps(vocab) + tokens = guard.perfect_hash_encode(vocab_json, "hello world test") + assert tokens == [1, 2, 3] + assert guard.perfect_hash_decode_sequence(vocab_json, tokens) == "hello world test" finally: - if os.path.exists(vocab_file): - os.remove(vocab_file) - - -def test_end_to_end() -> Dict[str, Any]: - """Test end-to-end perfect hash tokenizer workflow""" - print("Testing end-to-end workflow...") - + for p in (vocab_in, out_path): + if os.path.exists(p): + os.remove(p) + + +@pytest.mark.integration +def test_nodejs_bindings_roundtrip(repo_root: Path) -> None: + node = subprocess.run(["node", "--version"], capture_output=True, text=True) + if node.returncode != 0: + pytest.skip("node not available") + + # Build a real vocab via Rust first + words = {"words": ["hello", "world", "test"]} + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, encoding="utf-8" + ) as f: + json.dump(words, f) + vocab_in = f.name + mph_fd, mph_path = tempfile.mkstemp(suffix=".json") + os.close(mph_fd) try: - # Step 1: Generate perfect hash vocabulary - test_vocab = create_test_vocab() - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(test_vocab, f) - input_vocab_file = f.name - - output_vocab_file = "e2e_perfect_hash.json" - - # Generate perfect hash - result = subprocess.run( + gen = subprocess.run( [ "cargo", "run", + "--quiet", + "--manifest-path", + str(repo_root / "src/rust/guardd/Cargo.toml"), "--bin", "gen_perfect_hash", - "--manifest-path", - "src/rust/guardd/Cargo.toml", - input_vocab_file, - "--output", - output_vocab_file, + "--", + vocab_in, + mph_path, ], capture_output=True, text=True, - timeout=30, + check=False, + cwd=repo_root, ) + assert gen.returncode == 0, gen.stderr + vocab_json = Path(mph_path).read_text(encoding="utf-8") + + node_guard = (repo_root / "bindings/nodejs/node_guard.js").resolve().as_posix() + script = f""" +const {{ ModelAssetGuard }} = require({json.dumps(node_guard)}); +const guard = new ModelAssetGuard(); +const vocabJson = {json.dumps(vocab_json)}; +const tokens = guard.perfectHashEncode(vocabJson, "hello world test"); +if (!Array.isArray(tokens) || tokens.length !== 3) {{ + console.error("ENCODE_FAIL", tokens); + process.exit(2); +}} +const decoded = guard.perfectHashDecodeSequence(vocabJson, tokens); +if (decoded !== "hello world test") {{ + console.error("DECODE_FAIL", decoded); + process.exit(3); +}} +console.log("OK"); +""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".js", delete=False, encoding="utf-8" + ) as sf: + sf.write(script) + script_path = sf.name + try: + run = subprocess.run( + ["node", script_path], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + ) + err = (run.stderr or "") + (run.stdout or "") + if ( + "Cannot find module" in err + or "koffi" in err + or "ffi-napi" in err + or "Could not find libguardd" in err + ): + pytest.skip(f"nodejs native deps / libguardd unavailable: {err}") + assert run.returncode == 0, run.stderr or run.stdout + assert "OK" in run.stdout + finally: + os.remove(script_path) + finally: + for p in (vocab_in, mph_path): + if os.path.exists(p): + os.remove(p) - if result.returncode != 0: - return { - "test": "end_to_end", - "success": False, - "error": "Failed to generate perfect hash", - } - - # Step 2: Test with Python bindings - from pytorch_guard import ModelAssetGuard - - guard = ModelAssetGuard() - - with open(output_vocab_file, "r") as f: - generated_vocab = json.load(f) - - vocab_json = json.dumps(generated_vocab) - test_text = "hello world perfect hash" - - # Encode - tokens = guard.perfect_hash_encode(vocab_json, test_text) - if not tokens: - return {"test": "end_to_end", "success": False, "error": "Encoding failed"} - - # Decode - decoded_text = guard.perfect_hash_decode_sequence(vocab_json, tokens) - if not decoded_text: - return {"test": "end_to_end", "success": False, "error": "Decoding failed"} - - # Verify determinism (simplified check) - determinism_ok = len(decoded_text.split()) == len(test_text.split()) - - print(f" End-to-end workflow: {'✓' if determinism_ok else '✗'}") - - # Cleanup - if os.path.exists(output_vocab_file): - os.remove(output_vocab_file) - if os.path.exists(input_vocab_file): - os.remove(input_vocab_file) - - return { - "test": "end_to_end", - "success": determinism_ok, - "original_text": test_text, - "tokens": tokens, - "decoded_text": decoded_text, - "determinism_ok": determinism_ok, - } - except Exception as e: - return {"test": "end_to_end", "success": False, "error": str(e)} - - -def main(): - """Run all perfect hash integration tests""" - print("Perfect Hash Tokenizer Integration Test") - print("=" * 50) - - tests = [ - test_rust_cli, - test_python_bindings, - test_nodejs_bindings, - test_lean_cli, - test_end_to_end, - ] - - results = [] - for test_func in tests: - result = test_func() - results.append(result) - print() - - # Summary - passed = sum(1 for r in results if r["success"]) - total = len(results) - - print("=" * 50) - print(f"Test Summary: {passed}/{total} tests passed") - if passed == total: - print("✓ All perfect hash integration tests passed!") - else: - print("✗ Some tests failed:") - for result in results: - if not result["success"]: - print(f" - {result['test']}: {result.get('error', 'Unknown error')}") +@pytest.mark.integration +def test_lean_perfecthash_cli_is_honest_stub(repo_root: Path) -> None: + """Lean perfecthash must not claim simulated success (nonzero exit).""" + lake = subprocess.run(["lake", "--version"], capture_output=True, text=True) + if lake.returncode != 0: + pytest.skip("lake not available") - # Save results - with open("perfect_hash_test_results.json", "w") as f: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, encoding="utf-8" + ) as f: json.dump( - {"total_tests": total, "passed_tests": passed, "results": results}, + { + "words": ["hello"], + "hash_table": [0], + "table_size": 1, + "vocab": [["hello", 1]], + }, f, - indent=2, ) - - print("\nResults saved to perfect_hash_test_results.json") - - # Exit with appropriate code - if passed == total: - exit(0) - else: - exit(1) - - -if __name__ == "__main__": - main() + vocab_file = f.name + try: + result = subprocess.run( + ["lake", "exe", "perfecthash", "encode", vocab_file, "hello"], + capture_output=True, + text=True, + check=False, + cwd=repo_root, + timeout=120, + ) + assert result.returncode != 0 + assert "not implemented" in (result.stdout + result.stderr).lower() + finally: + os.remove(vocab_file)