From d3aed719687acc66ecfbbc04f758a6850f1ade61 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 20:19:46 +0000 Subject: [PATCH 01/11] fix(pslq): close the four value-preserving conversions that switched the trust gate off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3.9.0's trust fix made `relation_confidence` tri-state and wired `E-PSLQ-004` into `guess_relation`, but `_supplied_bits` classified the input by *type* — and four ordinary conversions change the type without changing the value (autoresearch run 2026-08-19, issue #4, High). `guess_relation([float(pi), float(e), float(log 2)])` refused; `guess_relation([mpf(x) for x in the same floats])` at `mp.dps = 300` returned the exact spurious relation the release was written to refuse, `[-60771139, 67263243, 11653676]`, with `credible=True` and 277 "spare" digits. And `relation_confidence([Fraction(str(x)) for x in floats], that relation)` reported `credible=True, precision_source='exact'` for a relation whose exact residual is 1.1e-9 — a refutable claim, certified. Three changes, in `python/alkahest/__init__.py`: - `mpmath.mpf` is now *unknown* precision rather than `value.context.prec`. That attribute is the ambient `mp.dps` at the moment of asking, not a property of the value, so the same objects judged before and after an unrelated `mp.dps = 300` got opposite verdicts. Accuracy is not recoverable from the object either — every `mpf` is exactly a dyadic rational — which is why the mantissa-bitcount fix is wrong: it reports 0.30 digits for `mpf(1), mpf(2), mpf(3)` and refutes the true relation `[1, 1, -1]`. An `mpf` is now treated exactly as a decimal string is, and `digits=` is how a caller declares better. - Exact inputs are *evaluated* rather than assumed. `available_digits` is `inf` on the `exact` branch, so no affordability test could fire and `credible=True` was unfalsifiable there. `Σ aᵢ·cᵢ` is now computed in exact `Fraction` arithmetic for `int`/`Fraction` constants; a nonzero residual refutes the relation (new `exact_residual` key on the verdict dict) and `guess_relation` raises the new `E-PSLQ-005`, a refutation rather than a precision complaint. - `guess_relation` gained the `digits=` escape hatch it was missing. `digits=` rescued `relation_confidence`, but on `guess_relation` `precision_bits` is the width of the *search*, so the entry point that raises `E-PSLQ-004` had no way for a caller who knows their input precision to be judged at all. Keyword-only and additive; no Rust surface changed, so semver-checks is unaffected. Also item 26n: the cost formula `n·log10(H)` collapses to 0 at `H = 1`, so a relation with unit coefficients was free however many constants it spanned. Coefficients bounded by `H` select one of `(2H+1)ⁿ` integer vectors, so the cost is `n·log10(2H+1)`; a 40-term ±1 relation now costs ~19 digits instead of 0. The refuted half of 26n — "it never evaluates the relation it is judging" — is left alone: affordability remains the contract, and evaluation now happens only on the exact branch, where it is the only thing that can decide. Item 26o: `E-PSLQ-*` had zero occurrences under `docs/`. All five codes are now in `docs/mdbook/src/errors.md`, in the per-class table, the refusals table and the subsystem axis. Regression tests in `tests/test_relation_precision_guard.py` (54 total, 23 of them failing before this change): the `mpf` lift at `mp.dps = 300`; the ambient-precision dependence, asserted as *equality* of the verdict across `mp.dps`; each of `Fraction(str(x))`, `Fraction(Decimal(repr(x)))` and a 20-digit `nstr` truncation reporting a false relation and being refused; the `mpf(1), mpf(2), mpf(3) → [1, 1, -1]` guard against the known-wrong fix; the `H = 1` cost formula; and the new escape hatch on `guess_relation`. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 41 +++ alkahest-skill/alkahest.md | 4 +- docs/mdbook/src/errors.md | 23 ++ examples/pslq_research_loop.py | 14 +- python/alkahest/__init__.py | 245 ++++++++++++----- python/alkahest/exceptions.py | 9 +- tests/test_relation_precision_guard.py | 360 ++++++++++++++++++++++++- 7 files changed, 608 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..ba054c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,47 @@ ## Unreleased +- **The PSLQ trust gate read the input's *type*, and four value-preserving + conversions changed the type without changing the value** (2026-08-19 + autoresearch run, issue #4). 3.9.0 made `relation_confidence` tri-state and + wired `E-PSLQ-004` into `guess_relation`, but `_supplied_bits` classified by + type, so `guess_relation([float(pi), float(e), float(log 2)])` refused while + `guess_relation([mpf(x) for x in the same floats])` returned the very relation + the release was written to refuse — `[-60771139, 67263243, 11653676]`, true + residual `3.8e8` — with `credible=True` and 277 "spare" digits. Three fixes: + + - **`mpmath.mpf` is now *unknown* precision, not its context's `prec`.** It + reported `value.context.prec`, which is the *ambient* `mp.dps` at the moment + of asking rather than a property of the value: the same objects judged before + and after an unrelated `mp.dps = 300` got opposite verdicts. Nor is accuracy + recoverable from the object — every `mpf` is exactly a dyadic rational — so + the mantissa-bitcount fix is wrong (it reports 0.30 digits for `mpf(1), + mpf(2), mpf(3)` and refutes the true relation `[1, 1, -1]`). An `mpf` is now + treated exactly as a decimal string is: unjudged unless the caller declares + `digits=`. + - **Exact inputs are now *evaluated* rather than assumed.** `available_digits` + is `inf` on the `exact` branch, so no affordability test could ever fire and + `credible=True` was unfalsifiable there — `Fraction(str(x))`, + `Fraction(Decimal(repr(x)))` and a 20-digit `nstr` truncation all reached it + with a relation that is *false for the very numbers supplied*. For `int` and + `Fraction` constants `Σ aᵢ·cᵢ` is now computed in exact `Fraction` + arithmetic; a nonzero residual refutes the relation (new `exact_residual` key + on `relation_confidence`'s dict) and `guess_relation` raises the new + **`E-PSLQ-005`**, which is a refutation rather than a precision complaint. + - **`guess_relation` gained the `digits=` escape hatch it was missing.** + `digits=` rescued `relation_confidence`, but on `guess_relation` + `precision_bits` means the width of the *search*, so the entry point that + raises `E-PSLQ-004` had no way for a caller who genuinely knows their input + precision to be judged at all. `guess_relation(constants, digits=…)` is + keyword-only and additive. + + Also fixed (item 26n): the cost formula `n·log10(H)` collapses to 0 at `H = 1`, + so a relation with unit coefficients was free however many constants it spanned. + A relation with coefficients bounded by `H` selects one of `(2H+1)ⁿ` integer + vectors, so the cost is `n·log10(2H+1)`; a 40-term ±1 relation now costs ~19 + digits instead of 0. `E-PSLQ-*` codes are documented in + `docs/mdbook/src/errors.md` for the first time (item 26o). + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index 7df084ab..30fc0d1d 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -1185,7 +1185,7 @@ All errors inherit `AlkahestError` and carry `.code`, `.remediation`, `.span`. | `JitError` | `E-JIT-*` | JIT compilation failed | | `SolverError` | `E-SOLVE-*` | Polynomial solver failed | | `SumError` / `ProductError` | `E-SUM-*` / `E-PROD-*` | Summation / product failed | -| `PslqError` | `E-PSLQ-*` | Integer relation not justified by the input precision (`E-PSLQ-004`) | +| `PslqError` | `E-PSLQ-*` | Integer relation not justified by the input precision (`E-PSLQ-004`), or false for the exact rationals supplied (`E-PSLQ-005`) | | `IoError` | `E-IO-*` | Pool checkpoint I/O | | `PoolError` | `E-POOL-*` | Cross-pool or closed-pool misuse | | `NumberTheoryError` | `E-NT-*` | Invalid input to number-theory helpers | @@ -1417,7 +1417,7 @@ reg.coverage_report_markdown() # same, rendered as a Markdown table 16. **Bound long calls with `context(budget=…)`, not `run_with_wall_fallback`.** The latter joins its worker and so does not bound wall time for an uncooperative callee. Only `integrate` and `limit` honour the cooperative budget and release the GIL. 17. **Do not export radical results to another CAS without evaluating them here first.** Casus-irreducibilis cube roots from `eigenvals()` are correct in Alkahest and wrong under a principal-branch evaluator. An `E-EVAL-009` or an infinite ball is the signal not to export as-is. 18. **`0 · 0⁻¹` is left unevaluated on purpose** (since 3.8). If you see `(0 * 0^-1)` in a result, that is Alkahest declining to give an indeterminate form a value — not a simplifier failure to work around. -19. **`relation_confidence` answers `None` when it cannot see the input precision** (since 3.8), which is the normal case: a decimal string may be an exact rational or a truncated constant, and nothing in it says which. `None` means *not checked*, never *passed* — branch on `if verdict["credible"]:`, not `is not False`. To get a real verdict on a `guess_relation` result computed from truncated decimal strings, pass the digits you trust: `relation_confidence(constants, coeffs, digits=60)`. Only `float` and `mpmath.mpf` inputs are judged without a declaration. +19. **`relation_confidence` answers `None` when it cannot see the input precision** (since 3.8), which is the normal case: a decimal string may be an exact rational or a truncated constant, and nothing in it says which. `None` means *not checked*, never *passed* — branch on `if verdict["credible"]:`, not `is not False`. To get a real verdict on a `guess_relation` result computed from truncated decimal strings, pass the digits you trust: `relation_confidence(constants, coeffs, digits=60)` — or, since 3.9.1, declare them on the search itself with `guess_relation(constants, digits=60)`, which is the only way to make it raise `E-PSLQ-004` on input whose precision it cannot see (`precision_bits` is the *search* width, not a claim about the data). Without a declaration only `float` inputs are judged: an `mpmath.mpf` reports the ambient `mp.dps` at the moment it is asked rather than anything about itself, so it is *unknown* like a decimal string. `int` and `Fraction` inputs are judged by **evaluating** the relation in exact arithmetic — `credible` is `False` and `E-PSLQ-005` is raised when `Σ aᵢ·cᵢ ≠ 0`, which is what catches `Fraction(str(x))`-style conversions of a rounded value. 20. **`zeilberger` does not claim its order is minimal** (since 3.9). The search visits `(order, degree)` cheapest-first, so it can reach a cheap order-2 probe before an expensive order-1 one; `cert.order_is_minimal` is `False` to say *not established*, never "a lower order exists". Pass `minimal=True` for an order-ascending search that does establish it — it costs the low-order sweep the default plan skips (Franel at `max_degree=16`: 0.23 s → 9.7 s), so claim minimality against the smallest `max_degree` you are willing to state. 21. **`guess_holonomic` returns `None` only for a swept grid** (since 3.9). It fits a P-recursive recurrence to exact `int`/`Fraction` terms, but only where the terms *over-determine* the ansatz — twice the unknowns by default — and reports `surplus_terms`, the equations that confirmed the fit without being needed. Too few terms to test the whole grid is `E-HOLO-005`, a refusal, not `None`; recording it as "not holonomic" closes a branch that was never explored. `float` terms are refused outright. diff --git a/docs/mdbook/src/errors.md b/docs/mdbook/src/errors.md index 37722253..071402ec 100644 --- a/docs/mdbook/src/errors.md +++ b/docs/mdbook/src/errors.md @@ -91,6 +91,25 @@ Raised when a mathematical side condition is violated. | `E-SOLVE-002` | High-degree univariate factor (> 2) | Symbolic solution not supported; use numerical solve | | `E-SOLVE-003` | Gröbner basis did not terminate | Increase node/iteration limits | +### PslqError (E-PSLQ-*) + +Raised by `alkahest.guess_relation`, the augmented-lattice integer-relation heuristic, and +reported without an exception by `alkahest.relation_confidence`. + +| Code | Cause | Remediation | +|---|---|---| +| `E-PSLQ-001` | Fewer than two constants supplied | Pass at least two constants that might admit a linear dependence | +| `E-PSLQ-002` | Every constant truncated to zero at the working precision | Use higher precision, or supply the constants as decimal strings | +| `E-PSLQ-003` | Working precision below the engine's 64-bit floor | Allocate at least 64 MPFR bits; ≈664 bits ≈ 200 decimal digits | +| `E-PSLQ-004` | The relation found is **larger than the inputs' precision can justify** — it was purchasable from the available digits and is evidence of nothing | Supply the constants at the precision they were computed to, declare their real accuracy with `digits=`, or pass `check_precision=False` to accept the relation unjudged | +| `E-PSLQ-005` | The constants are exact rationals and the relation is **false for them**: `Σ aᵢ·cᵢ` evaluated in exact arithmetic is not zero | The constants are probably truncations of a numerical computation rather than the values you mean — declare their accuracy with `digits=`, or supply more of them | + +`E-PSLQ-004` and `E-PSLQ-005` are raised from Python (`alkahest.guess_relation`), so they +are not in the Rust `REGISTRY`; both are subclasses of `PslqError` and are caught by +`except alkahest.PslqError`. The two are deliberately distinct: `004` is a statement about +how much precision the inputs carry, `005` is a statement about the relation itself and +does not depend on precision at all. + ### PrimaryDecompositionError (E-IDEAL-*) | Code | Cause | Remediation | @@ -116,6 +135,8 @@ loop must record as **undecided**, never as a negative result. | `E-IDEAL-006` | `IdealRefusal` | `primary_decomposition` reached a component it cannot show is primary, so it will not report the ideal itself with an unjustified `associated_prime` | | `E-SOLVE-004` | `TriangularizeRefusal` | `triangularize` extracted a chain that does not generate an ideal containing the input, i.e. one that cuts out a larger variety than the system. Splitting on the initials (Lazard–Kalkbrener) is not implemented | | `E-SERIES-003` | `SeriesError` | `series` ran past its work ceiling (or an active `Budget`) before reaching the requested order. Coefficients are formed by repeated differentiation without re-simplifying, so a nested radical's derivatives grow by a constant factor each time; a *shorter* series would carry an `O(h^order)` label nothing bounded | +| `E-PSLQ-004` | `PslqError` | `guess_relation` found an integer relation the inputs' precision cannot justify — pinning down `n` coefficients bounded by `H` costs about `n·log10(2H+1)` digits of agreement, and the inputs do not carry that many. **Record it as `undecided`, not as "no relation exists":** the same constants at higher precision may well admit one. `relation_confidence` reports the same judgement as data, including a three-valued `credible` whose `None` means *the inputs' precision is not knowable*, never a pass | +| `E-PSLQ-005` | `PslqError` | The constants are exact rationals and `Σ aᵢ·cᵢ` is not zero in exact arithmetic. **This one is a verdict, not a refusal** — the relation is refuted for the numbers supplied | | `E-INT-004` | `IntegrationError` | Proven non-elementary. **This one is a verdict, not a refusal** — keep it apart from the rest | | `E-BUDGET-001..003` | `BudgetExceededError` | Ran out of the time/steps it was given, or was cancelled | @@ -203,6 +224,8 @@ Every error is classified on two independent axes: **subsystem** (determines the | `E-ODE-*` | `OdeError` | ODE construction, lowering, event handling | | `E-DAE-*` | `DaeError` | DAE structural analysis (Pantelides, index reduction) | | `E-SOLVE-*` | `SolverError` | Polynomial system solving, Gröbner basis | +| `E-LAT-*` | `LatticeError` | Exact LLL lattice reduction over ℤ | +| `E-PSLQ-*` | `PslqError` | Integer-relation search (`guess_relation`); `E-PSLQ-004` is the input-precision refusal and `E-PSLQ-005` the exact refutation | | `E-JIT-*` | `JitError` | LLVM/Cranelift codegen and linking | | `E-CUDA-*` | `CudaError` | NVPTX compile, kernel launch, driver/runtime failures | | `E-POOL-*` | `PoolError` | `ExprPool` misuse (closed, cross-pool, persisted-handle mismatch) | diff --git a/examples/pslq_research_loop.py b/examples/pslq_research_loop.py index 1f6a8858..3ee8c567 100644 --- a/examples/pslq_research_loop.py +++ b/examples/pslq_research_loop.py @@ -99,15 +99,17 @@ def run_loop() -> ClaimGraph: # -- stage 2: integer-relation detection ------------------------------ constants = [str(integral), str(log_two)] - relation = ak.guess_relation(constants, 180, 10_000) + # These are decimal strings, so nothing in them tells an exact rational from + # a truncation, and both `guess_relation` and `relation_confidence` answer + # "unknown" on their own. Only this loop knows the strings are quadrature + # output good to DIGITS places, so only this loop can declare it -- `digits=` + # is that declaration, and `180` is the width of the search, not a claim + # about the data. Without it the search cannot refuse, and `None` is not a + # pass. + relation = ak.guess_relation(constants, 180, 10_000, digits=DIGITS) if relation is None: raise SystemExit("no integer relation found; nothing to conjecture") - # These are decimal strings, so `relation_confidence` cannot tell an exact - # rational from a truncation on its own and answers `credible: None`. Only - # this loop knows the strings are quadrature output good to DIGITS places, - # so only this loop can declare it. Without the declaration there is no - # verdict to promote on -- and `None` is not a pass. confidence = ak.relation_confidence(constants, relation, digits=DIGITS) if not confidence["credible"]: raise SystemExit( diff --git a/python/alkahest/__init__.py b/python/alkahest/__init__.py index dd95e7ce..c049039c 100644 --- a/python/alkahest/__init__.py +++ b/python/alkahest/__init__.py @@ -1,8 +1,8 @@ import functools as _functools import math import numbers as _numbers -import sys as _sys from contextlib import suppress as _suppress +from fractions import Fraction as _Fraction from importlib.metadata import PackageNotFoundError as _PackageNotFoundError from importlib.metadata import version as _meta_version @@ -1375,29 +1375,33 @@ def _supplied_bits(value) -> "tuple[float | None, str]": which. That is not the same as exact, and conflating the two is what made :func:`relation_confidence` a gate that could not fail. - ============================ ================================== - Input Verdict - ============================ ================================== - ``float`` 53 bits, whatever it is printed as - ``mpmath.mpf`` its context's working precision - ``int``, ``Fraction`` exact - ``str``, ``Decimal``, other unknown - ============================ ================================== + ================================== ================================== + Input Verdict + ================================== ================================== + ``float`` 53 bits, whatever it is printed as + ``int``, ``Fraction`` exact + ``mpmath.mpf``, ``str``, other unknown + ================================== ================================== A decimal string is the case that matters. ``"3.14159265358979"`` is a perfectly exact rational *and* the way every PSLQ caller spells a truncated numerical constant; the string cannot distinguish the two, so neither can this function. Pass ``digits=`` / ``precision_bits=`` to declare which. + + **An ``mpmath.mpf`` is the same case wearing a type.** It reports + ``value.context.prec`` — the *ambient* working precision at the moment it is + asked, not a property of the value — so a set of ``mpf`` objects computed at + 16 digits and judged after an unrelated ``mp.dps = 300`` claimed 301 digits + and blessed the very relation this guard exists to refuse + (``temp-alkahest/testing/autoresearch-issues-2026-08-19.md`` §4). Nor is the + accuracy recoverable from the object: every ``mpf`` is *exactly* a dyadic + rational, so ``mpf(1)`` and a 300-digit ``mpf`` of π are equally "exact" and + equally silent about how many of their digits mean anything. Unknown is the + only honest verdict, and ``digits=`` is how a caller who knows better says + so — the same decision this gate already makes for a decimal string. """ if isinstance(value, float): return float(_FLOAT_SIGNIFICAND_BITS), "float" - mpmath = _sys.modules.get("mpmath") - if mpmath is not None: - mpf = getattr(mpmath, "mpf", None) - if mpf is not None and isinstance(value, mpf): - context = getattr(value, "context", None) or getattr(mpmath, "mp", None) - prec = getattr(context, "prec", None) - return (float(prec) if prec else float(_FLOAT_SIGNIFICAND_BITS)), "mpmath" if isinstance(value, _numbers.Rational): # An int or a Fraction *is* the rational it spells; there is no # truncation for it to be hiding. @@ -1405,6 +1409,25 @@ def _supplied_bits(value) -> "tuple[float | None, str]": return None, "unknown" +def _exact_residual(constants, coeffs): + """``Σ aᵢ·cᵢ`` as an exact :class:`~fractions.Fraction`, or ``None``. + + ``None`` means at least one constant is not an exact rational, so the sum + would be an approximation of an approximation and could not refute anything. + A ``float`` and an ``mpmath.mpf`` are *representable* as exact dyadic + rationals but are not exact *values*: a nonzero residual is what a genuine + relation among rounded inputs looks like, so they are deliberately excluded. + """ + if len(coeffs) != len(constants): + return None + total = _Fraction(0) + for coefficient, constant in zip(coeffs, constants): + if not isinstance(constant, _numbers.Rational): + return None + total += _Fraction(coefficient) * _Fraction(constant) + return total + + def _relation_available_digits(constants, digits=None, precision_bits=None): """Decimal digits of precision the inputs are *known* to carry. @@ -1464,15 +1487,29 @@ def __init__(self, message: str, remediation: str): self.remediation = remediation +class _RelationFalseError(PslqError): + """``E-PSLQ-005`` — the relation is false for the exact values supplied. + + Distinct from ``E-PSLQ-004``: nothing here is short of precision. The inputs + are exact rationals and ``Σ aᵢ·cᵢ`` was evaluated in exact arithmetic and is + not zero, so the relation is refuted rather than merely unaffordable. + """ + + def __init__(self, message: str, remediation: str): + super().__init__(message) + self.code = "E-PSLQ-005" + self.remediation = remediation + + def _relation_is_credible(constants, coeffs, digits=None, precision_bits=None, margin_digits=None): """Judge a *found* relation against the precision its inputs actually have. This is the standard experimental-mathematics criterion, and it uses evidence rather than guessing what the caller meant. Pinning down `n` - coefficients of magnitude `H` consumes about `n·log10(H)` digits of - agreement. If that (plus a safety margin) exceeds the digits the inputs - carry, the relation was *purchasable* from the available precision and is - evidence of nothing. + coefficients bounded by `H` picks one of the `(2H+1)ⁿ` integer vectors in + that box, so it consumes about `n·log10(2H+1)` digits of agreement. If that + (plus a safety margin) exceeds the digits the inputs carry, the relation was + *purchasable* from the available precision and is evidence of nothing. ``credible`` is ``None`` — *unknown*, never ``True`` — when the inputs' precision cannot be established; see :func:`_supplied_bits`. Unknown is not @@ -1480,9 +1517,15 @@ def _relation_is_credible(constants, coeffs, digits=None, precision_bits=None, m bounds the whole set (precision is a ``min``), so a relation that already exceeds that bound is refuted outright rather than reported as unknown. + Exact inputs are the mirror image: precision is infinite, so no affordability + test can ever fire and ``credible`` was unconditionally ``True`` — the one + branch of this gate that could not fail. It is now decided by *evaluating* + the relation in exact arithmetic, which is what "exact" was always claiming. + Returns ``(credible, available_digits, consumed_digits, margin_digits, - source)``. + source, exact_residual)``. """ + constants = tuple(constants) available, source, ceiling = _relation_available_digits(constants, digits, precision_bits) margin = _DEFAULT_MARGIN_DIGITS if margin_digits is None else float(margin_digits) # NB: `abs`/`min`/`max` in this module's namespace are the *symbolic* @@ -1492,16 +1535,31 @@ def _relation_is_credible(constants, coeffs, digits=None, precision_bits=None, m magnitude = -a if a < 0 else a if magnitude > biggest: biggest = magnitude - consumed = len(coeffs) * math.log10(biggest) if biggest > 1 else 0.0 + # `(2H+1)ⁿ` vectors, not `Hⁿ`: coefficients run over ±H *and zero*, and the + # `Hⁿ` spelling collapses to 0 digits at `H = 1`, calling every relation with + # unit coefficients free whatever its length + # (``autoresearch-issues-2026-08-19.md`` item 26n). + consumed = len(coeffs) * math.log10(2 * biggest + 1) + residual = None + if source == "exact": + # `available` is infinite here, so the affordability test below is + # vacuous — `credible` would be `True` for *any* coefficients, including + # a relation that is simply false for the numbers supplied. + # `Fraction(str(float(pi)))` and friends reached exactly that branch + # (``autoresearch-issues-2026-08-19.md`` §4). Precision cannot be the + # fault for an exact input, but arithmetic can be, so check it. + residual = _exact_residual(constants, coeffs) + if residual is not None and residual != 0: + return False, available, consumed, margin, source, residual if available is None: # Some input's precision is unknown, so we cannot say how much room the # relation has. We can still say it has none: `ceiling` is a genuine # upper bound on the available precision, and unknown neighbours can # only drag the `min` lower, never raise it. if consumed + margin > ceiling: - return False, None, consumed, margin, source - return None, None, consumed, margin, source - return consumed + margin <= available, available, consumed, margin, source + return False, None, consumed, margin, source, residual + return None, None, consumed, margin, source, residual + return consumed + margin <= available, available, consumed, margin, source, residual def relation_confidence( @@ -1509,25 +1567,29 @@ def relation_confidence( ) -> dict: """Judge a found relation against the precision its inputs actually carry. - Pinning down ``n`` coefficients of magnitude ``H`` takes about - ``n·log10(H)`` digits of agreement. When that exceeds the digits the inputs - have, the relation was *purchasable* from the available precision — the - search would have found something whatever the constants were — and it is - evidence of nothing. Because PSLQ returns the *smallest* relation the + Pinning down ``n`` coefficients bounded by ``H`` selects one of the + ``(2H+1)ⁿ`` integer vectors in that box, so it takes about + ``n·log10(2H+1)`` digits of agreement. When that exceeds the digits the + inputs have, the relation was *purchasable* from the available precision — + the search would have found something whatever the constants were — and it + is evidence of nothing. Because PSLQ returns the *smallest* relation the precision can buy, a purchased one lands just under the bound rather than over it, so a relation must clear it by ``margin_digits`` (default 10) to count as credible. **The input's precision has to be knowable, and usually it is not.** A - ``float`` is 53 bits however it is printed, and an ``mpmath.mpf`` reports - its working precision, so those are judged. ``int`` and ``Fraction`` are - exactly the rationals they spell, so precision cannot be the reason a - relation among them is spurious. A **decimal string** — the way every - high-precision constant reaches this library — is *unknown*: the digits - ``"3.14159265358979"`` are equally the exact rational 314159265358979/10¹⁴ - and π truncated to 15 places, and nothing in the string says which. Pass - ``digits=`` (decimal) or ``precision_bits=`` (binary) to say how many of - them are trustworthy; a declaration is a cap, so declaring 200 digits for a + ``float`` is 53 bits however it is printed, so it is judged. ``int`` and + ``Fraction`` are exactly the rationals they spell, so precision cannot be + the reason a relation among them is spurious — instead the relation itself + is evaluated (see ``exact_residual`` below). A **decimal string** — the way + every high-precision constant reaches this library — is *unknown*: the + digits ``"3.14159265358979"`` are equally the exact rational + 314159265358979/10¹⁴ and π truncated to 15 places, and nothing in the string + says which. An ``mpmath.mpf`` is unknown for the same reason: it reports the + *ambient* ``mp.dps`` at the moment it is asked rather than anything about + itself, and its accuracy is not recoverable from the object. Pass + ``digits=`` (decimal) or ``precision_bits=`` (binary) to say how many digits + are trustworthy; a declaration is a cap, so declaring 200 digits for a ``float`` still yields ~16. Returns a dict: @@ -1544,14 +1606,27 @@ def relation_confidence( Digits the inputs are known to carry (``inf`` for exact inputs), or ``None`` when unknown. ``consumed_digits`` - Digits of agreement the relation costs, ``n·log10(H)``. + Digits of agreement the relation costs, ``n·log10(2H+1)``. ``spare_digits`` ``available - consumed``, or ``None`` when unknown. ``margin_digits`` Spare digits demanded of a credible relation. ``precision_source`` - Where ``available_digits`` came from: ``"float"``, ``"mpmath"``, - ``"exact"``, ``"declared"``, or ``"unknown"``. + Where ``available_digits`` came from: ``"float"``, ``"exact"``, + ``"declared"``, or ``"unknown"``. + ``exact_residual`` + ``Σ aᵢ·cᵢ`` as an exact :class:`~fractions.Fraction` when every constant + is an exact rational and no precision was declared, otherwise ``None``. + A nonzero value **refutes** the relation, and is the one way + ``credible`` can be ``False`` while ``available_digits`` is ``inf``: + nothing is short of precision, the relation is just false. + + Note what ``credible: True`` means on the exact branch: the relation holds + for **the numbers supplied**. If those numbers are rounded stand-ins for + something else — ``Fraction(*float(pi).as_integer_ratio())`` is exactly the + double, not π — then so is the verdict, and no gate reading only the + constants can tell. Declare ``digits=`` when they are stand-ins; that is + what turns them back into approximations with a precision to be judged. >>> import alkahest as ak >>> ak.relation_confidence([1.0, 2.0, 3.0], [1, 1, -1])["credible"] @@ -1559,6 +1634,13 @@ def relation_confidence( >>> ak.relation_confidence([0.1, 0.2, 0.7], [60771139, 67263243, 11653676])["credible"] False + Exact inputs are judged by evaluating the relation, not by counting digits: + + >>> ak.relation_confidence([1, 2, 3], [1, 1, -1])["credible"] + True + >>> ak.relation_confidence([1, 2, 3], [1, 1, 1])["credible"] + False + A decimal string is not judged at all until its precision is declared, and a ten-digit-per-coefficient relation does not survive a 20-digit declaration — it is exactly what 20 digits can buy: @@ -1573,7 +1655,7 @@ def relation_confidence( >>> ak.relation_confidence([pi_20, e_20], big, digits=60)["credible"] True """ - credible, available, consumed, margin, source = _relation_is_credible( + credible, available, consumed, margin, source, residual = _relation_is_credible( constants, coeffs, digits, precision_bits, margin_digits ) return { @@ -1583,15 +1665,19 @@ def relation_confidence( "margin_digits": margin, "credible": credible, "precision_source": source, + "exact_residual": residual, } -def guess_relation(constants, precision_bits=664, max_abs_coeff=None, check_precision=True): +def guess_relation( + constants, precision_bits=664, max_abs_coeff=None, check_precision=True, *, digits=None +): """Search for integers ``aᵢ`` with ``Σ aᵢ·constantsᵢ ≈ 0``. Raises :class:`PslqError` (``E-PSLQ-004``) when the relation found is larger than the inputs' precision can justify, rather than returning a number that - is an artifact of how the inputs were written. + is an artifact of how the inputs were written, and ``E-PSLQ-005`` when the + inputs are exact rationals and the relation is simply false for them. Passing ``float`` values — 53 bits, ~16 digits — while searching at the 664-bit default zero-pads them into exact rationals, among which exact @@ -1609,14 +1695,23 @@ def guess_relation(constants, precision_bits=664, max_abs_coeff=None, check_prec still come back: ``[1.0, 2.0, 3.0]`` needs almost no precision to pin down and is returned normally. See :func:`relation_confidence` for the numbers. - **The guard can only fire on inputs whose precision is knowable** — floats - and ``mpmath.mpf`` values. A decimal string may be exact or may be a - truncation, so a relation among strings is returned *unjudged*, not - endorsed. When the strings are truncations of a numerical computation, put - the result through - :func:`relation_confidence(constants, coeffs, digits=…) - ` with the number of digits they are actually accurate - to; that is where a purchasable relation gets caught. + **The guard can only fire on inputs whose precision is knowable** — floats, + exact rationals, and anything the caller declares with ``digits=``. A + decimal string or an ``mpmath.mpf`` may be exact or may be a truncation, so + a relation among them is returned *unjudged*, not endorsed. + + ``digits=`` is the escape hatch, and is what a caller who *knows* the + accuracy of their inputs should reach for. Note that ``precision_bits`` is + the width of the *search*, not a claim about the data; ``digits=`` declares + how many decimal digits of the constants are trustworthy, exactly as in + :func:`relation_confidence`:: + + guess_relation([mpf(x) for x in floats], digits=16) # raises E-PSLQ-004 + guess_relation(two_hundred_digit_strings, digits=200) # judged, and passes + + Without it, a relation among strings or ``mpf`` values comes back unjudged; + put such a result through :func:`relation_confidence(constants, coeffs, + digits=…) ` if you prefer a verdict to an exception. Pass ``check_precision=False`` to accept a relation among the supplied values themselves. @@ -1624,21 +1719,41 @@ def guess_relation(constants, precision_bits=664, max_abs_coeff=None, check_prec coeffs = _native_guess_relation(constants, precision_bits, max_abs_coeff) if coeffs is None or not check_precision: return coeffs - credible, available, consumed, margin, _source = _relation_is_credible(constants, coeffs) - if credible is False: - raise _RelationPrecisionError( - f"the relation {coeffs} needs ~{consumed:.0f} digits of agreement to pin " - f"down (plus a {margin:.0f}-digit margin), but the inputs carry only " - f"~{available:.0f}; a relation this size is purchasable from the available " - "precision and is evidence of nothing", + credible, available, consumed, margin, _source, residual = _relation_is_credible( + constants, coeffs, digits=digits + ) + if credible is not False: + return coeffs + if residual is not None and residual != 0: + raise _RelationFalseError( + f"the relation {coeffs} is false for the values supplied: evaluated in exact " + f"rational arithmetic, Σ aᵢ·cᵢ = {float(residual):.6g}, not 0. The constants are " + "exact rationals, so this is not a shortfall of precision — the relation simply " + "does not hold for them", ( - "supply the constants as high-precision decimal strings — a float " - f"carries only ~{_FLOAT_SIGNIFICAND_BITS} bits (~{available:.0f} digits) " - "however it is printed; pass check_precision=False to accept a relation " - "among the supplied values themselves" + "if the constants are truncations of a numerical computation rather than the " + "values you mean, declare the accuracy they really carry with digits=, or " + "supply more of them; pass check_precision=False to accept the relation " + "unjudged" ), ) - return coeffs + if available is None: + # Refuted by the *ceiling* — one input's precision bounded the whole + # set even though another's is unknown; report the bound, not "None". + available = _relation_available_digits(constants, digits)[2] + raise _RelationPrecisionError( + f"the relation {coeffs} needs ~{consumed:.0f} digits of agreement to pin " + f"down (plus a {margin:.0f}-digit margin), but the inputs carry only " + f"~{available:.0f}; a relation this size is purchasable from the available " + "precision and is evidence of nothing", + ( + "supply the constants as high-precision decimal strings — a float " + f"carries only ~{_FLOAT_SIGNIFICAND_BITS} bits " + f"(~{_FLOAT_SIGNIFICAND_BITS / _BITS_PER_DIGIT:.0f} digits) however it is " + "printed — and declare their accuracy with digits=; pass check_precision=False " + "to accept a relation among the supplied values themselves" + ), + ) _native_poly_normal = poly_normal diff --git a/python/alkahest/exceptions.py b/python/alkahest/exceptions.py index 10d954ea..b2a347f5 100644 --- a/python/alkahest/exceptions.py +++ b/python/alkahest/exceptions.py @@ -19,7 +19,10 @@ E-SOLVE-010 … E-SOLVE-011 SolverError (GPU Gröbner) E-JIT-001 … E-JIT-003 JitError E-LAT-001 … E-LAT-004 LatticeError - E-PSLQ-001 … E-PSLQ-004 PslqError (004 = input precision below requested) + E-PSLQ-001 … E-PSLQ-005 PslqError (004 = input precision below requested, + 005 = the relation is false for the exact rationals + supplied; both raised from Python, so both are absent + from the Rust REGISTRY) E-CAD-001 CadError E-ROOT-001 … E-ROOT-002 RealRootError (V2-4 VAS real root isolation) E-RES-001 … E-RES-003 ResultantError (V2-2) @@ -431,7 +434,9 @@ class PslqError(AlkahestError): """Integer-relation heuristic failed (input, coefficient bound, or lattice step). ``E-PSLQ-004`` is raised when the supplied constants carry less precision - than the search requests — see :func:`alkahest.guess_relation`. + than the relation found needs, and ``E-PSLQ-005`` when the constants are + exact rationals and the relation is simply false for them — see + :func:`alkahest.guess_relation`. """ def __init__( diff --git a/tests/test_relation_precision_guard.py b/tests/test_relation_precision_guard.py index 0cecd59c..cafb4c95 100644 --- a/tests/test_relation_precision_guard.py +++ b/tests/test_relation_precision_guard.py @@ -18,9 +18,10 @@ false lemma. The guard judges the *relation that was found*, not the input alone: pinning -down `n` coefficients of magnitude `H` takes about `n·log10(H)` digits of -agreement, and when that exceeds the digits the inputs carry, the relation was -purchasable from the available precision and is evidence of nothing. +down `n` coefficients bounded by `H` picks one of the `(2H+1)ⁿ` integer vectors +in that box, so it takes about `n·log10(2H+1)` digits of agreement, and when that +exceeds the digits the inputs carry, the relation was purchasable from the +available precision and is evidence of nothing. Testing the result rather than guessing the caller's intent is what lets `[1.0, 2.0, 3.0]` keep working: that relation costs almost no precision to pin @@ -35,13 +36,24 @@ (``temp-alkahest/testing/autoresearch-issues-2026-08-13.md`` §2). A gate that cannot fail is worse than no gate, because loop authors wire it into promotion logic. It now answers ``credible: None`` — *unknown* — unless the input's -precision is knowable (``float``, ``mpmath.mpf``) or declared (``digits=`` / -``precision_bits=``). +precision is knowable (``float``) or declared (``digits=`` / ``precision_bits=``). + +The 2026-08-19 run then found that the gate read the input's *type*, and that +four value-preserving conversions change the type without changing the value +(``autoresearch-issues-2026-08-19.md`` §4). ``mpmath.mpf`` reported the *ambient* +``mp.dps`` rather than anything about itself, so it now answers *unknown* like a +decimal string; and ``int``/``Fraction`` inputs, whose infinite ``available`` +made ``credible: True`` unfalsifiable, are now decided by evaluating +``Σ aᵢ·cᵢ`` in exact arithmetic. ``guess_relation`` gained the ``digits=`` +declaration it was missing, so the entry point that raises the refusal finally +has an escape hatch. """ from __future__ import annotations import math +from decimal import Decimal +from fractions import Fraction import alkahest as ak import pytest @@ -153,11 +165,17 @@ def test_relation_confidence_accepts_a_cheap_relation(self): assert verdict["spare_digits"] > 0 def test_exact_inputs_are_not_doubted_on_precision_grounds(self): - """An int *is* the rational it spells, so precision cannot be the fault.""" - verdict = ak.relation_confidence([1, 2, 3], [10**9, -(10**9), 1]) + """An int *is* the rational it spells, so precision cannot be the fault. + + What decides the exact branch instead is whether the relation is *true* + — see :class:`TestExactInputsAreEvaluatedNotAssumed`. A relation with + coefficients far larger than the constants is fine as long as it holds. + """ + verdict = ak.relation_confidence([1, 2, 3], [10**9, 10**9, -(10**9)]) assert verdict["credible"] is True assert verdict["available_digits"] == math.inf assert verdict["precision_source"] == "exact" + assert verdict["exact_residual"] == 0 class TestUnknownPrecisionIsNotAPass: @@ -226,15 +244,26 @@ def test_the_margin_is_what_catches_a_purchased_relation(self): assert raw["credible"] is True assert raw["consumed_digits"] < raw["available_digits"] - def test_mpmath_values_carry_their_working_precision(self): + def test_mpmath_values_are_unknown_until_their_accuracy_is_declared(self): + """An ``mpf`` is a decimal string with a type on it. + + It used to report ``value.context.prec``, which is the *ambient* + ``mp.dps`` when it is asked rather than anything about the value — see + :class:`TestAmbientPrecisionCannotDecideTheVerdict`. Declaring the + accuracy is what turns it back into a judgement. + """ mpmath = pytest.importorskip("mpmath") with mpmath.workprec(80): # ~24 digits constants = [+mpmath.pi, +mpmath.e] - verdict = ak.relation_confidence(constants, [5144503108, -5945642943]) - assert verdict["precision_source"] == "mpmath" - assert verdict["available_digits"] == pytest.approx(24.1, abs=0.5) - assert verdict["credible"] is False - assert ak.relation_confidence(constants, [1, -1])["credible"] is True + expensive = [5144503108, -5945642943] + verdict = ak.relation_confidence(constants, expensive) + assert verdict["precision_source"] == "unknown" + assert verdict["available_digits"] is None + assert verdict["credible"] is None + declared = ak.relation_confidence(constants, expensive, digits=24) + assert declared["precision_source"] == "declared" + assert declared["credible"] is False + assert ak.relation_confidence(constants, [1, -1], digits=24)["credible"] is True def test_one_known_input_can_refute_without_the_rest_being_known(self): """Unknown is not a licence to give up. @@ -273,3 +302,308 @@ def test_guess_relation_still_returns_unjudged_string_relations(self): assert ( ak.relation_confidence([SQRT2_60, TWO_SQRT2_60], coeffs, digits=60)["credible"] is True ) + + +# --------------------------------------------------------------------------- +# The 2026-08-19 autoresearch run, issue #4: four value-preserving conversions +# each switched this guard off, and the `exact` branch certified a relation that +# is false for the very numbers supplied. +# --------------------------------------------------------------------------- + +#: The spurious relation the 2026-08-13 run produced from `float(pi)`, +#: `float(e)`, `float(log 2)` — the one this whole guard exists to refuse. It +#: holds exactly among the three *doubles*, because 664-bit zero-padding makes +#: them exact rationals; re-evaluated against pi, e and log 2 themselves at 300 +#: digits the residual is ~4.8e-9, so it is not a relation among the constants +#: anyone meant to supply. +SPURIOUS_FLOAT_RELATION = [-60771139, 67263243, 11653676] + + +def _float_constants() -> list[float]: + """`pi`, `e`, `log 2` as plain doubles — ~16 digits, however they print.""" + return [float(math.pi), float(math.e), float(math.log(2))] + + +def _exact_spellings() -> dict[str, list[Fraction]]: + """The same three doubles, spelled as exact rationals three ordinary ways. + + Each is a conversion a caller writes without a second thought — and each one + changes the input's *type* without changing what the caller believes the + numbers are, routing them to ``precision_source: "exact"`` and an + unfalsifiable ``credible: True``. + """ + mpmath = pytest.importorskip("mpmath") + floats = _float_constants() + with mpmath.workdps(40): + truncated = [ + Fraction(Decimal(mpmath.nstr(v, 20))) for v in (mpmath.pi, mpmath.e, mpmath.log(2)) + ] + return { + "Fraction(str(x))": [Fraction(str(x)) for x in floats], + "Fraction(Decimal(repr(x)))": [Fraction(Decimal(repr(x))) for x in floats], + "20-digit nstr truncation": truncated, + } + + +class TestConversionsThatPreserveTheValueMustNotSwitchTheGuardOff: + """The guard read the input's *type*, and ``mpf(x)`` changes the type without + changing the value. At ``mp.dps = 300`` the lifted floats reported 301 + available digits — read off the ambient working precision — and the relation + the release was written to refuse came back ``credible: True`` with 277 + "spare" digits. + """ + + def test_lifting_floats_to_mpf_does_not_buy_precision(self): + mpmath = pytest.importorskip("mpmath") + with mpmath.workdps(300): + lifted = [mpmath.mpf(x) for x in _float_constants()] + verdict = ak.relation_confidence(lifted, SPURIOUS_FLOAT_RELATION) + assert verdict["credible"] is not True + assert verdict["precision_source"] == "unknown" + assert verdict["available_digits"] is None + + def test_guess_relation_does_not_endorse_the_lifted_floats(self): + """It may still *return* the relation — unknown precision is not a + refusal — but nothing downstream may read it as endorsed.""" + mpmath = pytest.importorskip("mpmath") + with mpmath.workdps(300): + lifted = [mpmath.mpf(x) for x in _float_constants()] + coeffs = ak.guess_relation(lifted) + assert coeffs is not None + assert ak.relation_confidence(lifted, coeffs)["credible"] is not True + + def test_declaring_the_real_accuracy_refuses_the_lifted_floats(self): + """The values came out of doubles, so 16 digits is the truth about them.""" + mpmath = pytest.importorskip("mpmath") + with mpmath.workdps(300): + lifted = [mpmath.mpf(x) for x in _float_constants()] + assert ( + ak.relation_confidence(lifted, SPURIOUS_FLOAT_RELATION, digits=16)["credible"] + is False + ) + + +class TestAmbientPrecisionCannotDecideTheVerdict: + """``_supplied_bits`` used ``value.context.prec`` — the ambient ``mp.dps`` + at the moment of asking, not a property of the value. The same objects + judged before and after an unrelated ``mp.dps = 300`` got opposite verdicts: + ``False`` at 16 digits, ``True`` at 300, with nothing about the numbers + changed. A gate whose answer moves with a global in another library is not + reporting evidence. + """ + + def test_the_verdict_does_not_move_with_mp_dps(self): + mpmath = pytest.importorskip("mpmath") + with mpmath.workdps(16): + constants = [+mpmath.pi, +mpmath.e, +mpmath.log(2)] + low = ak.relation_confidence(constants, SPURIOUS_FLOAT_RELATION) + with mpmath.workdps(300): + high = ak.relation_confidence(constants, SPURIOUS_FLOAT_RELATION) + assert low["credible"] == high["credible"] + assert low["available_digits"] == high["available_digits"] + assert low["precision_source"] == high["precision_source"] + assert high["credible"] is not True + + def test_only_a_declaration_moves_it_and_it_moves_the_same_way(self): + mpmath = pytest.importorskip("mpmath") + for dps in (16, 300): + with mpmath.workdps(dps): + constants = [+mpmath.pi, +mpmath.e, +mpmath.log(2)] + with mpmath.workdps(300): + verdict = ak.relation_confidence(constants, SPURIOUS_FLOAT_RELATION, digits=16) + assert verdict["credible"] is False, f"computed at dps={dps}" + + +class TestExactInputsAreEvaluatedNotAssumed: + """``precision_source: "exact"`` was an unfalsifiable pass. + + ``available_digits`` is ``inf`` on that branch, so no affordability test can + fire and ``credible`` was ``True`` for *any* coefficients — including a + relation whose exact residual is not zero. Precision genuinely cannot be the + fault for an exact rational, but *arithmetic* can be, and one line of + ``Fraction`` arithmetic settles it. This is what makes the exact branch + falsifiable for the first time. + """ + + @pytest.mark.parametrize("spelling", ["Fraction(str(x))", "Fraction(Decimal(repr(x)))"]) + def test_a_false_relation_among_exact_rationals_is_refuted(self, spelling): + constants = _exact_spellings()[spelling] + residual = sum(Fraction(a) * c for a, c in zip(SPURIOUS_FLOAT_RELATION, constants)) + assert residual != 0, "the premise: this relation is false for these very numbers" + verdict = ak.relation_confidence(constants, SPURIOUS_FLOAT_RELATION) + assert verdict["credible"] is False + assert verdict["precision_source"] == "exact" + assert verdict["exact_residual"] == residual + + def test_a_truncated_decimal_read_back_as_a_fraction_is_refuted(self): + """`Fraction(Decimal(nstr(pi, 20)))` is exactly a 20-digit truncation — + an exact rational that is *not* the constant it stands for, and the + relation found among such truncations does not hold for them.""" + constants = _exact_spellings()["20-digit nstr truncation"] + with pytest.raises(ak.PslqError) as excinfo: + ak.guess_relation(constants) + assert excinfo.value.code == "E-PSLQ-005" + assert "false for the values supplied" in str(excinfo.value) + coeffs = ak.guess_relation(constants, check_precision=False) + verdict = ak.relation_confidence(constants, coeffs) + assert verdict["credible"] is False + assert verdict["exact_residual"] != 0 + + def test_the_refusal_says_what_to_do_about_it(self): + constants = _exact_spellings()["Fraction(str(x))"] + with pytest.raises(ak.PslqError) as excinfo: + ak.guess_relation(constants) + assert excinfo.value.code == "E-PSLQ-005" + assert "digits=" in excinfo.value.remediation + + def test_a_true_relation_among_exact_rationals_still_passes(self): + """The control. Refusing every exact input would be the same + can't-fail gate with the sign flipped. + + Dyadic denominators so that the search — which reaches the constants + through ``f64`` — sees the same numbers the verdict is computed from. + """ + constants = [Fraction(1, 2), Fraction(1, 4), Fraction(3, 4)] + verdict = ak.relation_confidence(constants, [1, 1, -1]) + assert verdict["credible"] is True + assert verdict["exact_residual"] == 0 + coeffs = ak.guess_relation(constants) + assert coeffs is not None + assert sum(Fraction(a) * c for a, c in zip(coeffs, constants)) == 0 + + def test_a_declaration_turns_the_exact_check_off(self): + """``digits=`` says "treat these rationals as approximations", so the + affordability test decides and no exact residual is reported.""" + constants = _exact_spellings()["Fraction(str(x))"] + verdict = ak.relation_confidence(constants, SPURIOUS_FLOAT_RELATION, digits=200) + assert verdict["exact_residual"] is None + assert verdict["precision_source"] == "declared" + assert verdict["credible"] is True + + +class TestTheKnownWrongFixStaysOut: + """`min(bitcount(mantissa), context.prec)` was the finder's proposed fix. + + It stops the attack and reports **0.30 digits** for ``mpf(1), mpf(2), + mpf(3)``, flipping the true relation ``[1, 1, -1]`` to ``credible: False``. + Every ``mpf`` is exactly a dyadic rational, so accuracy is not recoverable + from the object and a mantissa bitcount measures how *round* a number is, + not how accurate. A gate that refutes a true relation is a worse failure + than one that shrugs at a false one, so this is an explicit guard: the + verdict here may be ``True`` or unknown, and must never be ``False``. + """ + + def test_mpf_one_two_three_is_not_refuted(self): + mpmath = pytest.importorskip("mpmath") + constants = [mpmath.mpf(1), mpmath.mpf(2), mpmath.mpf(3)] + verdict = ak.relation_confidence(constants, [1, 1, -1]) + assert verdict["credible"] is not False + # Unknown, on the same grounds as a decimal string: nothing in an `mpf` + # says how many of its digits mean anything. + assert verdict["credible"] is None + assert ak.guess_relation(constants) is not None + + def test_declaring_any_plausible_precision_makes_it_credible(self): + """And the escape hatch confirms the relation really is cheap: 3 + coefficients bounded by 1 cost 1.4 digits, so any declaration from ~12 + digits up clears the 10-digit margin.""" + mpmath = pytest.importorskip("mpmath") + constants = [mpmath.mpf(1), mpmath.mpf(2), mpmath.mpf(3)] + for digits in (16, 50, 300): + assert ak.relation_confidence(constants, [1, 1, -1], digits=digits)["credible"] is True + + def test_the_integer_spelling_of_the_same_relation_is_credible(self): + """`[1, 2, 3]` as ints has no precision question at all, and the exact + evaluation added for issue #4 confirms the relation rather than + assuming it.""" + verdict = ak.relation_confidence([1, 2, 3], [1, 1, -1]) + assert verdict["credible"] is True + assert verdict["exact_residual"] == 0 + + +class TestTheCostFormulaDoesNotCollapseAtUnitCoefficients: + """Item 26n. ``n·log10(H)`` is 0 for every relation with ``H = 1``, however + many constants it spans, so a 40-term ±1 relation was free. The count is + ``(2H+1)ⁿ`` — coefficients run over ±H *and zero* — so the cost is + ``n·log10(2H+1)``, which is ``n·log10(3)`` at ``H = 1``. + """ + + @pytest.mark.parametrize("n", [2, 3, 8, 40]) + def test_unit_coefficients_are_not_free(self, n): + coeffs = [1 if k % 2 else -1 for k in range(n)] + consumed = ak.relation_confidence(_decimal_strings(n, 60), coeffs)["consumed_digits"] + assert consumed == pytest.approx(n * math.log10(3)) + assert consumed > 0 + + def test_a_long_unit_relation_is_refused_at_low_precision(self): + """The end the collapsed formula could not reach: 40 ±1 coefficients + cost ~19 digits, which 16 digits of float cannot buy.""" + n = 40 + coeffs = [1 if k % 2 else -1 for k in range(n)] + verdict = ak.relation_confidence([0.1 * k + 0.5 for k in range(n)], coeffs) + assert verdict["consumed_digits"] == pytest.approx(40 * math.log10(3)) + assert verdict["credible"] is False + + def test_the_general_form_is_still_the_box_count(self): + n, h = 5, 65 + coeffs = [h, -h, 1, 0, -1] + consumed = ak.relation_confidence(_decimal_strings(n, 60), coeffs)["consumed_digits"] + assert consumed == pytest.approx(n * math.log10(2 * h + 1)) + + +class TestGuessRelationHasAnEscapeHatch: + """`digits=` rescued `relation_confidence`, but `guess_relation` — the entry + point that actually raises `E-PSLQ-004` — had no way to say what the inputs + are worth: its `precision_bits` is the width of the *search*. A caller who + genuinely knew their input precision had no move except turning the guard + off entirely with `check_precision=False`. + """ + + def test_digits_lets_a_high_precision_caller_through(self): + """200-digit strings judged as such: the relation is affordable and is + returned, where without the declaration it comes back unjudged.""" + coeffs = ak.guess_relation([SQRT2_60, TWO_SQRT2_60], precision_bits=BITS_60_DIGITS) + assert coeffs is not None + assert ( + ak.guess_relation([SQRT2_60, TWO_SQRT2_60], precision_bits=BITS_60_DIGITS, digits=60) + == coeffs + ) + + def test_digits_makes_the_guard_fire_on_otherwise_unjudged_input(self): + """The other direction, and the one that matters: declaring the real + accuracy of `mpf` values turns an unjudged return into `E-PSLQ-004`.""" + mpmath = pytest.importorskip("mpmath") + with mpmath.workdps(300): + lifted = [mpmath.mpf(x) for x in _float_constants()] + assert ak.guess_relation(lifted) is not None + with pytest.raises(ak.PslqError) as excinfo: + ak.guess_relation(lifted, digits=16) + assert excinfo.value.code == "E-PSLQ-004" + + def test_digits_refuses_a_purchased_string_relation(self): + """The case the docstring used to send the caller elsewhere for. + + pi, e and log 2 truncated to 20 digits buy a 10-digit-per-coefficient + relation at the 664-bit default. Undeclared, `guess_relation` returns it + unjudged — a decimal string may be exact — and `digits=20` is how the + caller says it is not, in the one call that used to require a second. + """ + constants = [PI_60[:21], E_60[:21], LOG2_60[:22]] + unjudged = ak.guess_relation(constants) + assert unjudged is not None + assert ak.relation_confidence(constants, unjudged)["credible"] is None + with pytest.raises(ak.PslqError) as excinfo: + ak.guess_relation(constants, digits=20) + assert excinfo.value.code == "E-PSLQ-004" + assert "purchasable" in str(excinfo.value) + + def test_digits_is_keyword_only_so_it_cannot_be_confused_with_the_search_width(self): + with pytest.raises(TypeError): + ak.guess_relation([1.0, 2.0, 3.0], 384, None, True, 16) + + def test_digits_and_precision_bits_do_not_collide(self): + """`precision_bits` still means the search width on `guess_relation`, so + passing both is not the mutual-exclusion error `relation_confidence` + raises — they are different quantities here.""" + coeffs = ak.guess_relation([1.0, 2.0, 3.0], precision_bits=384, digits=16) + assert coeffs is not None From 481826ac53eed5765a25e7c6c25f3931f3e5d8ec Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 20:21:51 +0000 Subject: [PATCH 02/11] fix(novelty): read OEIS's real notation, page terms= searches, carry a q-recurrence, cross-check terms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four false-red / coverage defects in the M11 novelty filter (autoresearch 2026-08-19, issues #23, #24, #25, #26q), all visible at once in a single symptom: the filter returned `not_found` for the Fibonacci recurrence against A000045, from 25 candidate lines of which 0 were usable. #24 — the recurrence lives in the entry's *name*, `Fibonacci numbers: F(n) = F(n-1) + F(n-2)`, which nothing read. `OeisEntry.candidate_lines()` now puts the name first; the parser accepts any single letter as the sequence, accepts an identifier passed as `RecurrenceClaim.from_text(..., names=…)` (the entry's own A-number, passed automatically by `_scanned`), and reads juxtaposition as multiplication (`2a(n-2)`). One line may still name only one sequence, so `a(n) = a(n-1) + A002026(n-1)` is refused exactly as before, and every parsed line is still held to reproducing the entry's own terms. Measured over the 377-entry live corpus the run used: 156 -> 252 lines parsed, 121 -> 195 usable statements, 114 -> 174 entries with at least one. #23 — `fmt=json` returns at most ten results with no total count, and `OeisWeb` reported `exhaustive=True` after one page, collapsing `unavailable` into `not_found`. A `terms=` search now continues at `&start=` until a short page arrives (exhaustive; recorded in the cache as a complete answer) or `max_results` is reached (not exhaustive; deliberately *not* recorded, since a truncated page list stored under that key becomes a false negative on every later offline run). `max_results` defaults to 50. An `ids=` lookup is not paged and stays exhaustive after one request, which was always correct. #25 — `QRecurrenceClaim`: normal form, `claim_hash` and equality for `Sum_i c_i(q, q^n)*u(n+i) = 0`, coefficients in Q(q, q^n) cleared to Laurent polynomials. Scale, index shift (which acts on the coefficients, since n -> n+1 sends q^n to q*q^n), a common monomial or polynomial factor, and zero padding are quotiented out; the tag is `q-recurrence/1` against `recurrence/1`, so the hash spaces cannot collide. No source here can state one, so `check_novelty` reports OEIS sources as `unavailable` for it rather than manufacturing a negative — sources declare what they can state with `CLAIM_KINDS`. #26q — `terms=` was lookup-only. It is now also checked against the claim, on the same lenient trailing-window rule a source's own formula line must pass: `NoveltyVerdict.terms_check` is "holds"/"fails"/"not_checked" and appears in `report()`. `check_novelty` takes `start=`, meaning exactly what `RecurrenceClaim.holds_for`'s `start` means; it is never sent to a source. Tests are offline. `tests/data/oeis_novelty_fixture.json` gains A000045 and its `id:A000045` query; `tests/data/oeis_paging_fixture.json` is new — raw `search?...&fmt=json` pages keyed "query|start", served to `OeisWeb` through a fake transport so the paging behaviour is covered without the network. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 49 ++ alkahest-skill/alkahest.md | 2 +- docs/mdbook/src/novelty.md | 88 ++- python/alkahest/experimental/__init__.py | 5 + python/alkahest/experimental/novelty.py | 806 +++++++++++++++++++++-- tests/data/oeis_novelty_fixture.json | 165 +++++ tests/data/oeis_paging_fixture.json | 648 ++++++++++++++++++ tests/test_novelty.py | 377 ++++++++++- 8 files changed, 2070 insertions(+), 70 deletions(-) create mode 100644 tests/data/oeis_paging_fixture.json diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..75e0dea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,55 @@ ## Unreleased +- **The M11 novelty filter reads more of what OEIS actually writes, pages its + searches, can represent a `q`-recurrence, and cross-checks the terms it is + given** (`alkahest.experimental.novelty`). Four false-red / coverage + defects, all found by pointing the filter at the Fibonacci recurrence and + watching it come back `not_found` against A000045: + + - **The recurrence in an entry's *name* is now read.** A000045's whole name + is *"Fibonacci numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and + F(1) = 1"*, which the parser was never pointed at; the entry produced 25 + candidate lines, **0 usable**, and a `not_found` verdict for its own + recurrence. `OeisEntry.candidate_lines()` now puts the name first, any + single letter may name the sequence (`F`, `L`, `T`, `b`), an identifier + passed as `RecurrenceClaim.from_text(..., names=…)` may too — the entry's + own A-number is passed automatically — and juxtaposition is read as + multiplication (`2a(n-2)`). One line may still name only **one** sequence, + so `a(n) = a(n-1) + A002026(n-1)` is refused as before, and a parsed line + is still only indexed once it reproduces the entry's own terms. Measured + over a 377-entry live sample: **156 → 252** lines parsed, **121 → 195** + usable statements, **114 → 174** entries with at least one. + - **A `terms=` search is paged; a full first page is no longer reported as + exhaustive.** `fmt=json` returns at most ten results and no total count, + so `OeisWeb` claimed `exhaustive=True` after one page and every negative + became a `not_found` — collapsing the `unavailable` half of the tri-state + the module exists to provide. It now continues at `&start=` until a short + page arrives (exhaustive, and recorded in the cache as a complete answer) + or `max_results` is reached (`exhaustive=False`, **not** recorded, and + `check_novelty` reports `unavailable`). `max_results` defaults to 50, five + pages. An `id:A…` lookup is *not* paged and stays exhaustive after one + request, which was always correct. + - **`QRecurrenceClaim`** — normal form, `claim_hash` and equality for + `Σ_i c_i(q, q^n)·u(n+i) = 0`, with coefficients in `ℚ(q, q^n)` cleared to + Laurent polynomials. `RecurrenceClaim` refused these outright + (*"coefficient mentions the symbol 'q'"*), so no `q_zeilberger` result had + any route to the promotion gate. Scale, index shift — which acts on the + coefficients, since `n → n+1` sends `q^n` to `q·q^n` — a common monomial + or polynomial factor, and zero padding are quotiented out. The normal form + is tagged `q-recurrence/1` against `recurrence/1`, so the hash spaces do + not collide. No source here can *state* a `q`-recurrence, so + `check_novelty` reports OEIS sources as `unavailable` for one rather than + manufacturing a negative; sources may declare what they can state with a + `CLAIM_KINDS` tuple. + - **`terms=` is checked against the claim**, not only used to drive the + search. `NoveltyVerdict.terms_check` reads `"holds"`, `"fails"` or + `"not_checked"`, on the same lenient trailing-window rule a source's own + formula line has to pass, and appears in `report()`. A `"fails"` means the + lookup was about a different sequence from the claim. `check_novelty` takes + `start=`, meaning exactly what `RecurrenceClaim.holds_for`'s `start` means; + it is never sent to a source. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index 7df084ab..77b968ec 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -1433,7 +1433,7 @@ reg.coverage_report_markdown() # same, rendered as a Markdown table 27. **`binomial_mod(a, b, p, k)` is Lucas at `k = 1` and Granville above it** (since 3.9). Cost is `O(p·k³ + log_p(a)·p·k)`, so `a` far larger than `p` is the ordinary case, not the hard one; `b > a` and `b < 0` return `0` rather than raising. Refuses with `E-HOLO-006` for a composite base or `p**k >= 2**62`, and `E-HOLO-008` when the one pass over `1 … p−1` is unaffordable. -28. **Check a fitted recurrence against OEIS with `experimental.novelty.check_novelty` before calling it new** (since 3.9). Build `RecurrenceClaim.from_recurrence(cert_or_guess, var=n)` from a `ZeilbergerCertificate` or `GuessedRecurrence`; it normalises away rescaling, sign flips, index shifts and a common polynomial factor, so `claim_hash` is equal for two presentations of the same relation and different for genuinely different ones. `check_novelty(claim, sources, terms=…)` returns a `NoveltyVerdict` whose `found` is **three-valued**, exactly like `relation_confidence`'s `credible`: `True` a source states the claim, `False` means *not found in the sources actually searched* — not "novel" — and `None` means no source could answer. There is no `novel` attribute anywhere on the type and `bool(verdict)` raises, so `if check_novelty(...):` cannot compile into the overclaim this API exists to prevent; branch on `verdict.status` (`"recorded"`, `"recorded_conjecturally"`, `"not_found"`, `"unavailable"`) or `verdict.found`. `verdict.hedged` is the difference between OEIS stating a recurrence as a theorem and marking it `Conjecture`/`Empirical` — restating the latter is not a result, proving it is. Sources are explicit and there is no default: `OeisCache` (file-backed, offline, what every test in this repository uses) or `OeisWeb` (opt-in, rate-limited, serves its cache first, degrades to `unavailable` rather than raising when there is no network) — pass `[cache]` or `[cache, web]` yourself. `RecurrenceClaim.from_text` parses OEIS's own `a(n) = …` formula lines and returns `None`, never a guess, for anything outside a homogeneous linear recurrence with polynomial coefficients (a sum, a generating function, another sequence, an inhomogeneous relation). +28. **Check a fitted recurrence against OEIS with `experimental.novelty.check_novelty` before calling it new** (since 3.9). Build `RecurrenceClaim.from_recurrence(cert_or_guess, var=n)` from a `ZeilbergerCertificate` or `GuessedRecurrence`; it normalises away rescaling, sign flips, index shifts and a common polynomial factor, so `claim_hash` is equal for two presentations of the same relation and different for genuinely different ones. `check_novelty(claim, sources, terms=…)` returns a `NoveltyVerdict` whose `found` is **three-valued**, exactly like `relation_confidence`'s `credible`: `True` a source states the claim, `False` means *not found in the sources actually searched* — not "novel" — and `None` means no source could answer. There is no `novel` attribute anywhere on the type and `bool(verdict)` raises, so `if check_novelty(...):` cannot compile into the overclaim this API exists to prevent; branch on `verdict.status` (`"recorded"`, `"recorded_conjecturally"`, `"not_found"`, `"unavailable"`) or `verdict.found`. `verdict.hedged` is the difference between OEIS stating a recurrence as a theorem and marking it `Conjecture`/`Empirical` — restating the latter is not a result, proving it is. Sources are explicit and there is no default: `OeisCache` (file-backed, offline, what every test in this repository uses) or `OeisWeb` (opt-in, rate-limited, serves its cache first, degrades to `unavailable` rather than raising when there is no network) — pass `[cache]` or `[cache, web]` yourself. `RecurrenceClaim.from_text` parses OEIS's own formula lines and returns `None`, never a guess, for anything outside a homogeneous linear recurrence with polynomial coefficients (a sum, a generating function, a relation between two sequences, an inhomogeneous relation). Four things it also does, each added because the filter came back `not_found` for the Fibonacci recurrence against A000045: (a) it reads the **name** of an entry, which is where OEIS puts the recurrence for the entries defined by one (`Fibonacci numbers: F(n) = F(n-1) + F(n-2)`), reads any single letter as the sequence and juxtaposition as multiplication, and still holds every parsed line to reproducing the entry's own terms; (b) an `OeisWeb` `terms=` search is **paged** — `fmt=json` returns at most ten results and no total count, so one full page gives `exhaustive=False` and hence `unavailable`, never `not_found`, while an `ids=` lookup is exhaustive after one request; (c) `QRecurrenceClaim` is the same normal form and hash for a `q`-recurrence `Σ_i c_i(q, q^n)·u(n+i) = 0` (tagged `q-recurrence/1`, so it cannot collide with the ordinary kind), and since no source here can *state* one, `check_novelty` reports OEIS sources as `unavailable` for it rather than manufacturing a negative; (d) `terms=` is checked against the claim as well as used to search — `verdict.terms_check` is `"holds"`/`"fails"`/`"not_checked"`, and a `"fails"` means the lookup was about a different sequence from the claim (pass `check_novelty(..., start=…)` if `terms[0]` is not `u(0)`). 29. **`cert.specialize_at_root_of_unity(d, n)` is the decision that carries a `q_zeilberger` verdict to `q = ζ_d`, and it is three-valued** (since 3.9). A proved `Q(q)` recurrence does not by itself license setting `q` to a primitive `d`-th root of unity — a coefficient or a sum value can have a pole there, and specialising anyway is the `q`-analogue of the A279013 failure (item 22): a certificate that re-checks perfectly while the specialised claim is false. The hypotheses (no pole in any `a_i(qⁿ)` or `S(n+i)` at `ζ_d`) are decided **exactly**, by polynomial divisibility by `Φ_d(q)` over `Q` in the cyclotomic field `Q(ζ_d) = Q[q]/(Φ_d(q))` — never numerically — and `cyclotomic_polynomial(pool, d)` exposes `Φ_d(q)` itself so a caller can redo the check by hand. `status` is `"specializes"` (proved, and re-checked as an exact identity in `Q(ζ_d)` before being returned), `"obstructed"` (a pole was **exhibited** — `sum_value`/`coefficient` raise, but `sum_valuation(i)` is still available since the negative valuation *is* the obstruction — and this is not a claim the specialised identity is false, only that this route is blocked), or `"unknown"` (the generic boundary verdict was already `"unknown"`, so there is nothing to specialise). Three things a `"specializes"` verdict does **not** by itself mean, each with its own accessor: `is_vacuous` (every coefficient died — always true at `d = 1`, the `q → 1` limit — so the recurrence is `0 = 0`, still true, but empty), `leading_coefficient_survives` (`False` means the specialised recurrence no longer determines the last value from the earlier ones), and `support_shrinks` (`q`-Lucas killing terms — `[2;1]_q = 1 + q` is non-zero in `Q(q)` and zero at `ζ_2` — reported via `effective_support`, which can shrink but never grow). `sum_valuation(i)` is the `q`-supercongruence content itself: the exact integer `v` with `Φ_d(q)^v ∥ S(n+i)`, so `v ≥ r` is precisely `Φ_d(q)^r | S(n)`. diff --git a/docs/mdbook/src/novelty.md b/docs/mdbook/src/novelty.md index 934249bb..a06a8b8d 100644 --- a/docs/mdbook/src/novelty.md +++ b/docs/mdbook/src/novelty.md @@ -46,15 +46,59 @@ guess = ak.guess_holonomic(terms, max_order=3, max_degree=4) claim = RecurrenceClaim.from_recurrence(guess) ``` -`RecurrenceClaim.from_text` reads OEIS's own `a(n) = …` formula lines by -recursive descent over `+ - * / ^ ( )`, `n` and `a(n±k)`. It refuses — returns -`None`, never a guess — anything outside that shape: a reference to another -sequence (`a(n) = a(n-1) + A002026(n-1)`), a sum, a generating function, an -inhomogeneous relation, a nonlinear one. A parser that guesses at prose +`RecurrenceClaim.from_text` reads OEIS's own formula lines by recursive +descent over `+ - * / ^ ( )`, `n` and a shifted sequence term. It refuses — +returns `None`, never a guess — anything outside that shape: a sum, a +generating function, an inhomogeneous relation, a nonlinear one, or a relation +between *two* sequences (`a(n) = a(n-1) + A002026(n-1)` is a statement about +two of them and a recurrence for neither). A parser that guesses at prose invents claims nobody made, so a line the parser does not fully cover is counted as unusable rather than truncated into a shorter claim that happens to parse. +The sequence need not be spelled `a(n)`. OEIS names a sequence after what it +counts, and an entry's **name** is where the recurrence lives for the entries +that are defined by one — A000045's whole name is *"Fibonacci numbers: F(n) = +F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1"*, and a filter reading only the +formula lines could not find the Fibonacci recurrence in the Fibonacci entry. +So `OeisEntry.candidate_lines()` puts the name first, any single letter may be +the sequence, an identifier passed as `names=` (the entry's own A-number) may +be too, and juxtaposition is read as multiplication (`2a(n-2)`). One line may +still only name **one** sequence, and — as always — a parsed line is only +indexed once it reproduces the entry's own terms, which is what stops a +comment's auxiliary `b(n)` from becoming a claim the entry never made. + +Over a 377-entry live sample (970 → 1276 candidate lines, since the name and +the other notations are now candidates) that widening takes the parser from +156 lines read to 252, from 121 usable statements to 195, and from 114 entries +with at least one usable statement to 174. + +### `q`-recurrences + +`QRecurrenceClaim` is the same three things — normal form, hash, equality — +for `Σ_i c_i(q, q^n)·u(n+i) = 0`, what +[`q_zeilberger`](./telescoping.md) produces. Its coefficients are Laurent +polynomials in `q` and `q^n` over `ℚ` (rational functions are accepted and +cleared), so they are not polynomials in `n` at all and `RecurrenceClaim` +refuses them outright. The same four things are quotiented out, read over +`ℚ[q^±1, (q^n)^±1]`; note that the index shift now *acts* on the +coefficients, because `n → n+1` sends `q^n` to `q·q^n`. The normal form is +tagged `q-recurrence/1` where the ordinary one is tagged `recurrence/1`, so +the two hash spaces cannot collide. + +```python +from alkahest.experimental.novelty import QRecurrenceClaim + +claim = QRecurrenceClaim.from_recurrence(certificate, var=n, q=q) +claim.normal_form # 'q-recurrence/1 (q^n - 1)*u(n+0) + (1)*u(n+1)' +``` + +**No source in this module can state a `q`-recurrence** — OEIS indexes integer +sequences — so `check_novelty` reports every OEIS source as `unavailable` for +a claim of this kind rather than manufacturing a `not_found` out of a search +that could not have matched. What it is good for today is the other half of +the job: a stable content address a loop can dedupe its own `q`-output with. + ### `holds_for` / `confirmations`, and what `start` means Both exactly re-check a claim's normal form against concrete terms — @@ -98,6 +142,16 @@ behalf. Two source types: requests, sends an identifying User-Agent, and **returns `unavailable` rather than raising** when the network is not there. +A `terms=` search is **paged**; an `ids=` lookup is not. `fmt=json` answers a +search with a bare list of at most ten results and no total count, so a single +full page is not evidence that there is nothing else: `OeisWeb` keeps asking +at `&start=` until a short page comes back (the search is over — the answer is +exhaustive, and is recorded in the cache as a complete answer) or until +`max_results` is reached (there may well be more — `exhaustive=False`, the +query is *not* recorded, and `check_novelty` reports `unavailable` rather than +`not_found`). An `id:A…` query asks for named entries and gets exactly them, +so it is exhaustive after one request. + ```python web = OeisWeb(cache=OeisCache()) web.lookup(ids=["A005259"]) @@ -146,6 +200,16 @@ verdict.hedged # True — OEIS has this, but never proved it visible next to it: a `"not_found"` against zero entries examined means something quite different from one against fifty. +`verdict.terms_check` is the other half of that honesty. `terms=` is used +twice: to identify the sequence to a source, and — since the two are supposed +to be about the same sequence — to re-check the claim itself, on the same +lenient trailing-window rule a source's own formula line has to pass. It reads +`"holds"`, `"fails"` or `"not_checked"`, and a `"fails"` means the lookup was +about a different sequence from the claim, so nothing it returned bears on the +claim: either the claim is wrong, the terms are, or `start` is (pass +`check_novelty(..., start=…)`, which means exactly what `holds_for`'s `start` +means and is never sent to a source). + ## Testing without the network `tests/test_novelty.py` never constructs `OeisWeb`; every OEIS-backed test @@ -154,7 +218,13 @@ from oeis.org (© The OEIS Foundation Inc., licensed CC BY-NC-SA 4.0 — the license travels with every cache this module saves) and committed. The fixture carries the sequences this project already certifies recurrences for — Apéry (A005259), Motzkin (A001006), Catalan (A000108), central binomial -coefficients (A000984) — plus A359643, a result this project's own search -found and which OEIS records only as an unproved `Conjecture`: the recorded -statement is `verdict.hedged is True`, and a claim one order lower that OEIS -does not have at all comes back `"not_found"`. +coefficients (A000984) — plus A000045, where the recurrence is in the name, +and A359643, a result this project's own search found and which OEIS records +only as an unproved `Conjecture`: the recorded statement is +`verdict.hedged is True`, and a claim one order lower that OEIS does not have +at all comes back `"not_found"`. + +The paging tests need raw HTTP pages rather than a cache, so +`tests/data/oeis_paging_fixture.json` holds recorded +`search?…&fmt=json` responses keyed `"query|start"` and the tests serve them +through a fake transport. `OeisWeb` is still never pointed at the network. diff --git a/python/alkahest/experimental/__init__.py b/python/alkahest/experimental/__init__.py index 3add0440..49504287 100644 --- a/python/alkahest/experimental/__init__.py +++ b/python/alkahest/experimental/__init__.py @@ -101,6 +101,8 @@ Novelty filtering (:mod:`alkahest.experimental.novelty`): - :class:`RecurrenceClaim` — a recurrence in a normal form two presentations of the same fact share, plus a stable ``claim_hash`` to dedupe on +- :class:`QRecurrenceClaim` — the same for a ``q``-recurrence, whose + coefficients live in ``Q(q, q^n)`` rather than ``Q[n]`` - :func:`check_novelty` / :class:`NoveltyVerdict` — was this claim already written down? Three-valued, and a negative is never reported as "novel" - :class:`OeisCache` (offline, the tested path) and :class:`OeisWeb` (opt-in @@ -196,6 +198,7 @@ OeisCache, OeisEntry, OeisWeb, + QRecurrenceClaim, RecurrenceClaim, check_novelty, ) @@ -237,6 +240,8 @@ # M9 — coefficient fields for elimination "ParametricGbPoly", "ParametricGroebnerBasis", + # M11 — novelty filtering + "QRecurrenceClaim", # M4 — root-of-unity specialisation "QRootOfUnitySpecialization", # M4(b) — q-analogue creative telescoping diff --git a/python/alkahest/experimental/novelty.py b/python/alkahest/experimental/novelty.py index ef65d92c..2549d110 100644 --- a/python/alkahest/experimental/novelty.py +++ b/python/alkahest/experimental/novelty.py @@ -18,6 +18,11 @@ (:class:`OeisCache` offline, :class:`OeisWeb` when explicitly opted into), returning a :class:`NoveltyVerdict`. +:class:`QRecurrenceClaim` is (1) and (2) for a ``q``-recurrence, whose +coefficients are Laurent polynomials in ``q`` and ``q^n`` and so are not +polynomials in ``n`` at all. No source here can state one, which +:func:`check_novelty` reports as *unavailable* rather than as a negative. + What a negative verdict is allowed to claim ------------------------------------------- @@ -34,6 +39,15 @@ * ``False`` — the sources searched do not state it. Not "novel". * ``None`` — no source could answer. Never a pass. +Two things feed that honesty and are easy to get wrong, so they are stated +here as well as at their definitions. A ``terms=`` search of OEIS is **paged**: +``fmt=json`` returns at most ten results with no total count, so a full page is +not an exhaustive answer and :class:`OeisWeb` keeps asking until it sees a short +one or gives up and says ``exhaustive=False``. And the *terms* a caller looks a +claim up by are **checked against the claim** — a claim that does not reproduce +them was never about the sequence that was searched for, and +:attr:`NoveltyVerdict.terms_check` says so. + There is deliberately no ``novel`` attribute anywhere in this module, and ``bool(verdict)`` raises rather than silently reading ``True``, because ``if check_novelty(...):`` is the exact sentence this file exists to prevent. @@ -63,8 +77,10 @@ :class:`OeisWeb` is opt-in, never constructed by default, serves from its cache before it touches the network, sleeps between requests, and returns ``unavailable`` rather than raising when the network is not there. **No test in -this repository requires the network**; the offline path is -:class:`OeisCache`, whose fixtures are recorded once and committed. OEIS data +this repository requires the network**: the offline path is :class:`OeisCache`, +whose fixtures are recorded once and committed, and the two tests that do +construct an :class:`OeisWeb` — for the paging in :meth:`OeisWeb.lookup` — +replace its transport with recorded pages. OEIS data is © The OEIS Foundation Inc., licensed CC BY-NC-SA 4.0 — a cache written by this module records that in the file. """ @@ -90,11 +106,13 @@ __all__ = [ "NOVELTY_STATUSES", "STATUS_MEANINGS", + "TERMS_CHECKS", "NoveltyMatch", "NoveltyVerdict", "OeisCache", "OeisEntry", "OeisWeb", + "QRecurrenceClaim", "RecordedRecurrence", "RecurrenceClaim", "SourceAnswer", @@ -120,8 +138,12 @@ "unavailable": ("no source could answer; nothing was established either way"), } +#: Every answer :attr:`NoveltyVerdict.terms_check` can give. +TERMS_CHECKS = ("holds", "fails", "not_checked") + #: Trailing windows of the entry's own data a parsed recurrence must satisfy -#: before it is believed to be what the formula line meant. +#: before it is believed to be what the formula line meant. The same rule +#: re-checks the caller's own claim against the terms it looked it up by. _MIN_CONFIRMATIONS = 3 # --------------------------------------------------------------------------- @@ -242,6 +264,173 @@ def _p_text(a: tuple) -> str: return body +# --------------------------------------------------------------------------- +# Laurent polynomials over ℚ in two variables, `q` and `Q = q^n`. +# +# This is what a `q`-recurrence coefficient is: `q_zeilberger` returns +# `1 + q*q^n - q*q^(2*n) - q^2*q^(3*n)`, which is neither a polynomial in `n` +# nor one in `q` alone. A term is a dict entry `(i, j) -> c` for `c*q^i*Q^j`, +# with `i` and `j` allowed to be negative; the zero polynomial is `{}`. +# --------------------------------------------------------------------------- + + +def _q_trim(terms: dict) -> dict: + return {m: c for m, c in terms.items() if c} + + +def _q_add(a: dict, b: dict) -> dict: + out = dict(a) + for m, c in b.items(): + out[m] = out.get(m, Fraction(0)) + c + return _q_trim(out) + + +def _q_mul(a: dict, b: dict) -> dict: + out: dict = {} + for (i, j), x in a.items(): + for (k, e), y in b.items(): + m = (i + k, j + e) + out[m] = out.get(m, Fraction(0)) + x * y + return _q_trim(out) + + +def _q_pow(a: dict, e: int) -> dict: + out = {(0, 0): Fraction(1)} + for _ in range(e): + out = _q_mul(out, a) + return out + + +def _q_substitute_shift(a: dict, s: int) -> dict: + """``a`` with ``n → n + s``, i.e. ``Q → q^s·Q``.""" + if s == 0: + return dict(a) + return {(i + s * j, j): c for (i, j), c in a.items()} + + +def _q_monomial_content(polys: Sequence[dict]) -> tuple: + """The largest ``(q^i, Q^j)`` dividing every term of every polynomial.""" + live = [m for p in polys for m in p] + if not live: + return (0, 0) + return (min(i for i, _ in live), min(j for _, j in live)) + + +def _q_columns(a: dict) -> list: + """*a* as a polynomial in ``Q`` whose coefficients are polynomials in ``q``. + + Requires non-negative exponents — divide the monomial content out first. + A list indexed by the power of ``Q``, each entry a ``_p_*`` tuple. + """ + if not a: + return [] + columns: list = [()] * (max(j for _, j in a) + 1) + for (i, j), c in a.items(): + column = list(columns[j]) + [Fraction(0)] * (i + 1 - len(columns[j])) + column[i] += c + columns[j] = _trim(column) + return columns + + +def _q_from_columns(columns: Sequence[tuple]) -> dict: + return _q_trim( + {(i, j): c for j, column in enumerate(columns) for i, c in enumerate(column) if c} + ) + + +def _q_col_trim(columns: Sequence[tuple]) -> list: + out = list(columns) + while out and not out[-1]: + out.pop() + return out + + +def _q_col_content(columns: Sequence[tuple]) -> tuple: + common: tuple = () + for column in columns: + common = _p_gcd(common, column) + return common + + +def _q_col_primitive(columns: Sequence[tuple]) -> list: + common = _q_col_content(columns) + if not common: + return list(columns) + return [_p_divmod(column, common)[0] for column in columns] + + +def _q_col_prem(a: Sequence[tuple], b: Sequence[tuple]) -> list: + """Pseudo-remainder of *a* by *b* in ``ℚ[q][Q]``.""" + rem = _q_col_trim(a) + b = _q_col_trim(b) + while rem and len(rem) >= len(b): + shift = len(rem) - len(b) + lead_a, lead_b = rem[-1], b[-1] + scaled = [_p_mul(c, lead_b) for c in rem] + for i, c in enumerate(b): + scaled[shift + i] = _p_sub(scaled[shift + i], _p_mul(lead_a, c)) + rem = _q_col_trim(scaled) + return rem + + +def _q_col_gcd(a: Sequence[tuple], b: Sequence[tuple]) -> list: + """gcd in ``ℚ[q][Q]`` by the primitive Euclidean algorithm.""" + a, b = _q_col_trim(a), _q_col_trim(b) + if not a: + return list(b) + if not b: + return list(a) + content = _p_gcd(_q_col_content(a), _q_col_content(b)) + a, b = _q_col_primitive(a), _q_col_primitive(b) + while b: + a, b = b, _q_col_primitive(_q_col_prem(a, b)) + return [_p_mul(column, content) for column in a] + + +def _q_col_divexact(a: Sequence[tuple], b: Sequence[tuple]) -> list | None: + """``a / b`` in ``ℚ[q][Q]`` when it is exact, else ``None``.""" + rem = _q_col_trim(a) + b = _q_col_trim(b) + quotient: list = [()] * max(1, len(rem) - len(b) + 1) + while rem and len(rem) >= len(b): + shift = len(rem) - len(b) + factor, residue = _p_divmod(rem[-1], b[-1]) + if residue or not factor: + return None + quotient[shift] = factor + scaled = list(rem) + for i, c in enumerate(b): + scaled[shift + i] = _p_sub(scaled[shift + i], _p_mul(factor, c)) + rem = _q_col_trim(scaled) + return None if rem else _q_col_trim(quotient) + + +def _q_text(a: dict) -> str: + """Canonical text for a ``q``-coefficient: descending in ``q^n``, then ``q``.""" + if not a: + return "0" + parts = [] + for i, j in sorted(a, key=lambda m: (m[1], m[0]), reverse=True): + c = a[(i, j)] + factors = [] + if i: + factors.append("q" if i == 1 else f"q^{i}") + if j: + factors.append("q^n" if j == 1 else f"q^({j}*n)") + if not factors: + parts.append(str(c)) + elif c == 1: + parts.append("*".join(factors)) + elif c == -1: + parts.append("-" + "*".join(factors)) + else: + parts.append("*".join([str(c), *factors])) + body = parts[0] + for part in parts[1:]: + body += f" - {part[1:]}" if part.startswith("-") else f" + {part}" + return body + + # --------------------------------------------------------------------------- # Linear forms in the shifts of one unknown sequence, over ℚ(n). # --------------------------------------------------------------------------- @@ -396,9 +585,10 @@ def _poly_from_expr(expr: Any, var: str) -> tuple: return _trim((Fraction(int(node[1]), int(node[2])),)) if head == "symbol": if node[1] != var: + hint = " — a q-recurrence is a QRecurrenceClaim, not this one" if node[1] == "q" else "" raise ValueError( f"coefficient mentions the symbol {node[1]!r}, but a recurrence " - f"coefficient must be a polynomial in {var!r} alone" + f"coefficient must be a polynomial in {var!r} alone{hint}" ) return (Fraction(0), Fraction(1)) if head == "add": @@ -523,19 +713,29 @@ def from_recurrence(cls, rec: Any, var: Any = None) -> RecurrenceClaim: return cls(list(rec) if coeffs is None else list(coeffs), var=var) @classmethod - def from_text(cls, text: str) -> RecurrenceClaim | None: + def from_text(cls, text: str, *, names: Sequence[str] = ()) -> RecurrenceClaim | None: """Parse one prose formula line, e.g. an OEIS ``a(n) = …`` statement. + The sequence may be written ``a(n)``, as any single letter (``F(n) = + F(n-1) + F(n-2)`` is how A000045 states the Fibonacci recurrence, in its + *name*), or under any identifier in *names* — pass the entry's own + A-number there so a line that spells it out is read as being about + itself. Whichever is used, **one** line may only name one sequence: a + relation between two of them is not a recurrence for either. + Returns ``None`` — never a guess — when the line is not a homogeneous - linear recurrence with polynomial coefficients in the single sequence - ``a``: a sum, a generating function, a nonlinear relation, a reference - to another sequence, an inhomogeneous relation, or a statement the - parser simply does not cover. Callers that need to know how often that - happened should count the ``None``s; :meth:`NoveltyVerdict.report` - does. + linear recurrence with polynomial coefficients in a single sequence: a + sum, a generating function, a nonlinear relation, a reference to another + sequence, an inhomogeneous relation, or a statement the parser simply + does not cover. Callers that need to know how often that happened should + count the ``None``s; :meth:`NoveltyVerdict.report` does. + + :param names: extra identifiers that denote the sequence the line is + about, e.g. ``names=("A000045",)``. """ + own = frozenset({"a", *names}) try: - relation = _parse_relation(text) + relation = _parse_relation(text, own) except _Unsupported: return None if relation is None: @@ -545,6 +745,16 @@ def from_text(cls, text: str) -> RecurrenceClaim | None: except ValueError: return None + @property + def claim_kind(self) -> str: + """``"recurrence"`` — what a source must be able to state to match this. + + See :class:`QRecurrenceClaim`, whose kind is ``"q-recurrence"``; a + source that cannot state a kind is ``unavailable`` for it, never + ``not_found``. + """ + return "recurrence" + @property def order(self) -> int: """``J`` — the span of the window in normal form.""" @@ -645,6 +855,292 @@ def __repr__(self) -> str: ) +# --------------------------------------------------------------------------- +# The `q`-analogue of the claim. +# --------------------------------------------------------------------------- + + +def _q_exponent(expr: Any, var: str) -> tuple | None: + """``(d, c)`` for an exponent ``d + c·n`` with integer ``d`` and ``c``.""" + try: + poly = _poly_from_expr(expr, var) + except (ValueError, _Unsupported): + return None + if len(poly) > 2: + return None + padded = [poly[i] if i < len(poly) else Fraction(0) for i in range(2)] + if any(c.denominator != 1 for c in padded): + return None + return int(padded[0]), int(padded[1]) + + +def _qpoly_from_expr(expr: Any, qname: str, var: str) -> tuple: + """``(numerator, denominator)`` in ``ℚ[q^±1, Q^±1]``, ``Q = q^n``, exactly.""" + node = expr.node() + head = node[0] + one = {(0, 0): Fraction(1)} + if head == "integer": + return _q_trim({(0, 0): Fraction(int(node[1]))}), one + if head == "rational": + return _q_trim({(0, 0): Fraction(int(node[1]), int(node[2]))}), one + if head == "symbol": + if node[1] == qname: + return {(1, 0): Fraction(1)}, one + raise ValueError( + f"coefficient mentions the symbol {node[1]!r}, but a q-recurrence " + f"coefficient must be a rational function of {qname!r} and " + f"{qname}^{var}" + ) + if head == "add": + num: dict = {} + den = one + for child in node[1]: + other_num, other_den = _qpoly_from_expr(child, qname, var) + num = _q_add(_q_mul(num, other_den), _q_mul(other_num, den)) + den = _q_mul(den, other_den) + return num, den + if head == "mul": + num, den = one, one + for child in node[1]: + other_num, other_den = _qpoly_from_expr(child, qname, var) + num, den = _q_mul(num, other_num), _q_mul(den, other_den) + return num, den + if head == "pow": + exponent = _q_exponent(node[2], var) + if exponent is None: + raise ValueError( + f"{expr} is not a rational function of {qname!r} and {qname}^{var}: " + f"an exponent must be an integer or an integer multiple of {var!r}" + ) + offset, slope = exponent + base = node[1].node() + if slope: + if base[0] != "symbol" or base[1] != qname: + raise ValueError( + f"{expr} raises something other than {qname!r} to a power in {var}" + ) + return {(offset, slope): Fraction(1)}, one + num, den = _qpoly_from_expr(node[1], qname, var) + if offset < 0: + num, den, offset = den, num, -offset + if not num: + raise ValueError(f"{expr} divides by zero") + return _q_pow(num, offset), _q_pow(den, offset) + raise ValueError(f"{expr} is not a rational function of {qname!r} and {qname}^{var}") + + +def _q_coerce(value: Any, qname: str | None, var: str | None) -> tuple: + """One ``q``-coefficient, from an ``Expr`` or a ``{(i, j): rational}`` map.""" + if hasattr(value, "node"): + if qname is None or var is None: + raise TypeError( + "coefficients given as Expr need both index variables: pass " + "var=n (the index) and q=q (the base), the two symbols a " + "q-recurrence coefficient is built from" + ) + return _qpoly_from_expr(value, qname, var) + one = {(0, 0): Fraction(1)} + if isinstance(value, (int, Fraction)): + return _q_trim({(0, 0): Fraction(value)}), one + return _q_trim({(int(i), int(j)): Fraction(c) for (i, j), c in dict(value).items()}), one + + +def _q_normalise(shifts: dict) -> tuple: + """Put ``Σ_j c_j(q, q^n)·u(n+j) = 0`` into normal form. + + The steps are those of :func:`_normalise`, read in ``ℚ[q^±1, Q^±1]`` + instead of ``ℚ[n]``: clear the coefficients' denominators, move the window + to ``u(n)`` — which acts on the coefficients, since ``n → n − low`` sends + ``Q`` to ``q^{−low}·Q`` — divide out the common monomial and the common + polynomial factor, clear rational denominators and the integer content, and + fix the sign. + """ + keys = sorted(shifts) + numerators = {} + for j in keys: + product = shifts[j][0] + for k in keys: + if k != j: + product = _q_mul(product, shifts[k][1]) + numerators[j] = product + live = {j: p for j, p in numerators.items() if p} + if len(live) < 2: + raise ValueError( + "a q-recurrence claim needs at least two sequence terms with " + f"nonzero coefficients, got {len(live)}" + ) + low, high = min(live), max(live) + polys = [_q_substitute_shift(live.get(low + i, {}), -low) for i in range(high - low + 1)] + shift_i, shift_j = _q_monomial_content(polys) + polys = [{(i - shift_i, j - shift_j): c for (i, j), c in p.items()} for p in polys] + columns = [_q_columns(p) for p in polys] + common: list = [] + for column in columns: + common = _q_col_gcd(common, column) + if len(common) > 1 or (common and len(common[0]) > 1): + divided = [_q_col_divexact(column, common) for column in columns] + if all(d is not None for d in divided): + columns = divided + polys = [_q_from_columns(column) for column in columns] + multiplier = 1 + for p in polys: + for c in p.values(): + multiplier = multiplier * c.denominator // gcd(multiplier, c.denominator) + integral = [{m: int(c * multiplier) for m, c in p.items()} for p in polys] + content = 0 + for p in integral: + for c in p.values(): + content = gcd(content, abs(c)) + if content > 1: + integral = [{m: c // content for m, c in p.items()} for p in integral] + for p in integral: + if not p: + continue + if p[max(p, key=lambda m: (m[1], m[0]))] < 0: + integral = [{m: -c for m, c in other.items()} for other in integral] + break + return tuple( + tuple(sorted(p.items(), key=lambda item: (item[0][1], item[0][0]))) for p in integral + ) + + +class QRecurrenceClaim: + """A ``q``-recurrence, in a normal form two presentations share. + + The claim is ``Σ_i c_i(q, q^n)·u(n+i) = 0`` — what + :func:`alkahest.experimental.q_zeilberger` produces, whose coefficients are + Laurent polynomials in ``q`` and ``q^n`` over ``ℚ`` and are therefore not + polynomials in ``n`` at all. :class:`RecurrenceClaim` refuses them + (*"coefficient mentions the symbol 'q'"*), which left every ``q``-result + with no route into :func:`check_novelty`; this is that route. + + Coefficients may be given as :class:`alkahest.Expr` (pass both ``var=n`` and + ``q=q``) or as ``{(i, j): rational}`` maps, where ``(i, j)`` is the monomial + ``q^i·(q^n)^j`` and either exponent may be negative. Rational functions are + accepted and cleared; the normal form is always Laurent-polynomial. + + What the normal form quotients out is exactly what :attr:`RecurrenceClaim` + quotients out, read over ``ℚ[q^±1, (q^n)^±1]``: scale, index shift — which + here acts on the coefficients, because ``n → n+1`` sends ``q^n`` to + ``q·q^n`` — a common monomial or polynomial factor, and zero padding. + + :attr:`claim_hash` is tagged ``q-recurrence/1`` where + :class:`RecurrenceClaim`'s is tagged ``recurrence/1``, so the two can share + a ``set`` without colliding even when the coefficients look alike. + + **No source in this module can state a ``q``-recurrence.** OEIS indexes + integer sequences, and the formula parser reads ``ℚ[n]`` coefficients only, + so :func:`check_novelty` reports every OEIS source as *unavailable* for a + claim of this kind rather than manufacturing a ``not_found`` out of a + search that could not have matched. What the class is good for today is the + other half of the job: a stable content address a loop can dedupe its own + ``q``-output with. + + >>> from alkahest.experimental.novelty import QRecurrenceClaim + >>> # (1 - q^n)·u(n) - u(n+1) = 0 + >>> a = QRecurrenceClaim([{(0, 0): 1, (0, 1): -1}, {(0, 0): -1}]) + >>> # the same relation about u(n+3), scaled by -2q + >>> b = QRecurrenceClaim( + ... [{(1, 0): -2, (4, 1): 2}, {(1, 0): 2}], offset=3) + >>> a.claim_hash == b.claim_hash + True + >>> a.order, a.degree, a.q_degree + (1, 1, 0) + """ + + __slots__ = ("_coefficients", "_hash", "_normal_form") + + def __init__( + self, + coefficients: Sequence[Any], + *, + offset: int = 0, + var: Any = None, + q: Any = None, + ): + """ + :param coefficients: ``[c_0, …, c_J]``, the coefficient of ``u(n+offset+i)``. + Each an :class:`alkahest.Expr` in *var* and *q*, or a + ``{(i, j): rational}`` map for ``Σ c_ij·q^i·(q^n)^j``. + :param offset: index of the first coefficient's shift. + :param var: the index symbol, required for ``Expr`` coefficients. + :param q: the base symbol, required for ``Expr`` coefficients. + :raises ValueError: when fewer than two coefficients are nonzero. + """ + name = None if var is None else (var if isinstance(var, str) else var.node()[1]) + base = None if q is None else (q if isinstance(q, str) else q.node()[1]) + shifts = {offset + i: _q_coerce(c, base, name) for i, c in enumerate(coefficients)} + self._coefficients = _q_normalise(shifts) + self._normal_form = "q-recurrence/1 " + " + ".join( + f"({_q_text({m: Fraction(c) for m, c in p})})*u(n+{i})" + for i, p in enumerate(self._coefficients) + ) + self._hash = _claim_id(self._normal_form, method="q-recurrence") + + @classmethod + def from_recurrence(cls, rec: Any, var: Any = None, q: Any = None) -> QRecurrenceClaim: + """From a :class:`~alkahest.QZeilbergerCertificate` or a coefficient list. + + Duck-typed on ``.coeffs``, exactly as + :meth:`RecurrenceClaim.from_recurrence` is. Both *var* and *q* are + required when the coefficients are expressions. + """ + coeffs = getattr(rec, "coeffs", None) + return cls(list(rec) if coeffs is None else list(coeffs), var=var, q=q) + + @property + def claim_kind(self) -> str: + """``"q-recurrence"`` — what a source must be able to state to match this.""" + return "q-recurrence" + + @property + def order(self) -> int: + """``J`` — the span of the window in normal form.""" + return len(self._coefficients) - 1 + + @property + def degree(self) -> int: + """Largest power of ``q^n`` in any coefficient in normal form.""" + return max((max(j for (_, j), _ in p) for p in self._coefficients if p), default=0) + + @property + def q_degree(self) -> int: + """Largest power of ``q`` alone in any coefficient in normal form.""" + return max((max(i for (i, _), _ in p) for p in self._coefficients if p), default=0) + + @property + def normal_form(self) -> str: + """The canonical text the hash is taken of, tagged ``q-recurrence/1``.""" + return self._normal_form + + @property + def claim_hash(self) -> str: + """Content address of :attr:`normal_form`, e.g. ``'clm_9f1c0b2a7d4e5f60'``.""" + return self._hash + + def coefficients(self) -> tuple: + """``(c_0, …, c_J)`` in normal form. + + Each is a tuple of ``((i, j), coefficient)`` pairs for + ``coefficient·q^i·(q^n)^j``, ascending in ``j`` then ``i``. + """ + return self._coefficients + + def __eq__(self, other: object) -> bool: + if not isinstance(other, QRecurrenceClaim): + return NotImplemented + return self._normal_form == other._normal_form + + def __hash__(self) -> int: + return hash(self._normal_form) + + def __repr__(self) -> str: + return ( + f"QRecurrenceClaim(order={self.order}, degree={self.degree}, " + f"q_degree={self.q_degree}, claim_hash={self._hash!r})" + ) + + # --------------------------------------------------------------------------- # Parsing prose formula lines. # --------------------------------------------------------------------------- @@ -654,7 +1150,14 @@ def __repr__(self) -> str: _OPENERS = frozenset("([{") _CLOSERS = frozenset(")]}") #: A statement worth handing to the parser at all: it mentions a shifted term. -_LOOKS_LIKE_RECURRENCE = re.compile(r"a\(\s*n\s*[-+]\s*\d+\s*\)") +#: OEIS does not only write ``a(n-1)``. An entry's own definition line names the +#: sequence after the objects it counts — ``F(n) = F(n-1) + F(n-2)`` is the whole +#: content of A000045's name — so any single letter counts as a candidate here. +#: Which of them the parser will accept as a term of *this* sequence is decided +#: in :class:`_Parser`, not here. An ``A123456(n-1)`` cross-reference is +#: deliberately *not* a candidate on its own: over the 377-entry sample this +#: module was measured against it added 256 lines and not one parse. +_LOOKS_LIKE_RECURRENCE = re.compile(r"\b[A-Za-z]\(\s*n\s*[-+]\s*\d+\s*\)") #: OEIS's own hedges. An entry that marks a formula this way is telling you the #: recurrence was fitted and never proved — which is the whole reason a novelty #: filter over OEIS is worth anything. @@ -672,17 +1175,34 @@ def _is_word(token: str) -> bool: return token[0].isalpha() or token[0] == "_" +#: Identifiers that may denote *the* sequence a line is about, beyond ``a`` and +#: whatever the caller adds: any single letter, because OEIS names a sequence +#: after what it counts (``F`` for Fibonacci, ``L`` for Lucas, ``T``, ``b``). +#: A multi-letter identifier is never one of these, which is what keeps +#: ``floor``, ``sqrt``, ``binomial``, ``Sum`` and ``A123456`` out. +def _is_sequence_name(token: str, own: frozenset) -> bool: + return token in own or (len(token) == 1 and token.isalpha()) + + class _Parser: - """Recursive descent over ``+ - * / ^ ( )``, integers, ``n`` and ``a(n±k)``. + """Recursive descent over ``+ - * / ^ ( )``, integers, ``n`` and ``F(n±k)``. - Everything else — another sequence's ``A123456(n)``, ``Sum_{…}``, a symbol + Everything else — a function that is not a sequence, ``Sum_{…}``, a symbol that is not the index — raises :class:`_Unsupported`. Refusing is the point: a parser that guesses at prose invents claims that were never made. + + The sequence identifiers a parse used are collected in :attr:`names`, and + :func:`_parse_relation` refuses the line unless they all denote the same + sequence — so ``a(n) = a(n-1) + A002026(n-1)`` is still refused, and so is + ``F(n) = L(n-1) + L(n-2)``, while ``F(n) = F(n-1) + F(n-2)`` is read. """ - def __init__(self, tokens: Sequence[str]): + def __init__(self, tokens: Sequence[str], own: frozenset = frozenset({"a"})): self.tokens = tokens self.pos = 0 + self.own = own + #: Sequence identifiers this parse applied to an index. + self.names: set = set() def peek(self) -> str | None: return self.tokens[self.pos] if self.pos < len(self.tokens) else None @@ -698,12 +1218,37 @@ def expression(self) -> _Form: def term(self) -> _Form: node = self.factor() - while self.peek() in ("*", "/"): - op = self.tokens[self.pos] - self.pos += 1 - rhs = self.factor() - node = node * rhs if op == "*" else node / rhs - return node + while True: + if self.peek() in ("*", "/"): + op = self.tokens[self.pos] + self.pos += 1 + node = node * self.factor() if op == "*" else node / self.factor() + elif self._starts_factor(): + # Juxtaposition is multiplication: `2a(n-2)`, `(n+1)a(n-1)`. + # OEIS's machine-written "D-finite with recurrence" lines always + # spell the `*` out, but the hand-written ones do not. + node = node * self.factor() + else: + return node + + def _starts_factor(self) -> bool: + """Whether an implicit ``*`` may be read before the next token. + + Only a bracket, a number or a sequence application counts. A bare word + never does — ``a(n) = a(n-1) + a(n-2) for n > 2`` must end at ``for`` + rather than read ``for`` as a factor, and a bare ``n`` is excluded for + the same reason: ``, n > 2`` is prose, not a coefficient. + """ + token = self.peek() + if token is None: + return False + if token == "(" or token.isdigit(): + return True + return ( + _is_sequence_name(token, self.own) + and self.pos + 1 < len(self.tokens) + and self.tokens[self.pos + 1] == "(" + ) def factor(self) -> _Form: if self.peek() in ("+", "-"): @@ -732,7 +1277,7 @@ def atom(self) -> _Form: return _Form.constant(Fraction(int(token))) if token == "n": return _Form.variable() - if token == "a": + if _is_sequence_name(token, self.own): if self.peek() != "(": raise _Unsupported("sequence name not applied to an index") self.pos += 1 @@ -740,6 +1285,7 @@ def atom(self) -> _Form: if self.peek() != ")": raise _Unsupported("unbalanced parenthesis in a sequence index") self.pos += 1 + self.names.add(token) return _Form.sequence_term(_shift_of(index)) raise _Unsupported(f"unsupported token {token!r}") @@ -778,13 +1324,14 @@ def _top_level_equals(tokens: Sequence[str]) -> int | None: return None -def _parse_all(tokens: Sequence[str]) -> _Form | None: - parser = _Parser(tokens) +def _parse_all(tokens: Sequence[str], own: frozenset) -> tuple | None: + """``(form, sequence identifiers used)`` for a whole token run, or ``None``.""" + parser = _Parser(tokens, own) try: form = parser.expression() except _Unsupported: return None - return form if parser.pos == len(tokens) else None + return (form, parser.names) if parser.pos == len(tokens) else None def _boundary_ok(token: str | None) -> bool: @@ -804,8 +1351,12 @@ def _boundary_ok(token: str | None) -> bool: return _is_word(token) or token in {".", ",", ";", ":", "=", "!"} -def _parse_relation(text: str) -> dict | None: - """``{shift: coefficient polynomial}`` for a prose linear recurrence, or ``None``.""" +def _parse_relation(text: str, own: frozenset = frozenset({"a"})) -> dict | None: + """``{shift: coefficient polynomial}`` for a prose linear recurrence, or ``None``. + + *own* is the set of identifiers known to name the sequence the line is about + — ``a`` always, plus the entry's own A-number when there is one. + """ tokens = _tokenise(_clean(text)) split = _top_level_equals(tokens) if split is None: @@ -814,7 +1365,7 @@ def _parse_relation(text: str) -> dict | None: for start in range(split): if not _boundary_ok(tokens[start - 1] if start else None): continue - lhs = _parse_all(tokens[start:split]) + lhs = _parse_all(tokens[start:split], own) if lhs is not None: break if lhs is None: @@ -823,12 +1374,17 @@ def _parse_relation(text: str) -> dict | None: for stop in range(len(tokens), split, -1): if not _boundary_ok(tokens[stop] if stop < len(tokens) else None): continue - rhs = _parse_all(tokens[split + 1 : stop]) + rhs = _parse_all(tokens[split + 1 : stop], own) if rhs is not None: break if rhs is None: return None - form = lhs - rhs + used = lhs[1] | rhs[1] + if len(used) > 1 and not used <= own: + # Two different sequences in one relation: `a(n) = a(n-1) + A002026(n-1)` + # is a statement about two sequences, not a recurrence for either. + return None + form = lhs[0] - rhs[0] if form.poly: # Inhomogeneous: `a(n) = a(n-1) + 1` is a different kind of claim and # is not silently truncated into a homogeneous one. @@ -876,9 +1432,11 @@ def __init__( def from_oeis_json(cls, payload: dict) -> OeisEntry: """From one element of ``https://oeis.org/search?…&fmt=json``. - Only the formula and comment lines that mention a shifted ``a(n±k)`` + Only the formula and comment lines that mention a shifted sequence term are kept: the rest cannot state a recurrence, and a cache that keeps - them is a cache nobody commits. + them is a cache nobody commits. The entry's ``name`` is stored whole and + scanned as a candidate line in its own right — see + :meth:`candidate_lines`. """ offset = 0 raw_offset = str(payload.get("offset", "0")).split(",")[0].strip() @@ -902,11 +1460,25 @@ def to_json(self) -> dict: "statements": list(self.statements), } + def candidate_lines(self) -> tuple: + """Every line that may state a recurrence: the name, then the statements. + + The **name** is here because that is where OEIS puts the recurrence for + the entries that are defined by one: A000045's whole name is *"Fibonacci + numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1"*, and a + filter that reads only the formula lines cannot find the Fibonacci + recurrence in the Fibonacci entry. + """ + name = self.name.strip() + if name and _LOOKS_LIKE_RECURRENCE.search(name): + return (name, *self.statements) + return self.statements + def _scanned(self) -> tuple: if self._scan is None: usable, unusable = [], [] - for statement in self.statements: - claim = RecurrenceClaim.from_text(statement) + for statement in self.candidate_lines(): + claim = RecurrenceClaim.from_text(statement, names=(self.id,)) if claim is None: unusable.append(statement) continue @@ -933,7 +1505,7 @@ def recurrences(self) -> tuple: return self._scanned()[0] def unusable_statements(self) -> tuple: - """Lines that mention ``a(n±k)`` but could not be turned into a claim. + """Candidate lines that could not be turned into a claim. Either the parser does not cover them or they failed the check against the entry's own data. They are counted into @@ -991,6 +1563,13 @@ class OeisCache: "licensed CC BY-NC-SA 4.0." ) + #: The kinds of claim this source is able to state at all. OEIS indexes + #: integer sequences and the formula parser reads ``ℚ[n]`` coefficients, so + #: a :class:`QRecurrenceClaim` is not something a search here could match — + #: :func:`check_novelty` reads this and reports *unavailable* rather than + #: turning a search that could not match into a ``not_found``. + CLAIM_KINDS: ClassVar[tuple] = ("recurrence",) + def __init__(self, path: Any = None): """:param path: a JSON file to load, if it exists.""" self.path = Path(path) if path is not None else None @@ -1129,13 +1708,30 @@ class OeisWeb: not there — so an offline run degrades to ``unavailable``, which is the honest verdict, rather than to an exception or, far worse, to a negative. - No test in this repository constructs one. Record a fixture once:: + **A ``terms=`` search is paged, an ``ids=`` lookup is not.** ``fmt=json`` + answers a search with a bare list of at most :data:`PAGE_SIZE` results and + no total count, so a single full page is not evidence that there is nothing + else: the search continues at ``&start=`` until a short page comes back + (there is no more) or *max_results* is reached (there may be, and the + answer says :attr:`SourceAnswer.exhaustive` is ``False``, which + :func:`check_novelty` turns into ``unavailable`` rather than ``not_found``). + An ``id:A…`` query asks for named entries and gets exactly them, so it is + exhaustive after one request. + + No test in this repository points one at the network. Record a fixture once:: web = OeisWeb(cache=OeisCache()) web.lookup(ids=["A005259"]) web.cache.save("tests/data/oeis_novelty_fixture.json") """ + #: Results one ``fmt=json`` request returns. OEIS's own page size; the JSON + #: form carries no total count, so this is the only signal a search is over. + PAGE_SIZE: ClassVar[int] = 10 + + #: As :attr:`OeisCache.CLAIM_KINDS` — the same encyclopaedia, live. + CLAIM_KINDS: ClassVar[tuple] = ("recurrence",) + #: Shared across instances so several sources cannot bypass the interval. _last_request: ClassVar[list] = [0.0] @@ -1146,7 +1742,7 @@ def __init__( min_interval: float = 1.0, timeout: float = 30.0, user_agent: str = "alkahest-novelty/1.0 (+https://github.com/alkahest-cas/alkahest)", - max_results: int = 10, + max_results: int = 50, ): self.cache = cache if cache is not None else OeisCache() self.min_interval = float(min_interval) @@ -1175,7 +1771,7 @@ def lookup( ) if not query: return None - payload = self._fetch(query) + payload, complete = self._fetch_all(query, paged=not ids) if payload is None: return cached entries = [] @@ -1186,15 +1782,50 @@ def lookup( continue self.cache.add(entry) entries.append(entry) - self.cache.record_query(terms=terms, ids=ids, found=[e.id for e in entries]) - return SourceAnswer(tuple(entries), exhaustive=True) - - def _fetch(self, query: str) -> list | None: + exhaustive = complete and len(payload) <= self.max_results + if exhaustive: + # Only a complete answer may be recorded as one: the cache reads a + # recorded query as "OEIS returned exactly this", and a truncated + # page list stored under that key would turn into a false negative + # on every later offline run. + self.cache.record_query(terms=terms, ids=ids, found=[e.id for e in entries]) + return SourceAnswer(tuple(entries), exhaustive=exhaustive) + + def _fetch_all(self, query: str, *, paged: bool) -> tuple: + """``(rows, complete)`` for *query*; ``(None, False)`` if nothing arrived. + + ``complete`` is ``True`` only when OEIS has been seen to run out of + results — a page shorter than :data:`PAGE_SIZE`, or a page that repeats + entries already collected (OEIS clamps ``start`` past the end rather + than returning nothing). Stopping at *max_results* instead gives + ``False``, and so does a request that fails partway through. + """ + rows: list = [] + seen: set = set() + start = 0 + while True: + page = self._fetch(query, start=start) + if page is None: + return (rows, False) if rows else (None, False) + numbered = [(raw, raw.get("number") if isinstance(raw, dict) else None) for raw in page] + fresh = [raw for raw, number in numbered if number is None or number not in seen] + seen.update(number for _, number in numbered if number is not None) + rows.extend(fresh) + if not paged or len(page) < self.PAGE_SIZE or not fresh: + return rows, True + start += len(page) + if len(rows) >= self.max_results: + return rows, False + + def _fetch(self, query: str, *, start: int = 0) -> list | None: self.last_error = None wait = self.min_interval - (time.monotonic() - self._last_request[0]) if wait > 0: time.sleep(wait) - url = "https://oeis.org/search?" + urllib.parse.urlencode({"q": query, "fmt": "json"}) + parameters: dict = {"q": query, "fmt": "json"} + if start: + parameters["start"] = start + url = "https://oeis.org/search?" + urllib.parse.urlencode(parameters) request = urllib.request.Request(url, headers={"User-Agent": self.user_agent}) try: with urllib.request.urlopen(request, timeout=self.timeout) as handle: @@ -1254,6 +1885,7 @@ class NoveltyVerdict: "_entries", "_matches", "_statements", + "_terms_check", "_unavailable", "_unusable", ) @@ -1268,6 +1900,7 @@ def __init__( entries: int, statements: int, unusable: int, + terms_check: str = "not_checked", ): self._claim_hash = claim_hash self._matches = tuple(matches) @@ -1276,6 +1909,7 @@ def __init__( self._entries = entries self._statements = statements self._unusable = unusable + self._terms_check = terms_check @property def status(self) -> str: @@ -1335,6 +1969,27 @@ def statements_unusable(self) -> int: """ return self._unusable + @property + def terms_check(self) -> str: + """Whether the claim survived the terms it was looked up by. + + One of :data:`TERMS_CHECKS`. ``check_novelty(claim, …, terms=…)`` uses + *terms* twice: to identify the sequence to a source, and — since the + two are supposed to be about the same sequence — to re-check the claim + itself, on the same lenient trailing-window rule a source's own formula + line has to pass (:meth:`RecurrenceClaim.confirmations`). + + * ``"holds"`` — the claim reproduces those terms. + * ``"fails"`` — it does not. **The lookup was then about a different + sequence from the claim**, so nothing it returned bears on the claim; + either the claim is wrong, the terms are, or *start* is (see + :meth:`RecurrenceClaim.holds_for` for what *start* must denote). + * ``"not_checked"`` — no *terms* were given, there were too few of them + to fill one window, or the claim is of a kind integer terms cannot + check (:class:`QRecurrenceClaim`). + """ + return self._terms_check + @property def means(self) -> str: """The one-line gloss of :attr:`status` from :data:`STATUS_MEANINGS`.""" @@ -1372,6 +2027,7 @@ def report(self) -> dict: "entries_examined": self._entries, "statements_compared": self._statements, "statements_unusable": self._unusable, + "terms_check": self._terms_check, } def __bool__(self) -> bool: @@ -1385,41 +2041,54 @@ def __bool__(self) -> bool: ) def __repr__(self) -> str: + disagreement = ", terms_check='fails'" if self._terms_check == "fails" else "" return ( f"NoveltyVerdict(status={self.status!r}, matches={len(self._matches)}, " f"entries_examined={self._entries}, sources_consulted=" - f"{list(self._consulted)})" + f"{list(self._consulted)}{disagreement})" ) def check_novelty( - claim: RecurrenceClaim, + claim: RecurrenceClaim | QRecurrenceClaim, sources: Sequence[Any], *, terms: Sequence[int] | None = None, ids: Sequence[str] | None = None, + start: int = 0, ) -> NoveltyVerdict: """Look *claim* up in *sources* and report what was found. :param claim: the normalised claim — build it with :meth:`RecurrenceClaim.from_recurrence` from a :class:`~alkahest.ZeilbergerCertificate` or a - :class:`~alkahest.GuessedRecurrence`. + :class:`~alkahest.GuessedRecurrence`, or with + :meth:`QRecurrenceClaim.from_recurrence` from a + :class:`~alkahest.QZeilbergerCertificate`. :param sources: objects with a ``name`` and a ``lookup(*, terms=None, ids=None)`` returning a :class:`SourceAnswer` or ``None``. :class:`OeisCache` offline, :class:`OeisWeb` live. **There is no default**: a check with no source returns ``unavailable``, and this module will not quietly reach for the network - on your behalf. + on your behalf. A source may declare a ``CLAIM_KINDS`` tuple; one that + cannot state ``claim.claim_kind`` is reported *unavailable* for it, + because a search that could not have matched is not a negative. :param terms: exact leading terms of the sequence, to identify it. Give enough that the identification is not accidental — ten is plenty for a - sequence that grows. + sequence that grows. They are **also checked against the claim**: see + :attr:`NoveltyVerdict.terms_check`, and *start* below. :param ids: source-specific identifiers to check instead, e.g. ``["A005259"]``. + :param start: the true index of ``terms[0]``, for that cross-check only — + it is never sent to a source. Exactly the parameter of + :meth:`RecurrenceClaim.holds_for`, and exactly as load-bearing: a + recurrence with polynomial coefficients evaluated at the wrong ``n`` + confirms nothing, so a wrong *start* shows up as + ``terms_check == "fails"``. :returns: a :class:`NoveltyVerdict`. Never raises for a missing source or a dead network; those are ``unavailable``. - :raises TypeError: when *claim* is not a :class:`RecurrenceClaim`. + :raises TypeError: when *claim* is not a claim type of this module. :raises ValueError: when neither *terms* nor *ids* is given. >>> from alkahest.experimental.novelty import ( @@ -1440,22 +2109,28 @@ def check_novelty( >>> check_novelty(claim, [], terms=[1, 2, 6, 20]).found is None True """ - if not isinstance(claim, RecurrenceClaim): + if not isinstance(claim, (RecurrenceClaim, QRecurrenceClaim)): raise TypeError( - "claim must be a RecurrenceClaim; build one with " - "RecurrenceClaim.from_recurrence(certificate, var=n) so that what " - "is looked up is the normal form, not one presentation of it" + "claim must be a RecurrenceClaim or a QRecurrenceClaim; build one " + "with RecurrenceClaim.from_recurrence(certificate, var=n) so that " + "what is looked up is the normal form, not one presentation of it" ) if not terms and not ids: raise ValueError( "give terms= (the sequence's leading terms) or ids= (source " "identifiers); there is nothing to look up otherwise" ) + terms_check = _cross_check_terms(claim, terms, start) matches, consulted, unavailable = [], [], [] seen_entries: dict = {} statements = unusable = 0 for source in sources: name = getattr(source, "name", type(source).__name__) + if claim.claim_kind not in getattr(source, "CLAIM_KINDS", ("recurrence",)): + # The source cannot state a claim of this kind at all, so its + # silence is not evidence of anything. + unavailable.append(name) + continue answer = source.lookup(terms=terms, ids=ids) if answer is None or not answer.exhaustive: unavailable.append(name) @@ -1487,4 +2162,27 @@ def check_novelty( entries=len(seen_entries), statements=statements, unusable=unusable, + terms_check=terms_check, ) + + +def _cross_check_terms(claim: Any, terms: Sequence[int] | None, start: int) -> str: + """Re-check *claim* against the terms it is being looked up by. + + ``terms`` drives the search; it is also, by construction, a statement about + the same sequence the claim is about, so the two can be held against each + other for free. The rule is the lenient one a source's own formula line has + to pass in :meth:`OeisEntry.recurrences`: the trailing windows must confirm, + because a recurrence is routinely stated only for ``n`` past some initial + segment. + """ + if not terms or not isinstance(claim, RecurrenceClaim): + return "not_checked" + windows = len(terms) - claim.order + if windows <= 0: + return "not_checked" + try: + confirmations = claim.confirmations(terms, start=start) + except (TypeError, ValueError, ZeroDivisionError): + return "not_checked" + return "holds" if confirmations >= min(_MIN_CONFIRMATIONS, windows) else "fails" diff --git a/tests/data/oeis_novelty_fixture.json b/tests/data/oeis_novelty_fixture.json index f3104eb3..c717dd52 100644 --- a/tests/data/oeis_novelty_fixture.json +++ b/tests/data/oeis_novelty_fixture.json @@ -3,6 +3,168 @@ "version": 1, "license": "Sequence data and formula lines are from the On-Line Encyclopedia of Integer Sequences (https://oeis.org), (c) The OEIS Foundation Inc., licensed CC BY-NC-SA 4.0.", "entries": { + "A000045": { + "id": "A000045", + "name": "Fibonacci numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1.", + "offset": 0, + "terms": [ + 0, + 1, + 1, + 2, + 3, + 5, + 8, + 13, + 21, + 34, + 55, + 89, + 144, + 233, + 377, + 610, + 987, + 1597, + 2584, + 4181, + 6765, + 10946, + 17711, + 28657, + 46368, + 75025, + 121393, + 196418, + 317811, + 514229, + 832040, + 1346269, + 2178309, + 3524578, + 5702887, + 9227465, + 14930352, + 24157817, + 39088169, + 63245986, + 102334155 + ], + "statements": [ + "F(n) = F(n-1) + F(n-2) = -(-1)^n F(-n).", + "F(n+1) = Sum_{j=0..floor(n/2)} binomial(n-j, j).", + "[0 1; 1 1]^n [0 1] = [F(n); F(n+1)]", + "a(n)=F(n) has the property: F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1). - _Miklos Kristof_, Nov 13 2003", + "Conjecture 1: for n >= 2, sqrt(F(2n+1) + F(2n+2) + F(2n+3) + F(2n+4) + 2*(-1)^n) = (F(2n+1) + 2*(-1)^n)/F(n-1). [For a proof see Comments section.]", + "Conjecture 2: for n >= 0, (F(n+2)*F(n+3)) - (F(n+1)*F(n+4)) + (-1)^n = 0.", + "Theorem 1: for n >= 0, (F(n+3)^ 2 - F(n+1)^ 2)/F(n+2) = (F(n+3)+ F(n+1)).", + "Theorem 2: for n >= 0, F(n+10) = 11*F(n+5) + F(n).", + "Theorem 3: for n >= 6, F(n) = 4*F(n-3) + F(n-6). (End)", + "Conjecture 2 of Rashid is actually a special case of the general law F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1) (take n <- n+1 and m <- -(n+4) in this law). - Harmel Nestra (harmel.nestra(AT)ut.ee), Apr 22 2005", + "Conjecture 2 of Rashid Kurmang simplified: F(n)*F(n+3) = F(n+1)*F(n+2)-(-1)^n. Follows from d'Ocagne's identity: m=n+2. - _Alex Ratushnyak_, May 06 2012", + "Conjecture: for all c such that 2-phi <= c < 2*(2-phi) we have F(n) = floor(phi*a(n-1)+c) for n > 2. - _Gerald McGarvey_, Jul 21 2004", + "F(n+1) = exponent of the n-th term in the series f(x, 1) determined by the equation f(x, y) = xy + f(xy, x). - _Jonathan Sondow_, Dec 19 2004", + "a(n-1) = Sum_{k=0..n} (-1)^k*binomial(n-ceiling(k/2), floor(k/2)). - _Benoit Cloitre_, May 05 2005", + "F(n+1) = Sum_{k=0..n} binomial((n+k)/2, (n-k)/2)(1+(-1)^(n-k))/2. - _Paul Barry_, Aug 28 2005", + "a(n) = (b(n+1) + b(n-1))/n where {b(n)} is the sequence A001629. - _Sergio Falcon_, Nov 22 2006", + "F(n*m) = Sum_{k = 0..m} binomial(m,k)*F(n-1)^k*F(n)^(m-k)*F(m-k). The generating function of F(n*m) (n fixed, m = 0,1,2,...) is G(x) = F(n)*x / ((1 - F(n-1)*x)^2 - F(n)*x*(1 - F(n-1)*x) - (F(n)*x)^2). E.g., F(15) = 610 = F(5*3) = binomial(3,0)* F(4)^0*F(5)^3*F(3) + binomial(3,1)* F(4)^1*F(5)^2*F(2) + binomial(3,2)* F(4)^2*F(5)^1*F(1) + binomial(3,3)* F(4)^3*F(5)^0*F(0) = 1*1*125*2 + 3*3*25*1 + 3*9*5*1 + 1*27*1*0 = 250 + 225 + 135 + 0 = 610. - _Miklos Kristof_, Feb 12 2007", + "F(n + 3) = 2F(n + 2) - F(n), F(n + 4) = 3F(n + 2) - F(n), F(n + 8) = 7F(n + 4) - F(n), F(n + 12) = 18F(n + 6) - F(n). - _Paul Curtz_, Feb 01 2008", + "a(n+1) = Sum_{k=0..n} A109466(n,k)*(-1)^(n-k). -_Philippe Del\u00e9ham_, Oct 26 2008", + "a(n+1) = 2^n sqrt(Product_{k=1..n} cos(k Pi/(n+1))^2+1/4) (Kasteleyn's formula specialized). - _Sarah-Marie Belcastro_, Jul 04 2009", + "a(n+1) = Sum_{k=floor(n/2) mod 5} C(n,k) - Sum_{k=floor((n+5)/2) mod 5} C(n,k) = A173125(n) - A173126(n) = |A054877(n)-A052964(n-1)|. - _Henry Bottomley_, Feb 10 2010", + "For n >= 1, F(n) = round(log_2(2^(phi*F(n-1)) + 2^(phi*F(n-2)))), where phi is the golden ratio. - _Vladimir Shevelev_, Jun 24 2010, Jun 27 2010", + "For n >= 1, a(n+1) = ceiling(phi*a(n)), if n is even and a(n+1) = floor(phi*a(n)), if n is odd (phi = golden ratio). - _Vladimir Shevelev_, Jul 01 2010", + "a(n) = 2*a(n-2) + a(n-3), n > 2. - _Gary Detlefs_, Sep 08 2010", + "a(n)^2 - a(n-1)^2 = a(n+1)*a(n-2), see A121646.", + "F(2*n) = F(n+2)^2 - F(n+1)^2 - 2*F(n)^2. - _Richard R. Forberg_, Jun 04 2011", + "F(n) = F(n+2) - 1 + (F(n+1))^4 + 2*(F(n+1)^3*F(n+2)) - (F(n+1)*F(n+2))^2 - 2*F(n+1)(F(n+2))^3 + (F(n+2))^4 - F(n+1). (End)", + "F(n) = 4*F(n-2) - 2*F(n-3) - F(n-6). - _Gary Detlefs_, Apr 01 2012", + "F(n) = Sum_{j=0..k} S(j+1,n-2j), where k = floor((n-1)/2) and the S(j,n) are the n-th j-simplex sums: S(1,n) = 1 is the 1-simplex sum, S(2,n) = Sum_{k=1..n} S(1,k) = 1+1+...+1 = n is the 2-simplex sum, S(3,n) = Sum_{k=1..n} S(2,k) = 1+2+3+...+n is the 3-simplex sum (= triangular numbers = A000217), S(4,n) = Sum_{k=1..n} S(3,k) = 1+3+6+...+n(n+1)/2 is the 4-simplex sum (= tetrahedral numbers = A000292) and so on.", + "Sum_{n >= 1} (-1)^(n-1)/(a(n)*a(n+1)) = 1/phi (phi=golden ratio). - _Vladimir Shevelev_, Feb 22 2013", + "(1) Expression a(n+1) via a(n): a(n+1) = (a(n) + sqrt(5*(a(n))^2 + 4*(-1)^n))/2;", + "(2) Sum_{k=1..n} (-1)^(k-1)/(a(k)*a(k+1)) = a(n)/a(n+1);", + "(3) a(n)/a(n+1) = 1/phi + r(n), where |r(n)| < 1/(a(n+1)*a(n+2)). (End)", + "F(n+1) = F(n)/2 + sqrt((-1)^n + 5*F(n)^2/4), n >= 0. F(n+1) = U_n(i/2)/i^n, (U:= Chebyshev polynomial of the 2nd kind, i=sqrt(-1)). - _Bill Gosper_, Mar 04 2013", + "Let b(n) = b(n-1) + b(n-2), with b(0) = 0, b(1) = phi. Then, for n >= 2, F(n) = floor(b(n-1)) if n is even, F(n) = ceiling(b(n-1)), if n is odd, with convergence. - _Richard R. Forberg_, Jan 19 2014", + "F(n) = round(sqrt(F(n-1)^2 + F(n)^2 + F(n+1)^2)/2), for n > 0. This rule appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Jul 27 2014", + "F(n) = round(2/(1/F(n) + 1/F(n+1) + 1/F(n+2))), for n > 0. This rule also appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Aug 03 2014", + "Limit_{n -> oo} (log F(n+1)/log F(n))^n = e. - _Thomas Ordowski_, Oct 06 2014", + "F(n) = (L(n+1)^2 - L(n-1)^2)/(5*L(n)), where L(n) is A000032(n), with a similar inverse relationship. - _Richard R. Forberg_, Nov 17 2014", + "Consider the graph G[1-vertex;1-loop,2-loop] in comment above. Construct the power matrix array T(n,j) = [A^*j]*[S^*(j-1)] where A=(1,1,0,...) and S=(0,1,0,...)(A063524). [* is convolution operation] Define S^*0=I with I=(1,0,...). Then T(n,j) counts n-walks containing (j) loops and a(n-1) = Sum_{j=1..n} T(n,j). - _David Neil McGrath_, Nov 21 2014", + "F(2*n) = F(n+1)^2 - F(n-1)^2, similar to Koshy (D) and Forberg 2011, but different. - _Hermann Stamm-Wilbrandt_, Aug 12 2015", + "F(n+1) = ceiling( (1/phi)*Sum_{k=0..n} F(k) ). - _Tom Edgar_, Sep 10 2015", + "a(n) = (L(n-3) + L(n+3))/10 where L(n)=A000032(n). - _J. M. Bergot_, Nov 25 2015", + "F(n) = (F(2n+k+1) - F(n+1)*F(n+k+1))/F(n+k), k >= 0.", + "Thus when k=0: F(n) = sqrt(F(2n+1) - F(n+1)^2).", + "F(n) = (F(3n) - F(n+1)^3 + F(n-1)^3)^(1/3).", + "f(n+1) = Sum_{j=0..floor(n/2)} Sum_{k=0..j} binomial(n-2j,k)*binomial(j,k). - _Tony Foster III_, Sep 04 2017", + "a(n) = (L(n-3) + L(n-2) + L(n-1) + L(n))/5 with L(n)=A000032(n). - _Art Baker_, Jan 04 2019", + "For n > 0, 1/F(n) = Sum_{k>=1} F(n*k)/(F(n+2)^(k+1)). - _Diego Rattaggi_, Oct 26 2022", + "F(n) = Sum_{i=0..n-1} F(i)^2 / F(n-1). - _Jules Beauchamp_, May 03 2025", + "In keeping with historical accounts (see the references by P. Singh and S. Kak), the generalized Fibonacci sequence a, b, a + b, a + 2b, 2a + 3b, 3a + 5b, ... can also be described as the Gopala-Hemachandra numbers H(n) = H(n-1) + H(n-2), with F(n) = H(n) for a = b = 1, and Lucas sequence L(n) = H(n) for a = 2, b = 1. - _Lekraj Beedassy_, Jan 11 2015", + "F(n+2) = number of binary sequences of length n that have no consecutive 0's.", + "F(n+2) = number of subsets of {1,2,...,n} that contain no consecutive integers.", + "F(n+1) = number of tilings of a 2 X n rectangle by 2 X 1 dominoes.", + "F(n+1) = number of matchings (i.e., Hosoya index) in a path graph on n vertices: F(5)=5 because the matchings of the path graph on the vertices A, B, C, D are the empty set, {AB}, {BC}, {CD} and {AB, CD}. - _Emeric Deutsch_, Jun 18 2001", + "Positive terms are the solutions to z = 2*x*y^4 + (x^2)*y^3 - 2*(x^3)*y^2 - y^5 - (x^4)*y + 2*y for x,y >= 0 (Ribenboim, page 193). When x=F(n), y=F(n + 1) and z > 0 then z=F(n + 1).", + "F(n+1) is the number of perfect matchings in ladder graph L_n = P_2 X P_n. - Sharon Sela (sharonsela(AT)hotmail.com), May 19 2002", + "F(n+1) = number of (3412,132)-, (3412,213)- and (3412,321)-avoiding involutions in S_n.", + "The number of sequences (s(0),s(1),...,s(n)) such that 0 < s(i) < 5, |s(i)-s(i-1)|=1 and s(0)=1 is F(n+1); e.g., F(5+1) = 8 corresponds to 121212, 121232, 121234, 123212, 123232, 123234, 123432, 123434. - _Clark Kimberling_, Jun 22 2004 [corrected by Neven Juric, Jan 09 2009]", + "F(n+1) (for n >= 1) = number of permutations p of 1,2,3,...,n such that |k-p(k)| <= 1 for k=1,2,...,n. (For <= 2 and <= 3, see A002524 and A002526.) - _Clark Kimberling_, Nov 28 2004", + "The ratios F(n+1)/F(n) for n > 0 are the convergents to the simple continued fraction expansion of the golden section. - _Jonathan Sondow_, Dec 19 2004", + "F(n+2) = Sum_{k=0..n} binomial(floor((n+k)/2),k), row sums of A046854. - _Paul Barry_, Mar 11 2003", + "F(n+1)/F(n) is also the Farey fraction sequence (see A097545 for explanation) for the golden ratio, which is the only number whose Farey fractions and continued fractions are the same. - _Joshua Zucker_, May 08 2006", + "a(n+2) is the number of paths through 2 plates of glass with n reflections (reflections occurring at plate/plate or plate/air interfaces). Cf. A006356-A006359. - _Mitch Harris_, Jul 06 2006", + "F(n+1) equals the number of downsets (i.e., decreasing subsets) of an n-element fence, i.e., an ordered set of height 1 on {1,2,...,n} with 1 > 2 < 3 > 4 < ... n and no other comparabilities. Alternatively, F(n+1) equals the number of subsets A of {1,2,...,n} with the property that, if an odd k is in A, then the adjacent elements of {1,2,...,n} belong to A, i.e., both k - 1 and k + 1 are in A (provided they are in {1,2,...,n}). - _Brian Davey_, Aug 25 2006", + "Inverse: floor(log_phi(sqrt(5)*F(n)) + 1/2) = n, for n > 1. Also for n > 0, floor((1/2)*log_phi(5*F(n)*F(n+1))) = n. Extension valid for integer n, except n=0,-1: floor((1/2)*sign(F(n)*F(n+1))*log_phi|5*F(n)*F(n+1)|) = n (where sign(x) = sign of x). - _Hieronymus Fischer_, May 02 2007", + "F(n+2) = the number of Khalimsky-continuous functions with a two-point codomain. - Shiva Samieinia (shiva(AT)math.su.se), Oct 04 2007", + "Let phi = A001622 then phi^n = (1/phi)*a(n) + a(n+1). - _Gary W. Adamson_, Dec 15 2007", + "The sequence of first differences, F(n+1)-F(n), is essentially the same sequence: 1, 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, ... - _Colm Mulcahy_, Mar 03 2008", + "(F(n),F(n+4)) satisfies the Diophantine equation: X^2 + Y^2 - 7XY = 9*(-1)^n. - _Mohamed Bouhamida_, Sep 06 2009", + "(F(n),F(n+2)) satisfies the Diophantine equation: X^2 + Y^2 - 3XY = (-1)^n. - _Mohamed Bouhamida_, Sep 08 2009", + "a(n+2) = A083662(A131577(n)). - _Reinhard Zumkeller_, Sep 26 2009", + "F(n+1) = number of Motzkin paths of length n having exactly one weak ascent. A Motzkin path of length n is a lattice path from (0,0) to (n,0) consisting of U=(1,1), D=(1,-1) and H=(1,0) steps and never going below the x-axis. A weak ascent in a Motzkin path is a maximal sequence of consecutive U and H steps. Example: a(5)=5 because we have (HHHH), (HHU)D, (HUH)D, (UHH)D, and (UU)DD (the unique weak ascent is shown between parentheses; see A114690). - _Emeric Deutsch_, Mar 11 2010", + "(F(n-1) + F(n+1))^2 - 5*F(n-2)*F(n+2) = 9*(-1)^n. - _Mohamed Bouhamida_, Mar 31 2010", + "F(n+1) = number of paths of length n starting at initial node on the path graph P_4. - _Johannes W. Meijer_, May 27 2010", + "As n->oo, (a(n)/a(n-1) - a(n-1)/a(n)) tends to 1.0. Example: a(12)/a(11) - a(11)/a(12) = 144/89 - 89/144 = 0.99992197.... - _Gary W. Adamson_, Jul 16 2010", + "Fibonacci numbers are those numbers m such that m*phi is closer to an integer than k*phi for all k, 1 <= k < m. More formally: a(0)=0, a(1)=1, a(2)=1, a(n+1) = minimal m > a(n) such that m*phi is closer to an integer than a(n)*phi.", + "F(n) = round(phi*F(n-1)) for n > 1. - _Joseph P. Shoulak_, Jan 13 2012", + "The sequence F(n+1)^(1/n) is increasing. The sequence F(n+2)^(1/n) is decreasing. - _Thomas Ordowski_, Apr 19 2012", + "Two conjectures: For n > 1, F(n+2)^2 mod F(n+1)^2 = F(n)*F(n+1) - (-1)^n. For n > 0, (F(2n) + F(2n+2))^2 = F(4n+3) + Sum_{k = 2..2n} F(2k). - _Alex Ratushnyak_, May 06 2012", + "Proof of Ratushnyak's first conjecture: For n > 1, F(n+2)^2 - F(n)*F(n+1) + (-1)^n = 2*F(n+1)^2.", + "Consider: F(n+2)^2 - F(n)*F(n+1) - 2*F(n+1)^2", + " = F(n+2)^2 - F(n+1)^2 - F(n+1)^2 - F(n)*F(n+1)", + " = (F(n+2) + F(n+1))*(F(n+2) - F(n+1)) - F(n+1)*(F(n+1) + F(n))", + " = F(n+3)*F(n) - F(n+1)*F(n+2) = -(-1)^n.", + "The sequence F(n) is the binomial transformation of the alternating sequence (-1)^(n-1)*F(n), whereas the sequence F(n+1) is the binomial transformation of the alternating sequence (-1)^n*F(n-1). Both of these facts follow easily from the equalities a(n;1)=F(n+1) and b(n;1)=F(n) where a(n;d) and b(n;d) are so-called \"delta-Fibonacci\" numbers as defined in comments to A014445 (see also the papers of Witula et al.). - _Roman Witula_, Jul 24 2012", + "For positive n, F(n+1) equals the determinant of the n X n tridiagonal matrix with 1's along the main diagonal, i's along the superdiagonal and along the subdiagonal where i = sqrt(-1). Example: Det([1,i,0,0; i,1,i,0; 0,i,1,i; 0,0,i,1]) = F(4+1) = 5. - _Philippe Del\u00e9ham_, Feb 24 2013", + "For n >= 1, number of compositions of n where there is a drop between every second pair of parts, starting with the first and second part; see example. Also, a(n+1) is the number of compositions where there is a drop between every second pair of parts, starting with the second and third part; see example. - _Joerg Arndt_, May 21 2013 [see the Hopkins/Tangboonduangjit reference for a proof, see also the Checa reference for alternative proofs and statistics]", + "For n >= 4, F(n-1) is the number of simple permutations in the geometric grid class given in A226433. - _Jay Pantone_, Sep 08 2013", + "a(n) are the pentagon (not pentagonal) numbers because the algebraic degree 2 number rho(5) = 2*cos(Pi/5) = phi (golden section), the length ratio diagonal/side in a pentagon, has minimal polynomial C(5,x) = x^2 - x - 1 (see A187360, n=5), hence rho(5)^n = a(n-1)*1 + a(n)*rho(5), n >= 0, in the power basis of the algebraic number field Q(rho(5)). One needs a(-1) = 1 here. See also the P. Steinbach reference under A049310. - _Wolfdieter Lang_, Oct 01 2013", + "The expression round(1/(F(k+1)/F(n) + F(k)/F(n+1))), for n > 0, yields a Fibonacci sequence with k-1 leading zeros (with rounding 0.5 to 0). - _Richard R. Forberg_, Aug 04 2014", + "a(n+1) counts closed walks on K_2, containing one loop on the other vertex. Equivalently the (1,1)_entry of A^(n+1) where the adjacency matrix of digraph is A=(0,1; 1,1). - _David Neil McGrath_, Oct 29 2014", + "a(n-1) counts closed walks on the graph G(1-vertex;l-loop,2-loop). - _David Neil McGrath_, Nov 26 2014", + "F(n+1) equals the number of binary words of length n avoiding runs of zeros of odd lengths. - _Milan Janjic_, Jan 28 2015", + "We use the following notation: F(n)=A000045(n), the Fibonacci numbers, and L(n) = A000032(n), the Lucas numbers. The fundamental Fibonacci-Lucas recursion asserts that G(n) = G(n-1) + G(n-2), with \"L\" or \"F\" replacing \"G\".", + "We need the following prerequisites which we label (A), (B), (C), (D). The prerequisites are formulas in the Koshy book listed in the References section. (A) F(m-1) + F(m+1) = L(m) (Koshy, p. 97, #32), (B) L(2m) + 2*(-1)^m = L(m)^2 (Koshy p. 97, #41), (C) F(m+k)*F(m-k) = (-1)^n*F(k)^2 (Koshy, p. 113, #24, Tagiuri's identity), and (D) F(n)^2 + F(n+1)^2 = F(2n+1) (Koshy, p. 97, #30).", + "We must also prove (E), L(n+2)*F(n-1) = F(2n+1)+2*(-1)^n. To prove (E), first note that by (A), proof of (E) is equivalent to proving that F(n+1)*F(n-1) + F(n+3)*F(n-1) = F(2n+1) + 2*(-1)^n. But by (C) with k=1, we have F(n+1)*F(n-1) = F(n)^2 + (-1)^n. Applying (C) again with k=2 and m=n+1, we have F(n+3)*F(n-1) = F(n+1) + (-1)^n. Adding these two applications of (C) together and using (D) we have F(n+1)*F(n-1) + F(n+3)*F(n-1) = F(n)^2 + F(n+1)^2 + 2*(-1)^n = F(2n+1) + 2(-1)^n, completing the proof of (E).", + "We now prove Conjecture 1. By (A) and the Fibonacci-Lucas recursion, we have F(2n+1) + F(2n+2) + F(2n+3) + F(2n+4) = (F(2n+1) + F(2n+3)) + (F(2n+2) + F(2n+4)) = L(2n+2) +L(2n+3) = L(2n+4). But then by (B), with m=2n+4, we have sqrt(L(2n+4) + 2(-1)^n) = L(n+2). Finally by (E), we have L(n+2)*F(n-1) = F(2n+1) + 2*(-1)^n. Dividing both sides by F(n-1), we have (F(2n+1) + 2*(-1)^n)/F(n-1) = L(n+2) = sqrt(F(2n+1) + F(2n+2) + F(2n+3) + F(2n+4) + 2(-1)^n), as required.", + "F(n+2) is the number of terms in p(n), where p(n)/q(n) is the n-th convergent of the formal infinite continued fraction [a(0),a(1),...]; e.g., p(3) = a(0)a(1)a(2)a(3) + a(0)a(1) + a(0)a(3) + a(2)a(3) + 1 has F(5) terms. Also, F(n+1) is the number of terms in q(n). - _Clark Kimberling_, Dec 23 2015", + "F(n+1) (for n >= 1) is the permanent of an n X n matrix M with M(i,j)=1 if |i-j| <= 1 and 0 otherwise. - _Dmitry Efimov_, Jan 08 2016", + "A trapezoid has three sides of lengths in order F(n), F(n+2), F(n). For increasing n a very close approximation to the maximum area will have the fourth side equal to 2*F(n+1). For a trapezoid with lengths of sides in order F(n+2), F(n), F(n+2), the fourth side will be F(n+3). - _J. M. Bergot_, Mar 17 2016", + "(1) Join two triangles with lengths of sides L(n), F(n+3), L(n+2) and F(n+2), L(n+1), L(n+2) (where L(n)=A000032(n)) along the common side of length L(n+2) to create an irregular quadrilateral. Its area is approximately 5*F(2*n-1) - (F(2*n-7) - F(2*n-13))/5. (2) Join two triangles with lengths of sides L(n), F(n+2), F(n+3) and L(n+1), F(n+1), F(n+3) along the common side F(n+3) to form an irregular quadrilateral. Its area is approximately 4*F(2*n-1) - 2*(F(2*n-7) + F(2*n-18)). - _J. M. Bergot_, Apr 06 2016", + "Consider the partitions of n, with all summands initially listed in nonincreasing order. Freeze all the 1's in place and then allow all the other summands to change their order, without displacing any of the 1's. The resulting number of arrangements is a(n+1). - _Gregory L. Simay_, Jun 14 2016", + "F(n) and Lucas numbers L(n), being related by the formulas F(n) = (F(n-1) + L(n-1))/2 and L(n) = 2 F(n+1) - F(n), are a typical pair of \"autosequences\" (see the link to OEIS Wiki). - _Jean-Fran\u00e7ois Alcover_, Jun 10 2017", + "F(n+1) is the number of fixed points of the Foata transformation on S_n. - _Kevin Long_, Oct 17 2018", + "F(n+2) is the dimension of the Hecke algebra of type A_n with independent parameters (0,1,0,1,...) or (1,0,1,0,...). See Corollary 1.5 in the link \"Hecke algebras with independent parameters\". - _Jia Huang_, Jan 20 2019", + "F(n+1) is the number of permutations in S_n whose principal order ideals in the weak order are Boolean lattices. - _Bridget Tenner_, Jan 16 2020", + "F(n+1) is the number of permutations w in S_n that form Boolean intervals [s, w] in the weak order for every simple reflection s in the support of w. - _Bridget Tenner_, Jan 16 2020", + "F(n+1) is the number of subsets of {1,2,.,.,n} in which all differences between successive elements of subsets are odd. For example, for n = 6, F(7) = 13 and the 13 subsets are {6}, {1,6}, {3,6}, {5,6}, {2,3,6}, {2,5,6}, {4,5,6}, {1,2,3,6}, {1,2,5,6}, {1,4,5,6}, {3,4,5,6}, {2,3,4,5,6}, {1,2,3,4,5,6}. For even differences between elements see Comment in A016116. - _Enrique Navarrete_, Jul 01 2020", + "F(n) is the number of subsets of {1,2,...,n} in which the smallest element of the subset equals the size of the subset (this type of subset is sometimes called extraordinary). For example, F(6) = 8 and the subsets are {1}, {2,3}, {2,4}, {2,5}, {3,4,5}, {2,6}, {3,4,6}, {3,5,6}. It is easy to see that these subsets follow the Fibonacci recursion F(n) = F(n-1) + F(n-2) since we get F(n) such subsets by keeping all F(n-1) subsets from the previous stage (in the example, the F(5)=5 subsets that don't include 6), and by adding one to all elements and appending an additional element n to each subset in F(n-2) subsets (in the example, by applying this to the F(4)=3 subsets {1}, {2,3}, {2,4} we obtain {2,6}, {3,4,6}, {3,5,6}). - _Enrique Navarrete_, Sep 28 2020", + "For n >= 1, number of compositions (c(1),c(2),...,c(k)) of n where c(1), c(3), c(5), ... are 1. To obtain such compositions K(n) of length n increase all parts c(2) by one in all of K(n-1) and prepend two parts 1 in all of K(n-2). - _Joerg Arndt_, Jan 05 2024", + "If n*(great) denotes n repetitions of \"great\", then F(n+4) is the number of n*(great)-grandparents a honeybee queen or worker has, and F(n+3) is the number of n*(great)-grandparents a honeybee drone has. This is because males hatch from unfertilized eggs and females hatch from fertilized ones. - _Johann Peters_, Apr 15 2026" + ] + }, "A000108": { "id": "A000108", "name": "Catalan numbers: C(n) = binomial(2n,n)/(n+1) = (2n)!/(n!(n+1)!).", @@ -627,6 +789,9 @@ } }, "queries": { + "id:A000045": [ + "A000045" + ], "seq:1,1,2,4,9,21,51,127,323,835,2188,5798": [ "A001006", "A086246", diff --git a/tests/data/oeis_paging_fixture.json b/tests/data/oeis_paging_fixture.json new file mode 100644 index 00000000..4ddd42f3 --- /dev/null +++ b/tests/data/oeis_paging_fixture.json @@ -0,0 +1,648 @@ +{ + "kind": "alkahest.oeis_raw_pages", + "version": 1, + "license": "Sequence data and formula lines are from the On-Line Encyclopedia of Integer Sequences (https://oeis.org), (c) The OEIS Foundation Inc., licensed CC BY-NC-SA 4.0.", + "note": "Raw https://oeis.org/search?...&fmt=json pages, keyed 'query|start', trimmed to the fields OeisEntry.from_oeis_json reads. Recorded once so the paging tests can run with no network.", + "pages": { + "1,1,2,3,5,8,13|0": [ + { + "number": 45, + "name": "Fibonacci numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1.", + "data": "0,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987,1597,2584,4181,6765,10946,17711,28657,46368,75025,121393,196418,317811,514229,832040,1346269,2178309,3524578,5702887,9227465,14930352,24157817,39088169,63245986,102334155", + "offset": "0,4", + "formula": [ + "G.f.: x / (1 - x - x^2).", + "G.f.: Sum_{n>=0} x^n * Product_{k=1..n} (k + x)/(1 + k*x). - _Paul D. Hanna_, Oct 26 2013", + "F(n) = ((1+sqrt(5))^n - (1-sqrt(5))^n)/(2^n*sqrt(5)).", + "Alternatively, F(n) = ((1/2+sqrt(5)/2)^n - (1/2-sqrt(5)/2)^n)/sqrt(5).", + "F(n) = F(n-1) + F(n-2) = -(-1)^n F(-n).", + "F(n) = round(phi^n/sqrt(5)).", + "F(n+1) = Sum_{j=0..floor(n/2)} binomial(n-j, j).", + "A strong divisibility sequence, that is, gcd(a(n), a(m)) = a(gcd(n, m)) for all positive integers n and m. - _Michael Somos_, Jan 03 2017", + "E.g.f.: (2/sqrt(5))*exp(x/2)*sinh(sqrt(5)*x/2). - _Len Smiley_, Nov 30 2001", + "[0 1; 1 1]^n [0 1] = [F(n); F(n+1)]", + "x | F(n) ==> x | F(kn).", + "A sufficient condition for F(m) to be divisible by a prime p is (p - 1) divides m, if p == 1 or 4 (mod 5); (p + 1) divides m, if p == 2 or 3 (mod 5); or 5 divides m, if p = 5. (This is essentially Theorem 180 in Hardy and Wright.) - Fred W. Helenius (fredh(AT)ix.netcom.com), Jun 29 2001", + "a(n)=F(n) has the property: F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1). - _Miklos Kristof_, Nov 13 2003", + "From _Kurmang. Aziz. Rashid_, Feb 21 2004: (Start)", + "Conjecture 1: for n >= 2, sqrt(F(2n+1) + F(2n+2) + F(2n+3) + F(2n+4) + 2*(-1)^n) = (F(2n+1) + 2*(-1)^n)/F(n-1). [For a proof see Comments section.]", + "Conjecture 2: for n >= 0, (F(n+2)*F(n+3)) - (F(n+1)*F(n+4)) + (-1)^n = 0.", + "[Two more conjectures removed by _Peter Luschny_, Nov 17 2017]", + "Theorem 1: for n >= 0, (F(n+3)^ 2 - F(n+1)^ 2)/F(n+2) = (F(n+3)+ F(n+1)).", + "Theorem 2: for n >= 0, F(n+10) = 11*F(n+5) + F(n).", + "Theorem 3: for n >= 6, F(n) = 4*F(n-3) + F(n-6). (End)", + "Conjecture 2 of Rashid is actually a special case of the general law F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1) (take n <- n+1 and m <- -(n+4) in this law). - Harmel Nestra (harmel.nestra(AT)ut.ee), Apr 22 2005", + "Conjecture 2 of Rashid Kurmang simplified: F(n)*F(n+3) = F(n+1)*F(n+2)-(-1)^n. Follows from d'Ocagne's identity: m=n+2. - _Alex Ratushnyak_, May 06 2012", + "Conjecture: for all c such that 2-phi <= c < 2*(2-phi) we have F(n) = floor(phi*a(n-1)+c) for n > 2. - _Gerald McGarvey_, Jul 21 2004", + "For x > phi, Sum_{n>=0} F(n)/x^n = x/(x^2 - x - 1). - _Gerald McGarvey_, Oct 27 2004", + "F(n+1) = exponent of the n-th term in the series f(x, 1) determined by the equation f(x, y) = xy + f(xy, x). - _Jonathan Sondow_, Dec 19 2004", + "a(n-1) = Sum_{k=0..n} (-1)^k*binomial(n-ceiling(k/2), floor(k/2)). - _Benoit Cloitre_, May 05 2005", + "a(n) = Sum_{k=0..n} abs(A108299(n, k)). - _Reinhard Zumkeller_, Jun 01 2005", + "a(n) = A001222(A000304(n)).", + "F(n+1) = Sum_{k=0..n} binomial((n+k)/2, (n-k)/2)(1+(-1)^(n-k))/2. - _Paul Barry_, Aug 28 2005", + "Fibonacci(n) = Product_{j=1..ceiling(n/2)-1} (1 + 4(cos(j*Pi/n))^2). [Bicknell and Hoggatt, pp. 47-48.] - _Emeric Deutsch_, Oct 15 2006", + "F(n) = 2^(-(n-1))*Sum_{k=0..floor((n-1)/2)} binomial(n,2*k+1)*5^k. - _Hieronymus Fischer_, Feb 07 2006", + "a(n) = (b(n+1) + b(n-1))/n where {b(n)} is the sequence A001629. - _Sergio Falcon_, Nov 22 2006", + "F(n*m) = Sum_{k = 0..m} binomial(m,k)*F(n-1)^k*F(n)^(m-k)*F(m-k). The generating function of F(n*m) (n fixed, m = 0,1,2,...) is G(x) = F(n)*x / ((1 - F(n-1)*x)^2 - F(n)*x*(1 - F(n-1)*x) - (F(n)*x)^2). E.g., F(15) = 610 = F(5*3) = binomial(3,0)* F(4)^0*F(5)^3*F(3) + binomial(3,1)* F(4)^1*F(5)^2*F(2) + binomial(3,2)* F(4)^2*F(5)^1*F(1) + binomial(3,3)* F(4)^3*F(5)^0*F(0) = 1*1*125*2 + 3*3*25*1 + 3*9*5*1 + 1*27*1*0 = 250 + 225 + 135 + 0 = 610. - _Miklos Kristof_, Feb 12 2007", + "From _Miklos Kristof_, Mar 19 2007: (Start)", + " Let L(n) = A000032(n) = Lucas numbers. Then:", + " For a >= b and odd b, F(a+b) + F(a-b) = L(a)*F(b).", + " For a >= b and even b, F(a+b) + F(a-b) = F(a)*L(b).", + " For a >= b and odd b, F(a+b) - F(a-b) = F(a)*L(b).", + " For a >= b and even b, F(a+b) - F(a-b) = L(a)*F(b).", + " F(n+m) + (-1)^m*F(n-m) = F(n)*L(m);", + " F(n+m) - (-1)^m*F(n-m) = L(n)*F(m);", + " F(n+m+k) + (-1)^k*F(n+m-k) + (-1)^m*(F(n-m+k) + (-1)^k*F(n-m-k)) = F(n)*L(m)*L(k);", + " F(n+m+k) - (-1)^k*F(n+m-k) + (-1)^m*(F(n-m+k) - (-1)^k*F(n-m-k)) = L(n)*L(m)*F(k);", + " F(n+m+k) + (-1)^k*F(n+m-k) - (-1)^m*(F(n-m+k) + (-1)^k*F(n-m-k)) = L(n)*F(m)*L(k);", + " F(n+m+k) - (-1)^k*F(n+m-k) - (-1)^m*(F(n-m+k) - (-1)^k*F(n-m-k)) = 5*F(n)*F(m)*F(k). (End)", + "A corollary to Kristof 2007 is 2*F(a+b) = F(a)*L(b) + L(a)*F(b). - _Graeme McRae_, Apr 24 2014", + "For n > m, the sum of the 2m consecutive Fibonacci numbers F(n-m-1) thru F(n+m-2) is F(n)*L(m) if m is odd, and L(n)*F(m) if m is even (see the McRae link). - _Graeme McRae_, Apr 24 2014.", + "F(n) = b(n) + (p-1)*Sum_{k=2..n-1} floor(b(k)/p)*F(n-k+1) where b(k) is the digital sum analog of the Fibonacci recurrence, defined by b(k) = ds_p(b(k-1)) + ds_p(b(k-2)), b(0)=0, b(1)=1, ds_p=digital sum base p. Example for base p=10: F(n) = A010077(n) + 9*Sum_{k=2..n-1} A059995(A010077(k))*F(n-k+1). - _Hieronymus Fischer_, Jul 01 2007", + "F(n) = b(n)+p*Sum_{k=2..n-1} floor(b(k)/p)*F(n-k+1) where b(k) is the digital product analog of the Fonacci recurrence, defined by b(k) = dp_p(b(k-1)) + dp_p(b(k-2)), b(0)=0, b(1)=1, dp_p=digital product base p. Example for base p=10: F(n) = A074867(n) + 10*Sum_{k=2..n-1} A059995(A074867(k))*F(n-k+1). - _Hieronymus Fischer_, Jul 01 2007", + "a(n) = denominator of continued fraction [1,1,1,...] (with n ones); e.g., 2/3 = continued fraction [1,1,1]; where barover[1] = [1,1,1,...] = 0.6180339.... - _Gary W. Adamson_, Nov 29 2007", + "F(n + 3) = 2F(n + 2) - F(n), F(n + 4) = 3F(n + 2) - F(n), F(n + 8) = 7F(n + 4) - F(n), F(n + 12) = 18F(n + 6) - F(n). - _Paul Curtz_, Feb 01 2008", + "a(2^n) = Product_{i=0..n-2} B(i) where B(i) is A001566. Example 3*7*47 = F(16). - _Kenneth J Ramsey_, Apr 23 2008", + "a(n+1) = Sum_{k=0..n} A109466(n,k)*(-1)^(n-k). -_Philippe Del\u00e9ham_, Oct 26 2008", + "a(n) = Sum_{l_1=0..n+1} Sum_{l_2=0..n}...Sum_{l_i=0..n-i}... Sum_{l_n=0..1} delta(l_1,l_2,...,l_i,...,l_n), where delta(l_1,l_2,...,l_i,...,l_n) = 0 if any l_i + l_(i+1) >= 2 for i=1..n-1 and delta(l_1,l_2,...,l_i,...,l_n) = 1 otherwise. - _Thomas Wieder_, Feb 25 2009", + "a(n+1) = 2^n sqrt(Product_{k=1..n} cos(k Pi/(n+1))^2+1/4) (Kasteleyn's formula specialized). - _Sarah-Marie Belcastro_, Jul 04 2009", + "a(n+1) = Sum_{k=floor(n/2) mod 5} C(n,k) - Sum_{k=floor((n+5)/2) mod 5} C(n,k) = A173125(n) - A173126(n) = |A054877(n)-A052964(n-1)|. - _Henry Bottomley_, Feb 10 2010", + "If p[i] = modp(i,2) and if A is Hessenberg matrix of order n defined by: A[i,j] = p[j-i+1], (i <= j), A[i,j]=-1, (i=j+1), and A[i,j]=0 otherwise. Then, for n >= 1, a(n)=det A. - _Milan Janjic_, May 02 2010", + "Limit_{k->oo} F(k+n)/F(k) = (L(n) + F(n)*sqrt(5))/2 with the Lucas numbers L(n) = A000032(n). - _Johannes W. Meijer_, May 27 2010", + "For n >= 1, F(n) = round(log_2(2^(phi*F(n-1)) + 2^(phi*F(n-2)))), where phi is the golden ratio. - _Vladimir Shevelev_, Jun 24 2010, Jun 27 2010", + "For n >= 1, a(n+1) = ceiling(phi*a(n)), if n is even and a(n+1) = floor(phi*a(n)), if n is odd (phi = golden ratio). - _Vladimir Shevelev_, Jul 01 2010", + "a(n) = 2*a(n-2) + a(n-3), n > 2. - _Gary Detlefs_, Sep 08 2010", + "a(2^n) = Product_{i=0..n-1} A000032(2^i). - _Vladimir Shevelev_, Nov 28 2010", + "a(n)^2 - a(n-1)^2 = a(n+1)*a(n-2), see A121646.", + "a(n) = sqrt((-1)^k*(a(n+k)^2 - a(k)*a(2n+k))), for any k. - _Gary Detlefs_, Dec 03 2010", + "F(2*n) = F(n+2)^2 - F(n+1)^2 - 2*F(n)^2. - _Richard R. Forberg_, Jun 04 2011", + "From _Artur Jasinski_, Nov 17 2011: (Start)", + "(-1)^(n+1) = F(n)^2 + F(n)*F(1+n) - F(1+n)^2.", + "F(n) = F(n+2) - 1 + (F(n+1))^4 + 2*(F(n+1)^3*F(n+2)) - (F(n+1)*F(n+2))^2 - 2*F(n+1)(F(n+2))^3 + (F(n+2))^4 - F(n+1). (End)", + "F(n) = 1 + Sum_{x=1..n-2} F(x). - _Joseph P. Shoulak_, Feb 05 2012", + "F(n) = 4*F(n-2) - 2*F(n-3) - F(n-6). - _Gary Detlefs_, Apr 01 2012", + "F(n) = round(phi^(n+1)/(phi+2)). - _Thomas Ordowski_, Apr 20 2012", + "From _Sergei N. Gladkovskii_, Jun 03 2012: (Start)", + "G.f.: A(x) = x/(1-x-x^2) = G(0)/sqrt(5) where G(k) = 1 - ((-1)^k)*2^k/(a^k - b*x*a^k*2^k/(b*x*2^k - 2*((-1)^k)*c^k/G(k+1))) and a=3+sqrt(5), b=1+sqrt(5), c=3-sqrt(5); (continued fraction, 3rd kind, 3-step).", + "Let E(x) be the e.g.f., i.e.,", + "E(x) = 1*x + (1/2)*x^2 + (1/3)*x^3 + (1/8)*x^4 + (1/24)*x^5 + (1/90)*x^6 + (13/5040)*x^7 + ...; then", + "E(x) = G(0)/sqrt(5); G(k) = 1 - ((-1)^k)*2^k/(a^k - b*x*a^k*2^k/(b*x*2^k - 2*((-1)^k)*(k+1)*c^k/G(k+1))), where a=3+sqrt(5), b=1+sqrt(5), c=3-sqrt(5); (continued fraction, 3rd kind, 3-step).", + "(End)", + "From _Hieronymus Fischer_, Nov 30 2012: (Start)", + "F(n) = 1 + Sum_{j_1=1..n-2} 1 + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} 1 + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} Sum_{j_3=1..j_2-2} 1 + ... + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} Sum_{j_3=1..j_2-2} ... Sum_{j_k=1..j_(k-1)-2} 1, where k = floor((n-1)/2).", + "Example: F(6) = 1 + Sum_{j=1..4} 1 + Sum_{j=1..4} Sum_{k=1..(j-2)} 1 + 0 = 1 + (1 + 1 + 1 + 1) + (1 + (1 + 1)) = 8.", + "F(n) = Sum_{j=0..k} S(j+1,n-2j), where k = floor((n-1)/2) and the S(j,n) are the n-th j-simplex sums: S(1,n) = 1 is the 1-simplex sum, S(2,n) = Sum_{k=1..n} S(1,k) = 1+1+...+1 = n is the 2-simplex sum, S(3,n) = Sum_{k=1..n} S(2,k) = 1+2+3+...+n is the 3-simplex sum (= triangular numbers = A000217), S(4,n) = Sum_{k=1..n} S(3,k) = 1+3+6+...+n(n+1)/2 is the 4-simplex sum (= tetrahedral numbers = A000292) and so on.", + "Since S(j,n) = binomial(n-2+j,j-1), the formula above equals the well-known binomial formula, essentially. (End)", + "G.f.: A(x) = x / (1 - x / (1 - x / (1 + x))). - _Michael Somos_, Jan 04 2013", + "Sum_{n >= 1} (-1)^(n-1)/(a(n)*a(n+1)) = 1/phi (phi=golden ratio). - _Vladimir Shevelev_, Feb 22 2013", + "From _Raul Prisacariu_, Oct 29 2023: (Start)", + "For odd k, Sum_{n >= 1} a(k)^2*(-1)^(n-1)/(a(k*n)*a(k*n+k)) = phi^(-k).", + "For even k, Sum_{n >= 1} a(k)^2/(a(k*n)*a(k*n+k)) = phi^(-k). (End)", + "From _Vladimir Shevelev_, Feb 24 2013: (Start)", + "(1) Expression a(n+1) via a(n): a(n+1) = (a(n) + sqrt(5*(a(n))^2 + 4*(-1)^n))/2;", + "(2) Sum_{k=1..n} (-1)^(k-1)/(a(k)*a(k+1)) = a(n)/a(n+1);", + "(3) a(n)/a(n+1) = 1/phi + r(n), where |r(n)| < 1/(a(n+1)*a(n+2)). (End)", + "F(n+1) = F(n)/2 + sqrt((-1)^n + 5*F(n)^2/4), n >= 0. F(n+1) = U_n(i/2)/i^n, (U:= Chebyshev polynomial of the 2nd kind, i=sqrt(-1)). - _Bill Gosper_, Mar 04 2013", + "G.f.: -Q(0) where Q(k) = 1 - (1+x)/(1 - x/(x - 1/Q(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Mar 06 2013", + "G.f.: x - 1 - 1/x + (1/x)/Q(0), where Q(k) = 1 - (k+1)*x/(1 - x/(x - (k+1)/Q(k+1))); (continued fraction). - _Sergei N. Gladkovskii_, Apr 23 2013", + "G.f.: x*G(0), where G(k) = 1 + x*(1+x)/(1 - x*(1+x)/(x*(1+x) + 1/G(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Jul 08 2013", + "G.f.: x^2 - 1 + 2*x^2/(W(0)-2), where W(k) = 1 + 1/(1 - x*(k + x)/( x*(k+1 + x) + 1/W(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Aug 28 2013", + "G.f.: Q(0) - 1, where Q(k) = 1 + x^2 + (k+2)*x - x*(k+1 + x)/Q(k+1); (continued fraction). - _Sergei N. Gladkovskii_, Oct 06 2013", + "Let b(n) = b(n-1) + b(n-2), with b(0) = 0, b(1) = phi. Then, for n >= 2, F(n) = floor(b(n-1)) if n is even, F(n) = ceiling(b(n-1)), if n is odd, with convergence. - _Richard R. Forberg_, Jan 19 2014", + "a(n) = Sum_{t1*g(1)+t2*g(2)+...+tn*g(n)=n} multinomial(t1+t2+...+tn,t1,t2,...,tn), where g(k)=2*k-1. - _Mircea Merca_, Feb 27 2014", + "F(n) = round(sqrt(F(n-1)^2 + F(n)^2 + F(n+1)^2)/2), for n > 0. This rule appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Jul 27 2014", + "F(n) = round(2/(1/F(n) + 1/F(n+1) + 1/F(n+2))), for n > 0. This rule also appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Aug 03 2014", + "F(n) = round(1/(Sum_{j>=n+2} 1/F(j))). - _Richard R. Forberg_, Aug 14 2014", + "a(n) = hypergeometric([-n/2+1/2, -n/2+1], [-n+1], -4) for n >= 2. - _Peter Luschny_, Sep 19 2014", + "Limit_{n -> oo} (log F(n+1)/log F(n))^n = e. - _Thomas Ordowski_, Oct 06 2014", + "F(n) = (L(n+1)^2 - L(n-1)^2)/(5*L(n)), where L(n) is A000032(n), with a similar inverse relationship. - _Richard R. Forberg_, Nov 17 2014", + "Consider the graph G[1-vertex;1-loop,2-loop] in comment above. Construct the power matrix array T(n,j) = [A^*j]*[S^*(j-1)] where A=(1,1,0,...) and S=(0,1,0,...)(A063524). [* is convolution operation] Define S^*0=I with I=(1,0,...). Then T(n,j) counts n-walks containing (j) loops and a(n-1) = Sum_{j=1..n} T(n,j). - _David Neil McGrath_, Nov 21 2014", + "Define F(-n) to be F(n) for n odd and -F(n) for n even. Then for all n and k, F(n) = F(k)*F(n-k+3) - F(k-1)*F(n-k+2) - F(k-2)*F(n-k) + (-1)^k*F(n-2k+2). - _Charlie Marion_, Dec 04 2014", + "F(n+k)^2 - L(k)*F(n)*F(n+k) + (-1)^k*F(n)^2 = (-1)^n*F(k)^2, if L(k) = A000032(k). - _Alexander Samokrutov_, Jul 20 2015", + "F(2*n) = F(n+1)^2 - F(n-1)^2, similar to Koshy (D) and Forberg 2011, but different. - _Hermann Stamm-Wilbrandt_, Aug 12 2015", + "F(n+1) = ceiling( (1/phi)*Sum_{k=0..n} F(k) ). - _Tom Edgar_, Sep 10 2015", + "a(n) = (L(n-3) + L(n+3))/10 where L(n)=A000032(n). - _J. M. Bergot_, Nov 25 2015", + "From _Bob Selcoe_, Mar 27 2016: (Start)", + "F(n) = (F(2n+k+1) - F(n+1)*F(n+k+1))/F(n+k), k >= 0.", + "Thus when k=0: F(n) = sqrt(F(2n+1) - F(n+1)^2).", + "F(n) = (F(3n) - F(n+1)^3 + F(n-1)^3)^(1/3).", + "F(n+2k) = binomial transform of any subsequence starting with F(n). Example F(6)=8: 1*8 = F(6)=8; 1*8 + 1*13 = F(8)=21; 1*8 + 2*13 + 1*21 = F(10)=55; 1*8 + 3*13 + 3*21 + 1*34 = F(12)=144, etc. This formula applies to Fibonacci-type sequences with any two seed values for a(0) and a(1) (e.g., Lucas sequence A000032: a(0)=2, a(1)=1).", + "(End)", + "F(n) = L(k)*F(n-k) + (-1)^(k+1)*F(n-2k) for all k >= 0, where L(k) = A000032(k). - _Anton Zakharov_, Aug 02 2016", + "From _Ilya Gutkovskiy_, Aug 03 2016: (Start)", + "a(n) = F_n(1), where F_n(x) are the Fibonacci polynomials.", + "Inverse binomial transform of A001906.", + "Number of zeros in substitution system {0 -> 11, 1 -> 1010} at step n from initial string \"1\" (1 -> 1010 -> 101011101011 -> ...) multiplied by 1/A000079(n). (End)", + "For n >= 2, a(n) = 2^(n^2+n) - (4^n-2^n-1)*floor(2^(n^2+n)/(4^n-2^n-1)) - 2^n*floor(2^(n^2) - (2^n-1-1/2^n)*floor(2^(n^2+n)/(4^n-2^n-1))). - _Benoit Cloitre_, Apr 17 2017", + "f(n+1) = Sum_{j=0..floor(n/2)} Sum_{k=0..j} binomial(n-2j,k)*binomial(j,k). - _Tony Foster III_, Sep 04 2017", + "F(n) = Sum_{k=0..floor((n-1)/2)} ( (n-k-1)! / ((n-2k-1)! * k!) ). - _Zhandos Mambetaliyev_, Nov 08 2017", + "For x even, F(n) = (F(n+x) + F(n-x))/L(x). For x odd, F(n) = (F(n+x) - F(n-x))/L(x) where n >= x in both cases. Therefore F(n) = F(2*n)/L(n) for n >= 0. - _David James Sycamore_, May 04 2018", + "From _Isaac Saffold_, Jul 19 2018: (Start)", + "Let [a/p] denote the Legendre symbol. Then, for an odd prime p:", + " F(p+n) == [5/p]*F([5/p]+n) (mod p), if [5/p] = 1 or -1.", + " F(p+n) == 3*F(n) (mod p), if [5/p] = 0 (i.e., p = 5).", + " This is true for negative-indexed terms as well, if this sequence is extended by the negafibonacci numbers (i.e., F(-n) = A039834(n)). (End)", + "a(n) = A094718(4, n). a(n) = A101220(0, j, n).", + "a(n) = A090888(0, n+1) = A118654(0, n+1) = A118654(1, n-1) = A109754(0, n) = A109754(1, n-1), for n > 0.", + "a(n) = (L(n-3) + L(n-2) + L(n-1) + L(n))/5 with L(n)=A000032(n). - _Art Baker_, Jan 04 2019", + "F(n) = F(k-1)*F(abs(n-k-2)) + F(k-1)*F(n-k-1) + F(k)*F(abs(n-k-2)) + 2*F(k)*F(n-k-1), for n > k > 0. - _Joseph M. Shunia_, Aug 12 2019", + "F(n) = F(n-k+2)*F(k-1) + F(n-k+1)*F(k-2) for all k such that 2 <= k <= n. - _Michael Tulskikh_, Oct 09 2019", + "F(n)^2 - F(n+k)*F(n-k) = (-1)^(n+k) * F(k)^2 for 2 <= k <= n [Catalan's identity]. - _Hermann Stamm-Wilbrandt_, May 07 2021", + "Sum_{n>=1} 1/a(n) = A079586 is the reciprocal Fibonacci constant. - _Gennady Eremin_, Aug 06 2021", + "a(n) = Product_{d|n} b(d) = Product_{k=1..n} b(gcd(n,k))^(1/phi(n/gcd(n,k))) = Product_{k=1..n} b(n/gcd(n,k))^(1/phi(n/gcd(n,k))) where b(n) = A061446(n) = primitive part of a(n), phi(n) = A000010(n). - _Richard L. Ollerton_, Nov 08 2021", + "a(n) = 2*i^(1-n)*sin(n*arccos(i/2))/sqrt(5), i=sqrt(-1). - _Bill Gosper_, May 05 2022", + "a(n) = i^(n-1)*sin(n*c)/sin(c) = i^(n-1)*sin(c*n)*csc(c), where c = Pi/2 + i*arccsch(2). - _Peter Luschny_, May 23 2022", + "F(2n) = Sum_{k=1..n} (k/5)*binomial(2n, n+k), where (k/5) is the Legendre or Jacobi Symbol; F(2n+1)= Sum_{k=1..n} (-(k+2)/5)*binomial(2n+1, n+k), where (-(k+2)/5) is the Legendre or Jacobi Symbol. For example, F(10) = 1*binomial(10,6) - 1*binomial(10,7) - 1*binomial(10,8) + 1*binomial(10,9) + 0*binomial(10,10), F(11) = 1*binomial(11,6) - 1*binomial(11,7) + 0*binomial(11,8) - 1*binomial(11,9) + 1*binomial(11,10) + 1*binomial(11,11). - _Yike Li_, Aug 21 2022", + "For n > 0, 1/F(n) = Sum_{k>=1} F(n*k)/(F(n+2)^(k+1)). - _Diego Rattaggi_, Oct 26 2022", + "From _Andrea Pinos_, Dec 02 2022: (Start)", + "For n == 0 (mod 4): F(n) = F((n+2)/2)*( F(n/2) + F((n/2)-2) ) + 1;", + "For n == 1 (mod 4): F(n) = F((n-1)/2)*( F((n-1)/2) + F(2+(n-1)/2) ) + 1;", + "For n == 2 (mod 4): F(n) = F((n-2)/2)*( F(n/2) + F((n/2)+2) ) + 1;", + "For n == 3 (mod 4): F(n) = F((n-1)/2)*( F((n-1)/2) + F(2+(n-1)/2) ) - 1. (End)", + "F(n) = Sum_{i=0..n-1} F(i)^2 / F(n-1). - _Jules Beauchamp_, May 03 2025", + "F(n) = Sum_{i=0..n-1} L(n-i)*(-2)^i. - _Greg Dresden_ and Xiaoya Gao, Oct 11 2025", + "For n>=2, a(n) = 2^(n-2)*hypergeometric([(1-n)/3,(2-n)/3,1-n/3,2-n],[(2-n)/2,(3-n)/2,1-n],27/32). - _Victor Petrescu_, Oct 25 2025", + "From _Friedjof Tellkamp_, Jun 10 2026: (Start)", + "Sum_{n>=1} 1/a(n)^s = 5^(s/2) * Sum_{k>=0} C(-s, k) * 1/(phi^(s+2k) - (-1)^k), see Eq. 5 in Navas.", + "Sum_{n>=1} (-1)^(n+1)/a(n)^s = 5^(s/2) * Sum_{k>=0} C(-s, k) * 1/(phi^(s+2k) + (-1)^k). (End)" + ] + }, + { + "number": 290689, + "name": "Number of transitive rooted trees with n nodes.", + "data": "1,1,1,2,3,5,8,13,21,34,55,88,143,229,370,592,955,1527,2457,3929,6304,10081,16147,25802,41265,65912,105279,168043,268164,427739,682026,1087180,1732295,2759660,4394579,6996857,11136152,17721389,28192021,44842171,71307030", + "offset": "1,4" + }, + { + "number": 212804, + "name": "Expansion of (1 - x)/(1 - x - x^2).", + "data": "1,0,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987,1597,2584,4181,6765,10946,17711,28657,46368,75025,121393,196418,317811,514229,832040,1346269,2178309,3524578,5702887,9227465,14930352,24157817,39088169,63245986,102334155,165580141,267914296,433494437,701408733,1134903170,1836311903,2971215073,4807526976", + "offset": "0,5", + "formula": [ + "G.f.: 1/(1 - (Sum_{k >= 2} x^k)). - _Joerg Arndt_, Aug 13 2012", + "a(n) = Fibonacci(n+1) - Fibonacci(n). - _Arkadiusz Wesolowski_, Oct 29 2012", + "G.f.: 1 - x*Q(0) where Q(k) = 1 - (1 + x)/(1 - x/(x - 1/Q(k+1))); (continued fraction). - _Sergei N. Gladkovskii_, Mar 06 2013", + "G.f.: 3*x^3/(3*x - Q(0)) - x^2 + 1, where Q(k) = 1 - 1/(4^k - x*16^k/(x*4^k - 1/(1 + 1/(2*4^k - 4*x*16^k/(2*x*4^k + 1/Q(k+1)))))); (continued fraction). - _Sergei N. Gladkovskii_, May 21 2013", + "G.f.: G(0)*(1 - x)/(2 - x), where G(k) = 1 + 1/(1 - (x*(5*k - 1))/((x*(5*k + 4)) - 2/G(k+1))); (continued fraction). - _Sergei N. Gladkovskii_, Jun 15 2013", + "G.f.: 1 + Q(0)*x^2/2, where Q(k) = 1 + 1/(1 - x*(2*k + 1 + x)/( x*(2*k + 2 + x) + 1/Q(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Aug 29 2013", + "a(n) = Sum_{k=0..n} (C(k, n-k) - C(k, n-k-1)). - _Peter Luschny_, Oct 01 2014", + "a(n) = (2^(-1-n)*((1 - sqrt(5))^n*(1 + sqrt(5)) + (-1 + sqrt(5))*(1 + sqrt(5))^n))/sqrt(5). - _Colin Barker_, Sep 25 2016", + "a(n) = A000045(n-1), n >= 1. - _R. J. Mathar_, Apr 14 2018", + "E.g.f.: exp((1 - sqrt(5))*x/2)*(3 + sqrt(5) + 2*exp(sqrt(5)*x))/(5 + sqrt(5)). - _Stefano Spezia_, Mar 09 2025" + ] + }, + { + "number": 27926, + "name": "Triangular array T read by rows: T(n,0) = T(n,2n) = 1 for n >= 0; T(n,1) = 1 for n >= 1; T(n,k) = T(n-1,k-2) + T(n-1,k-1) for k = 2..2n-1, n >= 2.", + "data": "1,1,1,1,1,1,2,2,1,1,1,2,3,4,3,1,1,1,2,3,5,7,7,4,1,1,1,2,3,5,8,12,14,11,5,1,1,1,2,3,5,8,13,20,26,25,16,6,1,1,1,2,3,5,8,13,21,33,46,51,41,22,7,1,1,1,2,3,5,8,13,21,34,54,79,97,92,63,29,8,1", + "offset": "0,7", + "formula": [ + "T(n, k) = Sum_{j=0..floor((2*n-k+1)/2)} binomial(n-j, 2*n-k-2*j). - _Len Smiley_, Oct 21 2001" + ] + }, + { + "number": 1129, + "name": "Iccanobif numbers: reverse digits of two previous terms and add.", + "data": "0,1,1,2,3,5,8,13,39,124,514,836,1053,4139,12815,61135,104937,792517,1454698,9679838,17354310,9735140,1760750,986050,621360,113815,581437,1252496,7676706,13019288,94367798,178067380,173537220,106496242,265429972,522619163", + "offset": "0,4" + }, + { + "number": 5347, + "name": "First differences of A005579.", + "data": "1,1,1,1,2,3,5,8,13,20,34,53,88,143,236,387,641,1061,1763,2937,4903,8202,13750,23095,38850,65461,110465,186665,315827,535011,907341,1540416,2617782,4452846,7581016,12917486,22027745,37591270,64196610", + "offset": "0,5", + "formula": [ + "a(n) = A005579(n+1) - A005579(n) - _T. D. Noe_, May 08 2006" + ] + }, + { + "number": 14260, + "name": "Iccanobif numbers: add a(n-1) to reversal of a(n-2).", + "data": "0,1,1,2,3,5,8,13,21,52,64,89,135,233,764,1096,1563,8464,12115,16763,67884,104645,153521,699922,825273,1055269,1427797,11053298,19030539,108265550,201768641,257331442,404198544,648332296,1094223700,1786457546,1859682447", + "offset": "0,4" + }, + { + "number": 374742, + "name": "Number of integer compositions of n whose leaders of weakly decreasing runs are identical.", + "data": "1,1,2,3,5,8,13,21,34,54,87,138,220,349,556,881,1403,2229,3551,5653,9019,14387,22988,36739,58785,94100,150765,241658,387617,622002,998658,1604032,2577512,4143243,6662520,10716931,17243904,27753518,44680121,71947123,115880662", + "offset": "0,3", + "formula": [ + "G.f.: 1 + Sum_{i>0} -1 + (1 + x^i/(1 - x^i))/(1 - B(i,x)) where B(i,x) = x^i/(1 - x^i) * Sum_{j=1..i-1} x^j * Product_{k=1..j} (1 - x^k)^(-1). - _John Tyler Rascoe_, Apr 29 2025" + ] + }, + { + "number": 225394, + "name": "Expansion of 1/(1 - x - x^2 + x^7 - x^9).", + "data": "1,1,2,3,5,8,13,20,32,51,81,129,205,326,519,826,1314,2091,3327,5294,8424,13404,21328,33937,54000,85924,136721,217548,346159,550803,876429,1394560,2219002,3530841,5618219,8939622,14224586,22633938,36014767,57306132,91184618", + "offset": "0,3", + "formula": [ + "a(n) = a(n-1) + a(n-2) - a(n-7) + a(n-9). - _Ilya Gutkovskiy_, Nov 16 2016" + ] + }, + { + "number": 181600, + "name": "Expansion of 1/(1 - x - x^2 + x^8 - x^10).", + "data": "1,1,2,3,5,8,13,21,33,53,85,136,218,349,559,895,1434,2297,3679,5893,9439,15119,24217,38790,62132,99520,159407,255331,408978,655083,1049283,1680695,2692063,4312028,6906816,11063033,17720278,28383559,45463532,72821479", + "offset": "0,3", + "formula": [ + "a(n) = a(n-1) + a(n-2) - a(n-8) + a(n-10). - _Franck Maminirina Ramaharo_, Oct 31 2018" + ] + } + ], + "1,1,2,3,5,8,13|10": [ + { + "number": 10077, + "name": "a(n) = sum of digits of a(n-1) + sum of digits of a(n-2); a(0) = 0, a(1) = 1.", + "data": "0,1,1,2,3,5,8,13,12,7,10,8,9,17,17,16,15,13,10,5,6,11,8,10,9,10,10,2,3,5,8,13,12,7,10,8,9,17,17,16,15,13,10,5,6,11,8,10,9,10,10,2,3,5,8,13,12,7,10,8,9,17,17,16,15,13,10,5,6,11,8,10,9,10,10", + "offset": "0,4", + "formula": [ + "Periodic from n=3 with period 24. - _Franklin T. Adams-Watters_, Mar 13 2006", + "a(n) = A030132(n-4) + A030132(n-3) for n>3. - _Reinhard Zumkeller_, Jul 04 2007", + "a(n) = a(n-1) + a(n-2) - 9*(floor(a(n-1)/10) + floor(a(n-2)/10)). - _Hieronymus Fischer_, Jun 27 2007", + "a(n) = floor(a(n-1)/10) + floor(a(n-2)/10) + (a(n-1) mod 10) + (a(n-2) mod 10). - _Hieronymus Fischer_, Jun 27 2007", + "a(n) = A059995(a(n-1)) + A059995(a(n-2)) + A010879(a(n-1)) + A010879(a(n-2)). - _Hieronymus Fischer_, Jun 27 2007", + "a(n) = Fibonacci(n) - 9*Sum_{k=2..n-1} Fibonacci(n-k+1)*floor(a(k)/10) where Fibonacci(n) = A000045(n). - _Hieronymus Fischer_, Jun 27 2007", + "G.f.: x*(1 + x + 2*x^2 + 3*x^3 + 5*x^4 + 8*x^5 + 13*x^6 + 12*x^7 + 7*x^8 + 10*x^9 + 8*x^10+ 9*x^11 + 17*x^12 + 17*x^13 + 16*x^14+ 15*x^15+ 13*x^16 + 10*x^17+ 5*x^18 + 6*x^19 + 11*x^20 + 8*x^21 + 10*x^22 + 9*x^23 + 9*x^24 + 9*x^25)/(1 - x^24). - _Stefano Spezia_, Nov 10 2025" + ] + }, + { + "number": 69638, + "name": "\"Sorted\" sum of two previous terms, beginning with 0,1. \"Sorted\" means to sort the digits of the sum in ascending order.", + "data": "0,1,1,2,3,5,8,13,12,25,37,26,36,26,26,25,15,4,19,23,24,47,17,46,36,28,46,47,39,68,17,58,57,115,127,224,135,359,449,88,357,445,28,347,357,47,44,19,36,55,19,47,66,113,179,229,48,277,235,125,36,116,125,124,249,337", + "offset": "0,4", + "formula": [ + "a(n) = SORT[a(n-1) + a(n-2)]." + ] + }, + { + "number": 324969, + "name": "Number of unlabeled rooted identity trees with n vertices whose non-leaf terminal subtrees are all different.", + "data": "1,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987,1597,2584,4181,6765,10946,17711,28657,46368,75025,121393,196418,317811,514229,832040,1346269,2178309,3524578,5702887,9227465,14930352,24157817,39088169,63245986,102334155", + "offset": "1,4", + "formula": [ + "From _Michael Somos_, Nov 22 2019: (Start)", + "G.f.: x*(1 - x^2) / (1 - x - x^2) = x*(1 + x/(1 - x/(1 - x/(1 + x)))).", + "a(n) = A000045(n-1) if n>=2. (End)", + "E.g.f.: -1 + x + exp(x/2)*(cosh(sqrt(5)*x/2) - (1/sqrt(5))*sinh(sqrt(5)*x/2)). - _G. C. Greubel_, Oct 24 2023" + ] + }, + { + "number": 104763, + "name": "Triangle read by rows: Fibonacci(1), Fibonacci(2), ..., Fibonacci(n) in row n.", + "data": "1,1,1,1,1,2,1,1,2,3,1,1,2,3,5,1,1,2,3,5,8,1,1,2,3,5,8,13,1,1,2,3,5,8,13,21,1,1,2,3,5,8,13,21,34,1,1,2,3,5,8,13,21,34,55,1,1,2,3,5,8,13,21,34,55,89,1,1,2,3,5,8,13,21,34,55,89,144,1,1,2,3,5,8,13,21,34,55,89,144,233", + "offset": "1,6", + "formula": [ + "F(1) through F(n) starting from the left in n-th row.", + "T(n,k) = A000045(k), 1<=k<=n. - _R. J. Mathar_, May 02 2008", + "a(n) = A000045(m), where m= n-t(t+1)/2, t=floor((-1+sqrt(8*n-7))/2). - _Boris Putievskiy_, Dec 13 2012", + "G.f.: (x*y)/((x-1)*(x^2*y^2+x*y-1)). - _Vladimir Kruchinin_, Jun 21 2025" + ] + }, + { + "number": 131297, + "name": "a(n) = ds_11(a(n-1))+ds_11(a(n-2)), a(0)=0, a(1)=1; where ds_11=digital sum base 11.", + "data": "0,1,1,2,3,5,8,13,11,4,5,9,14,13,7,10,17,17,14,11,5,6,11,7,8,15,13,8,11,9,10,19,19,18,17,15,12,7,9,16,15,11,6,7,13,10,13,13,6,9,15,14,9,13,12,5,7,12,9,11,10,11,11,2,3,5,8,13,11,4,5,9,14,13,7,10,17,17,14,11", + "offset": "0,4", + "formula": [ + "a(n) = a(n-1)+a(n-2)-10*(floor(a(n-1)/11)+floor(a(n-2)/11)).", + "a(n) = floor(a(n-1)/11)+floor(a(n-2)/11)+(a(n-1)mod 11)+(a(n-2)mod 11).", + "a(n) = Fib(n)-10*sum{1=1. - _Alois P. Heinz_, Jun 25 2023" + ] + }, + { + "number": 14259, + "name": "Iccanobif numbers: add reversal of a(n-1) to a(n-2).", + "data": "0,1,1,2,3,5,8,13,39,106,640,152,891,350,944,799,1941,2290,2863,5972,5658,14537,79199,113734,516510,129349,1460431,1469990,2460072,4170632,4820786,11040916,66724797,90783682,95363506,151320041,235386657,908003573,610687466", + "offset": "0,4" + }, + { + "number": 240733, + "name": "a(n) = floor(6^n/(2+2*cos(Pi/9))^n).", + "data": "1,1,2,3,5,8,13,21,32,50,78,121,187,289,448,693,1072,1658,2564,3966,6134,9487,14673,22695,35101,54288,83964,129862,200850,310643,480452,743085,1149282,1777523,2749182,4251987,6576279,10171116,15731022,24330178,37629950", + "offset": "0,3" + }, + { + "number": 326594, + "name": "Sum of the fifth largest parts of the partitions of n into 10 parts.", + "data": "0,0,0,0,0,0,0,0,0,0,1,1,2,3,5,8,13,19,29,42,62,85,121,164,226,303,407,534,706,912,1184,1511,1930,2433,3072,3831,4776,5900,7281,8909,10898,13223,16031,19312,23231,27787,33194,39444,46806,55292,65219,76603", + "offset": "0,13", + "formula": [ + "a(n) = Sum_{r=1..floor(n/10)} Sum_{q=r..floor((n-r)/9)} Sum_{p=q..floor((n-q-r)/8)} Sum_{o=p..floor((n-p-q-r)/7)} Sum_{m=o..floor((n-o-p-q-r)/6)} Sum_{l=m..floor((n-m-o-p-q-r)/5)} Sum_{k=l..floor((n-l-m-o-p-q-r)/4)} Sum_{j=k..floor((n-k-l-m-o-p-q-r)/3)} Sum_{i=j..floor((n-j-k-l-m-o-p-q-r)/2)} l.", + "a(n) = A326588(n) - A326589(n) - A326590(n) - A326591(n) - A326592(n) - A326593(n) - A326595(n) - A326596(n) - A326597(n) - A326598(n)." + ] + } + ], + "1,1,2,3,5,8,13|20": [ + { + "number": 243063, + "name": "Numbers generated by a Fibonacci-like sequence in which zeros are suppressed.", + "data": "1,1,2,3,5,8,13,21,34,55,89,144,233,377,61,438,499,937,1436,2373,389,2762,3151,5913,964,6877,7841,14718,22559,37277,59836,97113,156949,25462,182411,27873,21284,49157,7441,56598,6439,6337,12776,19113,31889,512,3241", + "offset": "1,3", + "formula": [ + "x(i) = no-zero(x(i-2) + x(i-1)). For example, no-zero(233 + 377) = no-zero(610) = 61." + ] + }, + { + "number": 326469, + "name": "Sum of the fifth largest parts of the partitions of n into 9 parts.", + "data": "0,0,0,0,0,0,0,0,0,1,1,2,3,5,8,13,19,29,42,60,83,117,158,216,288,383,500,655,840,1080,1371,1734,2172,2718,3364,4157,5099,6235,7574,9184,11059,13294,15895,18955,22501,26657,31432,36991,43368,50731,59138,68811", + "offset": "0,12", + "formula": [ + "a(n) = Sum_{q=1..floor(n/9)} Sum_{p=q..floor((n-q)/8)} Sum_{o=p..floor((n-p-q)/7)} Sum_{m=o..floor((n-o-p-q)/6)} Sum_{l=m..floor((n-m-o-p-q)/5)} Sum_{k=l..floor((n-l-m-o-p-q)/4)} Sum_{j=k..floor((n-k-l-m-o-p-q)/3)} Sum_{i=j..floor((n-j-k-l-m-o-p-q)/2)} l.", + "a(n) = A326464(n) - A326465(n) - A326466(n) - A326467(n) - A326468(n) - A326470(n) - A326471(n) - A326472(n) - A326473(n)." + ] + }, + { + "number": 374765, + "name": "Number of integer compositions of n whose leaders of strictly decreasing runs are weakly decreasing.", + "data": "1,1,2,3,5,8,13,21,34,55,88,141,225,357,565,891,1399,2191,3420,5321,8256,12774,19711,30339,46584,71359,109066,166340,253163,384539,582972,882166,1332538,2009377,3024969,4546562,6822926,10223632,15297051,22855872,34103117", + "offset": "0,3" + }, + { + "number": 78414, + "name": "a(n) = (a(n-1)+a(n-2))/7^k, where 7^k is the highest power of 7 dividing a(n-1)+a(n-2).", + "data": "1,1,2,3,5,8,13,3,16,19,5,24,29,53,82,135,31,166,197,363,80,443,523,138,661,799,1460,2259,3719,122,3841,3963,7804,1681,1355,3036,4391,1061,5452,6513,11965,18478,4349,3261,7610,1553,187,1740,1927,3667,5594,27", + "offset": "1,3", + "formula": [ + "a(n) = A242603(a(n-1)+a(n-2)). - _R. J. Mathar_, Mar 13 2024" + ] + }, + { + "number": 177194, + "name": "Fibonacci numbers whose decimal expansion does not contain any digit 0.", + "data": "1,1,2,3,5,8,13,21,34,55,89,144,233,377,987,1597,2584,4181,6765,17711,28657,46368,121393,196418,317811,514229,1346269,3524578,9227465,24157817,63245986,267914296,433494437,53316291173,86267571272", + "offset": "1,3", + "formula": [ + "a(n) = A000045(A076564(n)). [From _R. J. Mathar_, Oct 18 2010]" + ] + }, + { + "number": 308994, + "name": "Sum of the fifth largest parts in the partitions of n into 8 parts.", + "data": "0,0,0,0,0,0,0,0,1,1,2,3,5,8,13,19,29,40,58,79,111,148,201,264,349,449,583,739,943,1181,1482,1833,2273,2780,3405,4126,5002,6006,7215,8593,10235,12101,14300,16795,19713,23003,26825,31124,36083,41638,48012", + "offset": "0,11", + "formula": [ + "a(n) = Sum_{p=1..floor(n/8)} Sum_{o=p..floor((n-p)/7)} Sum_{m=o..floor((n-o-p)/6)} Sum_{l=m..floor((n-m-o-p)/5)} Sum_{k=l..floor((n-l-m-o-p)/4)} Sum_{j=k..floor((n-k-l-m-o-p)/3)} Sum_{i=j..floor((n-j-k-l-m-o-p)/2)} l.", + "a(n) = A308989(n) - A308990(n) - A308991(n) - A308992(n) - A308995(n) - A308996(n) - A308997(n) - A308998(n)." + ] + }, + { + "number": 55801, + "name": "Triangle T read by rows: T(i,0)=T(i,i)=1, T(i,j) = Sum_{k=1..floor(n/2)} T(i-2k, j-2k+1) for 1<=j<=i-1, where T(m,n) := 0 if m<0 or n<0.", + "data": "1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,2,2,1,1,1,1,2,3,3,1,1,1,1,2,3,4,3,1,1,1,1,2,3,5,6,4,1,1,1,1,2,3,5,7,7,4,1,1,1,1,2,3,5,8,11,10,5,1,1,1,1,2,3,5,8,12,14,11,5,1,1,1,1,2,3,5,8,13,19,21,15,6,1", + "offset": "0,14" + }, + { + "number": 55805, + "name": "a(n) = T(n,n-5), array T as in A055801.", + "data": "1,1,1,2,3,5,8,13,20,32,46,72,97,148,189,281,344,499,591,838,967,1343,1518,2069,2300,3082,3380,4460,4837,6294,6763,8689,9264,11765,12461,15658,16491,20521,21508,26525,27684,33860", + "offset": "5,4", + "formula": [ + "From _Colin Barker_, Nov 28 2014: (Start)", + "a(n) = ((2*n^5 -45*n^4 +450*n^3 -2070*n^2 +4873*n -3585) +5*(-1)^n*(n^4 -34*n^3 +446*n^2 -2741*n +6861))/7680 for n>5.", + "G.f.: x^5*(1 -5*x^2 +x^3 +11*x^4 -3*x^5 -12*x^6 +5*x^7 +7*x^8 -3*x^9 -2*x^10 + x^11)/((1-x)^6*(1+x)^5). (End)" + ] + }, + { + "number": 55806, + "name": "a(n) = T(n,n-6), array T as in A055801.", + "data": "1,1,1,2,3,5,8,13,21,33,53,79,125,176,273,365,554,709,1053,1300,1891,2267,3234,3785,5303,6085,8385,9465,12845,14302,19139,21065,27828,30329,39593,42790,55251,59281,75772,80789,102297,108473,136157,143683,178893", + "offset": "6,4", + "formula": [ + "From _G. C. Greubel_, Jan 24 2020: (Start)", + "a(n) = (48915 -58884*n +29723*n^2 -7200*n^3 +965*n^4 -66*n^5 +2*n^6 + 3*(-1)^n*(-231345 +98988*n -18505*n^2 +1840*n^3 -95*n^4 +2*n^5))/92160, n > 6.", + "G.f.: x^6*(1 -6*x^2 +x^3 +16*x^4 -4*x^5 -23*x^6 +8*x^7 +20*x^8 -8*x^9 -9*x^10 + 4*x^11 +2*x^12 -x^13)/((1-x)^7*(1+x)^6). (End)" + ] + }, + { + "number": 227374, + "name": "G.f.: 1/(1 - x*(1-x^5)/(1 - x^2*(1-x^6)/(1 - x^3*(1-x^7)/(1 - x^4*(1-x^8)/(1 - x^5*(1-x^9)/(1 - ...)))))), a continued fraction.", + "data": "1,1,1,2,3,5,8,13,22,36,61,101,169,283,473,793,1325,2220,3715,6220,10413,17431,29185,48856,81797,136937,229257,383813,642564,1075762,1800995,3015171,5047886,8451001,14148368,23686705,39655467,66389797,111147511,186079299,311527531,521548600", + "offset": "0,4", + "formula": [ + "G.f.: T(0), where T(k) = 1 - x^(k+1)*(1-x^(k+5))/(x^(k+1)*(1-x^(k+5)) - 1/T(k+1) ); (continued fraction). - _Sergei N. Gladkovskii_, Oct 18 2013" + ] + } + ], + "1,5,73,1445,33001,819005,21460825,584307365,16367912425,468690849005|0": [ + { + "number": 5259, + "name": "Apery (Ap\u00e9ry) numbers: Sum_{k=0..n} (binomial(n,k)*binomial(n+k,k))^2.", + "data": "1,5,73,1445,33001,819005,21460825,584307365,16367912425,468690849005,13657436403073,403676083788125,12073365010564729,364713572395983725,11111571997143198073,341034504521827105445,10534522198396293262825,327259338516161442321485", + "offset": "0,2", + "formula": [ + "D-finite with recurrence (n+1)^3*a(n+1) = (34*n^3 + 51*n^2 + 27*n + 5)*a(n) - n^3*a(n-1), n >= 1.", + "Representation as a special value of the hypergeometric function 4F3, in Maple notation: a(n)=hypergeom([n+1, n+1, -n, -n], [1, 1, 1], 1), n=0, 1, ... - _Karol A. Penson_ Jul 24 2002", + "a(n) = Sum_{k >= 0} A063007(n, k)*A000172(k). A000172 = Franel numbers. - _Philippe Del\u00e9ham_, Aug 14 2003", + "G.f.: (-1/2)*(3*x - 3 + (x^2-34*x+1)^(1/2))*(x+1)^(-2)*hypergeom([1/3,2/3],[1],(-1/2)*(x^2 - 7*x + 1)*(x+1)^(-3)*(x^2 - 34*x + 1)^(1/2)+(1/2)*(x^3 + 30*x^2 - 24*x + 1)*(x+1)^(-3))^2. - _Mark van Hoeij_, Oct 29 2011", + "Let g(x, y) = 4*cos(2*x) + 8*sin(y)*cos(x) + 5 and let P(n,z) denote the Legendre polynomial of degree n. Then G. A. Edgar posted a conjecture of Alexandru Lupas that a(n) equals the double integral 1/(4*Pi^2)*int {y = -Pi..Pi} int {x = -Pi..Pi} P(n,g(x,y)) dx dy. (Added Jan 07 2015: Answered affirmatively in Math Overflow question 178790) - _Peter Bala_, Mar 04 2012; edited by _G. A. Edgar_, Dec 10 2016", + "a(n) ~ (1+sqrt(2))^(4*n+2)/(2^(9/4)*Pi^(3/2)*n^(3/2)). - _Vaclav Kotesovec_, Nov 01 2012", + "a(n) = Sum_{k=0..n} C(n,k)^2 * C(n+k,k)^2. - _Joerg Arndt_, May 11 2013", + "0 = (-x^2+34*x^3-x^4)*y''' + (-3*x+153*x^2-6*x^3)*y'' + (-1+112*x-7*x^2)*y' + (5-x)*y, where y is g.f. - _Gheorghe Coserea_, Jul 14 2016", + "From _Peter Bala_, Jan 18 2020: (Start)", + "a(n) = Sum_{0 <= j, k <= n} (-1)^(n+j) * C(n,k)^2 * C(n+k,k)^2 * C(n,j) * C(n+k+j,k+j).", + "a(n) = Sum_{0 <= j, k <= n} C(n,k) * C(n+k,k) * C(k,j)^3 (see Koepf, p. 55).", + "a(n) = Sum_{0 <= j, k <= n} C(n,k)^2 * C(n,j)^2 * C(3*n-j-k,2*n) (see Koepf, p. 119).", + "Diagonal coefficients of the rational function 1/((1 - x - y)*(1 - z - t) - x*y*z*t) (Straub, 2014). (End)", + "a(n) = [x^n] 1/(1 - x)*( Legendre_P(n,(1 + x)/(1 - x)) )^m at m = 2. At m = 1 we get the Ap\u00e9ry numbers A005258. - _Peter Bala_, Dec 22 2020", + "a(n) = Sum_{k = 0..n} (-1)^(n+k)*binomial(n, k)*binomial(n+k, k)*A108625(n, k). - _Peter Bala_, Jul 18 2024", + "a(n) = Sum_{k=0..n} Sum_{j=0..n} C(n,k)^2 * C(n,j)^2 * C(k+j,k), see Labelle et al. link. - _Max Alekseyev_, Mar 12 2025" + ] + } + ], + "id:A000045|0": [ + { + "number": 45, + "name": "Fibonacci numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1.", + "data": "0,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987,1597,2584,4181,6765,10946,17711,28657,46368,75025,121393,196418,317811,514229,832040,1346269,2178309,3524578,5702887,9227465,14930352,24157817,39088169,63245986,102334155", + "offset": "0,4", + "formula": [ + "G.f.: x / (1 - x - x^2).", + "G.f.: Sum_{n>=0} x^n * Product_{k=1..n} (k + x)/(1 + k*x). - _Paul D. Hanna_, Oct 26 2013", + "F(n) = ((1+sqrt(5))^n - (1-sqrt(5))^n)/(2^n*sqrt(5)).", + "Alternatively, F(n) = ((1/2+sqrt(5)/2)^n - (1/2-sqrt(5)/2)^n)/sqrt(5).", + "F(n) = F(n-1) + F(n-2) = -(-1)^n F(-n).", + "F(n) = round(phi^n/sqrt(5)).", + "F(n+1) = Sum_{j=0..floor(n/2)} binomial(n-j, j).", + "A strong divisibility sequence, that is, gcd(a(n), a(m)) = a(gcd(n, m)) for all positive integers n and m. - _Michael Somos_, Jan 03 2017", + "E.g.f.: (2/sqrt(5))*exp(x/2)*sinh(sqrt(5)*x/2). - _Len Smiley_, Nov 30 2001", + "[0 1; 1 1]^n [0 1] = [F(n); F(n+1)]", + "x | F(n) ==> x | F(kn).", + "A sufficient condition for F(m) to be divisible by a prime p is (p - 1) divides m, if p == 1 or 4 (mod 5); (p + 1) divides m, if p == 2 or 3 (mod 5); or 5 divides m, if p = 5. (This is essentially Theorem 180 in Hardy and Wright.) - Fred W. Helenius (fredh(AT)ix.netcom.com), Jun 29 2001", + "a(n)=F(n) has the property: F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1). - _Miklos Kristof_, Nov 13 2003", + "From _Kurmang. Aziz. Rashid_, Feb 21 2004: (Start)", + "Conjecture 1: for n >= 2, sqrt(F(2n+1) + F(2n+2) + F(2n+3) + F(2n+4) + 2*(-1)^n) = (F(2n+1) + 2*(-1)^n)/F(n-1). [For a proof see Comments section.]", + "Conjecture 2: for n >= 0, (F(n+2)*F(n+3)) - (F(n+1)*F(n+4)) + (-1)^n = 0.", + "[Two more conjectures removed by _Peter Luschny_, Nov 17 2017]", + "Theorem 1: for n >= 0, (F(n+3)^ 2 - F(n+1)^ 2)/F(n+2) = (F(n+3)+ F(n+1)).", + "Theorem 2: for n >= 0, F(n+10) = 11*F(n+5) + F(n).", + "Theorem 3: for n >= 6, F(n) = 4*F(n-3) + F(n-6). (End)", + "Conjecture 2 of Rashid is actually a special case of the general law F(n)*F(m) + F(n+1)*F(m+1) = F(n+m+1) (take n <- n+1 and m <- -(n+4) in this law). - Harmel Nestra (harmel.nestra(AT)ut.ee), Apr 22 2005", + "Conjecture 2 of Rashid Kurmang simplified: F(n)*F(n+3) = F(n+1)*F(n+2)-(-1)^n. Follows from d'Ocagne's identity: m=n+2. - _Alex Ratushnyak_, May 06 2012", + "Conjecture: for all c such that 2-phi <= c < 2*(2-phi) we have F(n) = floor(phi*a(n-1)+c) for n > 2. - _Gerald McGarvey_, Jul 21 2004", + "For x > phi, Sum_{n>=0} F(n)/x^n = x/(x^2 - x - 1). - _Gerald McGarvey_, Oct 27 2004", + "F(n+1) = exponent of the n-th term in the series f(x, 1) determined by the equation f(x, y) = xy + f(xy, x). - _Jonathan Sondow_, Dec 19 2004", + "a(n-1) = Sum_{k=0..n} (-1)^k*binomial(n-ceiling(k/2), floor(k/2)). - _Benoit Cloitre_, May 05 2005", + "a(n) = Sum_{k=0..n} abs(A108299(n, k)). - _Reinhard Zumkeller_, Jun 01 2005", + "a(n) = A001222(A000304(n)).", + "F(n+1) = Sum_{k=0..n} binomial((n+k)/2, (n-k)/2)(1+(-1)^(n-k))/2. - _Paul Barry_, Aug 28 2005", + "Fibonacci(n) = Product_{j=1..ceiling(n/2)-1} (1 + 4(cos(j*Pi/n))^2). [Bicknell and Hoggatt, pp. 47-48.] - _Emeric Deutsch_, Oct 15 2006", + "F(n) = 2^(-(n-1))*Sum_{k=0..floor((n-1)/2)} binomial(n,2*k+1)*5^k. - _Hieronymus Fischer_, Feb 07 2006", + "a(n) = (b(n+1) + b(n-1))/n where {b(n)} is the sequence A001629. - _Sergio Falcon_, Nov 22 2006", + "F(n*m) = Sum_{k = 0..m} binomial(m,k)*F(n-1)^k*F(n)^(m-k)*F(m-k). The generating function of F(n*m) (n fixed, m = 0,1,2,...) is G(x) = F(n)*x / ((1 - F(n-1)*x)^2 - F(n)*x*(1 - F(n-1)*x) - (F(n)*x)^2). E.g., F(15) = 610 = F(5*3) = binomial(3,0)* F(4)^0*F(5)^3*F(3) + binomial(3,1)* F(4)^1*F(5)^2*F(2) + binomial(3,2)* F(4)^2*F(5)^1*F(1) + binomial(3,3)* F(4)^3*F(5)^0*F(0) = 1*1*125*2 + 3*3*25*1 + 3*9*5*1 + 1*27*1*0 = 250 + 225 + 135 + 0 = 610. - _Miklos Kristof_, Feb 12 2007", + "From _Miklos Kristof_, Mar 19 2007: (Start)", + " Let L(n) = A000032(n) = Lucas numbers. Then:", + " For a >= b and odd b, F(a+b) + F(a-b) = L(a)*F(b).", + " For a >= b and even b, F(a+b) + F(a-b) = F(a)*L(b).", + " For a >= b and odd b, F(a+b) - F(a-b) = F(a)*L(b).", + " For a >= b and even b, F(a+b) - F(a-b) = L(a)*F(b).", + " F(n+m) + (-1)^m*F(n-m) = F(n)*L(m);", + " F(n+m) - (-1)^m*F(n-m) = L(n)*F(m);", + " F(n+m+k) + (-1)^k*F(n+m-k) + (-1)^m*(F(n-m+k) + (-1)^k*F(n-m-k)) = F(n)*L(m)*L(k);", + " F(n+m+k) - (-1)^k*F(n+m-k) + (-1)^m*(F(n-m+k) - (-1)^k*F(n-m-k)) = L(n)*L(m)*F(k);", + " F(n+m+k) + (-1)^k*F(n+m-k) - (-1)^m*(F(n-m+k) + (-1)^k*F(n-m-k)) = L(n)*F(m)*L(k);", + " F(n+m+k) - (-1)^k*F(n+m-k) - (-1)^m*(F(n-m+k) - (-1)^k*F(n-m-k)) = 5*F(n)*F(m)*F(k). (End)", + "A corollary to Kristof 2007 is 2*F(a+b) = F(a)*L(b) + L(a)*F(b). - _Graeme McRae_, Apr 24 2014", + "For n > m, the sum of the 2m consecutive Fibonacci numbers F(n-m-1) thru F(n+m-2) is F(n)*L(m) if m is odd, and L(n)*F(m) if m is even (see the McRae link). - _Graeme McRae_, Apr 24 2014.", + "F(n) = b(n) + (p-1)*Sum_{k=2..n-1} floor(b(k)/p)*F(n-k+1) where b(k) is the digital sum analog of the Fibonacci recurrence, defined by b(k) = ds_p(b(k-1)) + ds_p(b(k-2)), b(0)=0, b(1)=1, ds_p=digital sum base p. Example for base p=10: F(n) = A010077(n) + 9*Sum_{k=2..n-1} A059995(A010077(k))*F(n-k+1). - _Hieronymus Fischer_, Jul 01 2007", + "F(n) = b(n)+p*Sum_{k=2..n-1} floor(b(k)/p)*F(n-k+1) where b(k) is the digital product analog of the Fonacci recurrence, defined by b(k) = dp_p(b(k-1)) + dp_p(b(k-2)), b(0)=0, b(1)=1, dp_p=digital product base p. Example for base p=10: F(n) = A074867(n) + 10*Sum_{k=2..n-1} A059995(A074867(k))*F(n-k+1). - _Hieronymus Fischer_, Jul 01 2007", + "a(n) = denominator of continued fraction [1,1,1,...] (with n ones); e.g., 2/3 = continued fraction [1,1,1]; where barover[1] = [1,1,1,...] = 0.6180339.... - _Gary W. Adamson_, Nov 29 2007", + "F(n + 3) = 2F(n + 2) - F(n), F(n + 4) = 3F(n + 2) - F(n), F(n + 8) = 7F(n + 4) - F(n), F(n + 12) = 18F(n + 6) - F(n). - _Paul Curtz_, Feb 01 2008", + "a(2^n) = Product_{i=0..n-2} B(i) where B(i) is A001566. Example 3*7*47 = F(16). - _Kenneth J Ramsey_, Apr 23 2008", + "a(n+1) = Sum_{k=0..n} A109466(n,k)*(-1)^(n-k). -_Philippe Del\u00e9ham_, Oct 26 2008", + "a(n) = Sum_{l_1=0..n+1} Sum_{l_2=0..n}...Sum_{l_i=0..n-i}... Sum_{l_n=0..1} delta(l_1,l_2,...,l_i,...,l_n), where delta(l_1,l_2,...,l_i,...,l_n) = 0 if any l_i + l_(i+1) >= 2 for i=1..n-1 and delta(l_1,l_2,...,l_i,...,l_n) = 1 otherwise. - _Thomas Wieder_, Feb 25 2009", + "a(n+1) = 2^n sqrt(Product_{k=1..n} cos(k Pi/(n+1))^2+1/4) (Kasteleyn's formula specialized). - _Sarah-Marie Belcastro_, Jul 04 2009", + "a(n+1) = Sum_{k=floor(n/2) mod 5} C(n,k) - Sum_{k=floor((n+5)/2) mod 5} C(n,k) = A173125(n) - A173126(n) = |A054877(n)-A052964(n-1)|. - _Henry Bottomley_, Feb 10 2010", + "If p[i] = modp(i,2) and if A is Hessenberg matrix of order n defined by: A[i,j] = p[j-i+1], (i <= j), A[i,j]=-1, (i=j+1), and A[i,j]=0 otherwise. Then, for n >= 1, a(n)=det A. - _Milan Janjic_, May 02 2010", + "Limit_{k->oo} F(k+n)/F(k) = (L(n) + F(n)*sqrt(5))/2 with the Lucas numbers L(n) = A000032(n). - _Johannes W. Meijer_, May 27 2010", + "For n >= 1, F(n) = round(log_2(2^(phi*F(n-1)) + 2^(phi*F(n-2)))), where phi is the golden ratio. - _Vladimir Shevelev_, Jun 24 2010, Jun 27 2010", + "For n >= 1, a(n+1) = ceiling(phi*a(n)), if n is even and a(n+1) = floor(phi*a(n)), if n is odd (phi = golden ratio). - _Vladimir Shevelev_, Jul 01 2010", + "a(n) = 2*a(n-2) + a(n-3), n > 2. - _Gary Detlefs_, Sep 08 2010", + "a(2^n) = Product_{i=0..n-1} A000032(2^i). - _Vladimir Shevelev_, Nov 28 2010", + "a(n)^2 - a(n-1)^2 = a(n+1)*a(n-2), see A121646.", + "a(n) = sqrt((-1)^k*(a(n+k)^2 - a(k)*a(2n+k))), for any k. - _Gary Detlefs_, Dec 03 2010", + "F(2*n) = F(n+2)^2 - F(n+1)^2 - 2*F(n)^2. - _Richard R. Forberg_, Jun 04 2011", + "From _Artur Jasinski_, Nov 17 2011: (Start)", + "(-1)^(n+1) = F(n)^2 + F(n)*F(1+n) - F(1+n)^2.", + "F(n) = F(n+2) - 1 + (F(n+1))^4 + 2*(F(n+1)^3*F(n+2)) - (F(n+1)*F(n+2))^2 - 2*F(n+1)(F(n+2))^3 + (F(n+2))^4 - F(n+1). (End)", + "F(n) = 1 + Sum_{x=1..n-2} F(x). - _Joseph P. Shoulak_, Feb 05 2012", + "F(n) = 4*F(n-2) - 2*F(n-3) - F(n-6). - _Gary Detlefs_, Apr 01 2012", + "F(n) = round(phi^(n+1)/(phi+2)). - _Thomas Ordowski_, Apr 20 2012", + "From _Sergei N. Gladkovskii_, Jun 03 2012: (Start)", + "G.f.: A(x) = x/(1-x-x^2) = G(0)/sqrt(5) where G(k) = 1 - ((-1)^k)*2^k/(a^k - b*x*a^k*2^k/(b*x*2^k - 2*((-1)^k)*c^k/G(k+1))) and a=3+sqrt(5), b=1+sqrt(5), c=3-sqrt(5); (continued fraction, 3rd kind, 3-step).", + "Let E(x) be the e.g.f., i.e.,", + "E(x) = 1*x + (1/2)*x^2 + (1/3)*x^3 + (1/8)*x^4 + (1/24)*x^5 + (1/90)*x^6 + (13/5040)*x^7 + ...; then", + "E(x) = G(0)/sqrt(5); G(k) = 1 - ((-1)^k)*2^k/(a^k - b*x*a^k*2^k/(b*x*2^k - 2*((-1)^k)*(k+1)*c^k/G(k+1))), where a=3+sqrt(5), b=1+sqrt(5), c=3-sqrt(5); (continued fraction, 3rd kind, 3-step).", + "(End)", + "From _Hieronymus Fischer_, Nov 30 2012: (Start)", + "F(n) = 1 + Sum_{j_1=1..n-2} 1 + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} 1 + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} Sum_{j_3=1..j_2-2} 1 + ... + Sum_{j_1=1..n-2} Sum_{j_2=1..j_1-2} Sum_{j_3=1..j_2-2} ... Sum_{j_k=1..j_(k-1)-2} 1, where k = floor((n-1)/2).", + "Example: F(6) = 1 + Sum_{j=1..4} 1 + Sum_{j=1..4} Sum_{k=1..(j-2)} 1 + 0 = 1 + (1 + 1 + 1 + 1) + (1 + (1 + 1)) = 8.", + "F(n) = Sum_{j=0..k} S(j+1,n-2j), where k = floor((n-1)/2) and the S(j,n) are the n-th j-simplex sums: S(1,n) = 1 is the 1-simplex sum, S(2,n) = Sum_{k=1..n} S(1,k) = 1+1+...+1 = n is the 2-simplex sum, S(3,n) = Sum_{k=1..n} S(2,k) = 1+2+3+...+n is the 3-simplex sum (= triangular numbers = A000217), S(4,n) = Sum_{k=1..n} S(3,k) = 1+3+6+...+n(n+1)/2 is the 4-simplex sum (= tetrahedral numbers = A000292) and so on.", + "Since S(j,n) = binomial(n-2+j,j-1), the formula above equals the well-known binomial formula, essentially. (End)", + "G.f.: A(x) = x / (1 - x / (1 - x / (1 + x))). - _Michael Somos_, Jan 04 2013", + "Sum_{n >= 1} (-1)^(n-1)/(a(n)*a(n+1)) = 1/phi (phi=golden ratio). - _Vladimir Shevelev_, Feb 22 2013", + "From _Raul Prisacariu_, Oct 29 2023: (Start)", + "For odd k, Sum_{n >= 1} a(k)^2*(-1)^(n-1)/(a(k*n)*a(k*n+k)) = phi^(-k).", + "For even k, Sum_{n >= 1} a(k)^2/(a(k*n)*a(k*n+k)) = phi^(-k). (End)", + "From _Vladimir Shevelev_, Feb 24 2013: (Start)", + "(1) Expression a(n+1) via a(n): a(n+1) = (a(n) + sqrt(5*(a(n))^2 + 4*(-1)^n))/2;", + "(2) Sum_{k=1..n} (-1)^(k-1)/(a(k)*a(k+1)) = a(n)/a(n+1);", + "(3) a(n)/a(n+1) = 1/phi + r(n), where |r(n)| < 1/(a(n+1)*a(n+2)). (End)", + "F(n+1) = F(n)/2 + sqrt((-1)^n + 5*F(n)^2/4), n >= 0. F(n+1) = U_n(i/2)/i^n, (U:= Chebyshev polynomial of the 2nd kind, i=sqrt(-1)). - _Bill Gosper_, Mar 04 2013", + "G.f.: -Q(0) where Q(k) = 1 - (1+x)/(1 - x/(x - 1/Q(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Mar 06 2013", + "G.f.: x - 1 - 1/x + (1/x)/Q(0), where Q(k) = 1 - (k+1)*x/(1 - x/(x - (k+1)/Q(k+1))); (continued fraction). - _Sergei N. Gladkovskii_, Apr 23 2013", + "G.f.: x*G(0), where G(k) = 1 + x*(1+x)/(1 - x*(1+x)/(x*(1+x) + 1/G(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Jul 08 2013", + "G.f.: x^2 - 1 + 2*x^2/(W(0)-2), where W(k) = 1 + 1/(1 - x*(k + x)/( x*(k+1 + x) + 1/W(k+1) )); (continued fraction). - _Sergei N. Gladkovskii_, Aug 28 2013", + "G.f.: Q(0) - 1, where Q(k) = 1 + x^2 + (k+2)*x - x*(k+1 + x)/Q(k+1); (continued fraction). - _Sergei N. Gladkovskii_, Oct 06 2013", + "Let b(n) = b(n-1) + b(n-2), with b(0) = 0, b(1) = phi. Then, for n >= 2, F(n) = floor(b(n-1)) if n is even, F(n) = ceiling(b(n-1)), if n is odd, with convergence. - _Richard R. Forberg_, Jan 19 2014", + "a(n) = Sum_{t1*g(1)+t2*g(2)+...+tn*g(n)=n} multinomial(t1+t2+...+tn,t1,t2,...,tn), where g(k)=2*k-1. - _Mircea Merca_, Feb 27 2014", + "F(n) = round(sqrt(F(n-1)^2 + F(n)^2 + F(n+1)^2)/2), for n > 0. This rule appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Jul 27 2014", + "F(n) = round(2/(1/F(n) + 1/F(n+1) + 1/F(n+2))), for n > 0. This rule also appears to apply to any sequence of the form a(n) = a(n-1) + a(n-2), for any two values of a(0) and a(1), if n is sufficiently large. - _Richard R. Forberg_, Aug 03 2014", + "F(n) = round(1/(Sum_{j>=n+2} 1/F(j))). - _Richard R. Forberg_, Aug 14 2014", + "a(n) = hypergeometric([-n/2+1/2, -n/2+1], [-n+1], -4) for n >= 2. - _Peter Luschny_, Sep 19 2014", + "Limit_{n -> oo} (log F(n+1)/log F(n))^n = e. - _Thomas Ordowski_, Oct 06 2014", + "F(n) = (L(n+1)^2 - L(n-1)^2)/(5*L(n)), where L(n) is A000032(n), with a similar inverse relationship. - _Richard R. Forberg_, Nov 17 2014", + "Consider the graph G[1-vertex;1-loop,2-loop] in comment above. Construct the power matrix array T(n,j) = [A^*j]*[S^*(j-1)] where A=(1,1,0,...) and S=(0,1,0,...)(A063524). [* is convolution operation] Define S^*0=I with I=(1,0,...). Then T(n,j) counts n-walks containing (j) loops and a(n-1) = Sum_{j=1..n} T(n,j). - _David Neil McGrath_, Nov 21 2014", + "Define F(-n) to be F(n) for n odd and -F(n) for n even. Then for all n and k, F(n) = F(k)*F(n-k+3) - F(k-1)*F(n-k+2) - F(k-2)*F(n-k) + (-1)^k*F(n-2k+2). - _Charlie Marion_, Dec 04 2014", + "F(n+k)^2 - L(k)*F(n)*F(n+k) + (-1)^k*F(n)^2 = (-1)^n*F(k)^2, if L(k) = A000032(k). - _Alexander Samokrutov_, Jul 20 2015", + "F(2*n) = F(n+1)^2 - F(n-1)^2, similar to Koshy (D) and Forberg 2011, but different. - _Hermann Stamm-Wilbrandt_, Aug 12 2015", + "F(n+1) = ceiling( (1/phi)*Sum_{k=0..n} F(k) ). - _Tom Edgar_, Sep 10 2015", + "a(n) = (L(n-3) + L(n+3))/10 where L(n)=A000032(n). - _J. M. Bergot_, Nov 25 2015", + "From _Bob Selcoe_, Mar 27 2016: (Start)", + "F(n) = (F(2n+k+1) - F(n+1)*F(n+k+1))/F(n+k), k >= 0.", + "Thus when k=0: F(n) = sqrt(F(2n+1) - F(n+1)^2).", + "F(n) = (F(3n) - F(n+1)^3 + F(n-1)^3)^(1/3).", + "F(n+2k) = binomial transform of any subsequence starting with F(n). Example F(6)=8: 1*8 = F(6)=8; 1*8 + 1*13 = F(8)=21; 1*8 + 2*13 + 1*21 = F(10)=55; 1*8 + 3*13 + 3*21 + 1*34 = F(12)=144, etc. This formula applies to Fibonacci-type sequences with any two seed values for a(0) and a(1) (e.g., Lucas sequence A000032: a(0)=2, a(1)=1).", + "(End)", + "F(n) = L(k)*F(n-k) + (-1)^(k+1)*F(n-2k) for all k >= 0, where L(k) = A000032(k). - _Anton Zakharov_, Aug 02 2016", + "From _Ilya Gutkovskiy_, Aug 03 2016: (Start)", + "a(n) = F_n(1), where F_n(x) are the Fibonacci polynomials.", + "Inverse binomial transform of A001906.", + "Number of zeros in substitution system {0 -> 11, 1 -> 1010} at step n from initial string \"1\" (1 -> 1010 -> 101011101011 -> ...) multiplied by 1/A000079(n). (End)", + "For n >= 2, a(n) = 2^(n^2+n) - (4^n-2^n-1)*floor(2^(n^2+n)/(4^n-2^n-1)) - 2^n*floor(2^(n^2) - (2^n-1-1/2^n)*floor(2^(n^2+n)/(4^n-2^n-1))). - _Benoit Cloitre_, Apr 17 2017", + "f(n+1) = Sum_{j=0..floor(n/2)} Sum_{k=0..j} binomial(n-2j,k)*binomial(j,k). - _Tony Foster III_, Sep 04 2017", + "F(n) = Sum_{k=0..floor((n-1)/2)} ( (n-k-1)! / ((n-2k-1)! * k!) ). - _Zhandos Mambetaliyev_, Nov 08 2017", + "For x even, F(n) = (F(n+x) + F(n-x))/L(x). For x odd, F(n) = (F(n+x) - F(n-x))/L(x) where n >= x in both cases. Therefore F(n) = F(2*n)/L(n) for n >= 0. - _David James Sycamore_, May 04 2018", + "From _Isaac Saffold_, Jul 19 2018: (Start)", + "Let [a/p] denote the Legendre symbol. Then, for an odd prime p:", + " F(p+n) == [5/p]*F([5/p]+n) (mod p), if [5/p] = 1 or -1.", + " F(p+n) == 3*F(n) (mod p), if [5/p] = 0 (i.e., p = 5).", + " This is true for negative-indexed terms as well, if this sequence is extended by the negafibonacci numbers (i.e., F(-n) = A039834(n)). (End)", + "a(n) = A094718(4, n). a(n) = A101220(0, j, n).", + "a(n) = A090888(0, n+1) = A118654(0, n+1) = A118654(1, n-1) = A109754(0, n) = A109754(1, n-1), for n > 0.", + "a(n) = (L(n-3) + L(n-2) + L(n-1) + L(n))/5 with L(n)=A000032(n). - _Art Baker_, Jan 04 2019", + "F(n) = F(k-1)*F(abs(n-k-2)) + F(k-1)*F(n-k-1) + F(k)*F(abs(n-k-2)) + 2*F(k)*F(n-k-1), for n > k > 0. - _Joseph M. Shunia_, Aug 12 2019", + "F(n) = F(n-k+2)*F(k-1) + F(n-k+1)*F(k-2) for all k such that 2 <= k <= n. - _Michael Tulskikh_, Oct 09 2019", + "F(n)^2 - F(n+k)*F(n-k) = (-1)^(n+k) * F(k)^2 for 2 <= k <= n [Catalan's identity]. - _Hermann Stamm-Wilbrandt_, May 07 2021", + "Sum_{n>=1} 1/a(n) = A079586 is the reciprocal Fibonacci constant. - _Gennady Eremin_, Aug 06 2021", + "a(n) = Product_{d|n} b(d) = Product_{k=1..n} b(gcd(n,k))^(1/phi(n/gcd(n,k))) = Product_{k=1..n} b(n/gcd(n,k))^(1/phi(n/gcd(n,k))) where b(n) = A061446(n) = primitive part of a(n), phi(n) = A000010(n). - _Richard L. Ollerton_, Nov 08 2021", + "a(n) = 2*i^(1-n)*sin(n*arccos(i/2))/sqrt(5), i=sqrt(-1). - _Bill Gosper_, May 05 2022", + "a(n) = i^(n-1)*sin(n*c)/sin(c) = i^(n-1)*sin(c*n)*csc(c), where c = Pi/2 + i*arccsch(2). - _Peter Luschny_, May 23 2022", + "F(2n) = Sum_{k=1..n} (k/5)*binomial(2n, n+k), where (k/5) is the Legendre or Jacobi Symbol; F(2n+1)= Sum_{k=1..n} (-(k+2)/5)*binomial(2n+1, n+k), where (-(k+2)/5) is the Legendre or Jacobi Symbol. For example, F(10) = 1*binomial(10,6) - 1*binomial(10,7) - 1*binomial(10,8) + 1*binomial(10,9) + 0*binomial(10,10), F(11) = 1*binomial(11,6) - 1*binomial(11,7) + 0*binomial(11,8) - 1*binomial(11,9) + 1*binomial(11,10) + 1*binomial(11,11). - _Yike Li_, Aug 21 2022", + "For n > 0, 1/F(n) = Sum_{k>=1} F(n*k)/(F(n+2)^(k+1)). - _Diego Rattaggi_, Oct 26 2022", + "From _Andrea Pinos_, Dec 02 2022: (Start)", + "For n == 0 (mod 4): F(n) = F((n+2)/2)*( F(n/2) + F((n/2)-2) ) + 1;", + "For n == 1 (mod 4): F(n) = F((n-1)/2)*( F((n-1)/2) + F(2+(n-1)/2) ) + 1;", + "For n == 2 (mod 4): F(n) = F((n-2)/2)*( F(n/2) + F((n/2)+2) ) + 1;", + "For n == 3 (mod 4): F(n) = F((n-1)/2)*( F((n-1)/2) + F(2+(n-1)/2) ) - 1. (End)", + "F(n) = Sum_{i=0..n-1} F(i)^2 / F(n-1). - _Jules Beauchamp_, May 03 2025", + "F(n) = Sum_{i=0..n-1} L(n-i)*(-2)^i. - _Greg Dresden_ and Xiaoya Gao, Oct 11 2025", + "For n>=2, a(n) = 2^(n-2)*hypergeometric([(1-n)/3,(2-n)/3,1-n/3,2-n],[(2-n)/2,(3-n)/2,1-n],27/32). - _Victor Petrescu_, Oct 25 2025", + "From _Friedjof Tellkamp_, Jun 10 2026: (Start)", + "Sum_{n>=1} 1/a(n)^s = 5^(s/2) * Sum_{k>=0} C(-s, k) * 1/(phi^(s+2k) - (-1)^k), see Eq. 5 in Navas.", + "Sum_{n>=1} (-1)^(n+1)/a(n)^s = 5^(s/2) * Sum_{k>=0} C(-s, k) * 1/(phi^(s+2k) + (-1)^k). (End)" + ] + } + ] + } +} diff --git a/tests/test_novelty.py b/tests/test_novelty.py index 5b5337f8..7b35d01c 100644 --- a/tests/test_novelty.py +++ b/tests/test_novelty.py @@ -2,16 +2,22 @@ Nothing here touches the network. The OEIS half runs against ``tests/data/oeis_novelty_fixture.json``, a cache recorded once from -https://oeis.org (© The OEIS Foundation Inc., CC BY-NC-SA 4.0) and committed; -:class:`~alkahest.experimental.novelty.OeisWeb` is never constructed by a test. +https://oeis.org (© The OEIS Foundation Inc., CC BY-NC-SA 4.0) and committed. To re-record it:: from alkahest.experimental.novelty import OeisCache, OeisWeb - web = OeisWeb(cache=OeisCache(), min_interval=1.5, max_results=8) + web = OeisWeb(cache=OeisCache(), min_interval=1.5) for terms in (...): # the exact term lists the tests query with web.lookup(terms=terms) + web.lookup(ids=["A000045"]) web.cache.save("tests/data/oeis_novelty_fixture.json") +:class:`~alkahest.experimental.novelty.OeisWeb` *is* constructed, by the paging +tests only, and never with a live transport: ``urlopen`` is replaced with one +that serves :data:`PAGING_FIXTURE`, the recorded raw pages, so that what a full +result page means — and does not mean — is covered offline like everything +else. + The recorded queries matter as much as the recorded entries: a cache that only stores hits cannot tell "OEIS was asked and had nothing" from "nobody asked", and reporting the second as the first is the overclaim this module exists to @@ -20,7 +26,9 @@ from __future__ import annotations +import json import math +import urllib.parse from fractions import Fraction from pathlib import Path @@ -31,11 +39,18 @@ NoveltyVerdict, OeisCache, OeisEntry, + OeisWeb, + QRecurrenceClaim, RecurrenceClaim, check_novelty, ) FIXTURE = Path(__file__).resolve().parent / "data" / "oeis_novelty_fixture.json" +#: Raw `search?...&fmt=json` pages, keyed `"query|start"`, recorded once from +#: oeis.org. `OeisWeb` is exercised against these through a fake transport, so +#: the paging behaviour is tested without the network the module promises never +#: to need. +PAGING_FIXTURE = Path(__file__).resolve().parent / "data" / "oeis_paging_fixture.json" # --------------------------------------------------------------------------- # The sequences this project has already certified recurrences for, computed @@ -471,6 +486,7 @@ def test_report_carries_the_scope_of_the_search(cache: OeisCache) -> None: "entries_examined", "statements_compared", "statements_unusable", + "terms_check", } assert report["sources_consulted"] == ["oeis-cache"] # OEIS says a great deal this parser cannot read, and the count of what it @@ -520,7 +536,14 @@ def test_web_source_is_opt_in_and_never_default() -> None: def test_experimental_exports() -> None: from alkahest import experimental - for name in ("RecurrenceClaim", "NoveltyVerdict", "OeisCache", "OeisWeb", "check_novelty"): + for name in ( + "RecurrenceClaim", + "QRecurrenceClaim", + "NoveltyVerdict", + "OeisCache", + "OeisWeb", + "check_novelty", + ): assert name in experimental.__all__ assert hasattr(experimental, name) assert "novelty" in experimental.__all__ @@ -530,7 +553,11 @@ def test_experimental_exports() -> None: def test_accessor_convention_holds_for_the_new_types() -> None: """Zero-argument O(1) scalars are properties; collections are methods.""" for cls, scalars, collections in ( - (RecurrenceClaim, ("order", "degree", "normal_form", "claim_hash"), ("coefficients",)), + ( + RecurrenceClaim, + ("order", "degree", "normal_form", "claim_hash", "claim_kind"), + ("coefficients",), + ), ( NoveltyVerdict, ( @@ -542,11 +569,21 @@ def test_accessor_convention_holds_for_the_new_types() -> None: "statements_compared", "statements_unusable", "means", + "terms_check", ), ("matches", "sources_consulted", "sources_unavailable", "report"), ), + ( + QRecurrenceClaim, + ("order", "degree", "q_degree", "normal_form", "claim_hash", "claim_kind"), + ("coefficients",), + ), (OeisCache, ("name", "n_entries", "n_queries"), ("lookup", "save", "load", "add")), - (OeisEntry, (), ("recurrences", "unusable_statements", "to_json")), + ( + OeisEntry, + (), + ("recurrences", "unusable_statements", "candidate_lines", "to_json"), + ), ): for name in scalars: assert isinstance(inspect_static(cls, name), property), f"{cls.__name__}.{name}" @@ -559,3 +596,331 @@ def inspect_static(cls: type, name: str) -> object: import inspect return inspect.getattr_static(cls, name) + + +# --------------------------------------------------------------------------- +# 6. The coverage the filter actually has (issues #23, #24, #25, #26q). +# --------------------------------------------------------------------------- + +#: The Fibonacci recurrence, `u(n+2) = u(n+1) + u(n)`, as a claim. +FIBONACCI = ((-1,), (-1,), (1,)) + + +def test_the_fibonacci_recurrence_is_found_in_the_fibonacci_entry(cache: OeisCache) -> None: + """#24. A filter that cannot find this clears almost anything. + + A000045 states its recurrence in its **name** — ``Fibonacci numbers: F(n) = + F(n-1) + F(n-2)`` — and nowhere in the formula lines the parser used to be + pointed at, so the whole entry came back with zero usable statements and the + verdict was ``not_found``: "not in the sources searched", read by a loop + author as novelty. + """ + claim = RecurrenceClaim(FIBONACCI) + verdict = check_novelty(claim, [cache], ids=["A000045"]) + assert verdict.status == "recorded", verdict.report() + assert verdict.found is True + assert verdict.hedged is False, "the name of an entry is not a conjecture" + assert verdict.statements_compared > 0 + assert {m.entry for m in verdict.matches()} == {"A000045"} + assert any(m.statement.startswith("Fibonacci numbers:") for m in verdict.matches()) + + +def test_the_name_of_an_entry_is_a_candidate_line(cache: OeisCache) -> None: + entry = cache.lookup(ids=["A000045"]).entries[0] + assert entry.candidate_lines()[0] == entry.name + assert entry.candidate_lines()[1:] == entry.statements + assert RecurrenceClaim.from_text(entry.name).claim_hash == RecurrenceClaim(FIBONACCI).claim_hash + + # A name that states nothing is not turned into a candidate. + plain = OeisEntry("A000027", "The positive integers.", terms=list(range(1, 20))) + assert plain.candidate_lines() == () + + +@pytest.mark.parametrize( + ("line", "why"), + [ + ( + "Fibonacci numbers: F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1.", + "the sequence under the letter it is named for, in an entry's name", + ), + ("a(n) = 2a(n-2) + a(n-3), n > 2.", "implicit multiplication"), + ("L(n) = L(n-1) + L(n-2).", "another single letter"), + ("a(n) = a(n-1) + A000045(n-2).", "the entry's own A-number spelled out"), + ], +) +def test_parses_the_other_notations_oeis_uses(line: str, why: str) -> None: + assert RecurrenceClaim.from_text(line, names=("A000045",)) is not None, why + + +@pytest.mark.parametrize( + "line", + [ + # Two sequences in one relation is not a recurrence for either of them, + # whichever way they are spelled. + "F(n) = L(n-1) + L(n-2).", + "a(n) = a(n-1) + A002026(n-1). - _R. J. Mathar_, Jul 25 2017", + "a(n) = 2a(n-1) + 3b(n-2).", + # A function that is not a sequence must not become one. + "a(n) = floor(a(n-1)*phi).", + # Two indices is a triangle, not a sequence; `A(x)` is a generating + # function, and `x` is not the running index. + "T(n,k) = T(n-1,k) + T(n-1,k-1).", + "A(x) = 1 + x*A(x)^2.", + ], +) +def test_the_widened_parser_still_refuses_what_it_should(line: str) -> None: + assert RecurrenceClaim.from_text(line, names=("A000045",)) is None, line + + +@pytest.mark.parametrize( + "line", + [ + # Prose after the formula is prose, not a factor: reading `n > 2` as an + # implicit multiplication would invent `(a(n-1) + a(n-2))*n`, a claim + # nobody made, and reading `for` as one would invent a different one. + "a(n) = a(n-1) + a(n-2) for n > 2, with a(0) = 0.", + "a(n) = a(n-1) + a(n-2), n >= 2.", + "F(n) = F(n-1) + F(n-2) with F(0) = 0 and F(1) = 1.", + ], +) +def test_implicit_multiplication_does_not_swallow_the_prose_after_a_formula(line: str) -> None: + claim = RecurrenceClaim.from_text(line) + assert claim is not None, line + assert claim.claim_hash == RecurrenceClaim(FIBONACCI).claim_hash, claim.normal_form + + +def test_a_single_letter_sequence_is_still_held_to_the_entrys_own_data() -> None: + """The letter is not trusted; the data is. + + ``b(n) = b(n-1) + b(n-2)`` in a comment may be about an auxiliary sequence, + so what licenses indexing it is the same thing that licenses an ``a(n)`` + line: it has to reproduce the terms the entry ships with. + """ + line = "Let b(n) = b(n-1) + b(n-2), with b(0) = 0, b(1) = 1." + fibonacci = OeisEntry("A000045", terms=[0, 1, 1, 2, 3, 5, 8, 13, 21, 34], statements=[line]) + assert len(fibonacci.recurrences()) == 1 + + elsewhere = OeisEntry("A000079", terms=[1, 2, 4, 8, 16, 32, 64, 128], statements=[line]) + assert elsewhere.recurrences() == () + assert len(elsewhere.unusable_statements()) == 1 + + +# --------------------------------------------------------------------------- +# Paging (#23), against recorded pages through a fake transport. +# --------------------------------------------------------------------------- + + +class _RecordedResponse: + """The two methods :class:`OeisWeb` uses of a ``urlopen`` result.""" + + def __init__(self, body: bytes) -> None: + self._body = body + + def read(self) -> bytes: + return self._body + + def __enter__(self) -> _RecordedResponse: + return self + + def __exit__(self, *exception: object) -> bool: + return False + + +@pytest.fixture +def oeis_transport(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Serve :data:`PAGING_FIXTURE`; return the list of ``"query|start"`` asked for.""" + pages = json.loads(PAGING_FIXTURE.read_text(encoding="utf-8"))["pages"] + asked: list[str] = [] + + def urlopen(request: object, timeout: float | None = None) -> _RecordedResponse: + parameters = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query) + key = f"{parameters['q'][0]}|{int(parameters.get('start', ['0'])[0])}" + asked.append(key) + assert key in pages, f"no page recorded for {key!r}; re-record the fixture" + return _RecordedResponse(json.dumps(pages[key]).encode("utf-8")) + + monkeypatch.setattr(novelty.urllib.request, "urlopen", urlopen) + return asked + + +def test_a_terms_search_is_paged_and_one_full_page_is_not_exhaustive( + oeis_transport: list[str], +) -> None: + """#23. ``fmt=json`` sends ten results and no count, so ten is not "all".""" + web = OeisWeb(cache=OeisCache(), min_interval=0.0, max_results=15) + answer = web.lookup(terms=[1, 1, 2, 3, 5, 8, 13]) + assert oeis_transport == ["1,1,2,3,5,8,13|0", "1,1,2,3,5,8,13|10"], ( + "a full first page must be followed by `&start=10`" + ) + assert answer.exhaustive is False, "OEIS never said these were all of them" + assert len(answer.entries) == 15 + + # And the partial answer must not be recorded as a complete one, or every + # later offline run would read it as a licence to say `not_found`. + assert web.cache.lookup(terms=[1, 1, 2, 3, 5, 8, 13]).exhaustive is False + + +def test_an_ids_lookup_stays_exhaustive_after_one_request(oeis_transport: list[str]) -> None: + """The other direction of #23: ``id:A…`` asks for named entries and gets them.""" + web = OeisWeb(cache=OeisCache(), min_interval=0.0) + answer = web.lookup(ids=["A000045"]) + assert oeis_transport == ["id:A000045|0"], "an identifier lookup is not paged" + assert answer.exhaustive is True + assert [entry.id for entry in answer.entries] == ["A000045"] + assert web.cache.lookup(ids=["A000045"]).exhaustive is True + + +def test_a_terms_search_that_runs_out_of_results_is_exhaustive( + oeis_transport: list[str], +) -> None: + """A short page *is* the end of the search, and may be recorded as one.""" + web = OeisWeb(cache=OeisCache(), min_interval=0.0) + terms = apery(10) + answer = web.lookup(terms=terms) + assert oeis_transport == [",".join(str(t) for t in terms) + "|0"] + assert answer.exhaustive is True + assert [entry.id for entry in answer.entries] == ["A005259"] + assert web.cache.lookup(terms=terms).exhaustive is True + + +def test_a_paged_out_search_is_unavailable_and_never_a_negative( + oeis_transport: list[str], +) -> None: + """The whole point of #23: it collapsed ``unavailable`` into ``not_found``.""" + web = OeisWeb(cache=OeisCache(), min_interval=0.0, max_results=15) + absent = RecurrenceClaim([(3, 3), (5, 2), (-4, -1)]) + verdict = check_novelty(absent, [web], terms=[1, 1, 2, 3, 5, 8, 13]) + assert verdict.matches() == () + assert verdict.status == "unavailable" + assert verdict.found is None + assert verdict.sources_unavailable() == ("oeis",) + + +# --------------------------------------------------------------------------- +# `q`-recurrences (#25). +# --------------------------------------------------------------------------- + +#: `(1 - q^n)·u(n) - u(n+1) = 0`, written five ways. `(i, j)` is `q^i·(q^n)^j`. +Q_PRESENTATIONS = { + "as stated": ([{(0, 0): 1, (0, 1): -1}, {(0, 0): -1}], 0), + "scaled by -2*q^5*(q^n)^2": ([{(5, 2): -2, (5, 3): 2}, {(5, 2): 2}], 0), + "times the polynomial 1 + q*q^n": ( + [{(0, 0): 1, (1, 1): 1, (0, 1): -1, (1, 2): -1}, {(0, 0): -1, (1, 1): -1}], + 0, + ), + "stated about u(n+3), scaled by -2q": ([{(1, 0): -2, (4, 1): 2}, {(1, 0): 2}], 3), + "stated about u(n-3), scaled by 3, padded": ( + [{}, {(0, 0): 3, (-3, 1): -3}, {(0, 0): -3}, {}], + -4, + ), +} + + +@pytest.mark.parametrize("label", sorted(Q_PRESENTATIONS)) +def test_presentations_of_one_q_recurrence_hash_equal(label: str) -> None: + """#25. The claim type M4's `q` half had no route into. + + The index shift is the interesting one: ``n → n+1`` sends ``q^n`` to + ``q·q^n``, so re-indexing a `q`-recurrence rewrites its coefficients rather + than leaving them alone. + """ + reference = QRecurrenceClaim(*Q_PRESENTATIONS["as stated"][:1]) + coefficients, offset = Q_PRESENTATIONS[label] + claim = QRecurrenceClaim(coefficients, offset=offset) + assert claim.claim_hash == reference.claim_hash, ( + f"{label!r} normalised to {claim.normal_form!r}, not {reference.normal_form!r}" + ) + assert claim == reference + + +def test_a_q_claim_accepts_rational_coefficients_and_expressions() -> None: + pool = ak.ExprPool() + one = pool.integer(1) + n, q = pool.symbol("n"), pool.symbol("q") + power = q**n + denominator = one + q * power + claim = QRecurrenceClaim([(one - power) / denominator, -one / denominator], var=n, q=q) + assert claim.claim_hash == QRecurrenceClaim(*Q_PRESENTATIONS["as stated"][:1]).claim_hash + assert claim.normal_form == "q-recurrence/1 (q^n - 1)*u(n+0) + (1)*u(n+1)" + assert (claim.order, claim.degree, claim.q_degree) == (1, 1, 0) + + +def test_a_q_certificate_becomes_a_claim() -> None: + """The exact call that used to raise ``coefficient mentions the symbol 'q'``.""" + from alkahest.experimental import q_zeilberger, qbinomial + + pool = ak.ExprPool() + n, k, q = pool.symbol("n"), pool.symbol("k"), pool.symbol("q") + binomial = qbinomial(pool, n, k) + certificate = q_zeilberger(binomial * binomial * q ** (k * k), q, n, k) + + with pytest.raises(ValueError, match="QRecurrenceClaim"): + RecurrenceClaim.from_recurrence(certificate, var=n) + + claim = QRecurrenceClaim.from_recurrence(certificate, var=n, q=q) + assert claim.order == 1 + assert claim.claim_kind == "q-recurrence" + assert claim.normal_form.startswith("q-recurrence/1 ") + assert claim.claim_hash.startswith("clm_") + + +def test_a_q_claim_does_not_collide_with_an_ordinary_one() -> None: + ordinary = RecurrenceClaim([(1,), (-1,)]) # u(n+1) = u(n) + q_analogue = QRecurrenceClaim([{(0, 0): 1}, {(0, 0): -1}]) + assert ordinary.normal_form.startswith("recurrence/1 ") + assert q_analogue.normal_form.startswith("q-recurrence/1 ") + assert ordinary.claim_hash != q_analogue.claim_hash + assert len({ordinary, q_analogue}) == 2 + assert ordinary != q_analogue + + +def test_a_source_that_cannot_state_a_q_recurrence_is_unavailable_for_one( + cache: OeisCache, +) -> None: + """Not ``not_found``: a search that could not have matched is not a negative.""" + claim = QRecurrenceClaim([{(0, 0): 1, (0, 1): -1}, {(0, 0): -1}]) + verdict = check_novelty(claim, [cache], terms=[1, 2, 6, 20, 70, 252, 924, 3432]) + assert verdict.status == "unavailable" + assert verdict.found is None + assert verdict.sources_consulted() == () + assert verdict.sources_unavailable() == ("oeis-cache",) + assert verdict.terms_check == "not_checked" + assert verdict.claim_hash == claim.claim_hash + + +# --------------------------------------------------------------------------- +# The `terms` cross-check (#26q). +# --------------------------------------------------------------------------- + + +def test_terms_are_checked_against_the_claim_not_only_used_to_search( + cache: OeisCache, +) -> None: + """#26q. *terms* said which sequence; the claim has to be about it.""" + claim = RecurrenceClaim([(-2, -4), (1, 1)]) # (n+1)u(n+1) = (4n+2)u(n) + + agrees = check_novelty(claim, [cache], terms=central_binomial(12)) + assert agrees.terms_check == "holds" + assert agrees.report()["terms_check"] == "holds" + assert agrees.status == "recorded" + + # The central binomial recurrence, looked up by the Motzkin numbers: the + # search was about one sequence and the claim about another, and saying so + # is the difference between a report and a silently misleading `not_found`. + disagrees = check_novelty(claim, [cache], terms=motzkin(12)) + assert disagrees.terms_check == "fails" + assert disagrees.report()["terms_check"] == "fails" + assert "terms_check='fails'" in repr(disagrees) + assert disagrees.terms_check in novelty.TERMS_CHECKS + + # Nothing to check it against is not a failure. + assert check_novelty(claim, [cache], ids=["A000984"]).terms_check == "not_checked" + assert check_novelty(claim, [cache], terms=[1]).terms_check == "not_checked" + + +def test_the_terms_cross_check_reads_start_the_way_holds_for_does(cache: OeisCache) -> None: + """The one way an honest caller can trip it, and the knob that fixes it.""" + claim = RecurrenceClaim([(-2, -4), (1, 1)]) + padded = [99, *central_binomial(12)] + assert check_novelty(claim, [cache], terms=padded, start=0).terms_check == "fails" + assert check_novelty(claim, [cache], terms=padded, start=-1).terms_check == "holds" From 8de5af985a9ce90bae41a83b3d72f2273509b57a Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 20:31:12 +0000 Subject: [PATCH 03/11] fix: five trust-boundary and false-red bugs in the research/claim-graph layer Autoresearch 2026-08-19 issues #18-#22 and 26m. #22 `RecurrenceClaim.from_text` collapsed shift gaps. `_parse_relation` returns a sparse `{shift: coefficient}` map and the constructor reads its list positionally, so passing the values at the *sorted* keys closed every gap: `a(n) = a(n-1) + a(n-3)` and `a(n) = a(n-2) + a(n-4)` both produced the Fibonacci `claim_hash` (`clm_8023cee0d17e234d`) verbatim. Fill the window densely with explicit zeros. Measured over the run's 377-entry live OEIS corpus: 33 gapped lines, all 33 recovered, one of them a Fibonacci collision. It was a 21% false-red loss of parsed statements, never a forging hazard -- a mangled reading still has to reproduce the entry's own terms to be indexed. #20 `verify()` bound every free symbol to the same sample value, putting the evaluation on the diagonal `x = y = z`, where `sin(x)cos(y) = sin(y)cos(x)` and `x + y = 2x` are both true and came back `numeric_ok` with `|residual| <= 0`. Offset each symbol by its rank in sorted name order, so the point is off the diagonal and the single-symbol case is unchanged. A sample an offset pushes out of a domain is skipped and reported; none evaluating is `inconclusive`, not a verdict. The detail string no longer implies the points were independent. #21 `verify()` cast high-precision decimal constants to `float`, so an exact relation given at 60 digits (alpha a root of 5144503108x^2 - 5945642943x + 1) picked up a 9.5e-7 rounding residual against a true 6.7e-52 and, with `mark_refuted=True` the default, was marked `refuted` -- the only place in the graph machinery where `verify()` destroyed a true claim. Compute the `numeric_relation` residual exactly in `Fraction`, propagate each input's own half-ulp, and decide from where the band `|R| +- U` sits relative to the tolerance: inside is `numeric_ok`, outside is `failed`, straddling is `inconclusive` ("not enough digits to decide"). This also catches the mirror false green (exact residual 1, float residual 0). #18 `record(result, statement=X)` transplanted a machine-checked status onto arbitrary prose: `record(integrate(sin(x), x), statement="0 = 1")` was `exactly_verified`, `machine_checked`, `[VERIFIED]`. A caller-supplied statement without a `check` recipe no longer inherits a machine-checked status; it is badged `asserted` / `[ASSERTED, UNCHECKED]` with the result's own status kept under `verification["result_status"]`. Assertions the engine renders itself (`_infer_assertion`, the capture path) are unaffected. `smt.md` and `tests/test_smt.py` shipped exactly the flagged pattern and are updated to record the result as itself. #19 Re-recording a statement dropped the second record's `check` recipe, so the one supported way to link a statement to its evidence was a no-op: `verify()` reported `skipped` in one order and `refuted` in the other. `ClaimGraph.add` now adopts a recipe the stored claim lacks; an existing one is never overwritten. 26m `_HEDGE_RE` missed "It appears that", "seems that", "Probably", "believed", "verified up to n" -- each the same statement about the same epistemic status as "conjecture". `claim-graphs.md` gains the `numeric_ok` sampling caveat, the precision caveat and its escape hatch, the `asserted` rule and the recipe-merge rule; `smt.md` gains the corrected recording pattern. Co-Authored-By: Claude Opus 5 --- docs/mdbook/src/claim-graphs.md | 74 +++++++ docs/mdbook/src/smt.md | 21 +- python/alkahest/experimental/novelty.py | 19 +- python/alkahest/research.py | 220 +++++++++++++++++++-- tests/test_novelty.py | 74 +++++++ tests/test_research_claim_graph.py | 245 ++++++++++++++++++++++++ tests/test_smt.py | 12 +- 7 files changed, 644 insertions(+), 21 deletions(-) diff --git a/docs/mdbook/src/claim-graphs.md b/docs/mdbook/src/claim-graphs.md index 6a63ae5e..f05c80a0 100644 --- a/docs/mdbook/src/claim-graphs.md +++ b/docs/mdbook/src/claim-graphs.md @@ -123,6 +123,12 @@ s.conjecture( - `conjecture()` always produces `"unverified"`. - `verify()` may only *lower* confidence: a failed re-check sets `"refuted"`; a successful one appends an audit entry and promotes nothing. +- `record(result, statement=...)` does not move a machine-checked status onto prose. The + status describes `result`; `statement=` is free text, and nothing relates the two. So a + re-worded claim is stored as `"asserted"` (`[ASSERTED, UNCHECKED]`, `machine_checked` + false), with the result's own status kept under `verification["result_status"]`. + Record the result as itself, or supply the `check` recipe that re-establishes the link, + to keep the machine-checked status. The renderers follow the same rule. An emitted-but-unchecked Lean certificate is marked `[CERT ONLY, UNCHECKED]`, never as a proof, and every document opens with the exact @@ -133,6 +139,12 @@ machine-checkable fraction: `Claim.machine_checked` is true only for `exactly_verified` and `lean_checked`. +Re-recording a statement merges into the stored claim rather than replacing it: it keeps +its original status and derivation and gains the new edges, tags and audit entries. It +also **adopts a `check` recipe** when it had none, so recording a statement bare and then +recording it again with its recipe attaches the recipe rather than dropping it. An +existing recipe is never overwritten. + ## Querying ```python @@ -206,6 +218,68 @@ report.summary() # {'ok': 2, 'numeric_ok': 1, 'skipped': 1} print(report.to_markdown()) ``` +`verify()` can only ever **lower** confidence: a `failed` re-check sets the claim +`"refuted"` (unless `mark_refuted=False`), and no outcome ever raises a status. + +### What `numeric_ok` does and does not mean + +The numeric fallback evaluates the residual at the points in `samples` (default +`(0.37, 1.23, 2.71)`). Each sample gives a **point**, not one value shared by every +symbol: a free symbol is bound to the sample offset by its rank in sorted name order, so +`x`, `y`, `z` take `0.37`, `0.48`, `0.59` and the evaluation is off the diagonal +`x = y = z`. That is deliberate — on the diagonal, `sin(x)cos(y) = sin(y)cos(x)` and +`x + y = 2x` are both true, so a symmetry error between two variables, the commonest bug +class this fallback exists to catch, would come back `numeric_ok`. + +Two consequences worth knowing: + +- Finitely many points is still evidence, not a proof. `numeric_ok` means "did not + contradict at the points sampled", which is why it never upgrades a status. +- An offset can push a sample out of an operation's domain. That point is skipped; the + detail string reports how many of the samples actually evaluated, and a recipe where + none of them do comes back `inconclusive` rather than `numeric_ok`. + +Pass your own `samples=` to move the points, and `tolerance=` to set the cutoff. + +### Precision, and `numeric_relation` + +`numeric_relation` does **not** narrow its inputs to `float`. Constants are the output of +a high-precision search — `guess_relation`'s docstring tells you to hand it 50 or more +digits — and a `float` holds about 16, so casting them would fabricate a residual out of +rounding. With coefficients around `5e9`, a *true* relation given at 60 digits picks up a +`9.5e-7` double-rounding residual, eight orders of magnitude past the default `1e-8` +tolerance, and `mark_refuted=True` is the default: the pass documented as one that can +only lower confidence would destroy a true claim. + +So the residual is computed exactly, in `fractions.Fraction`, from the numerals as +written, and each input contributes the precision its own notation implies: `"1"` is the +exact integer, `"1.15572734962273134279"` is known to half a unit in its last decimal +place, a `float` to half its own ulp. The true residual therefore lies in a band +`|R| ± U`, and the outcome follows from where that band sits: + +| Band vs. `tolerance` | Outcome | +| --- | --- | +| entirely inside | `numeric_ok` | +| entirely outside | `failed` | +| straddles it | `inconclusive` — "you did not give me the digits to decide this" | + +The escape hatch is the recipe's own `"tolerance"` key, which overrides the `tolerance=` +argument for that claim: + +```python +check = { + "kind": "numeric_relation", + # Strings, not floats: the digits past the 17th are the evidence. + "constants": ["1.33571181795176524...", "1.15572734962273134...", "1"], + "coefficients": [5144503108, -5945642943, 1], + "tolerance": 1e-40, +} +``` + +Supply the constants as strings (or `Decimal`, or `Fraction`). A `float` in that list has +already lost the digits before `verify()` ever sees it, and the reported uncertainty will +say so. + ## A long session accumulates, by design Two things grow monotonically for as long as a session is open, and neither is a leak — diff --git a/docs/mdbook/src/smt.md b/docs/mdbook/src/smt.md index 4492ae9b..91e8bfb0 100644 --- a/docs/mdbook/src/smt.md +++ b/docs/mdbook/src/smt.md @@ -379,12 +379,31 @@ a real answer ("I could not decide this"), distinct from a resource verdict. ```python with ak.research.session(title="mixed feasibility", pool=pool) as s: result = ak.smt.solve(f, budget=ak.Budget(wall_ms=5000)) - claim = s.record(result, statement="the system is feasible", method="smt.solve") + claim = s.record(result, method="smt.solve") +claim.statement # the formula itself — the thing the model was checked against claim.status # 'exactly_verified' for sat, 'externally_asserted' for unsat claim.machine_checked # True only for the checked sat case ``` +Record it **as itself**, as above. A `statement=` argument is free text and nothing +relates it to the formula the solver was handed, so a re-worded claim does not inherit +the machine-checked status: + +```python +claim = s.record(result, statement="the system is feasible", method="smt.solve") + +claim.status # 'asserted', not 'exactly_verified' +claim.machine_checked # False +claim.verification["result_status"] # 'exactly_verified' — the *result* was checked +``` + +`"the system is feasible"` may well be the right English for `f`, but that is a +translation nobody checked, and `machine_checked` is read by tooling as "a machine +verified this sentence". To keep the status on a re-worded claim, attach the `check` +recipe that re-establishes the link and let +[`ClaimGraph.verify()`](./claim-graphs.md) run it. + ## What is *not* here, and why - **No vendored solver.** No libz3 in the Rust build: it keeps the wheel small, the diff --git a/python/alkahest/experimental/novelty.py b/python/alkahest/experimental/novelty.py index ef65d92c..fced2c66 100644 --- a/python/alkahest/experimental/novelty.py +++ b/python/alkahest/experimental/novelty.py @@ -540,8 +540,13 @@ def from_text(cls, text: str) -> RecurrenceClaim | None: return None if relation is None: return None + # The window must be filled densely: ``relation`` is a sparse map from + # shift to coefficient and the constructor reads its list positionally, + # so handing it the values at the *sorted* keys would close every gap + # and read ``a(n) = a(n-1) + a(n-3)`` as ``a(n) = a(n-1) + a(n-2)``. + low, high = min(relation), max(relation) try: - return cls([relation[j] for j in sorted(relation)], offset=min(relation)) + return cls([relation.get(j, ()) for j in range(low, high + 1)], offset=low) except ValueError: return None @@ -658,8 +663,18 @@ def __repr__(self) -> str: #: OEIS's own hedges. An entry that marks a formula this way is telling you the #: recurrence was fitted and never proved — which is the whole reason a novelty #: filter over OEIS is worth anything. +#: +#: Deliberately wider than the obvious four words: contributors write "It appears +#: that", "seems that", "Probably", "believed to hold" and "verified up to n=1000" +#: as often as they write "conjecture", and every one of those is the same +#: statement about the same epistemic status. Missing one silently promotes a +#: fitted formula to an unqualified one. _HEDGE_RE = re.compile( - r"\b(conjectur\w*|empirical\w*|apparently|seems? to|guessed|unproved|unproven)\b", + r"\b(" + r"conjectur\w*|empirical\w*|apparently|appears?|seems?|probabl\w+|" + r"believ\w+|presumab\w+|observ\w+|guessed|unproved|unproven|" + r"(checked|verified) (up )?(to|for)" + r")\b", re.IGNORECASE, ) diff --git a/python/alkahest/research.py b/python/alkahest/research.py index 297221ff..64bcd03e 100644 --- a/python/alkahest/research.py +++ b/python/alkahest/research.py @@ -55,10 +55,13 @@ import hashlib import inspect import json +import math import threading from contextlib import ExitStack, contextmanager, suppress from dataclasses import dataclass, field, replace from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from fractions import Fraction from typing import TYPE_CHECKING, Any if TYPE_CHECKING: # pragma: no cover - typing only @@ -105,6 +108,10 @@ "externally_asserted": ( "an external solver asserted this; no proof was checked and none was produced" ), + "asserted": ( + "the caller wrote this statement; the result it was recorded from was checked, " + "but nothing checked that the statement is what was checked" + ), "unverified": "recorded without verification evidence", "refuted": "re-verification contradicted this claim", } @@ -115,6 +122,7 @@ "certificate_available": "[CERT ONLY, UNCHECKED]", "numerically_checked": "[NUMERIC ONLY]", "externally_asserted": "[EXTERNAL, UNCHECKED]", + "asserted": "[ASSERTED, UNCHECKED]", "unverified": "[UNVERIFIED]", "refuted": "[REFUTED]", } @@ -184,6 +192,12 @@ "e": 2.718281828459045, } +#: Gap between the values consecutive free symbols are bound to in the numeric +#: residual fallback. Anything nonzero takes the evaluation off the diagonal; +#: this is small enough to stay inside the usual domains and not a round binary +#: fraction, so it is unlikely to sit on a zero of the residual by accident. +_SYMBOL_SPACING = 0.11 + # --------------------------------------------------------------------------- # Errors @@ -388,7 +402,16 @@ def _infer_assertion( latex = rf"{_tex(exprs[0])} = {_tex(value)}" else: return None - return {"kind": "relation", "statement": _canonical_text(text), "latex": latex} + # ``inferred`` marks this as the *engine's* rendering of what the operation + # asserts, not caller prose. :meth:`ResearchSession.record` reads it to + # decide whether a machine-checked status may be carried over; it is dropped + # by :func:`_normalize_statement` and never reaches the stored claim. + return { + "kind": "relation", + "statement": _canonical_text(text), + "latex": latex, + "inferred": True, + } def claim_id(statement: str, hypotheses: Sequence[str] = (), method: str = "") -> str: @@ -886,6 +909,14 @@ def add(self, claim: Claim) -> Claim: (a claim that cites an identically-addressed claim, which happens when an operation is a no-op) are dropped. + A **re-verification recipe on the later claim is adopted** when the + stored claim carries none. Attaching a ``check`` is the one supported + way to link a statement to evidence, so recording a statement bare and + then recording it again with a recipe has to work; dropping the recipe + made :meth:`verify` report ``skipped`` for a claim that was in fact + checkable. An existing recipe is never overwritten — the first + recorded evidence wins, as the status does. + Raises ------ MissingClaimError @@ -912,10 +943,12 @@ def add(self, claim: Claim) -> Claim: merged_deps = tuple(dict.fromkeys((*existing.depends_on, *deps))) merged_tags = tuple(dict.fromkeys((*existing.tags, *claim.tags))) + merged_check = existing.check if existing.check else claim.check stored = replace( existing, depends_on=merged_deps, tags=merged_tags, + check=dict(merged_check) if merged_check else None, audit=(*existing.audit, *claim.audit), ) self._claims[claim.id] = stored @@ -1170,10 +1203,19 @@ def verify( pool : ExprPool, optional Pool to parse into. A fresh pool is created when omitted. tolerance : float - Absolute tolerance for the numeric residual fallback. + Absolute tolerance for the numeric residual fallback. It does not + apply to a ``numeric_relation`` recipe whose constants are supplied + at a precision a float cannot hold: those are evaluated exactly and + judged against the precision the caller actually gave (see below). samples : sequence of float - Sample points used for the numeric fallback, one value bound to - every free symbol at a time. + Sample points used for the numeric fallback. Each sample gives a + *point*, not a single value: free symbols are bound to the sample + offset by their rank in sorted name order, so they take distinct + values and the evaluation is off the diagonal ``x = y = z``. A + ``numeric_ok`` outcome is still finitely many points — evidence, + never a proof — and a symbol whose offset leaves an operation's + domain makes that point unevaluable, which the detail string + reports and which can leave the outcome ``inconclusive``. mark_refuted : bool When true (default), failed claims have their status set to ``"refuted"`` in place. @@ -1306,14 +1348,33 @@ def _residual_is_zero(residual: Any) -> bool: def _numeric_residual( expr: Any, symbols: Mapping[str, Any], samples: Sequence[float] -) -> float | None: - """Largest absolute value of *expr* over the sample points, or ``None``.""" +) -> tuple[float | None, int]: + """Largest ``|expr|`` over the sample points, and how many were evaluable. + + Every free symbol used to be bound to the *same* value, which put the + evaluation on the diagonal ``x = y = z``. A residual that vanishes only + there — ``sin(x)cos(y) - sin(y)cos(x)``, ``x - y``, any symmetry error + between two variables, the commonest bug class this fallback exists to + catch — came back indistinguishable from an identity. Each symbol is + therefore offset by :data:`_SYMBOL_SPACING` times its rank in the sorted + symbol names, so the point is off the diagonal in every coordinate while + the single-symbol case (rank 0) evaluates exactly where it always did. + + An offset can push a sample out of an operation's domain; ``eval_expr`` + then raises and the point is skipped. The count of points that *did* + evaluate is returned alongside the worst value so the caller can say + ``inconclusive`` rather than read a verdict off a residual it could not + sample. + """ ak = _ak() worst: float | None = None + evaluated = 0 + ranks = {name: rank for rank, name in enumerate(sorted(symbols))} for sample in samples: bindings = {} for name, sym in symbols.items(): - bindings[sym] = _NUMERIC_CONSTANTS.get(name, float(sample)) + offset = ranks[name] * _SYMBOL_SPACING + bindings[sym] = _NUMERIC_CONSTANTS.get(name, float(sample) + offset) try: value = ak.eval_expr(expr, bindings) except Exception: @@ -1322,20 +1383,102 @@ def _numeric_residual( magnitude = abs(float(value)) except (TypeError, ValueError): # pragma: no cover - complex results continue + evaluated += 1 worst = magnitude if worst is None else max(worst, magnitude) - return worst + return worst, evaluated def _decide(residual: Any, symbols: dict[str, Any], tolerance: float, samples) -> tuple[str, str]: """Classify a residual that ought to be identically zero.""" if _residual_is_zero(residual): return "ok", "symbolic residual simplified to 0" - worst = _numeric_residual(residual, symbols, samples) + worst, evaluated = _numeric_residual(residual, symbols, samples) if worst is None: return "inconclusive", f"residual did not simplify to 0 (got {residual}); no numeric sample" + where = f"{evaluated} of {len(samples)} sample point(s)" if worst <= tolerance: - return "numeric_ok", f"|residual| <= {worst:.3g} over {len(samples)} sample point(s)" - return "failed", f"|residual| = {worst:.6g} exceeds tolerance {tolerance:g}" + return "numeric_ok", ( + f"|residual| <= {worst:.3g} at {where}, free symbols at distinct " + f"values (numeric evidence, not a proof)" + ) + return "failed", f"|residual| = {worst:.6g} at {where} exceeds tolerance {tolerance:g}" + + +def _exact_and_uncertainty(value: Any) -> tuple[Fraction, Fraction]: + """*value* as an exact rational, with the half-ulp its notation implies. + + A ``numeric_relation`` recipe carries its constants as text — the form + :func:`alkahest.guess_relation`'s own docstring tells callers to use — and + that text carries its precision with it. ``"1"`` is the integer one and is + exact; ``"1.15572734962273134279187535795567192711"`` names a value known + to half a unit in its last decimal place, which is *far* more than a + ``float`` holds. Narrowing either to 53 bits throws that away, so an exact + relation with coefficients around ``5e9`` picks up a ``9.5e-7`` rounding + residual and gets refuted. + + :raises ValueError: when *value* is not a recognisable exact numeral. + """ + if isinstance(value, bool): # bool is an int; refuse it explicitly + raise ValueError(f"not a numeric constant: {value!r}") + if isinstance(value, Fraction): + return value, Fraction(0) + if isinstance(value, int): + return Fraction(value), Fraction(0) + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"not a finite constant: {value!r}") + # A float names itself exactly, but only to its own resolution. + return Fraction(value), Fraction(math.ulp(value)) / 2 + if isinstance(value, Decimal): + decimal_value = value + elif isinstance(value, str): + try: + decimal_value = Decimal(value.strip()) + except InvalidOperation: + raise ValueError(f"not a decimal numeral: {value!r}") from None + elif hasattr(value, "__float__"): + # An in-process numeric of some other type (``mpmath.mpf``, ``numpy``). + # Narrowing is what the old code did to everything; here it is the last + # resort, and the ulp it reports says the digits were lost. + try: + narrowed = float(value) + except (TypeError, ValueError, OverflowError): + raise ValueError(f"not a numeric constant: {value!r}") from None + if not math.isfinite(narrowed): + raise ValueError(f"not a finite constant: {value!r}") + return Fraction(narrowed), Fraction(math.ulp(narrowed)) / 2 + else: + raise ValueError(f"unsupported constant type {type(value).__name__}") + if not decimal_value.is_finite(): + raise ValueError(f"not a finite constant: {value!r}") + exponent = decimal_value.as_tuple().exponent + text = str(value).strip() if isinstance(value, str) else str(decimal_value) + if "." not in text and "e" not in text.lower(): + # An integer written as an integer is exact, not "±0.5". + return Fraction(decimal_value), Fraction(0) + ulp = Fraction(10) ** int(exponent) + return Fraction(decimal_value), ulp / 2 + + +def _relation_residual( + constants: Sequence[Any], coefficients: Sequence[Any] +) -> tuple[Fraction, Fraction]: + """``(|sum a_i c_i|, uncertainty)`` computed exactly from the given numerals. + + The uncertainty is the first-order propagation of each input's own half-ulp, + so it is zero when every constant and coefficient is exact. The true + residual lies in ``[|R| - U, |R| + U]``, which is what lets + :func:`_recheck` distinguish "this relation is false" from "you did not give + me the digits to tell". + """ + residual = Fraction(0) + uncertainty = Fraction(0) + for raw_constant, raw_coefficient in zip(constants, coefficients): + constant, constant_ulp = _exact_and_uncertainty(raw_constant) + coefficient, coefficient_ulp = _exact_and_uncertainty(raw_coefficient) + residual += coefficient * constant + uncertainty += abs(coefficient) * constant_ulp + abs(constant) * coefficient_ulp + return abs(residual), uncertainty def _recheck(claim: Claim, pool: Any, tolerance: float, samples: Sequence[float]) -> RecheckOutcome: @@ -1387,23 +1530,40 @@ def _recheck(claim: Claim, pool: Any, tolerance: float, samples: Sequence[float] outcome, detail = _decide(residual, symbols, tolerance, samples) return RecheckOutcome(claim.id, outcome, kind, detail) if kind == "numeric_relation": - constants = [float(c) for c in check["constants"]] - coefficients = [float(c) for c in check["coefficients"]] + constants = list(check["constants"]) + coefficients = list(check["coefficients"]) if len(constants) != len(coefficients): return RecheckOutcome( claim.id, "inconclusive", kind, "constant/coefficient length mismatch" ) - residual = sum(a * c for a, c in zip(coefficients, constants)) + try: + residual, uncertainty = _relation_residual(constants, coefficients) + except ValueError as exc: + return RecheckOutcome(claim.id, "inconclusive", kind, str(exc)) bound = float(check.get("tolerance", tolerance)) - if abs(residual) <= bound: + # The residual is exact; the *inputs* are only as precise as their + # own notation, so the true value lies in [residual +- uncertainty]. + upper = float(residual + uncertainty) + lower = float(max(Fraction(0), residual - uncertainty)) + at = f"at the supplied precision (+-{float(uncertainty):.3g})" + if upper <= bound: return RecheckOutcome( claim.id, "numeric_ok", kind, - f"|sum a_i c_i| = {abs(residual):.3g} <= {bound:g} (numeric evidence only)", + f"|sum a_i c_i| <= {upper:.3g} <= {bound:g} {at} (numeric evidence only)", + ) + if lower > bound: + return RecheckOutcome( + claim.id, "failed", kind, f"|sum a_i c_i| >= {lower:.6g} > {bound:g} {at}" ) return RecheckOutcome( - claim.id, "failed", kind, f"|sum a_i c_i| = {abs(residual):.6g} > {bound:g}" + claim.id, + "inconclusive", + kind, + f"|sum a_i c_i| = {float(residual):.6g} {at}, which straddles the " + f"tolerance {bound:g}: the constants were not supplied to enough " + f"digits to decide this relation", ) except Exception as exc: return RecheckOutcome(claim.id, "inconclusive", kind, f"{type(exc).__name__}: {exc}") @@ -1637,6 +1797,15 @@ def record( mapping of the form ``{"kind": ..., "statement": ..., "latex": ...}`` is used verbatim, which is how relations such as ``∫ f dx = F`` are supplied. + + It is **free text, and nothing checks that it describes** + ``result``. So a machine-checked status is not carried over onto + it: when *statement* is supplied without a *check* recipe and the + result's status is in :data:`MACHINE_CHECKED_STATUSES`, the claim + is stored as ``"asserted"`` instead, with the result's own status + preserved under ``verification["result_status"]``. Supply *check* + — the recipe :meth:`ClaimGraph.verify` re-runs — to keep the + machine-checked status, or record the result without *statement*. method : str, optional Operation name. Defaults to ``"record"``. label : str, optional @@ -1688,6 +1857,23 @@ def record( status = str(verification.get("status", "unverified")) evidence = str(verification.get("evidence", "none")) + # A caller-supplied *statement* is free text: nothing relates it to the + # result whose status is being copied, so `record(integrate(...), + # statement="0 = 1")` must not inherit `exactly_verified`. A `check` + # recipe re-establishes the link — it is the recipe `verify()` runs + # against the statement — so it, and an assertion the engine rendered + # itself (`_infer_assertion`), keep the status. Everything else is + # badged `"asserted"` until a recipe is attached. + if ( + statement is not None + and not (isinstance(statement, dict) and statement.get("inferred")) + and not check + and status in MACHINE_CHECKED_STATUSES + ): + verification = dict(verification) + verification["result_status"] = status + verification["statement_source"] = "caller" + status = "asserted" certificate_format = verification.get("artifact_format") if certificate is not None and certificate_format is None: certificate_format = "lean4" diff --git a/tests/test_novelty.py b/tests/test_novelty.py index 5b5337f8..8b1c25f9 100644 --- a/tests/test_novelty.py +++ b/tests/test_novelty.py @@ -238,6 +238,80 @@ def test_refuses_lines_it_does_not_understand(line: str) -> None: assert RecurrenceClaim.from_text(line) is None +def test_a_shift_gap_is_not_closed_by_the_parser() -> None: + """``from_text`` must not compact a sparse shift map into a dense window. + + ``_parse_relation`` returns ``{shift: coefficient}``, and the constructor + reads its list *positionally*. Handing it the values at the sorted keys + closed every gap, so ``a(n) = a(n-1) + a(n-3)`` and + ``a(n) = a(n-2) + a(n-4)`` were both read as Fibonacci — the same + ``claim_hash``, verbatim. Measured over 377 live OEIS entries this lost + 21% of parsed statements to the data guard; it never forged a claim, + because a mangled reading has to reproduce the entry's own terms to be + indexed, but a false red is still a loss. + """ + fibonacci = RecurrenceClaim.from_text("a(n) = a(n-1) + a(n-2)") + gapped = { + # u(n+3) - u(n+2) - u(n) = 0: a gap at n-2. + "a(n) = a(n-1) + a(n-3)": RecurrenceClaim([(-1,), (0,), (-1,), (1,)]), + # u(n+4) - u(n+2) - u(n) = 0: gaps at n-1 and n-3. + "a(n) = a(n-2) + a(n-4)": RecurrenceClaim([(-1,), (0,), (-1,), (0,), (1,)]), + # u(n+2) - 2u(n) = 0: a gap at n-1. + "a(n) = 2*a(n-2)": RecurrenceClaim([(-2,), (0,), (1,)]), + } + for line, expected in gapped.items(): + parsed = RecurrenceClaim.from_text(line) + assert parsed is not None, line + assert parsed.claim_hash == expected.claim_hash, line + assert parsed.claim_hash != fibonacci.claim_hash, line + assert parsed.order == expected.order, line + # Distinct gapped lines stay distinct from each other, too. + hashes = {RecurrenceClaim.from_text(line).claim_hash for line in gapped} + assert len(hashes) == len(gapped) + + +def test_a_gapped_recurrence_confirms_against_its_own_data() -> None: + """The false red the gap collapse caused, end to end. + + Padovan-like: a(n) = a(n-1) + a(n-3). Read as Fibonacci it does not + reproduce the entry's terms, so the data guard threw the whole statement + away rather than record a wrong one. + """ + terms = [1, 1, 1] + while len(terms) < 16: + terms.append(terms[-1] + terms[-3]) + entry = OeisEntry("A000930", terms=terms, statements=["a(n) = a(n-1) + a(n-3)."]) + recurrences = entry.recurrences() + assert len(recurrences) == 1 + assert entry.unusable_statements() == () + assert recurrences[0].claim.order == 3 + + +def test_hedges_oeis_actually_writes_are_recognised() -> None: + """ "Conjecture" is not the only way a contributor says "unproved".""" + hedged = [ + "It appears that a(n) = a(n-1) + a(n-2).", + "It seems that a(n) = 2*a(n-1).", + "Probably a(n) = a(n-1) + a(n-3).", + "This is believed to hold: a(n) = a(n-1) + a(n-2).", + "Presumably a(n) = a(n-1) + a(n-2).", + "Observed: a(n) = a(n-1) + a(n-2).", + "a(n) = a(n-1) + a(n-2), verified up to n = 1000.", + # The four that already worked, so widening cannot have dropped them. + "Conjecture: a(n) = a(n-1) + a(n-2).", + "Empirical g.f.: a(n) = a(n-1) + a(n-2).", + "Apparently a(n) = a(n-1) + a(n-2).", + "Unproved: a(n) = a(n-1) + a(n-2).", + ] + for line in hedged: + assert novelty._HEDGE_RE.search(line), line + for line in [ + "D-finite with recurrence: n*a(n) + 2*(1-2*n)*a(n-1)=0.", + "a(n) = a(n-1) + a(n-2).", + ]: + assert not novelty._HEDGE_RE.search(line), line + + def test_a_parsed_recurrence_is_checked_against_the_entrys_own_data() -> None: """A line that does not reproduce the entry's terms is not indexed. diff --git a/tests/test_research_claim_graph.py b/tests/test_research_claim_graph.py index 882b47da..0a4237f5 100644 --- a/tests/test_research_claim_graph.py +++ b/tests/test_research_claim_graph.py @@ -608,3 +608,248 @@ def test_renderers_handle_an_empty_graph(): graph = ClaimGraph(title="Nothing yet") assert "No claims recorded" in graph.to_markdown() assert graph.to_latex().startswith("\\documentclass") + + +# --------------------------------------------------------------------------- +# Trust boundaries the re-verification pass has to hold +# --------------------------------------------------------------------------- + + +def _identity_claim(name: str, lhs: str, rhs: str) -> Claim: + return Claim( + id=claim_id(name, (), "test"), + statement=name, + kind="text", + method="test", + status="unverified", + check={"kind": "identity", "lhs": lhs, "rhs": rhs}, + ) + + +@pytest.mark.parametrize( + ("name", "lhs", "rhs"), + [ + # True on the diagonal x = y, false everywhere else. This is the + # commonest bug class there is: a symmetry error between two variables. + ("sin(x)cos(y) = sin(y)cos(x)", "sin(x)*cos(y)", "sin(y)*cos(x)"), + ("x + y = 2x", "x + y", "2*x"), + ("x*y = x^2", "x*y", "x^2"), + ], +) +def test_verify_refutes_an_identity_that_only_holds_on_the_diagonal(name, lhs, rhs): + """Free symbols must not all be bound to the same sample value. + + Binding every symbol to one value put the evaluation on ``x = y = z``, + where each of these is true, so all three came back ``numeric_ok`` with + ``|residual| <= 0`` and ``report.ok``. + """ + graph = ClaimGraph() + graph.add(_identity_claim(name, lhs, rhs)) + report = graph.verify() + assert report.summary() == {"failed": 1} + assert not report.ok + assert graph.claims[0].status == "refuted" + + +def test_verify_still_accepts_a_genuine_multivariate_identity(): + """Offsetting the symbols must not manufacture a false red.""" + graph = ClaimGraph() + graph.add(_identity_claim("(x+y)^2 expanded", "(x + y)^2", "x^2 + 2*x*y + y^2")) + report = graph.verify() + assert report.outcomes[0].outcome in {"ok", "numeric_ok"} + assert report.ok + assert graph.claims[0].status == "unverified" + + +def test_numeric_ok_detail_does_not_claim_independent_points(): + """The detail string used to read "over 3 sample point(s)" alone.""" + graph = ClaimGraph() + graph.add(_identity_claim("sin^2 + cos^2 = 1", "sin(x)^2 + cos(x)^2 - 1", "0")) + report = graph.verify() + outcome = report.outcomes[0] + assert outcome.outcome in {"ok", "numeric_ok"} + if outcome.outcome == "numeric_ok": + assert "of 3 sample point(s)" in outcome.detail + assert "free symbols at distinct values" in outcome.detail + assert "not a proof" in outcome.detail + + +def test_a_sample_outside_the_domain_is_skipped_not_counted_against_the_claim(): + """An offset can leave a domain; that point is skipped, not read as a failure. + + ``log(x - 5)`` is undefined at every default sample, so nothing evaluates + and the honest answer is ``inconclusive`` — never ``failed``, and never a + ``numeric_ok`` that silently rested on zero points. + """ + graph = ClaimGraph() + graph.add(_identity_claim("log(x-5) = log(x-5) + 1", "log(x - 5)", "log(x - 5) + 1")) + report = graph.verify() + assert report.outcomes[0].outcome == "inconclusive" + assert "no numeric sample" in report.outcomes[0].detail + assert graph.claims[0].status == "unverified" + + +def _relation_claim(name: str, constants, coefficients, **extra) -> Claim: + check = {"kind": "numeric_relation", "constants": constants, "coefficients": coefficients} + check.update(extra) + return Claim( + id=claim_id(name, (), "test"), + statement=name, + kind="text", + method="guess_relation", + status="unverified", + check=check, + ) + + +#: alpha is the root of 5144503108 x^2 - 5945642943 x + 1 near 1.1557, so +#: 5144503108*alpha^2 - 5945642943*alpha + 1 = 0 exactly. At 60 digits the true +#: residual is 6.7e-52; narrowed to float it is 9.5e-7. +_ALPHA = "1.15572734962273134279187535795567192711118619980130442852708" +_ALPHA_SQUARED = "1.33570570666598308927592305767244172500786340378426566550083" + + +def test_verify_does_not_refute_an_exact_relation_given_at_60_digits(): + """Casting 60-digit decimal strings to float refuted a true relation. + + ``mark_refuted=True`` is the default, so this was the one place in the + graph machinery where ``verify()`` actively destroyed a true claim. + """ + graph = ClaimGraph() + graph.add( + _relation_claim( + "5144503108 a^2 - 5945642943 a + 1 = 0", + [_ALPHA_SQUARED, _ALPHA, "1"], + [5144503108, -5945642943, 1], + ) + ) + report = graph.verify() + assert report.outcomes[0].outcome == "numeric_ok" + assert report.ok + assert graph.claims[0].status == "unverified" + assert "supplied precision" in report.outcomes[0].detail + + +def test_verify_refutes_a_relation_a_float_would_round_to_zero(): + """The mirror: an exact residual of 1 that double precision cannot see.""" + graph = ClaimGraph() + graph.add( + _relation_claim( + "10^18 - 10^18 * 0.999999999999999999 = 0", + ["1", "0.999999999999999999"], + [10**18, -(10**18)], + ) + ) + report = graph.verify() + assert report.outcomes[0].outcome == "failed" + assert graph.claims[0].status == "refuted" + + +def test_relation_at_too_few_digits_is_inconclusive_not_refuted(): + """Six digits cannot decide a 1e-8 tolerance, and saying so is the answer.""" + graph = ClaimGraph() + graph.add( + _relation_claim( + "pi - e - 0.423311 = 0", + ["3.141593", "2.718282", "0.423311"], + [1, -1, -1], + ) + ) + report = graph.verify() + assert report.outcomes[0].outcome == "inconclusive" + assert report.ok + assert graph.claims[0].status == "unverified" + + +def test_relation_tolerance_key_is_the_precision_escape_hatch(): + graph = ClaimGraph() + graph.add( + _relation_claim( + "pi - e - 0.423311 = 0 to 1e-5", + ["3.141593", "2.718282", "0.423311"], + [1, -1, -1], + tolerance=1e-5, + ) + ) + assert graph.verify().outcomes[0].outcome == "numeric_ok" + + +def test_re_recording_a_statement_attaches_a_check_recipe_it_lacked(): + """The one supported way to link a statement to its evidence was a no-op. + + Recorded bare and then recorded again *with* a recipe, ``verify()`` + reported ``skipped``; recorded the other way round the same pair yielded + ``refuted``. It must not depend on the order. + """ + recipe = {"kind": "identity", "lhs": "x + y", "rhs": "2*x"} + bare = _claim("x + y = 2 x") + with_check = replace(bare, check=recipe) + for order in ((bare, with_check), (with_check, bare)): + graph = ClaimGraph() + for claim in order: + graph.add(claim) + assert graph.claims[0].check == recipe + report = graph.verify() + assert report.summary() == {"failed": 1} + assert graph.claims[0].status == "refuted" + + +def test_an_attached_recipe_is_never_overwritten_by_a_later_record(): + first = replace(_claim("x = x"), check={"kind": "identity", "lhs": "x", "rhs": "x"}) + second = replace(first, check={"kind": "identity", "lhs": "x", "rhs": "x + 1"}) + graph = ClaimGraph() + graph.add(first) + graph.add(second) + assert graph.claims[0].check == {"kind": "identity", "lhs": "x", "rhs": "x"} + + +# --------------------------------------------------------------------------- +# A caller-supplied statement is not what the machine checked +# --------------------------------------------------------------------------- + + +def test_a_reworded_statement_does_not_inherit_a_machine_checked_status(pool): + """``record(result, statement="0 = 1")`` used to report ``[VERIFIED]``.""" + x = pool.symbol("x") + with session(pool=pool) as s: + result = ak.integrate(ak.sin(x), x) + assert result.verification["status"] in MACHINE_CHECKED_STATUSES + claim = s.record(result, statement="0 = 1") + assert claim.status == "asserted" + assert claim.machine_checked is False + assert claim.mark == "[ASSERTED, UNCHECKED]" + assert s.graph.machine_checkable() == () + # Nothing about the result itself is lost. + assert claim.verification["result_status"] == "exactly_verified" + assert claim.verification["statement_source"] == "caller" + assert "nothing checked that the statement is what was checked" in claim.badge + + +def test_a_check_recipe_re_establishes_the_link_to_the_statement(pool): + x = pool.symbol("x") + with session(pool=pool) as s: + result = ak.integrate(ak.sin(x), x) + claim = s.record( + result, + statement="the antiderivative of sin is -cos", + check={ + "kind": "antiderivative", + "integrand": "sin(x)", + "var": "x", + "antiderivative": "-cos(x)", + }, + ) + assert claim.status == "exactly_verified" + assert claim.machine_checked is True + assert s.graph.verify().ok + + +def test_captured_operations_keep_their_machine_checked_status(pool): + """The gate is about caller prose, not the assertion the engine renders.""" + x = pool.symbol("x") + with session(pool=pool, capture=True) as s: + ak.integrate(ak.cos(x), x) + claim = s.graph.claims[0] + assert claim.statement == "integral(cos(x), dx) = sin(x)" + assert claim.status == "exactly_verified" + assert claim.machine_checked is True diff --git a/tests/test_smt.py b/tests/test_smt.py index dd5aa917..cba0937e 100644 --- a/tests/test_smt.py +++ b/tests/test_smt.py @@ -963,10 +963,20 @@ def test_result_records_into_a_claim_graph_with_an_honest_status(pool): sat = smt.solve( ak.And(pool.gt(x, pool.integer(0)), pool.lt(x, pool.integer(2))), budget=BUDGET ) - claim = session.record(sat, statement="0 < x < 2 is satisfiable", method="smt.solve") + # Recorded as itself, the claim *is* the formula the model was checked + # against, so the machine-checked status carries over. + claim = session.record(sat, method="smt.solve") assert claim.status == "exactly_verified" assert claim.machine_checked is True + # Re-worded into prose, it is not: nothing relates "0 < x < 2 is + # satisfiable" to the formula z3 was handed, so the status drops to + # "asserted" and the result's own status is kept where it belongs. + reworded = session.record(sat, statement="0 < x < 2 is satisfiable", method="smt.solve") + assert reworded.status == "asserted" + assert reworded.machine_checked is False + assert reworded.verification["result_status"] == "exactly_verified" + unsat = smt.solve( ak.And(pool.gt(x, pool.integer(0)), pool.lt(x, pool.integer(0))), budget=BUDGET ) From 251333b7c757a79627a5127e0d3b9bb1da57bd1e Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 20:50:34 +0000 Subject: [PATCH 04/11] fix(holonomic): report singular indices on a guessed recurrence, make its verdict tri-state, and stop refusing a basis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `guess_holonomic` absorbed a corrupted term into roots of the leading coefficient and still reported `confirmed=True`. A single typo in an otherwise order-2/degree-1 sequence is fitted, at the *default* `max_degree = 4`, by multiplying the true operator by the cubic vanishing at exactly the three indices whose equations the typo breaks — and the result came back with `dimension` 1, 55 surplus equations and no untested candidates, every number a caller reads to judge a fit looking perfect. Scoped fairly: this is not unsoundness. The returned relation satisfies every equation the terms supplied, and it holds on the *clean* sequence too, being a left multiple of the true operator, so no re-check can catch it. What it is not is the sequence's recurrence, and the tell is that its leading coefficient has roots inside the data, where every coefficient vanishes at once, the equation reads `0 = 0` and the fit was therefore unconstrained. * `GuessedRecurrence.singular_indices` is a first-class field carrying exactly those indices — the same name and meaning as `ModularEvaluation.singular_indices`, with the difference that a modular evaluation must refuse (`E-HOLO-007`) where a fit can be returned and flagged. Computed by exact Horner evaluation over the indices the fit's own equations were written at. * `GuessedRecurrence.status` names the verdict from the closed vocabulary `GUESS_STATUSES` (`confirmed` / `singular` / `underdetermined` / `unconfirmed`), glossed by `GUESS_STATUS_MEANINGS` and `.means`, in the shape `experimental.NoveltyVerdict.status` uses. `confirmed` is correspondingly `True` / `False` / `None`, the discipline `relation_confidence`'s `credible` already had — and which `novelty.py`'s module docstring already claimed this attribute had. * `dimension > 1` returns `GuessedRecurrence.basis`, the whole solution space, instead of raising. That refusal made the `(order, degree)` cell unusable and closed OEIS A277060 entirely, though `zeilberger` decides it immediately. Only a fit that consumed its own evidence is still refused (`E-HOLO-005`). The guard is unchanged in the direction it was built for: ten non-P-recursive sequences still answer `None` after a full sweep, and too few terms still raises `E-HOLO-005` naming the shortfall. Also in the same surface: * `supercongruence_sweep` records `E-HOLO-006` in `skipped()` instead of raising it out of the sweep, when it is `p**(k + extra_precision)` past the machine-word ceiling — the same "out of reach of this backend" that `E-HOLO-008` was already recorded for. Propagating it destroyed every residue already computed. The *other* `E-HOLO-006`, a composite in `primes`, is a fact about the call and still raises; it is now decided up front so the two halves can be told apart. * `experimental.asymptotics_from_recurrence` takes `n` optionally, from the new `ZeilbergerCertificate.n` getter or from a pool made on the spot for integer coefficients. A foreign `n` was an uncoded `PoolError` from several frames inside the coefficient walk; it is now `E-POOL-001` naming the argument and the fix. * `zeilberger(minimal=True)` is documented as establishing minimality *at certificate degree `<= max_degree`* rather than "genuinely minimal". Documentation only; a sweep of seven families found no counterexample. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 65 +++++ alkahest-py/src/lib.rs | 38 ++- alkahest-skill/alkahest.md | 4 +- docs/features.md | 2 +- docs/mdbook/src/asymptotics.md | 12 + docs/mdbook/src/guessing.md | 69 ++++- docs/mdbook/src/supercongruences.md | 14 +- docs/mdbook/src/telescoping.md | 6 + python/alkahest/__init__.py | 11 +- python/alkahest/_guess_holonomic.py | 297 +++++++++++++++++---- python/alkahest/_recurrence_asymptotics.py | 89 +++++- python/alkahest/_supercongruence.py | 75 ++++-- tests/test_guess_holonomic.py | 291 ++++++++++++++++++++ tests/test_modular_holonomic.py | 37 +++ tests/test_recurrence_asymptotics.py | 68 +++++ 15 files changed, 995 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..2bbda8a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,71 @@ ## Unreleased +- **`guess_holonomic` reports the leading coefficient's roots inside the data, + and its verdict is three-valued.** A single wrong term in an otherwise + order-2/degree-1 sequence was fitted, at the *default* `max_degree = 4`, by + multiplying the true operator by the cubic that vanishes at exactly the three + indices whose equations the typo breaks — and came back `confirmed=True` with + `dimension` 1, 55 surplus equations and no untested candidates. The returned + relation is not unsound: it satisfies every equation the terms supplied, and + it holds on the *clean* sequence too, being a left multiple of the true + operator, so no re-check can catch it. What it is not is the sequence's + recurrence, and the tell is that its leading coefficient has roots inside the + data, where every coefficient vanishes at once, the equation reads `0 = 0` + and the fit was therefore unconstrained. + + `GuessedRecurrence.singular_indices` is now a first-class field carrying + exactly those indices — the same name and meaning as + `ModularEvaluation.singular_indices`, with the difference that a modular + evaluation must refuse (`E-HOLO-007`) where a fit can be returned and + flagged. `GuessedRecurrence.status` names the verdict from the closed + vocabulary `GUESS_STATUSES` (`"confirmed"`, `"singular"`, + `"underdetermined"`, `"unconfirmed"`), glossed by `GUESS_STATUS_MEANINGS` and + by `.means`, in the shape `experimental.NoveltyVerdict.status` uses. + `confirmed` is correspondingly `True` / `False` / `None` rather than a bare + boolean, the discipline `relation_confidence`'s `credible` already had: + `False` is *the data says nothing about this fit*, `None` is *the relation + holds and is still not the sequence's recurrence*, and only `True` is a + result. Two typos at `max_degree=8` produce six roots, so the field is a + diagnostic rather than a flag. The guard is unchanged in the direction it was + built for: ten non-P-recursive sequences (primes, partitions, Bell, `σ`, `τ`, + `π` digits, pseudorandom, two Beatty sequences, digit sums) still answer + `None` after a full sweep, and too few terms still raises `E-HOLO-005`. + +- **`guess_holonomic` returns the solution space instead of refusing when + `dimension > 1`.** A probe wider than the sequence's annihilator makes the + terms admit several independent relations, which used to raise and make the + whole `(order, degree)` cell unusable — it closed OEIS A277060 entirely, + though `zeilberger` decides it immediately. `GuessedRecurrence.basis` now + carries every independent relation (`basis[0]` is `coeffs`), the result comes + back with `status == "underdetermined"` and `confirmed is None`, and only a + fit that consumed its own evidence is still refused. + +- **`supercongruence_sweep` records `E-HOLO-006` in `skipped()` instead of + raising it out of the sweep.** `p**(k + extra_precision)` past the + machine-word ceiling of `2**62` is a fact about one prime — the same "out of + reach of this backend" that `E-HOLO-008` was already recorded for — and + letting it propagate destroyed every residue already computed, leaving the + caller to pre-filter the prime list by `int((2**62) ** (1 / (k + 1)))` by + hand. The *other* `E-HOLO-006`, a composite in `primes`, is a fact about the + call and still raises; it is now decided up front, before any evaluation, so + that the two halves can be told apart. + +- **`experimental.asymptotics_from_recurrence` derives its index symbol.** `n` + is now optional: it is taken from `ZeilbergerCertificate.n` (a new getter) + when *rec* is a certificate, and from a pool created on the spot when the + coefficients are plain integers, as they are for a `GuessedRecurrence`. + Passing a symbol from a foreign pool used to surface as an uncoded + `PoolError` from several frames inside the coefficient walk; it is now + `E-POOL-001` naming the argument and the fix. + +- **`zeilberger(minimal=True)` is documented as minimal at certificate degree + `<= max_degree`**, which is what it establishes, rather than "genuinely + minimal". A lower-order relation whose certificate needs a higher degree than + the bound is never probed, and order–degree trade-offs are real in creative + telescoping. Documentation only — a sweep of seven families found no + counterexample and the behaviour is unchanged. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-py/src/lib.rs b/alkahest-py/src/lib.rs index 49e65f17..5a697f8b 100644 --- a/alkahest-py/src/lib.rs +++ b/alkahest-py/src/lib.rs @@ -4803,13 +4803,20 @@ impl PyZeilbergerCertificate { self.order } - /// Whether the search **established** that no lower-order relation exists. + /// Whether the search **established** that no lower-order relation exists + /// *at certificate degree* ``<= max_degree``. /// /// ``True`` means every order below :attr:`order` was refused at every /// certificate degree up to ``max_degree``. ``False`` means *not /// established* — never "a lower order exists", since a lower-order /// relation that had been found would have been returned instead. /// + /// The degree bound is part of the claim and not a detail: a lower-order + /// relation whose certificate needs a degree above ``max_degree`` was + /// never probed, so ``True`` is minimality within the grid that was swept + /// rather than minimality for the summand. Order–degree trade-offs are a + /// real phenomenon here, so state the bound alongside the flag. + /// /// The default search visits the ``(order, degree)`` grid cheapest-first, /// so it can reach a cheap order-2 probe before an expensive order-1 one; /// an order-2 result therefore does not rule out order 1 and this flag is @@ -4834,6 +4841,22 @@ impl PyZeilbergerCertificate { .collect() } + /// The index symbol ``n`` the coefficient polynomials are written in. + /// + /// The symbol *this* certificate was built with, out of *this* + /// certificate's :class:`ExprPool` — the only ``n`` that can be combined + /// with :attr:`coeffs`, since expressions from two pools cannot meet. + /// Anything downstream that needs the index variable should take it from + /// here rather than make one of its own; that is why ``n`` is optional on + /// :func:`alkahest.experimental.asymptotics_from_recurrence`. + #[getter] + fn n(&self, py: Python<'_>) -> PyExpr { + PyExpr { + id: self.n_id, + pool: self.pool.clone_ref(py), + } + } + /// ``R(n, k)`` — the rational certificate, with ``G(n,k) = R(n,k)·F(n,k)``. #[getter] fn certificate(&self, py: Python<'_>) -> PyExpr { @@ -5097,9 +5120,16 @@ fn format_range(pool: &ExprPool, lo: ExprId, hi: ExprId) -> String { /// :attr:`~alkahest.ZeilbergerCertificate.order_is_minimal` is ``False`` to say /// so rather than leaving it to be assumed. Pass ``minimal=True`` to search /// **order-ascending** instead — every degree ``0..=max_degree`` at order ``J`` -/// is refused before order ``J+1`` is tried — which makes a returned order -/// genuinely minimal and sets the flag. That is the hopeless low-order sweep -/// the default plan exists to avoid, and it costs accordingly; ask for it when +/// is refused before order ``J+1`` is tried — and the flag is set. +/// +/// What that establishes is **minimality at certificate degree +/// ``<= max_degree``**, not minimality outright: a lower-order relation whose +/// certificate needs a higher degree than the bound is never probed, and +/// order–degree trade-offs are real in creative telescoping. So +/// ``order_is_minimal`` is a statement about the grid that was swept, and +/// ``max_degree`` is part of it — raise the bound if the minimality is what +/// the caller is after. Order-ascending is the hopeless low-order sweep the +/// default plan exists to avoid, and it costs accordingly; ask for it when /// minimality is the result, not as a habit. /// /// Raises :exc:`alkahest.HolonomicError` rather than guessing when ``term`` is diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index 7df084ab..165cf0a5 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -1418,8 +1418,8 @@ reg.coverage_report_markdown() # same, rendered as a Markdown table 17. **Do not export radical results to another CAS without evaluating them here first.** Casus-irreducibilis cube roots from `eigenvals()` are correct in Alkahest and wrong under a principal-branch evaluator. An `E-EVAL-009` or an infinite ball is the signal not to export as-is. 18. **`0 · 0⁻¹` is left unevaluated on purpose** (since 3.8). If you see `(0 * 0^-1)` in a result, that is Alkahest declining to give an indeterminate form a value — not a simplifier failure to work around. 19. **`relation_confidence` answers `None` when it cannot see the input precision** (since 3.8), which is the normal case: a decimal string may be an exact rational or a truncated constant, and nothing in it says which. `None` means *not checked*, never *passed* — branch on `if verdict["credible"]:`, not `is not False`. To get a real verdict on a `guess_relation` result computed from truncated decimal strings, pass the digits you trust: `relation_confidence(constants, coeffs, digits=60)`. Only `float` and `mpmath.mpf` inputs are judged without a declaration. -20. **`zeilberger` does not claim its order is minimal** (since 3.9). The search visits `(order, degree)` cheapest-first, so it can reach a cheap order-2 probe before an expensive order-1 one; `cert.order_is_minimal` is `False` to say *not established*, never "a lower order exists". Pass `minimal=True` for an order-ascending search that does establish it — it costs the low-order sweep the default plan skips (Franel at `max_degree=16`: 0.23 s → 9.7 s), so claim minimality against the smallest `max_degree` you are willing to state. -21. **`guess_holonomic` returns `None` only for a swept grid** (since 3.9). It fits a P-recursive recurrence to exact `int`/`Fraction` terms, but only where the terms *over-determine* the ansatz — twice the unknowns by default — and reports `surplus_terms`, the equations that confirmed the fit without being needed. Too few terms to test the whole grid is `E-HOLO-005`, a refusal, not `None`; recording it as "not holonomic" closes a branch that was never explored. `float` terms are refused outright. +20. **`zeilberger` does not claim its order is minimal** (since 3.9). The search visits `(order, degree)` cheapest-first, so it can reach a cheap order-2 probe before an expensive order-1 one; `cert.order_is_minimal` is `False` to say *not established*, never "a lower order exists". Pass `minimal=True` for an order-ascending search that does establish it — *at certificate degree `<= max_degree`*, which is part of the claim and not a detail, since a lower-order relation needing a higher degree is never probed — and it costs the low-order sweep the default plan skips (Franel at `max_degree=16`: 0.23 s → 9.7 s), so claim minimality against the smallest `max_degree` you are willing to state. +21. **`guess_holonomic` returns `None` only for a swept grid, and its verdict is three-valued** (since 3.9). It fits a P-recursive recurrence to exact `int`/`Fraction` terms, but only where the terms *over-determine* the ansatz — twice the unknowns by default — and reports `surplus_terms`, the equations that confirmed the fit without being needed. Too few terms to test the whole grid is `E-HOLO-005`, a refusal, not `None`; recording it as "not holonomic" closes a branch that was never explored. `float` terms are refused outright. **Branch on `guess.status`, and treat `confirmed` as `True`/`False`/`None` rather than a boolean** — `"confirmed"` is the only result. `"singular"` means `guess.singular_indices` is non-empty: the leading coefficient vanishes at those indices *inside the data*, so the fit was unconstrained there, and the overwhelmingly likely cause is a corrupted term the fit absorbed into a root (one typo in an order-2/degree-1 sequence comes back at degree 4 with three roots, `dimension` 1 and 55 surplus equations — every other number perfect). The relation does hold on the terms, and on the clean sequence too, so no re-check finds it; recompute the terms at those indices. `"underdetermined"` means `dimension > 1` — read `guess.basis`, which is every relation the terms admit, rather than `coeffs`, which is an arbitrary member of it. 22. **A `zeilberger` certificate is about the *summand*; `cert.boundary` is what makes it about the *sum*** (since 3.9). `"vanishes"` licenses the homogeneous `Σ_i a_i(n)·S(n+i) = 0`; `"nonzero"` licenses the inhomogeneous `Σ_i a_i(n)·S(n+i) = b(n)` with `b(n)` in `cert.boundary_rhs` — a result, not a refusal; `"unknown"` licenses **nothing** about the sum, and recording the recurrence anyway is how a verified certificate becomes a false theorem (it did, on OEIS A279013). The verdict is about the range in `cert.limits`, which defaults to `k = 0..n` and is echoed back rather than inferred — pass `limits=(k_lo, k_hi)` when you are summing over anything else, because truncating a sum by one term generally flips `"vanishes"` to `"nonzero"`. `cert.boundary_at(k_lo, k_hi)` asks about another range without re-running the search. diff --git a/docs/features.md b/docs/features.md index 74fcb480..03a8733f 100644 --- a/docs/features.md +++ b/docs/features.md @@ -72,7 +72,7 @@ Current stable feature surface. - Creative telescoping / Zeilberger's algorithm (`zeilberger`): P-recursive recurrence for a proper hypergeometric term plus a rational certificate, re-checked as an exact `Q(n)(k)` identity before it is returned; refuses (`E-HOLO-*`) rather than guessing outside the class or beyond the search bounds. `order_is_minimal` reports whether the search established that no lower-order relation exists — the default cost-ordered search usually cannot, and says so; `minimal=True` searches order-ascending and can establish it, at a cost that grows with `max_degree` (free at `max_degree=4`, ~13 s versus 0.08 s at 16 on Apéry), so it is opt-in rather than the default - Boundary verdict for creative telescoping (`ZeilbergerCertificate.boundary`): whether the certificate implies a recurrence for the **sum** over the range in `limits` (default `k = 0..n`, echoed back rather than inferred) — `"vanishes"` (homogeneous recurrence proved by exact order counting in `Q(n)`), `"nonzero"` (inhomogeneous recurrence proved, with `b(n)` in `boundary_rhs`) or `"unknown"` (nothing may be claimed). `boundary_at(k_lo, k_hi)` re-decides for another range without re-running the search - `q`-analogue creative telescoping (`experimental.q_zeilberger`): `q`-Zeilberger for `q`-hypergeometric summands (Gaussian binomials `qbinomial(N, K)`, `q`-Pochhammer symbols `qpochhammer(u, d, v)`, powers of `q` with a degree-≤2 exponent in `n, k`), which the classical engine cannot express at all. The certificate is re-checked as an exact `Q(q)(q**n)(q**k)` identity before return; `sum_term(n0)` gives the exact `q`-series value from the definition of the `q`-Pochhammer symbol, so the returned recurrence can be checked independently of the machinery that produced it. The boundary verdict is two-valued — `"vanishes"` (proved for `S(n) = Σ_{k ∈ Z} F(n,k)`, with the proved support window in `support`) or `"unknown"` — and `q` is treated as transcendental throughout, so a verdict does not license specialising `q` to a root of unity. Refuses with `E-HOLO-020` (outside the class), `E-HOLO-021` (bounds exhausted), `E-HOLO-023` (malformed call) or `E-HOLO-024` (in the shape of the class but with a non-rational shift quotient, e.g. `(q; q**2)_k` shifted in `k`) -- Recurrence guessing (`guess_holonomic`): fit a P-recursive recurrence to the first terms of a sequence in exact rational arithmetic, the guessing half of *guess then prove*. Only fits candidates the terms over-determine, reports how many surplus terms confirmed the fit, and refuses (`E-HOLO-005`) rather than returning an interpolation or reporting an untested grid as a negative +- Recurrence guessing (`guess_holonomic`): fit a P-recursive recurrence to the first terms of a sequence in exact rational arithmetic, the guessing half of *guess then prove*. Only fits candidates the terms over-determine, reports how many surplus terms confirmed the fit, and refuses (`E-HOLO-005`) rather than returning an interpolation or reporting an untested grid as a negative. The verdict is three-valued (`status` / `confirmed` as `True`/`False`/`None`, the discipline `relation_confidence.credible` uses): `singular_indices` reports the indices inside the data where the fitted leading coefficient vanishes — the signature of a corrupted term absorbed into a root, which every other statistic misses — and `basis` returns the whole solution space when the terms admit more than one relation instead of refusing the candidate - Modular / `p`-adic evaluation of a holonomic sequence (`ModularRecurrence`): `S(N) mod p^k` straight from `Σ_i a_i(n)·S(n+i) = b(n)`, in machine-word modular arithmetic and `O(1)` memory, without ever forming `S(N)` over `ℤ`. Indices where the leading coefficient `a_J(n)` is not a unit mod `p` are handled by a first pass that measures the total `p`-adic precision loss and a forward pass that runs at `p^(k+loss)`; a step that cannot be justified refuses (`E-HOLO-007`) and a working precision past the 64-bit modulus refuses (`E-HOLO-008`), so no path returns a residue that is silently short of the precision it claims. `supercongruence_sweep` drives it over a range of primes and reports counterexamples, the `v_p(LHS − RHS)` histogram and whether the claimed modulus is sharp - `binomial(a, b) mod p^k` (`binomial_mod`): Lucas at `k = 1`, Andrew Granville / Davis–Webb for prime powers, with the `p`-free factorial taken by a product tree over blocks of `p` so the cost is `O(p·k³ + log_p(a)·p·k)` rather than `O(p^k)`; `a` far larger than `p` is the ordinary case diff --git a/docs/mdbook/src/asymptotics.md b/docs/mdbook/src/asymptotics.md index 25fc7b2c..620bf3e5 100644 --- a/docs/mdbook/src/asymptotics.md +++ b/docs/mdbook/src/asymptotics.md @@ -135,6 +135,18 @@ r.verdict # "single_dominant_root" coefficient polynomials `[p_0, …, p_J]` for `Σ_i p_i(n)·u(n+i) = 0`; each `p_i` is an `Expr` in `n` or a tuple of ascending integer coefficients. +**`n` is optional, and leaving it out is the safe way to call this.** +Expressions from two pools cannot meet, so a certificate's coefficients combine +only with the certificate's own `cert.n` — a symbol made in a fresh pool for the +occasion is a pool mismatch (`E-POOL-001`), not an answer. Omitted, `n` is taken +from `rec` when `rec` has one, and from a pool made here when the coefficients +are plain integers, as they are for a `GuessedRecurrence`: + +```python +asymptotics_from_recurrence(cert, terms=[1, 2]) # uses cert.n +asymptotics_from_recurrence(guess, terms=motzkin[:2]) # fresh pool +``` + ### What is derived and what is fitted Write `D = max_i deg p_i`, take the coefficient of `n^D` in each `p_i` to build diff --git a/docs/mdbook/src/guessing.md b/docs/mdbook/src/guessing.md index 4df50dcd..138ee932 100644 --- a/docs/mdbook/src/guessing.md +++ b/docs/mdbook/src/guessing.md @@ -49,8 +49,11 @@ What survives is then reported with the evidence attached: | `equations_used` | independent equations the fit consumed (the matrix rank) | | `surplus_terms` | equations that were *not* needed and agreed anyway | | `dimension` | dimension of the solution space; `1` for a genuine fit | +| `basis` | every independent relation the terms admit; `basis[0]` is `coeffs` | +| `singular_indices` | indices inside the data where the leading coefficient vanishes | | `untested_candidates` | lower `(order, degree)` candidates the terms could not test | -| `confirmed` | enough surplus **and** dimension exactly 1 | +| `status` | one of `GUESS_STATUSES`; `means` glosses it | +| `confirmed` | `True` / `False` / `None` — see below | `untested_candidates` is the minimality caveat, and it is the same discipline as `ZeilbergerCertificate.order_is_minimal`. `0` means the returned order is the @@ -63,6 +66,59 @@ of it as a dict for logging next to the result. This is `relation_confidence`'s discipline applied to sequences: a fit is judged against what the data can actually support, rather than endorsed because the arithmetic came out even. +## The verdict is three-valued + +`confirmed` is `True`, `False`, or `None`, and `status` names which: + +| `status` | `confirmed` | what it means | +|---|---|---| +| `confirmed` | `True` | over-determined, unique, and non-singular in the data | +| `singular` | `None` | the operator vanishes identically at `singular_indices` | +| `underdetermined` | `None` | several independent relations; read `basis` | +| `unconfirmed` | `False` | the fit consumed the equations that would have confirmed it | + +Neither `False` nor `None` is a pass, and they are different: `False` is *the +data says nothing about this fit*, `None` is *the relation holds and is still +not the sequence's recurrence*. This is `relation_confidence`'s `credible` and +`NoveltyVerdict.found` for sequences — the same three values and the same rule +that only the first is a result. + +`GUESS_STATUSES` is the closed vocabulary and `GUESS_STATUS_MEANINGS` glosses +each entry; `guess.means` is the gloss for the one at hand. + +## Singular indices, and the corrupted term + +This is the failure `surplus_terms` and `dimension` do not catch. A single +wrong term in an otherwise clean sequence does not stop a fit — it is absorbed: + +```python +spoiled = motzkin_71_terms.copy() +spoiled[30] += 1 # one typo + +fit = ak.guess_holonomic(spoiled) # default max_degree = 4 +fit.order, fit.degree # (2, 4) — one degree up from the truth +fit.dimension, fit.surplus_terms # (1, 55) — every number still perfect +fit.singular_indices # (28, 29, 30) +fit.status, fit.confirmed # ('singular', None) +``` + +The fit is the true operator multiplied by the cubic `(n−28)(n−29)(n−30)`, +which vanishes at exactly the three indices whose equations the typo breaks. +Every coefficient polynomial vanishes there at once, so those three equations +read `0 = 0` and constrained nothing — and the relation that comes back +satisfies every equation the terms supplied. **No re-check can catch this**: it +holds on the clean sequence too, being a left multiple of the true operator. +The roots inside the data are the only tell, which is why they are a field. + +Two typos need `max_degree=8` and produce six roots — the count scales with the +corruption, so the field is a diagnostic and not just a flag. The first move on +a non-empty `singular_indices` is to recompute the terms at those indices. + +`ModularRecurrence.value_mod` meets the same phenomenon and *refuses* +(`E-HOLO-007`), because a modular evaluation genuinely cannot step through a +singular index. A fit can be returned and flagged, because the relation is true +on the data — it is only untrue that it is the sequence's recurrence. + ## What it refuses, and what `None` means Two different negative answers, kept apart on purpose: @@ -107,11 +163,18 @@ ak.guess_holonomic(terms, max_order=4, max_degree=4, *, - `min_surplus` overrides the surplus demanded. `0` turns the requirement off while leaving the reporting intact. - `check_evidence=False` fits every candidate regardless of surplus and returns - the first fit with `confirmed` set honestly. It is the escape hatch, in the + the first fit with `status` set honestly. It is the escape hatch, in the same role `check_precision=False` plays on `guess_relation` — useful when the candidate is going somewhere else to be checked, never a way to make a weak fit look strong. +Only `status == "unconfirmed"` — a fit with no surplus left — is refused. +`"singular"` and `"underdetermined"` are *returned*, carrying the reason: in +both the relation genuinely holds on the terms, so there is something for the +caller to act on. `dimension > 1` used to raise, which made the whole +`(order, degree)` cell unusable on sequences whose annihilator is narrower than +the probe that reached them first. + Terms must be exact: Python `int` of any size, or `fractions.Fraction`. A `float` is refused rather than converted, because every step after this one is exact and would happily certify a recurrence for the sequence you rounded to. @@ -120,7 +183,7 @@ exact and would happily certify a recurrence for the sequence you rounded to. ```python guess = ak.guess_holonomic(terms) -if guess is not None and guess.confirmed: +if guess is not None and guess.confirmed is True: assert guess.holds_for(more_terms) # exact, on data it never saw cert = ak.zeilberger(F, n, k, minimal=True) # …and now prove it cert.order == guess.order diff --git a/docs/mdbook/src/supercongruences.md b/docs/mdbook/src/supercongruences.md index 3bccafec..79c8baeb 100644 --- a/docs/mdbook/src/supercongruences.md +++ b/docs/mdbook/src/supercongruences.md @@ -123,9 +123,17 @@ these sweeps live in — one index per prime, at or near `p` — there are at mo one or two singular steps and the headroom is free. `supercongruence_sweep` records `E-HOLO-007` and `E-HOLO-008` in `skipped()` and -carries on, because those are facts about one prime. `E-HOLO-006` is a fact -about the *call*, so it propagates — a list of composites must not come back -`holds=True` over zero primes. +carries on, because those are facts about one prime — and `E-HOLO-006` for the +same reason, when it is `p**(k + extra_precision)` past `2**62`. That one is the +same "out of reach of this backend"; letting it propagate destroyed every +residue already computed and left the caller to work out the per-`k` cap +`int((2**62) ** (1 / (k + 1)))` by hand. + +The *other* `E-HOLO-006` — a composite in `primes` — is a fact about the +**call**, and still raises: a list of composites must not come back +`holds=True` over zero primes. The sweep decides that half itself, before any +evaluation, which is what lets the reachable-modulus half be skipped without +taking this half with it. ## Binomial coefficients diff --git a/docs/mdbook/src/telescoping.md b/docs/mdbook/src/telescoping.md index e4ff4ea8..184a2d22 100644 --- a/docs/mdbook/src/telescoping.md +++ b/docs/mdbook/src/telescoping.md @@ -256,6 +256,12 @@ exact verification, same certificate — only what was ruled out along the way differs. The flag is computed from the probes that actually happened rather than from the mode, so it cannot drift away from what the search did. +**What it establishes is minimality at certificate degree `≤ max_degree`**, not +minimality outright. A lower-order relation whose certificate needs a higher +degree than the bound was never probed, and order–degree trade-offs are a real +phenomenon in creative telescoping — so quote the bound alongside the claim, and +raise it when minimality is the result being published. + The price is the whole hopeless low-order sweep the default plan exists to avoid, and it is charged against `max_degree` because that is the bound minimality is claimed relative to. Measured on this machine at `max_order=4`: diff --git a/python/alkahest/__init__.py b/python/alkahest/__init__.py index dd95e7ce..833c7f65 100644 --- a/python/alkahest/__init__.py +++ b/python/alkahest/__init__.py @@ -50,7 +50,12 @@ symbol, ) from ._dlpack import _call_batch, _to_numpy -from ._guess_holonomic import GuessedRecurrence, guess_holonomic +from ._guess_holonomic import ( + GUESS_STATUS_MEANINGS, + GUESS_STATUSES, + GuessedRecurrence, + guess_holonomic, +) from ._parse import parse from ._plot import ( plot, @@ -2100,6 +2105,10 @@ def wrapper(*args, **kwargs): __all__ = [ # Phase 17 "DAE", + # M2 — the vocabulary GuessedRecurrence.status is drawn from + "GUESS_STATUSES", + "GUESS_STATUS_MEANINGS", + # Phase 17 "HAS_EGRAPH", # Phase 16 "ODE", diff --git a/python/alkahest/_guess_holonomic.py b/python/alkahest/_guess_holonomic.py index bb548059..8b87249e 100644 --- a/python/alkahest/_guess_holonomic.py +++ b/python/alkahest/_guess_holonomic.py @@ -44,17 +44,34 @@ never explored. 2. **Reported evidence.** The result carries :attr:`GuessedRecurrence.surplus_terms` — how many equations the fit did not - need and satisfies anyway — and :attr:`GuessedRecurrence.dimension`, the - dimension of the solution space. Confirmation means surplus above the - threshold *and* dimension exactly one; a fit the caller can choose from - several is not a fit. + need and satisfies anyway — :attr:`GuessedRecurrence.dimension`, the + dimension of the solution space, and + :attr:`GuessedRecurrence.singular_indices`, the indices at which the fitted + operator is singular. Confirmation means surplus above the threshold, a + solution space of dimension one, *and* no singular index. + +The third of those exists because of the failure the first two miss. A +corrupted term does not stop a fit: at ``max_degree`` 4 a single typo in an +order-2/degree-1 sequence is absorbed by multiplying the true operator by the +cubic that vanishes at exactly the three indices whose equations the typo +breaks. The relation that comes back genuinely holds on the data supplied — +this is not unsoundness — it is simply not the sequence's recurrence, and the +tell is that its leading coefficient has roots inside the data, where the +operator determines nothing and the fit was therefore unconstrained. Those +roots are reported, and a fit that has any is never ``confirmed``. It is the +same fact :class:`alkahest.ModularRecurrence` refuses on with ``E-HOLO-007``, +reported here rather than raised, because the relation *does* hold on the +terms. This is :func:`alkahest.relation_confidence`'s discipline for sequences: -credibility is judged against what the data can actually support, and a fit -that fails is refused rather than returned with a caveat nobody reads. Pass -``check_evidence=False`` to get the raw fit with :attr:`~GuessedRecurrence. -confirmed` set honestly, the way ``check_precision=False`` works on -:func:`alkahest.guess_relation`. +credibility is judged against what the data can actually support, and +:attr:`~GuessedRecurrence.confirmed` is three-valued for the same reason +``credible`` is — ``True``, ``False``, and ``None`` for a fit whose evidence is +*undecided* rather than absent. :attr:`~GuessedRecurrence.status` names which +of those it is, in the vocabulary of +:attr:`alkahest.experimental.NoveltyVerdict.status`. Pass +``check_evidence=False`` to get the raw fit however weak, the way +``check_precision=False`` works on :func:`alkahest.guess_relation`. """ from __future__ import annotations @@ -71,7 +88,41 @@ from .alkahest import Expr -__all__ = ["GuessedRecurrence", "guess_holonomic"] +__all__ = [ + "GUESS_STATUSES", + "GUESS_STATUS_MEANINGS", + "GuessedRecurrence", + "guess_holonomic", +] + +#: Every verdict a fit can reach, in the spirit of +#: :data:`alkahest.experimental.novelty.NOVELTY_STATUSES`: the judgement is a +#: name from a closed vocabulary, not a bare boolean a caller can misread. +GUESS_STATUSES = ("confirmed", "singular", "underdetermined", "unconfirmed") + +#: Deliberately unflattering glosses, so that no reading of a non-``confirmed`` +#: status can be mistaken for "this is the sequence's recurrence". +GUESS_STATUS_MEANINGS = { + "confirmed": ( + "the terms over-determined the fit, singled it out, and the operator is " + "non-singular at every index they constrain — still a conjecture about " + "the sequence, never a proof" + ), + "singular": ( + "the fit holds on the terms supplied, but its leading coefficient " + "vanishes at singular_indices, where the recurrence determines nothing " + "and the fit was unconstrained; a corrupted term absorbed into a root " + "looks exactly like this" + ), + "underdetermined": ( + "the terms admit several independent relations of this shape and do not " + "single one out; read basis, not coeffs, and narrow the ansatz" + ), + "unconfirmed": ( + "the fit consumed the equations it was checked against — interpolation " + "wearing a recurrence's clothes, and evidence of nothing" + ), +} # The *native* PyO3 class, not the pure-Python one in `exceptions.py`: # `alkahest/__init__.py` overlays the native classes over the module namespace, @@ -90,8 +141,11 @@ class HolonomicEvidenceError(_HolonomicError): Raised in two situations, both of which are *undecided*, not *negative*: the ``(order, degree)`` grid could not be swept in full because the terms - supplied too few equations, or a fit was found that the surplus/uniqueness - guard will not endorse. + supplied too few equations, or the only fit found had no surplus equations + left to confirm it. The other two ways to miss confirmation — + ``"underdetermined"`` and ``"singular"`` — are *returned* with + :attr:`GuessedRecurrence.confirmed` ``None``, because there the relation + does hold on the terms and there is something for the caller to act on. """ def __init__(self, message: str, remediation: str): @@ -194,9 +248,14 @@ class GuessedRecurrence: asked to; hand the result to :func:`alkahest.zeilberger` when the sequence has a hypergeometric summand, and to :meth:`holds_for` when more terms turn up. + + Read :attr:`status` rather than ``bool(guess.confirmed)`` when the + difference between "the data does not support this" and "the data cannot + decide" matters — :attr:`confirmed` is ``True`` / ``False`` / ``None``. """ __slots__ = ( + "_basis", "_coeffs", "_degree", "_dimension", @@ -205,6 +264,7 @@ class GuessedRecurrence: "_n_terms", "_order", "_rank", + "_singular", "_start", "_untested", ) @@ -222,6 +282,7 @@ def __init__( dimension: int, min_surplus: int, untested: int, + basis: tuple[tuple[tuple[int, ...], ...], ...] | None = None, ): self._order = order self._degree = degree @@ -233,6 +294,8 @@ def __init__( self._dimension = dimension self._min_surplus = min_surplus self._untested = untested + self._basis = (coeffs,) if basis is None else basis + self._singular = _singular_indices(coeffs[order], start, n_equations) @property def order(self) -> int: @@ -297,12 +360,54 @@ def dimension(self) -> int: """Dimension of the solution space at this ``(order, degree)``. ``1`` for a genuine fit. Larger means the data admits several - independent relations of this shape and does not single one out; the - vector reported is then an arbitrary choice among them, which is why - :attr:`confirmed` is ``False``. + independent relations of this shape and does not single one out, so + :attr:`coeffs` is an arbitrary choice among them and :attr:`confirmed` + is ``None``. The whole space is :attr:`basis`; ``dimension == + len(basis)`` always. """ return self._dimension + @property + def basis(self) -> tuple[tuple[tuple[int, ...], ...], ...]: + """Every independent relation the terms admit at this ``(order, degree)``. + + A tuple of :attr:`coeffs`-shaped vectors, of which ``basis[0]`` *is* + :attr:`coeffs`. Length one for a fit the data singles out; longer means + the probe was wider than the sequence's annihilator, which is + information rather than a dead end — the minimal operator is a right + divisor of everything in here, and :func:`alkahest.zeilberger` will + often produce it outright when the sequence has a hypergeometric + summand. + """ + return self._basis + + @property + def singular_indices(self) -> tuple[int, ...]: + """Indices in the fitted range where the leading coefficient vanishes. + + The relation solves for ``u(n+order)`` by dividing through by + ``p_J(n)``, so at a root of ``p_J`` it determines nothing: the equation + there is satisfied whatever the terms are, and the fit was + *unconstrained*. Reported for the indices the fit's own equations were + written at, ``start <= n < start + n_equations`` — a root outside that + window constrains nothing the terms could have tested either way. + + **A non-empty list is the signature of corrupted data.** A single wrong + term in an otherwise clean sequence is fitted, at any generous + ``max_degree``, by multiplying the true operator by a polynomial + vanishing at exactly the indices whose equations that term breaks; the + result satisfies every equation supplied and is not the sequence's + recurrence. Recompute the terms at these indices before doing anything + else with the fit. + + Same name and same meaning as + :meth:`alkahest.ModularEvaluation.singular_indices`, which is where the + other half of this library meets the same phenomenon — with the + difference that a modular evaluation must refuse (``E-HOLO-007``) while + a fit can be returned and flagged, the relation being true on the data. + """ + return self._singular + @property def min_surplus(self) -> int: """Surplus equations demanded of a confirmed fit at this candidate.""" @@ -327,21 +432,66 @@ def untested_candidates(self) -> int: return self._untested @property - def confirmed(self) -> bool: - """Whether the data supports the fit: enough surplus, and unique. + def status(self) -> str: + """One of :data:`GUESS_STATUSES`; :data:`GUESS_STATUS_MEANINGS` glosses it. + + ``"unconfirmed"`` when the fit consumed its own evidence + (``surplus_terms < min_surplus``), ``"underdetermined"`` when + ``dimension > 1``, ``"singular"`` when :attr:`singular_indices` is + non-empty, and ``"confirmed"`` only when none of those applies. The + order is a precedence: a fit can fail more than one test and is named + for the most damning. + """ + if self.surplus_terms < self._min_surplus: + return "unconfirmed" + if self._dimension > 1: + return "underdetermined" + if self._singular: + return "singular" + return "confirmed" - ``True`` requires ``surplus_terms >= min_surplus`` **and** - ``dimension == 1``. It is never a claim that the recurrence holds for - the whole sequence — only that these terms are entitled to suggest it. + @property + def means(self) -> str: + """The one-line gloss of :attr:`status` from :data:`GUESS_STATUS_MEANINGS`.""" + return GUESS_STATUS_MEANINGS[self.status] + + @property + def confirmed(self) -> bool | None: + """Whether the data supports the fit — ``True`` / ``False`` / ``None``. + + Three-valued for the reason + :func:`alkahest.relation_confidence`'s ``credible`` is, and neither + ``False`` nor ``None`` is a pass: + + ``True`` + ``surplus_terms >= min_surplus``, ``dimension == 1``, and no + :attr:`singular_indices`. Never a claim that the recurrence holds + for the whole sequence — only that these terms are entitled to + suggest it. + ``False`` + the fit consumed the equations that would have confirmed it, so + the data says nothing about it either way (``"unconfirmed"``). + ``None`` + *undecided*: the relation holds on the terms, but the terms did + not single it out (``"underdetermined"``) or the operator is + singular where they were meant to constrain it (``"singular"``). + The fit is returned rather than refused because it is genuinely + true on the data — what it is not is the sequence's recurrence. + + :attr:`status` says which, and is the attribute to branch on. """ - return self.surplus_terms >= self._min_surplus and self._dimension == 1 + status = self.status + if status == "confirmed": + return True + return False if status == "unconfirmed" else None def evidence(self) -> dict: """The confirmation numbers as a dict, for logging next to the result. - Sibling of :func:`alkahest.relation_confidence`'s return value: the - judgement plus everything that went into it, so a research loop can - record *why* a fit was believed rather than only that it was. + Sibling of :func:`alkahest.relation_confidence`'s return value and of + :meth:`alkahest.experimental.NoveltyVerdict.report`: the judgement plus + everything that went into it, so a research loop can record *why* a fit + was believed rather than only that it was. """ return { "n_terms": self._n_terms, @@ -350,7 +500,10 @@ def evidence(self) -> dict: "surplus_terms": self.surplus_terms, "min_surplus": self._min_surplus, "dimension": self._dimension, + "singular_indices": list(self._singular), "untested_candidates": self._untested, + "status": self.status, + "means": self.means, "confirmed": self.confirmed, } @@ -405,7 +558,9 @@ def __repr__(self) -> str: return ( f"GuessedRecurrence(order={self._order}, degree={self._degree}, " f"coeffs={self._coeffs}, surplus_terms={self.surplus_terms}, " - f"dimension={self._dimension}, confirmed={self.confirmed})" + f"dimension={self._dimension}, " + f"singular_indices={list(self._singular)}, " + f"status={self.status!r}, confirmed={self.confirmed})" ) @@ -416,6 +571,19 @@ def _horner(poly: Sequence[int], x: int) -> Fraction: return total +def _singular_indices(leading: Sequence[int], start: int, n_equations: int) -> tuple[int, ...]: + """Integer roots of the leading polynomial among the fitted indices. + + Evaluated rather than solved for: the coefficients are exact integers of + arbitrary size, the window is the ``n_equations`` indices the fit was + written at, and one Horner pass per index is both exact and cheaper than + factoring a constant term that can run to hundreds of digits. Roots outside + the window are not looked for — no equation was written there, so nothing + about the fit was unconstrained by them. + """ + return tuple(start + row for row in range(n_equations) if _horner(leading, start + row) == 0) + + def guess_holonomic( terms: Sequence[Any], max_order: int = 4, @@ -449,6 +617,18 @@ def guess_holonomic( candidates need, because a loop that reads "not holonomic" off a grid it never swept has closed a branch it never explored. + **Check :attr:`GuessedRecurrence.status` on what comes back.** A returned + fit satisfies every equation the terms supplied; that it is the sequence's + recurrence is a separate question, and only ``status == "confirmed"`` + (equivalently :attr:`~GuessedRecurrence.confirmed` ``is True``) says the + data is entitled to suggest it. The two undecided outcomes are returned + rather than refused, because in both the relation genuinely holds and the + caller can act on it: ``"underdetermined"`` (several independent relations + — read :attr:`~GuessedRecurrence.basis`) and ``"singular"`` (the leading + coefficient vanishes inside the data at + :attr:`~GuessedRecurrence.singular_indices`, which is what a corrupted term + looks like — recompute those terms). + :param terms: the first terms of the sequence, exact ``int`` / ``Fraction``. :param max_order: largest recurrence order to try. :param max_degree: largest coefficient-polynomial degree to try. @@ -460,12 +640,12 @@ def guess_holonomic( disabling the reporting. :param check_evidence: when ``False``, every candidate is fitted regardless of surplus and the first fit is returned with - :attr:`GuessedRecurrence.confirmed` set honestly. This is the escape + :attr:`GuessedRecurrence.status` set honestly. This is the escape hatch, not the default — the same role ``check_precision=False`` plays on :func:`alkahest.guess_relation`. :raises HolonomicError: ``E-HOLO-005`` when the terms cannot support the - search, or when a fit was found that the evidence does not justify. + search, or when the only fit found had no surplus left to confirm it. :raises TypeError: when a term is not an exact rational. :raises ValueError: when the bounds are not positive. @@ -501,6 +681,21 @@ def guess_holonomic( ... 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, ... 263, 269, 271, 277, 281]) is None True + + One wrong term does not stop a fit — it is absorbed into roots of the + leading coefficient, which is what ``status`` and ``singular_indices`` + exist to say. The relation returned holds on every equation these terms + provide and is not Motzkin's recurrence: + + >>> spoiled = [1, 1] + >>> for i in range(1, 35): + ... spoiled.append(((2 * i + 3) * spoiled[-1] + 3 * i * spoiled[-2]) // (i + 3)) + >>> spoiled[12] += 1 + >>> fit = ak.guess_holonomic(spoiled) + >>> fit.status, fit.confirmed + ('singular', None) + >>> fit.singular_indices + (10, 11, 12) """ if max_order < 1: raise ValueError("max_order must be at least 1") @@ -536,7 +731,13 @@ def guess_holonomic( fitted = _fit(values, order, degree, start, threshold, len(skipped)) if fitted is None: continue - if check_evidence and not fitted.confirmed: + # Only an outright `False` — a fit that ate its own evidence — is + # refused. An *undecided* one (`None`: several relations, or an + # operator singular inside the data) is returned carrying the + # reason, because unlike the interpolating case the relation does + # hold on the terms and the caller can act on it: read `basis`, or + # recompute the terms at `singular_indices`. + if check_evidence and fitted.confirmed is False: raise _unjustified(fitted) return fitted @@ -584,11 +785,20 @@ def _fit( if not basis: return None - flat = [_fraction_from_expr(basis[0].get(i, 0)) for i in range((order + 1) * (degree + 1))] - integers = _primitive(flat) - coeffs = tuple( - tuple(integers[i * (degree + 1) : (i + 1) * (degree + 1)]) for i in range(order + 1) - ) + # Every basis vector is normalised, not only the one reported: the extra + # ones are what `GuessedRecurrence.basis` hands a caller whose probe was + # wider than the sequence's annihilator, and an un-normalised vector there + # would not be comparable to anything. + vectors = [] + for vector in basis: + flat = [_fraction_from_expr(vector.get(i, 0)) for i in range((order + 1) * (degree + 1))] + integers = _primitive(flat) + vectors.append( + tuple( + tuple(integers[i * (degree + 1) : (i + 1) * (degree + 1)]) for i in range(order + 1) + ) + ) + coeffs = vectors[0] if not any(coeffs[order]): # The leading polynomial vanished identically, so this is a relation of # lower order dressed up as one of order `order` — and the ascending @@ -609,22 +819,19 @@ def _fit( dimension=len(basis), min_surplus=min_surplus, untested=untested, + basis=tuple(vectors), ) def _unjustified(fitted: GuessedRecurrence) -> HolonomicEvidenceError: - if fitted.dimension > 1: - reason = ( - f"the terms admit {fitted.dimension} independent relations of order " - f"{fitted.order} and degree {fitted.degree}, so they do not single one " - "out and the vector reported would be an arbitrary choice among them" - ) - else: - reason = ( - f"it consumed {fitted.equations_used} of the {fitted.n_equations} " - f"equations the terms provide and only {fitted.surplus_terms} were left " - f"to confirm it, short of the {fitted.min_surplus} required" - ) + # Reached only for `status == "unconfirmed"`. The other two ways to miss + # confirmation are *undecided* rather than empty and are returned with + # `confirmed=None`, not raised. + reason = ( + f"it consumed {fitted.equations_used} of the {fitted.n_equations} " + f"equations the terms provide and only {fitted.surplus_terms} were left " + f"to confirm it, short of the {fitted.min_surplus} required" + ) return HolonomicEvidenceError( f"a recurrence of order {fitted.order} and degree {fitted.degree} fits the " f"{fitted.n_terms} terms supplied, but {reason}; a fit the data cannot " diff --git a/python/alkahest/_recurrence_asymptotics.py b/python/alkahest/_recurrence_asymptotics.py index 1f4a63fa..a3b69d2f 100644 --- a/python/alkahest/_recurrence_asymptotics.py +++ b/python/alkahest/_recurrence_asymptotics.py @@ -40,7 +40,8 @@ from numbers import Rational from typing import TYPE_CHECKING, Any -from .alkahest import RecurrenceAsymptotics +from .alkahest import ExprPool, RecurrenceAsymptotics +from .alkahest import PoolError as _PoolError from .alkahest import asymptotics_from_recurrence as _native if TYPE_CHECKING: # pragma: no cover - typing only @@ -51,8 +52,24 @@ __all__ = ["RecurrenceAsymptotics", "asymptotics_from_recurrence"] -def _coefficients(rec: Any) -> tuple[Any, int | None]: - """The coefficient polynomials of *rec*, and the index its terms start at. +class RecurrencePoolError(_PoolError): + """``E-POOL-001`` — *n* is not from the pool *rec*'s coefficients live in. + + A subclass of the *native* :class:`alkahest.PoolError`, so ``except + ak.PoolError`` catches it, carrying a code and a remediation that name the + fix. The bare mismatch the kernel raises is correct but arrives from + several frames down without saying which argument caused it or that there + is now a way not to pass one. + """ + + def __init__(self, message: str, remediation: str): + super().__init__(message) + self.code = "E-POOL-001" + self.remediation = remediation + + +def _coefficients(rec: Any) -> tuple[Any, int | None, Any]: + """The coefficients of *rec*, the index its terms start at, and its own ``n``. Accepts the two objects that produce recurrences in this library plus the raw form. Duck-typed rather than ``isinstance``-checked so that a wrapper @@ -65,9 +82,31 @@ def _coefficients(rec: Any) -> tuple[Any, int | None]: coeffs = getattr(rec, "coeffs", None) if coeffs is None: # A plain sequence of coefficient polynomials. - return list(rec), None + return list(rec), None, None start = getattr(rec, "start", None) - return list(coeffs), start + # `ZeilbergerCertificate.n` is the symbol its coefficients are written in, + # and the only one they can be combined with. A `GuessedRecurrence` has no + # pool at all — its coefficients are plain integers — so it has no `n`. + return list(coeffs), start, getattr(rec, "n", None) + + +def _index_symbol(rec: Any, coeffs: Sequence[Any], own_n: Any) -> Expr: + """The index variable to use when the caller did not supply one.""" + if own_n is not None: + return own_n + if all(hasattr(p, "__iter__") and not isinstance(p, str) for p in coeffs): + # Every coefficient is a sequence of integers, so nothing in *rec* + # belongs to a pool and the result is free to be built in a fresh one. + return ExprPool().symbol("n") + raise RecurrencePoolError( + f"n must be given: the coefficients of this {type(rec).__name__} are " + "expressions, and only the pool they were built in can say which " + "symbol is the index", + remediation=( + "pass the symbol the coefficients were built with — for a " + "ZeilbergerCertificate that is cert.n, and omitting n uses it" + ), + ) def _exact(value: Any, where: str) -> int | tuple[int, int]: @@ -92,7 +131,7 @@ def _exact(value: Any, where: str) -> int | tuple[int, int]: def asymptotics_from_recurrence( rec: Any, - n: Expr, + n: Expr | None = None, *, terms: Sequence[Any] | None = None, start: int | None = None, @@ -107,7 +146,15 @@ def asymptotics_from_recurrence( ``Σ_{i=0}^{J} p_i(n) · u(n+i) = 0``. *n* is the index variable; it also says which - :class:`~alkahest.ExprPool` the result is built in. + :class:`~alkahest.ExprPool` the result is built in. **It is optional**, and + leaving it out is the safe way to call this: expressions from two different + pools cannot meet, so a certificate's coefficients only combine with the + certificate's own :attr:`~alkahest.ZeilbergerCertificate.n`, and a symbol + made in a fresh pool for the occasion is a pool mismatch several frames + down rather than an answer. Omitted, *n* comes from *rec* when *rec* has + one, and from a pool created here when its coefficients are plain integers + (a :class:`~alkahest.GuessedRecurrence`, or raw integer tuples), which + belong to no pool at all. :param terms: exact leading terms of the sequence, ``terms[0] = u(start)``. ``int`` or :class:`fractions.Fraction`; a ``float`` is refused. Without @@ -125,6 +172,9 @@ def asymptotics_from_recurrence( coefficients, a coefficient that is not a polynomial in *n* over ``ℚ``, or a characteristic polynomial all of whose roots are zero. A recurrence whose hypotheses fail is *reported* through ``verdict``, not refused. + :raises alkahest.PoolError: ``E-POOL-001`` when *n* does not come from the + pool *rec*'s coefficients live in, or when it was omitted and *rec* is + a raw list of expressions that cannot say what its index symbol is. :raises TypeError: when a term is not an exact rational. Central binomial coefficients, ``C(2n,n) ~ 4ⁿ/√(πn)``: @@ -156,9 +206,30 @@ def asymptotics_from_recurrence( >>> osc = asymptotics_from_recurrence([(-4,), (0,), (1,)], n, terms=[1, 2]) >>> osc.verdict, osc.growth_rate ('equal_modulus_roots', None) + + Integer coefficients belong to no pool, so ``n`` can simply be left out: + + >>> asymptotics_from_recurrence([(-2, -4), (1, 1)], terms=[1]).growth_rate + 4.0 """ - coeffs, rec_start = _coefficients(rec) + coeffs, rec_start, own_n = _coefficients(rec) if start is None: start = rec_start if rec_start is not None else 0 + if n is None: + n = _index_symbol(rec, coeffs, own_n) exact = [_exact(t, "every term of the sequence") for t in (terms or ())] - return _native(coeffs, n, terms=exact, start=start) + try: + return _native(coeffs, n, terms=exact, start=start) + except _PoolError as exc: + # The kernel is right to refuse — two pools cannot meet — but it + # refuses from inside the coefficient walk, naming neither the argument + # at fault nor the fact that `rec` was carrying the right symbol all + # along. Re-raise with both. + if own_n is None: + raise + raise RecurrencePoolError( + f"n belongs to a different ExprPool than this {type(rec).__name__}, " + "whose coefficient polynomials can only be combined with the symbol " + "it was built with", + remediation=("omit n — it is taken from rec — or pass rec.n, which is that symbol"), + ) from exc diff --git a/python/alkahest/_supercongruence.py b/python/alkahest/_supercongruence.py index 080381df..6b877cd3 100644 --- a/python/alkahest/_supercongruence.py +++ b/python/alkahest/_supercongruence.py @@ -35,6 +35,7 @@ from .alkahest import HolonomicError as _HolonomicError from .alkahest import ModularRecurrence +from .number_theory import isprime as _isprime if TYPE_CHECKING: # pragma: no cover - typing only from collections.abc import Iterable @@ -43,8 +44,18 @@ #: Refusals that are a fact about one prime, not about the call. These are #: recorded in :meth:`CongruenceSweep.skipped` and the sweep carries on; -#: everything else (a composite base, a malformed recurrence) propagates. -_PER_PRIME_REFUSALS = frozenset({"E-HOLO-007", "E-HOLO-008"}) +#: everything else (a malformed recurrence, a bad index) propagates. +#: +#: ``E-HOLO-006`` is one of them because the two ways to earn it are not alike. +#: A composite base is a fact about the *call*, and the sweep rules that out +#: itself before evaluating anything (see :func:`supercongruence_sweep`), so +#: the only ``E-HOLO-006`` that can reach the loop is ``p**k`` past the +#: machine-word ceiling — the same "this prime is out of reach of this backend" +#: that ``E-HOLO-008`` is, and it gets the same treatment. Letting it propagate +#: destroyed every residue already computed and left the caller to work out the +#: per-``k`` cap ``int((2**62) ** (1 / (k + 1)))`` by hand, which is exactly the +#: accounting :meth:`CongruenceSweep.skipped` exists to do. +_PER_PRIME_REFUSALS = frozenset({"E-HOLO-006", "E-HOLO-007", "E-HOLO-008"}) class CongruenceSweep: @@ -153,12 +164,14 @@ def skipped(self) -> list[tuple[int, str]]: """``[(p, reason)]`` for primes the evaluation refused. A refusal is *undecided*, not *satisfied*: a sweep that silently - dropped these would be reporting a range it never covered. The two - causes are a run of singular indices demanding more working precision - than a machine-word modulus can hold (``E-HOLO-008``) and a sequence - that is not ``p``-integral at that prime (``E-HOLO-007``). A refusal - about the *call* — a composite base, a malformed recurrence — is not - skipped, it is raised. + dropped these would be reporting a range it never covered. The three + causes are all "this backend cannot reach this prime": ``p**(k+extra)`` + past the machine-word ceiling of ``2**62`` (``E-HOLO-006``), a run of + singular indices demanding more working precision than a machine-word + modulus can hold (``E-HOLO-008``), and a sequence that is not + ``p``-integral there (``E-HOLO-007``). A refusal about the *call* — a + composite in *primes*, a malformed recurrence — is not skipped, it is + raised. """ return list(self._skipped) @@ -179,6 +192,31 @@ def _as_callable(value: Any, name: str) -> Callable[[int], int]: raise TypeError(f"{name} must be an int or a callable of p, got {type(value).__name__}") +def _require_prime(p: Any) -> None: + """Refuse a composite in *primes* before any residue is computed. + + The kernel refuses it too, with the same ``E-HOLO-006``, but it does so + from inside the loop where this module can no longer tell that refusal + apart from "this prime is past the machine-word ceiling" — which is a fact + about one prime and belongs in :meth:`CongruenceSweep.skipped`. Deciding + the *call*-level half here is what lets the other half be skipped rather + than fatal. The message is the kernel's, so the two cannot drift. + """ + if isinstance(p, int) and p >= 2 and _isprime(p): + return + error = _HolonomicError( + f"holonomic: unsupported modulus: {p} is not prime; the lifting " + "argument this module rests on needs a prime power modulus, and v_p " + "is not defined otherwise" + ) + error.code = "E-HOLO-006" + error.remediation = ( + "the modulus must be p**k with p prime, k >= 1 and p**k < 2**62; for a " + "composite modulus, evaluate at each prime power and recombine by CRT" + ) + raise error + + def supercongruence_sweep( recurrence: ModularRecurrence, primes: Iterable[int], @@ -199,10 +237,13 @@ def supercongruence_sweep( :param recurrence: the sequence, as a :class:`alkahest.ModularRecurrence`. - :param primes: the primes to test. Not checked for primality here — the - evaluation checks, and a composite raises ``HolonomicError`` - (``E-HOLO-006``) rather than being skipped, because a sweep that - silently drops its inputs reports a range it did not cover. + :param primes: the primes to test. Checked for primality here, and a + composite raises ``HolonomicError`` (``E-HOLO-006``) rather than being + skipped, because a sweep that silently drops its inputs reports a range + it did not cover. The check is up front so that the *other* + ``E-HOLO-006`` — ``p**(k + extra_precision)`` past the machine-word + ceiling, a fact about one prime rather than about the call — can be + recorded in :meth:`CongruenceSweep.skipped` and the sweep continue. :param k: the claimed exponent. ``a(p-1) ≡ 1 (mod p**4)`` is ``k=4``. :param index: ``p -> n``, the index to evaluate at. Defaults to ``p - 1``, which is the shape of nearly every Apéry-like supercongruence. @@ -249,14 +290,18 @@ def supercongruence_sweep( skipped: list[tuple[int, str]] = [] for p in primes: + # The one `E-HOLO-006` that is a fact about the *call* rather than + # about one prime, decided here rather than left to the evaluation, so + # that the code can be skipped below without a list of composites + # coming back `holds=True` over zero primes — the sweep lying about a + # range it never covered. + _require_prime(p) modulus = p**precision try: value = recurrence.value_mod(index_of(p), p, precision) except _HolonomicError as exc: # A refusal about *this prime* is recorded and reported; a refusal - # about the call itself is re-raised. Skipping `E-HOLO-006` would - # let a list of composites come back `holds=True` over zero primes, - # which is the sweep lying about a range it never covered. + # about the call itself is re-raised. if getattr(exc, "code", None) not in _PER_PRIME_REFUSALS: raise skipped.append((p, str(exc))) diff --git a/tests/test_guess_holonomic.py b/tests/test_guess_holonomic.py index 38654505..d5144ba7 100644 --- a/tests/test_guess_holonomic.py +++ b/tests/test_guess_holonomic.py @@ -12,6 +12,7 @@ import decimal import doctest +import math from fractions import Fraction import alkahest as ak @@ -95,7 +96,10 @@ def test_surplus_is_reported_and_is_the_number_that_justifies_the_fit(): "surplus_terms": 14, "min_surplus": 6, "dimension": 1, + "singular_indices": [], "untested_candidates": 0, + "status": "confirmed", + "means": ak.GUESS_STATUS_MEANINGS["confirmed"], "confirmed": True, } @@ -359,3 +363,290 @@ def test_module_docstrings_have_runnable_doctests(): optionflags=doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL, ) assert failures == 0 + + +# --------------------------------------------------------------------------- +# Corrupted data: the fit that holds on the terms and is not the recurrence +# --------------------------------------------------------------------------- + + +def _at(poly, index): + """``p(index)`` for an ascending integer coefficient tuple.""" + return sum(c * index**j for j, c in enumerate(poly)) + + +def _motzkin(count): + """The first *count* Motzkin numbers, from their own recurrence.""" + terms = [1, 1] + while len(terms) < count: + i = len(terms) - 1 + terms.append(((2 * i + 3) * terms[-1] + 3 * i * terms[-2]) // (i + 3)) + return terms + + +def test_a_clean_sequence_has_no_singular_index_and_stays_confirmed(): + """The control the corrupted cases are read against. + + Motzkin's own leading coefficient is ``p_J(n) = n + 4``, whose only root is + ``−4`` — outside the fitted range, so nothing is reported and the verdict + is unchanged by any of this. + """ + guess = ak.guess_holonomic(_motzkin(71)) + assert guess.coeffs[-1] == (4, 1), "p_J(n) = n + 4" + assert guess.singular_indices == () + assert guess.status == "confirmed" + assert guess.confirmed is True + + +def test_one_corrupted_term_is_reported_as_singular_and_never_confirmed(): + """A single typo is absorbed into three roots of the leading coefficient. + + At the default ``max_degree`` the fit multiplies the true operator by the + cubic vanishing at exactly the three indices whose equations the typo + breaks. Everything a caller reads to judge a fit looked perfect — + ``dimension`` 1, 55 surplus equations, no untested candidates — and the + relation really does hold on the terms supplied. It is simply not Motzkin's + recurrence, and the roots inside the data are the only tell. + """ + spoiled = _motzkin(71) + spoiled[30] += 1 + + guess = ak.guess_holonomic(spoiled) + assert guess is not None + assert guess.singular_indices == (28, 29, 30) + assert guess.status == "singular" + assert guess.confirmed is None, "never a bare True on corrupted data" + # The evidence that used to be the whole story is still exactly as strong. + assert guess.dimension == 1 + assert guess.surplus_terms == 55 + assert guess.untested_candidates == 0 + assert guess.holds_for(spoiled), "the relation does hold on what it was shown" + # It is the true operator multiplied by that cubic, so it also holds on the + # *clean* sequence — which is precisely why no re-check can catch this and + # why the roots have to be reported. What it does not do is determine the + # sequence at 28, 29 and 30, where every coefficient vanishes at once. + assert guess.holds_for(_motzkin(71)) + for index in guess.singular_indices: + assert all(_at(poly, index) == 0 for poly in guess.coeffs), ( + f"every coefficient vanishes at n = {index}, so the equation there " + "is 0 = 0 and constrained nothing" + ) + + +def test_two_corrupted_terms_are_reported_as_six_roots(): + """Two typos need ``max_degree=8``, and produce two triples of roots. + + Same mechanism one degree up: a sextic factor vanishing at the six indices + the two wrong terms break. The count of roots scales with the corruption, + which is what makes the field usable as a diagnostic rather than a flag. + """ + spoiled = _motzkin(71) + spoiled[30] += 1 + spoiled[50] += 1 + + guess = ak.guess_holonomic(spoiled, 4, 8) + assert guess.singular_indices == (28, 29, 30, 48, 49, 50) + assert guess.status == "singular" + assert guess.confirmed is None + + +def test_the_evidence_dict_carries_the_verdict_and_the_roots(): + """``evidence()`` is what a research loop logs, so it must carry both.""" + spoiled = _motzkin(71) + spoiled[30] += 1 + evidence = ak.guess_holonomic(spoiled).evidence() + + assert evidence["singular_indices"] == [28, 29, 30] + assert evidence["status"] == "singular" + assert evidence["confirmed"] is None + assert "singular" in evidence["means"] + + +def test_the_status_vocabulary_is_closed_and_glossed(): + """Every status is nameable and has a meaning, as for ``NoveltyVerdict``.""" + assert set(ak.GUESS_STATUSES) == set(ak.GUESS_STATUS_MEANINGS) + for name in ("GUESS_STATUSES", "GUESS_STATUS_MEANINGS"): + assert name in ak.__all__ + + spoiled = _motzkin(71) + spoiled[30] += 1 + for guess in (ak.guess_holonomic(_motzkin(71)), ak.guess_holonomic(spoiled)): + assert guess.status in ak.GUESS_STATUSES + assert guess.means == ak.GUESS_STATUS_MEANINGS[guess.status] + + +def test_a_corrupted_fit_says_so_in_its_repr(): + """The one-line form a loop prints must not read as a clean answer.""" + spoiled = _motzkin(71) + spoiled[30] += 1 + text = repr(ak.guess_holonomic(spoiled)) + assert "singular_indices=[28, 29, 30]" in text + assert "status='singular'" in text + assert "confirmed=None" in text + + +# --------------------------------------------------------------------------- +# The guard still refuses everything it was built to refuse +# --------------------------------------------------------------------------- + + +def _partitions(count): + """``p(n)``, by Euler's pentagonal-number recurrence. Not P-recursive.""" + values = [1] + for n in range(1, count): + total = 0 + k = 1 + while True: + for pentagonal in ((3 * k * k - k) // 2, (3 * k * k + k) // 2): + if pentagonal > n: + break + total += (-1) ** (k + 1) * values[n - pentagonal] + if (3 * k * k - k) // 2 > n: + break + k += 1 + values.append(total) + return values + + +def _bell(count): + """Bell numbers off the Bell triangle. Not P-recursive.""" + row = [1] + values = [1] + for _ in range(count - 1): + nxt = [row[-1]] + for value in row: + nxt.append(nxt[-1] + value) + row = nxt + values.append(row[0]) + return values + + +def _divisor(count, power): + """``σ(n)`` for *power* 1 and ``τ(n)`` for *power* 0, from ``n = 1``.""" + return [sum(d**power for d in range(1, n + 1) if n % d == 0) for n in range(1, count + 1)] + + +def _digits_of_pi(count): + """Decimal digits of ``π``, by the unbounded spigot. Not P-recursive.""" + out = [] + q, r, t, k, m, x = 1, 0, 1, 1, 3, 3 + while len(out) < count: + if 4 * q + r - t < m * t: + out.append(m) + q, r, t, k, m, x = 10 * q, 10 * (r - m * t), t, k, (10 * (3 * q + r)) // t - 10 * m, x + else: + q, r, t, k, m, x = ( + q * k, + (2 * q + r) * x, + t * x, + k + 1, + (q * (7 * k + 2) + r * x) // (t * x), + x + 2, + ) + return out + + +def _pseudorandom(count): + """A fixed pseudo-random sequence: the control with no structure at all.""" + values = [] + state = 20260820 + for _ in range(count): + state = (state * 6364136223846793005 + 1442695040888963407) % 2**64 + values.append(state % 101 - 50) + return values + + +def _beatty_sqrt(radicand, count): + """``floor(n·√radicand)``, computed exactly. Not P-recursive.""" + decimal.getcontext().prec = 200 + root = decimal.Decimal(radicand).sqrt() + return [int(decimal.Decimal(n) * root) for n in range(count)] + + +def _digit_sums(count): + return [sum(int(c) for c in str(n)) for n in range(count)] + + +NOT_P_RECURSIVE = { + "primes": PRIMES, + "partitions": _partitions(60), + "bell": _bell(60), + "sigma": _divisor(60, 1), + "tau": _divisor(60, 0), + "pi_digits": _digits_of_pi(60), + "pseudorandom": _pseudorandom(60), + "beatty_phi": _beatty(60), + "beatty_sqrt2": _beatty_sqrt(2, 60), + "digit_sums": _digit_sums(60), +} + + +@pytest.mark.parametrize("name", sorted(NOT_P_RECURSIVE)) +def test_ten_non_p_recursive_sequences_still_come_back_none(name): + """The guard is real in the direction it was designed for, and stays real. + + Reporting singular indices and returning an underdetermined fit both loosen + what comes *back*; neither may loosen what gets returned at all. Ten + sequences that are not P-recursive — none of which the literature gives a + P-recursive relation — must still answer ``None`` after a full sweep, not a + fit with a caveat attached. + """ + assert ak.guess_holonomic(NOT_P_RECURSIVE[name]) is None + + +def test_too_few_terms_still_refuses_and_names_the_shortfall(): + """``E-HOLO-005`` is untouched: undecided is still not a negative.""" + with pytest.raises(ak.HolonomicError) as excinfo: + ak.guess_holonomic(MOTZKIN[:7]) + assert excinfo.value.code == "E-HOLO-005" + assert "7 terms are not enough" in str(excinfo.value) + + +# --------------------------------------------------------------------------- +# dimension > 1 returns the solution space instead of refusing +# --------------------------------------------------------------------------- + + +def _a277060(count): + """OEIS A277060, ``a(n) = (1/2)·Σ_k (C(n,k)·C(n+k,k+1))²``.""" + return [ + sum((math.comb(n, k) * math.comb(n + k, k + 1)) ** 2 for k in range(n + 1)) // 2 + for n in range(count) + ] + + +def test_a_wider_probe_than_the_annihilator_returns_the_basis(): + """A277060: ``dimension`` 2 is information, and used to be a dead end. + + The probe that succeeds first is wider than the sequence's annihilator, so + the terms admit two independent relations of that shape. Refusing made the + whole ``(order, degree)`` cell unusable and closed a sequence that + ``zeilberger`` decides immediately; the space is returned instead, and the + verdict says the terms did not single a member of it out. + """ + terms = _a277060(80) + guess = ak.guess_holonomic(terms, 4, 6) + + assert guess is not None, "used to raise E-HOLO-005" + assert guess.dimension == 2 + assert len(guess.basis) == 2 + assert guess.basis[0] == guess.coeffs + assert guess.status == "underdetermined" + assert guess.confirmed is None + + # Every element of the basis is a relation on the data, which is the whole + # reason returning it is better than refusing. + for vector in guess.basis: + member = ak.GuessedRecurrence( + order=guess.order, + degree=guess.degree, + start=guess.start, + coeffs=vector, + n_terms=guess.n_terms, + n_equations=guess.n_equations, + rank=guess.equations_used, + dimension=1, + min_surplus=guess.min_surplus, + untested=guess.untested_candidates, + ) + assert member.holds_for(terms) diff --git a/tests/test_modular_holonomic.py b/tests/test_modular_holonomic.py index cae10946..d5f9a234 100644 --- a/tests/test_modular_holonomic.py +++ b/tests/test_modular_holonomic.py @@ -623,6 +623,43 @@ def test_sweep_records_refusals_instead_of_counting_them_as_successes(): assert not sweep.sharp +def test_a_modulus_past_the_word_ceiling_is_skipped_not_fatal(): + """``E-HOLO-006`` from ``p**k`` past ``2**62`` must not kill the sweep. + + It used to propagate, destroying every residue already computed and leaving + the caller to pre-filter the prime list by ``int((2**62)**(1/(k+1)))`` — the + exact accounting ``skipped()`` exists to do, and the same "out of reach of + this backend" that ``E-HOLO-008`` is already recorded for. At ``k=6`` with + one digit of extra precision the ceiling lands at ``p**7 < 2**62``, i.e. + just past 460, so this range straddles it. + """ + rec = ak.ModularRecurrence(*SEQUENCES[0][1:3]) + primes = [p for p in small_primes(700) if p >= 5] + + sweep = ak.supercongruence_sweep(rec, primes, k=6, expect=1, max_counterexamples=len(primes)) + assert sweep.n_skipped > 0 + assert sweep.n_tested > 0, "the partial result survives" + assert sweep.n_tested + sweep.n_skipped == len(primes) + assert all("E-HOLO-006" in reason for _p, reason in sweep.skipped()) + assert all(p > 460 for p, _reason in sweep.skipped()) + + +def test_a_composite_in_the_prime_list_is_still_fatal(): + """The *other* ``E-HOLO-006`` is a fact about the call and must still raise. + + A sweep that skipped its way through a list of composites would report + ``holds`` over zero primes — a range it never covered. The check is made up + front so that the reachable-modulus half can be skipped without taking this + half with it. + """ + rec = ak.ModularRecurrence(*SEQUENCES[0][1:3]) + with pytest.raises(ak.HolonomicError) as excinfo: + ak.supercongruence_sweep(rec, [5, 7, 9, 11], k=3, expect=1) + assert excinfo.value.code == "E-HOLO-006" + assert "9 is not prime" in str(excinfo.value) + assert excinfo.value.remediation + + def test_sweep_without_extra_precision_cannot_claim_sharpness(): rec = ak.ModularRecurrence(*SEQUENCES[0][1:3]) primes = [p for p in small_primes(100) if p >= 5] diff --git a/tests/test_recurrence_asymptotics.py b/tests/test_recurrence_asymptotics.py index 1c736d8d..35c96c68 100644 --- a/tests/test_recurrence_asymptotics.py +++ b/tests/test_recurrence_asymptotics.py @@ -473,6 +473,74 @@ def test_repr_does_not_leak_rust_option_syntax(): assert "fitted" in text +# --------------------------------------------------------------------------- +# The index symbol: derived rather than demanded +# --------------------------------------------------------------------------- + + +def test_n_is_optional_for_a_certificate_that_carries_its_own(): + """``asymptotics_from_recurrence(cert)`` — no pool bookkeeping at all. + + A certificate's coefficients live in the certificate's pool and can be + combined with nothing else, so requiring the caller to supply a matching + ``n`` was requiring them to reconstruct something ``cert`` already had. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + one = pool.integer(1) + binomial = ak.gamma(n + one) / (ak.gamma(k + one) * ak.gamma(n - k + one)) + cert = ak.zeilberger(binomial, n, k) + + assert cert.n == n + r = asymptotics_from_recurrence(cert, terms=[1, 2]) + assert str(r.growth_rate_exact) == "2" + assert r.polynomial_exponent == 0.0 + + +def test_n_is_optional_when_the_coefficients_are_plain_integers(): + """A ``GuessedRecurrence`` belongs to no pool, so one is made for it.""" + motzkin = [ + 1, 1, 2, 4, 9, 21, 51, 127, 323, 835, 2188, + 5798, 15511, 41835, 113634, 310572, 853467, + 2356779, 6536382, 18199284, 50852019, + ] # fmt: skip + r = asymptotics_from_recurrence(ak.guess_holonomic(motzkin), terms=motzkin[:2]) + assert str(r.growth_rate_exact) == "3" + + raw = asymptotics_from_recurrence(CENTRAL_BINOMIAL, terms=[1]) + assert raw.growth_rate == 4.0 + + +def test_a_foreign_n_is_a_coded_error_naming_the_argument(): + """The bare pool mismatch said nothing about which argument was wrong. + + It arrived from several frames inside the coefficient walk as an + uncoded ``PoolError``, with no hint that ``rec`` was carrying the right + symbol all along. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + one = pool.integer(1) + binomial = ak.gamma(n + one) / (ak.gamma(k + one) * ak.gamma(n - k + one)) + cert = ak.zeilberger(binomial, n, k) + + with pytest.raises(ak.PoolError) as excinfo: + asymptotics_from_recurrence(cert, _n(), terms=[1, 2]) + assert excinfo.value.code == "E-POOL-001" + assert "different ExprPool" in str(excinfo.value) + assert "rec.n" in excinfo.value.remediation + + +def test_omitting_n_on_raw_expressions_says_why_it_cannot_be_derived(): + """A bare list of ``Expr`` cannot name its own index variable.""" + pool = ak.ExprPool() + n = pool.symbol("n") + with pytest.raises(ak.PoolError) as excinfo: + asymptotics_from_recurrence([n * pool.integer(-2), n], terms=[1]) + assert excinfo.value.code == "E-POOL-001" + assert "n must be given" in str(excinfo.value) + + def test_docstring_examples(): import alkahest._recurrence_asymptotics as module From cfc1994ca161699b2226c9504e5c79f1f9cc8755 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 21:29:34 +0000 Subject: [PATCH 05/11] fix: give the Zeilberger boundary verdict the domain of n it is a theorem on The verdict was a bare tag with an implied "for every n" attached to it, and that quantifier was false in two ways. Empty ranges. F = C(n,k)^2 over limits=(5, 3) -- an empty range, so every S(n) is 0 -- returned boundary="nonzero", implies_sum_recurrence=True and a degree-9 b(n) whose residual sum_i a_i(n)*S(n+i) - b(n) ran 4, 107, 800, 2725, 2450, -23716, -162288 at n = 3..9: a valid certificate implying a false recurrence for the sum, the class of defect this verdict exists to close. The cause is that k_hi < k_lo - 1 makes the declared range run backwards, where a sum is 0 under the empty-sum reading and a signed sum under the reversed-sum one, and every piece of the boundary analysis silently used the second. The realistic form is n-dependent and worse: limits=(3, n-3) is empty at n = 3, 4 and a range after, so the returned b(n) was wrong at exactly the n a loop reaches first. `boundary_verdict` now returns the verdict together with `valid_from`, the smallest n it is claimed for -- 5 for k = 3..n-3, checked against the sum -- and a range that runs backwards at every n, or at every large n, is Unknown: there is no domain left to claim it on. k_hi = k_lo - 1 (k = 0..-1, k = n+1..n), the one empty range both readings agree is 0, keeps its "vanishes", which is what the two treatments disagreeing was. `boundary_status` keeps its signature and fails safe: a verdict false at some n >= 0 comes back as Unknown naming the restriction. Interior poles. C(n,k)/(n-2k+1) over k = 0..n returned "vanishes" although S(n) is undefined for every odd n and the certificate itself has a pole at k = (n+3)/2, an integer strictly inside the range -- so the telescoping breaks in the middle of the sum, where neither boundary value can see it. Those points are searched for (rational roots of the summand's and G's denominators on a bounded grid at two sample n, then verified exactly over Q(n), then order-counted to drop the ones a gamma zero cancels), reported in `certificate_poles`, and the verdict is Unknown when there are any. The same closes C(n,k)/(k-3) over k = 0..n, whose sum does not exist for n >= 3. Verdicts that were already right are unchanged and now have guard tests: the 0-times-infinity endpoint continuation C(n,k)/(n-k+1) over k = 0..n (a certificate pole exactly at k = k_hi+1, cancelled by a zero of the summand), negative-slope lower limits, and every classical natural-boundary identity. Both new fields are additive: BoundaryVerdict is a new non_exhaustive struct alongside the unchanged BoundaryStatus, and cargo semver-checks reports no semver update required. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 40 ++ alkahest-core/src/holonomic/boundary.rs | 751 ++++++++++++++++++++-- alkahest-core/src/holonomic/mod.rs | 4 +- alkahest-core/src/holonomic/zeilberger.rs | 8 +- alkahest-py/src/lib.rs | 116 +++- alkahest-skill/alkahest.md | 2 +- docs/features.md | 2 +- docs/mdbook/src/telescoping.md | 41 ++ tests/test_holonomic_boundary.py | 155 ++++- 9 files changed, 1052 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..b91c84cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,46 @@ ## Unreleased +- **A Zeilberger `boundary` verdict now carries the `n` it is a theorem on** + (`ZeilbergerCertificate.boundary_valid_from`, `.certificate_poles`). The + verdict was a bare tag with an implied "for every `n`" attached to it, and + that quantifier was false in two ways. + + **Empty ranges.** `F = C(n,k)²` over `limits=(5, 3)` — an empty range, so + every `S(n)` is `0` — returned `boundary="nonzero"`, + `implies_sum_recurrence=True` and a degree-9 `b(n)` whose residual + `Σ_i a_i(n)·S(n+i) − b(n)` ran `4, 107, 800, 2725, 2450, −23716, −162288` at + `n = 3..9`: a valid certificate implying a false recurrence for the sum, the + same class of defect the verdict was built to close. The cause is that + `κ₁ < κ₀ − 1` makes the declared range run *backwards*, where a sum is `0` + under the "empty sum" reading and a signed sum under the reversed-sum one, + and every piece of the boundary analysis silently used the second. The + realistic form is `n`-dependent and worse: `limits=(3, n−3)` is empty at + `n = 3, 4` and a range afterwards, so the returned `b(n)` was wrong at + exactly the `n` a loop reaches first. Such a verdict is now returned *with* + its domain — `boundary_valid_from == 5` — and a range that is backwards at + every `n` (or at every large `n`) is `"unknown"`. `κ₁ = κ₀ − 1` (`k = 0..−1`, + `k = n+1..n`), the one empty range both readings agree is `0`, keeps its + `"vanishes"`, which is what the two treatments disagreeing was. + + **Interior poles.** `C(n,k)/(n−2k+1)` over `k = 0..n` returned + `"vanishes"` although `S(n)` is undefined for every odd `n` *and* the + certificate itself has a pole at `k = (n+3)/2`, an integer strictly inside + the range — so the telescoping `Σ_k (G(n,k+1) − G(n,k))` breaks in the middle + of the sum, where neither boundary value can see it. Those points are now + searched for, reported in `certificate_poles` as expressions in `n`, and the + verdict is `"unknown"` when there are any. The same closes + `C(n,k)/(k−3)` over `k = 0..n`, whose sum does not exist for `n ≥ 3`. + + The Rust `holonomic::boundary_status` keeps its signature and fails safe: a + verdict that is false at some `n ≥ 0` comes back as `Unknown` naming the + restriction, and the new `holonomic::boundary_verdict` returns the same + verdict with `BoundaryVerdict::valid_from` and + `BoundaryVerdict::certificate_poles`. Verdicts that were already right are + unchanged, including the `0·∞` endpoint continuation `C(n,k)/(n−k+1)` over + `k = 0..n` (a certificate pole exactly at `k = k_hi+1`, cancelled by a zero of + the summand) and negative-slope lower limits. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-core/src/holonomic/boundary.rs b/alkahest-core/src/holonomic/boundary.rs index a7289fdd..0b404aa1 100644 --- a/alkahest-core/src/holonomic/boundary.rs +++ b/alkahest-core/src/holonomic/boundary.rs @@ -71,6 +71,29 @@ //! arithmetic to something other than zero. Sampling that finds only zeros //! proves nothing, and yields `Unknown` — never `Vanishes`. //! +//! # The domain a verdict is a theorem on +//! +//! A bare [`BoundaryStatus`] carries an implied "for every `n`", and that +//! quantifier is not always true. Two things bound it, and +//! [`boundary_verdict`] returns both rather than leaving them implicit: +//! +//! * **The declared range has to be a range.** `k = κ₀(n)..κ₁(n)` with +//! `κ₁(n) < κ₀(n) − 1` runs *backwards*: `Σ` over it is `0` under the "empty +//! sum" reading and `−Σ_{κ₁+1}^{κ₀−1}` under the reversed-sum reading, and the +//! telescoping above silently uses the second. Every piece of `b(n)` is then a +//! statement about a sum the caller did not write, and the verdict is false at +//! exactly those `n` — including for an ordinary `k = 3..n−3`, which is +//! backwards at `n = 3, 4`. [`BoundaryVerdict::valid_from`] is the smallest +//! `n` from which the range runs forwards; a range that runs backwards at +//! *every* `n`, or at every large `n`, gets [`BoundaryStatus::Unknown`] rather +//! than a verdict. +//! * **`G` has to be finite at every integer `k` in the range.** The telescoping +//! collapses `Σ_k (G(n,k+1) − G(n,k))` term by term, so a pole of the +//! certificate at an interior integer `k` — `k = (n+3)/2` for +//! `C(n,k)/(n−2k+1)` — breaks it in the middle, not at the two endpoints this +//! analysis evaluates. [`BoundaryVerdict::certificate_poles`] reports those +//! points, and the verdict is `Unknown` whenever there are any. +//! //! # The residual hypothesis //! //! `q ∈ Q(n)` may have poles, and a `Γ` argument may land on a non-positive @@ -164,6 +187,93 @@ impl BoundaryStatus { } } +/// A [`BoundaryStatus`] together with the set of `n` it is claimed for. +/// +/// The bare status is a verdict with an implied universal quantifier over `n`. +/// That quantifier is what made a `Nonzero` verdict false on `k = 5..3` and on +/// `k = 3..n−3` at `n = 3, 4`, so it is stated here instead: see the +/// [module documentation](self) for the two things that bound it. +/// +/// Returned by [`boundary_verdict`]. Non-exhaustive so that a further +/// restriction can be added without breaking callers. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BoundaryVerdict { + /// The verdict, over the `n` this struct's other fields delimit. + pub status: BoundaryStatus, + /// The verdict is claimed for integers `n ≥ n_min` and for no smaller `n`, + /// because the declared range runs backwards below it. + /// + /// `None` when the range runs forwards — or is exactly empty, `κ₁ = κ₀ − 1`, + /// which both readings agree is `0` — at *every* integer `n`, so nothing is + /// excluded on that ground. + /// + /// It bounds a verdict, so it only says anything next to one: a + /// [`BoundaryStatus::Unknown`] claims nothing at any `n`, whatever this + /// field happens to hold. + pub valid_from: Option, + /// Integer points `k` inside the declared range at which the telescoped + /// `G = R·F`, or the summand `F` itself, could not be shown to be finite, + /// as expressions in `n`. + /// + /// A pole of the certificate here breaks the telescoping at an *interior* + /// point rather than at an endpoint, which is why it cannot be read off the + /// two boundary values. Non-empty therefore implies that [`status`] is + /// [`BoundaryStatus::Unknown`]. + /// + /// [`status`]: BoundaryVerdict::status + pub certificate_poles: Vec, +} + +impl BoundaryVerdict { + /// `"vanishes"`, `"nonzero"` or `"unknown"` — the stable tag to record. + pub fn tag(&self) -> &'static str { + self.status.tag() + } + + /// Whether a recurrence for the *sum* may be read off at all, on + /// [`valid_from`](BoundaryVerdict::valid_from) and above. + pub fn implies_sum_recurrence(&self) -> bool { + self.status.implies_sum_recurrence() + } + + /// What is still assumed after this verdict, as plain strings: the status's + /// own conditions, then the domain, then the interior poles. + pub fn side_conditions(&self, range: &str, pool: &ExprPool) -> Vec { + let mut out = self.status.side_conditions(range); + if let Some(n_min) = self.valid_from { + out.push(format!( + "this verdict is claimed for integers n >= {n_min} only: the range {range} runs \ + backwards below that, where a sum over it is 0 under one reading and a signed \ + sum under the other, and the relation above is FALSE" + )); + } + if !self.certificate_poles.is_empty() { + let list: Vec = self + .certificate_poles + .iter() + .map(|&p| pool.display(p).to_string()) + .collect(); + out.push(format!( + "the telescoped G(n,k) = R(n,k)*F(n,k) was not shown to be finite at the interior \ + point(s) k = {} of {range}, so the telescoping breaks inside the range and not \ + at an endpoint", + list.join(", ") + )); + } + out + } + + /// The verdict for a range this analysis declined to place at all. + fn unknown(reason: String) -> Self { + BoundaryVerdict { + status: BoundaryStatus::Unknown { reason }, + valid_from: None, + certificate_poles: Vec::new(), + } + } +} + /// The summation range `k = 0 .. n` — the convention `Σ_{k=0}^{n}` that the /// classical identities and the OEIS formula field both use. /// @@ -186,6 +296,12 @@ pub fn natural_limits(n: ExprId, pool: &ExprPool) -> (ExprId, ExprId) { /// Anything else — a second symbol, an infinity, a non-integer offset — is /// reported as [`BoundaryStatus::Unknown`], which is the honest answer for a /// range this analysis cannot place. +/// +/// This returns the verdict *without* its domain, so it **fails safe**: when the +/// declared range runs backwards at some `n ≥ 0` the verdict is not a theorem +/// there, and a shape with nowhere to say so reports [`BoundaryStatus::Unknown`] +/// instead. [`boundary_verdict`] returns the same verdict with the domain +/// attached, and keeps it where it is valid. pub fn boundary_status( result: &ZeilbergerResult, term: ExprId, @@ -194,54 +310,141 @@ pub fn boundary_status( limits: Option<(ExprId, ExprId)>, pool: &ExprPool, ) -> BoundaryStatus { - match collect_terms(result, term, n, k, limits, pool) { - Err(reason) => BoundaryStatus::Unknown { reason }, - Ok(terms) => decide(&terms, n, pool), + let verdict = boundary_verdict(result, term, n, k, limits, pool); + match verdict.valid_from { + Some(n_min) if n_min > 0 && verdict.status.implies_sum_recurrence() => { + BoundaryStatus::Unknown { + reason: format!( + "the declared summation range runs backwards for n < {n_min}, where the \ + relation is false; the verdict holds from n = {n_min} on and \ + `boundary_verdict` returns it with that domain attached" + ), + } + } + _ => verdict.status, } } -/// Assemble every signed piece of `b(n)`; see the module docs for the formula. -fn collect_terms( +/// Decide the boundary hypothesis, with the domain of `n` the verdict is a +/// theorem on and any interior pole that breaks the telescoping. +/// +/// Same analysis and same arguments as [`boundary_status`]; the difference is +/// that nothing about the domain has to be dropped to fit the return type. See +/// [`BoundaryVerdict`]. +pub fn boundary_verdict( result: &ZeilbergerResult, term: ExprId, n: ExprId, k: ExprId, limits: Option<(ExprId, ExprId)>, pool: &ExprPool, -) -> Result, String> { - let Some((lo, hi)) = limits else { - return Err( - "the summation limits were not supplied, so there is no range over which \ - to evaluate the boundary; pass (k_lo, k_hi) — the usual choice is \ - k = 0..n" - .into(), - ); +) -> BoundaryVerdict { + let setup = match Setup::new(result, term, n, k, limits, pool) { + Err(reason) => return BoundaryVerdict::unknown(reason), + Ok(s) => s, }; - let f = ProperTerm::parse(term, n, k, pool) - .map_err(|_| "the summand did not re-parse as a proper hypergeometric term".to_string())?; - let r = as_ratk(result.certificate, n, k, pool, 0) - .ok_or("the certificate did not re-parse as an element of Q(n)(k)")?; + // Where the declared range is a range at all. + let valid_from = match range_domain(setup.lo, setup.hi) { + Err(reason) => return BoundaryVerdict::unknown(reason), + Ok(v) => v, + }; - let lo_pt = endpoint_point(lo, n, k, pool).map_err(|e| format!("lower limit k_lo: {e}"))?; - let hi_pt = endpoint_point(hi, n, k, pool).map_err(|e| format!("upper limit k_hi: {e}"))?; + // Where `G` is finite: an interior pole breaks the telescoping in the middle + // of the range, which no boundary value can detect. + let poles = interior_poles(&setup); + if !poles.is_empty() { + let list: Vec = poles.iter().map(|p| p.to_string()).collect(); + return BoundaryVerdict { + status: BoundaryStatus::Unknown { + reason: format!( + "the telescoped G(n,k) = R(n,k)*F(n,k) was not shown to be finite at the \ + interior point(s) k = {}, which are integers inside the declared range for \ + infinitely many n; the telescoping breaks there and not at an endpoint", + list.join(", ") + ), + }, + valid_from, + certificate_poles: poles.iter().map(|p| p.to_expr(pool, n)).collect(), + }; + } - let order = result.order; - let extras = (lo_pt.alpha.unsigned_abs() + hi_pt.alpha.unsigned_abs()) * order as u64; - if extras > MAX_CORRECTION_TERMS { - return Err(format!( - "the summation limits move with n fast enough to need {extras} correction terms, \ - past the supported limit of {MAX_CORRECTION_TERMS}" - )); + let status = match collect_terms(&setup, result, n, k, pool) { + Err(reason) => BoundaryStatus::Unknown { reason }, + Ok(terms) => decide(&terms, n, valid_from.unwrap_or(1).max(1), pool), + }; + BoundaryVerdict { + status, + valid_from, + certificate_poles: Vec::new(), } +} +/// The parsed form every stage of the verdict works from: the summand, the +/// certificate, and the two limits as `α·n + β`. +struct Setup { + f: ProperTerm, + r: RatK, + lo: Point, + hi: Point, +} + +impl Setup { + fn new( + result: &ZeilbergerResult, + term: ExprId, + n: ExprId, + k: ExprId, + limits: Option<(ExprId, ExprId)>, + pool: &ExprPool, + ) -> Result { + let Some((lo, hi)) = limits else { + return Err( + "the summation limits were not supplied, so there is no range over which \ + to evaluate the boundary; pass (k_lo, k_hi) — the usual choice is \ + k = 0..n" + .into(), + ); + }; + + let f = ProperTerm::parse(term, n, k, pool).map_err(|_| { + "the summand did not re-parse as a proper hypergeometric term".to_string() + })?; + let r = as_ratk(result.certificate, n, k, pool, 0) + .ok_or("the certificate did not re-parse as an element of Q(n)(k)")?; + + let lo = endpoint_point(lo, n, k, pool).map_err(|e| format!("lower limit k_lo: {e}"))?; + let hi = endpoint_point(hi, n, k, pool).map_err(|e| format!("upper limit k_hi: {e}"))?; + + let order = result.order; + let extras = (lo.alpha.unsigned_abs() + hi.alpha.unsigned_abs()) * order as u64; + if extras > MAX_CORRECTION_TERMS { + return Err(format!( + "the summation limits move with n fast enough to need {extras} correction terms, \ + past the supported limit of {MAX_CORRECTION_TERMS}" + )); + } + Ok(Setup { f, r, lo, hi }) + } +} + +/// Assemble every signed piece of `b(n)`; see the module docs for the formula. +fn collect_terms( + setup: &Setup, + result: &ZeilbergerResult, + n: ExprId, + k: ExprId, + pool: &ExprPool, +) -> Result, String> { + let (f, r, lo_pt, hi_pt) = (&setup.f, &setup.r, setup.lo, setup.hi); + let order = result.order; let mut terms: Vec = Vec::new(); // The telescoped part: + G(n, k_hi+1) − G(n, k_lo). let at_hi = hi_pt.offset(1); - push_value(&mut terms, value_at(&r, &f, 0, at_hi), &rn_one()) + push_value(&mut terms, value_at(r, f, 0, at_hi), &rn_one()) .map_err(|e| format!("G(n, k_hi+1) could not be evaluated: {e}"))?; - push_value(&mut terms, value_at(&r, &f, 0, lo_pt), &rn_neg(&rn_one())) + push_value(&mut terms, value_at(r, f, 0, lo_pt), &rn_neg(&rn_one())) .map_err(|e| format!("G(n, k_lo) could not be evaluated: {e}"))?; // The range-shift corrections: Σ_i a_i(n)·D_i(n). @@ -257,14 +460,14 @@ fn collect_terms( for (t, sign) in signed_window(1, hi_pt.alpha * i64_i) { let weight = scale_sign(&a_i, sign); let at = hi_pt.offset(t); - push_value(&mut terms, value_at(&one, &f, i64_i, at), &weight) + push_value(&mut terms, value_at(&one, f, i64_i, at), &weight) .map_err(|e| format!("the upper range-shift correction failed: {e}"))?; } // Lower window: Σ_{k=κ₀(n+i)}^{κ₀(n)−1} F(n+i, k). for (t, sign) in signed_window(lo_pt.alpha * i64_i, -1) { let weight = scale_sign(&a_i, sign); let at = lo_pt.offset(t); - push_value(&mut terms, value_at(&one, &f, i64_i, at), &weight) + push_value(&mut terms, value_at(&one, f, i64_i, at), &weight) .map_err(|e| format!("the lower range-shift correction failed: {e}"))?; } } @@ -272,8 +475,9 @@ fn collect_terms( } /// Group the pieces into hypergeometric-similarity classes and read off the -/// verdict. -fn decide(terms: &[HypTerm], n: ExprId, pool: &ExprPool) -> BoundaryStatus { +/// verdict. `witness_from` is the smallest `n` a `Nonzero` witness may be taken +/// at, so that the witness lies in the domain the verdict is claimed on. +fn decide(terms: &[HypTerm], n: ExprId, witness_from: i64, pool: &ExprPool) -> BoundaryStatus { let mut classes: Vec<(Rational, Vec, Rn)> = Vec::new(); for t in terms { let Some((coeff, base, sig)) = t.canonical() else { @@ -295,7 +499,7 @@ fn decide(terms: &[HypTerm], n: ExprId, pool: &ExprPool) -> BoundaryStatus { // Otherwise b(n) is explicit — but "the classes did not cancel" is not a // proof that b ≢ 0, because two classes can still be equal as functions. // Reporting `Nonzero` needs a witness. - match nonzero_witness(terms) { + match nonzero_witness(terms, witness_from) { Some(witness_n) => BoundaryStatus::Nonzero { // Report the *collected* classes: one summand per similarity class, // with the Γ ladder already worked off, which is both shorter and @@ -363,6 +567,254 @@ fn endpoint_point(e: ExprId, n: ExprId, k: ExprId, pool: &ExprPool) -> Result Result, String> { + // κ₁(n) − κ₀(n) = d·n + e, and the range runs backwards where d·n + e < −1. + let (Some(d), Some(e)) = (hi.alpha.checked_sub(lo.alpha), hi.beta.checked_sub(lo.beta)) else { + return Err("the summation limits are too large to compare".into()); + }; + let backwards = "the declared summation range runs backwards there (k_hi < k_lo - 1), where a \ + sum over it is 0 under the empty-sum reading and a signed sum under the \ + reversed-sum reading, so no recurrence for it is claimed"; + match d.cmp(&0) { + std::cmp::Ordering::Equal => { + if e >= -1 { + Ok(None) + } else { + Err(format!("the range is empty at every n: {backwards}")) + } + } + std::cmp::Ordering::Greater => { + // d·n + e ≥ −1 ⟺ n ≥ ⌈(−1 − e)/d⌉. + let num = (-1_i64) + .checked_sub(e) + .ok_or("the summation limits are too large to compare")?; + Ok(Some(ceil_div(num, d))) + } + std::cmp::Ordering::Less => Err(format!( + "the range is empty for every sufficiently large n: {backwards}" + )), + } +} + +/// `⌈a/b⌉` for `b > 0`. +fn ceil_div(a: i64, b: i64) -> i64 { + let q = a.div_euclid(b); + if a.rem_euclid(b) == 0 { + q + } else { + q + 1 + } +} + +// --------------------------------------------------------------------------- +// Poles inside the range +// --------------------------------------------------------------------------- + +/// A location `k = s·n + t` with `s, t ∈ Q` — where a pole of `G` or of `F` sits. +#[derive(Clone, Debug, PartialEq, Eq)] +struct PolePoint { + s: Rational, + t: Rational, +} + +impl PolePoint { + fn as_rn(&self) -> Rn { + rn_add( + &rn_mul(&rn_rat(self.s.clone()), &rn_var()), + &rn_rat(self.t.clone()), + ) + } + + /// `Some(point)` when the location is `α·n + β` with integer `α, β`, which + /// is what [`value_at`] can do order counting at. + fn as_integer_point(&self) -> Option { + if *self.s.clone().denom() != 1 || *self.t.clone().denom() != 1 { + return None; + } + Some(Point { + alpha: self.s.numer().to_i64()?, + beta: self.t.numer().to_i64()?, + }) + } + + /// Whether `s·n + t` is an integer for infinitely many integer `n`: with + /// `k = (P·n + Q)/c`, `P·n ≡ −Q (mod c)` is solvable exactly when + /// `gcd(P, c) | Q`. A location that is never an integer is never summed at. + fn hits_integers(&self) -> bool { + let c = Integer::from(self.s.denom().lcm_ref(self.t.denom())); + let p = self.s.numer().clone() * (c.clone() / self.s.denom().clone()); + let q = self.t.numer().clone() * (c.clone() / self.t.denom().clone()); + (q % Integer::from(p.gcd_ref(&c))) == 0 + } + + /// Whether `κ₀(n) ≤ s·n + t ≤ κ₁(n)` for every sufficiently large `n`. + fn inside(&self, lo: Point, hi: Point) -> bool { + let (a_lo, b_lo) = (Rational::from(lo.alpha), Rational::from(lo.beta)); + let (a_hi, b_hi) = (Rational::from(hi.alpha), Rational::from(hi.beta)); + (self.s > a_lo || (self.s == a_lo && self.t >= b_lo)) + && (self.s < a_hi || (self.s == a_hi && self.t <= b_hi)) + } + + fn to_expr(&self, pool: &ExprPool, n: ExprId) -> ExprId { + rn_to_expr(pool, n, &self.as_rn()) + } +} + +impl std::fmt::Display for PolePoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.s == 0 { + write!(f, "{}", self.t) + } else if self.t == 0 { + write!(f, "{}*n", self.s) + } else { + write!(f, "{}*n + {}", self.s, self.t) + } + } +} + +/// Where the search for interior poles samples, and how fine its grid is. +/// +/// A location is looked for at two consecutive integers `n`, far enough out that +/// one which is only *eventually* inside the range is already inside it there, +/// and near enough that the grid over the range stays short. A location with a +/// denominator past [`POLE_MAX_DEN`], one that enters the range only for `n` past +/// the samples, or one that is inside it for finitely many `n` only, is not +/// found: this is a search for the poles that bite, and finding none is not a +/// proof that there are none. +const POLE_SAMPLE_N: i64 = 12; +const POLE_MAX_DEN: i64 = 4; +const POLE_SLACK: i64 = 4; + +/// The largest grid this will scan before giving up on the search. +const POLE_MAX_GRID: i64 = 4096; + +/// Integer points inside the declared range at which the telescoped `G = R·F` — +/// or the summand `F` on its own, which would leave `S(n)` undefined — could not +/// be shown to be finite. +/// +/// The endpoints `κ₀` and `κ₁+1` are not what this is for: [`collect_terms`] +/// evaluates `G` at both and refuses when either is unbounded. What this adds is +/// the interior, where a pole is invisible to the two boundary values and breaks +/// the telescoping in the middle of the sum. +fn interior_poles(setup: &Setup) -> Vec { + let g = setup.r.mul(&setup.f.rat).normalize(); + let mut out: Vec = Vec::new(); + for den in [&setup.f.rat.den, &g.den] { + for cand in affine_roots(den, setup.lo, setup.hi) { + if out.contains(&cand) || !cand.hits_integers() || !cand.inside(setup.lo, setup.hi) { + continue; + } + // A pole of the rational part that a Γ zero cancels is no pole of the + // term; order counting settles that exactly, when the location is + // integer-affine in n. When it is not — `k = (n+3)/2` — this analysis + // cannot place it and says so rather than assuming it away. + let regular = match cand.as_integer_point() { + Some(at) => { + is_finite(value_at(&setup.r, &setup.f, 0, at)) + && is_finite(value_at(&RatK::one(), &setup.f, 0, at)) + } + None => false, + }; + if !regular { + out.push(cand); + } + } + } + out +} + +fn is_finite(v: Value) -> bool { + matches!(v, Value::Zero | Value::Finite(_)) +} + +/// Roots of `den ∈ Q(n)[k]` of the form `k = s·n + t` with `s, t ∈ Q`. +/// +/// Found by sampling — the rational roots of `den(n₀, ·)` on a bounded grid at +/// two consecutive `n₀`, which pins `s` and `t` — and then **verified exactly** +/// over `Q(n)`, so a returned root is a root and not a numerical coincidence. +fn affine_roots(den: &PolyK, lo: Point, hi: Point) -> Vec { + if den.degree() < 1 || lo.alpha > hi.alpha { + return Vec::new(); + } + let (Some(at0), Some(at1)) = ( + specialize(den, POLE_SAMPLE_N), + specialize(den, POLE_SAMPLE_N + 1), + ) else { + return Vec::new(); + }; + let w_lo = lo.alpha * POLE_SAMPLE_N + lo.beta - POLE_SLACK; + let w_hi = hi.alpha * POLE_SAMPLE_N + hi.beta + POLE_SLACK; + if w_hi < w_lo || (w_hi - w_lo).saturating_mul(POLE_MAX_DEN) > POLE_MAX_GRID { + return Vec::new(); + } + + let mut out: Vec = Vec::new(); + for c in 1..=POLE_MAX_DEN { + for a in (w_lo * c)..=(w_hi * c) { + let v0 = Rational::from((a, c)); + if poly_at(&at0, &v0) != 0 { + continue; + } + // Only a slope that keeps the location inside the range for large n + // can matter, which is what makes this grid small. + for sc in 1..=POLE_MAX_DEN { + for sn in (lo.alpha * sc)..=(hi.alpha * sc) { + let s = Rational::from((sn, sc)); + if poly_at(&at1, &(v0.clone() + s.clone())) != 0 { + continue; + } + let cand = PolePoint { + t: v0.clone() - s.clone() * Rational::from(POLE_SAMPLE_N), + s, + }; + // Exact over Q(n): the sampling only proposed it. + if !out.contains(&cand) && rn_is_zero(&poly_eval(den, &cand.as_rn())) { + out.push(cand); + } + } + } + } + } + out +} + +/// `p ∈ Q(n)[k]` at `n = m`, or `None` when a coefficient has a pole there or +/// the whole polynomial degenerates. +fn specialize(p: &PolyK, m: i64) -> Option> { + let d = p.degree(); + if d < 0 { + return None; + } + let m = Rational::from(m); + let coeffs: Vec = (0..=d as usize) + .map(|i| rn_eval(&p.coeff(i), &m)) + .collect::>>()?; + if coeffs.iter().all(|c| *c == 0) { + return None; + } + Some(coeffs) +} + +/// Horner evaluation of a specialised polynomial. +fn poly_at(coeffs: &[Rational], x: &Rational) -> Rational { + let mut acc = Rational::from(0); + for c in coeffs.iter().rev() { + acc = acc * x.clone() + c.clone(); + } + acc +} + /// The integer offsets in `Σ_{t=from}^{to}`, with `Σ_{t=from}^{to} = −Σ_{t=to+1}^{from−1}` /// when the range runs backwards — so a limit that *decreases* with `n` is /// handled with the right sign rather than silently dropped. @@ -660,10 +1112,10 @@ impl HypTerm { /// Small on purpose: a witness is normally found at the first usable point, and /// the values involve factorials of `O(n)`. Finding none is reported as /// `Unknown`, never as "it vanishes". -const WITNESS_POINTS: std::ops::RangeInclusive = 1..=16; +const WITNESS_POINTS: i64 = 16; -fn nonzero_witness(terms: &[HypTerm]) -> Option { - 'points: for n0 in WITNESS_POINTS { +fn nonzero_witness(terms: &[HypTerm], from: i64) -> Option { + 'points: for n0 in from..from.saturating_add(WITNESS_POINTS) { let mut acc = Rational::from(0); for t in terms { let Some(v) = t.eval(n0) else { @@ -1066,6 +1518,233 @@ mod tests { assert_eq!(signed_window(-1, -1), vec![(-1, 1)]); } + // ----------------------------------------------------------------------- + // The domain a verdict is claimed on (issue: false verdicts on empty ranges) + // ----------------------------------------------------------------------- + + fn full_verdict( + f: ExprId, + n: ExprId, + k: ExprId, + pool: &ExprPool, + limits: (ExprId, ExprId), + ) -> BoundaryVerdict { + let cert = zeilberger(f, n, k, pool, &ZeilbergerOpts::default()).expect("certificate"); + boundary_verdict(&cert.value, f, n, k, Some(limits), pool) + } + + /// `F = C(n,k)²` over a constant range that runs backwards — `k = 5..3` and + /// friends. Every `S(n)` is `0`, so no `b(n) ≢ 0` can be right; the engine + /// used to return `"nonzero"` with an order-9 polynomial whose residual at + /// `n = 3..9` was `4, 107, 800, 2725, 2450, −23716, −162288`. + #[test] + fn a_range_that_runs_backwards_is_never_a_recurrence() { + for (lo, hi) in [(5, 3), (3, 1), (2, 0), (4, 2)] { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let c = binom(&pool, n, k); + let f = pool.mul(vec![c, c]); + let limits = (pool.integer(lo), pool.integer(hi)); + let status = verdict(f, n, k, &pool, Some(limits)); + let BoundaryStatus::Unknown { reason } = &status else { + panic!("k = {lo}..{hi} is empty: expected unknown, got {status:?}"); + }; + assert!(reason.contains("backwards"), "{reason}"); + assert!(!status.implies_sum_recurrence()); + + let full = full_verdict(f, n, k, &pool, limits); + assert_eq!(full.tag(), "unknown"); + // Nothing is claimed at any n, so the bound has nothing to bound. + assert_eq!(full.valid_from, None); + } + } + + /// The realistic form: `k = 3..n−3` is empty at `n = 3, 4` and a range from + /// `n = 5` on. The verdict is kept — it is a theorem for large `n` — but it + /// is carried with the domain, and the bare-status entry point refuses. + #[test] + fn an_n_dependent_range_carries_the_n_it_is_claimed_for() { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let f = binom(&pool, n, k); + let three = pool.integer(3_i32); + let limits = (three, pool.add(vec![n, pool.integer(-3_i32)])); + + let full = full_verdict(f, n, k, &pool, limits); + assert_eq!( + full.valid_from, + Some(5), + "k = 3..n-3 runs backwards at n = 3, 4 and is a range from n = 5 on: {:?}", + full.status + ); + assert!(full.implies_sum_recurrence(), "got {:?}", full.status); + let conds = full.side_conditions("k = 3..n - 3", &pool); + assert!( + conds + .iter() + .any(|c| c.contains("n >= 5") && c.contains("FALSE")), + "the domain must be stated: {conds:?}" + ); + + // A shape with nowhere to put the domain must not imply a recurrence. + let status = verdict(f, n, k, &pool, Some(limits)); + let BoundaryStatus::Unknown { reason } = &status else { + panic!("expected unknown from the bare status, got {status:?}"); + }; + assert!(reason.contains("n < 5"), "{reason}"); + } + + /// `k = 0..−1` and `k = n+1..n` are empty too, and were already answered + /// `"vanishes"` — correctly, because `κ₁ = κ₀ − 1` is the one empty range + /// both readings agree is `0`. The fix must not make the two treatments + /// disagree in the other direction. + #[test] + fn an_exactly_empty_range_is_still_a_proved_zero() { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let c = binom(&pool, n, k); + let f = pool.mul(vec![c, c]); + let cases = [ + (pool.integer(0_i32), pool.integer(-1_i32)), + (pool.add(vec![n, pool.integer(1_i32)]), n), + ]; + for limits in cases { + let full = full_verdict(f, n, k, &pool, limits); + assert_eq!(full.status, BoundaryStatus::Vanishes, "{:?}", full.status); + assert_eq!( + full.valid_from, None, + "no n is excluded by an adjacent-empty range" + ); + assert!(full.certificate_poles.is_empty()); + assert_eq!(verdict(f, n, k, &pool, Some(limits)).tag(), "vanishes"); + } + } + + /// `C(n,k)/(n−2k+1)` over `k = 0..n`: the certificate has a pole at + /// `k = (n+3)/2`, an integer strictly inside the range for every odd `n`, so + /// the telescoping breaks in the *middle* of the sum. The old verdict was + /// `"vanishes"` on a sum that is not even defined for odd `n`. + #[test] + fn an_interior_certificate_pole_is_reported_and_refused() { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let one = pool.integer(1_i32); + let den = pool.add(vec![n, pool.mul(vec![pool.integer(-2_i32), k]), one]); + let f = pool.mul(vec![ + binom(&pool, n, k), + pool.pow(den, pool.integer(-1_i32)), + ]); + let full = full_verdict(f, n, k, &pool, natural_limits(n, &pool)); + assert_eq!(full.tag(), "unknown", "{:?}", full.status); + assert!( + !full.certificate_poles.is_empty(), + "the interior pole must be reported, not merely refused" + ); + let shown: Vec = full + .certificate_poles + .iter() + .map(|&p| pool.display(p).to_string()) + .collect(); + assert!( + shown.iter().any(|s| s.contains("/2") || s.contains("1/2")), + "expected a half-integer location, got {shown:?}" + ); + let conds = full.side_conditions("k = 0..n", &pool); + assert!(conds.iter().any(|c| c.contains("interior")), "{conds:?}"); + } + + /// `C(n,k)/(k−3)` over `k = 0..n`: the *summand* is undefined at `k = 3`, so + /// `S(n)` does not exist for `n ≥ 3`. The verdict was `"vanishes"` — the same + /// string a genuine theorem gets. + #[test] + fn a_summand_pole_inside_the_range_is_refused() { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let den = pool.add(vec![k, pool.integer(-3_i32)]); + let f = pool.mul(vec![ + binom(&pool, n, k), + pool.pow(den, pool.integer(-1_i32)), + ]); + let full = full_verdict(f, n, k, &pool, natural_limits(n, &pool)); + assert_eq!(full.tag(), "unknown", "{:?}", full.status); + let shown: Vec = full + .certificate_poles + .iter() + .map(|&p| pool.display(p).to_string()) + .collect(); + assert_eq!(shown, vec!["3".to_string()], "got {shown:?}"); + } + + /// Cases the analysis got right before and must keep getting right: a + /// negative-slope lower limit that cannot be placed stays `"unknown"`, one + /// that can stays `"nonzero"`, and a `0·∞` endpoint continuation — a pole of + /// the certificate exactly at `k = κ₁+1`, cancelled by a zero of `F` — stays + /// `"nonzero"` rather than being caught by the interior-pole search. + #[test] + fn the_verdicts_that_were_already_right_stay_right() { + let pool = ExprPool::new(); + let (n, k) = nk(&pool); + let c = binom(&pool, n, k); + let sq = pool.mul(vec![c, c]); + let neg_n = pool.mul(vec![pool.integer(-1_i32), n]); + + // [-n..n]: honest "unknown" (too many correction terms), not "vanishes". + let wide = full_verdict(sq, n, k, &pool, (neg_n, n)); + assert_eq!(wide.tag(), "unknown", "{:?}", wide.status); + + // [-n..0]: the boundary really does not vanish. + let half = full_verdict(c, n, k, &pool, (neg_n, pool.integer(0_i32))); + assert_eq!(half.tag(), "nonzero", "{:?}", half.status); + + // C(n,k)/(n-k+1) over 0..n: R has a pole at k = n+1 = κ₁+1 and F a zero + // there, so the endpoint value is finite and the verdict stands. + let one = pool.integer(1_i32); + let den = pool.add(vec![n, pool.mul(vec![pool.integer(-1_i32), k]), one]); + let g = pool.mul(vec![c, pool.pow(den, pool.integer(-1_i32))]); + let cont = full_verdict(g, n, k, &pool, natural_limits(n, &pool)); + assert_eq!(cont.tag(), "nonzero", "{:?}", cont.status); + assert!(cont.certificate_poles.is_empty()); + } + + /// The domain arithmetic on its own: `κ₁ − κ₀ = d·n + e ≥ −1`. + #[test] + fn the_range_domain_is_the_first_n_that_is_not_backwards() { + let pt = |alpha, beta| Point { alpha, beta }; + // k = 0..n is a range from n = -1 on. + assert_eq!(range_domain(pt(0, 0), pt(1, 0)), Ok(Some(-1))); + // k = 3..n-3 is backwards at n = 3, 4. + assert_eq!(range_domain(pt(0, 3), pt(1, -3)), Ok(Some(5))); + // Adjacent-empty and single-point constant ranges exclude nothing. + assert_eq!(range_domain(pt(1, 1), pt(1, 0)), Ok(None)); + assert_eq!(range_domain(pt(0, 0), pt(0, 0)), Ok(None)); + // Backwards everywhere, and backwards for every large n. + assert!(range_domain(pt(0, 5), pt(0, 3)).is_err()); + assert!(range_domain(pt(1, 0), pt(0, 0)).is_err()); + // A steeper upper limit reaches the lower one sooner: 2n-3 >= 4-1 at n = 3. + assert_eq!(range_domain(pt(0, 4), pt(2, -3)), Ok(Some(3))); + } + + /// A location that is never an integer is never summed at, and one that is + /// outside the range does not break a telescoping inside it. + #[test] + fn only_integer_locations_inside_the_range_count() { + let p = |s: (i64, i64), t: (i64, i64)| PolePoint { + s: Rational::from(s), + t: Rational::from(t), + }; + // (n+1)/2 is an integer for odd n; n/2 + 1/3 never is. + assert!(p((1, 2), (1, 2)).hits_integers()); + assert!(!p((1, 2), (1, 3)).hits_integers()); + assert!(p((0, 1), (3, 1)).hits_integers()); + + let (lo, hi) = (Point { alpha: 0, beta: 0 }, Point { alpha: 1, beta: 0 }); + assert!(p((1, 2), (1, 2)).inside(lo, hi)); + assert!(p((0, 1), (3, 1)).inside(lo, hi)); + // k = n+1 is the endpoint κ₁+1, which the boundary values already cover. + assert!(!p((1, 1), (1, 1)).inside(lo, hi)); + assert!(!p((0, 1), (-1, 1)).inside(lo, hi)); + } + /// `⌊·⌋`, not truncation — the `Γ` ladder is wrong by one for negative /// arguments otherwise. #[test] diff --git a/alkahest-core/src/holonomic/mod.rs b/alkahest-core/src/holonomic/mod.rs index 1285a766..38ca8fad 100644 --- a/alkahest-core/src/holonomic/mod.rs +++ b/alkahest-core/src/holonomic/mod.rs @@ -76,7 +76,9 @@ pub use asymptotics::{ asymptotics_from_recurrence, CharacteristicAnalysis, CharacteristicRoot, ConnectionConstant, PerronVerdict, RecurrenceAsymptotics, }; -pub use boundary::{boundary_status, natural_limits, BoundaryStatus}; +pub use boundary::{ + boundary_status, boundary_verdict, natural_limits, BoundaryStatus, BoundaryVerdict, +}; pub use hyperterm::{GammaFactor, ProperTerm}; pub use modular::{binomial_mod, ModularError, ModularEvaluation, ModularRecurrence}; pub use qfield::{PolyK, RatK, Rn}; diff --git a/alkahest-core/src/holonomic/zeilberger.rs b/alkahest-core/src/holonomic/zeilberger.rs index 8619f470..d5dfae36 100644 --- a/alkahest-core/src/holonomic/zeilberger.rs +++ b/alkahest-core/src/holonomic/zeilberger.rs @@ -34,9 +34,11 @@ //! **[`super::boundary::boundary_status`] decides it** over a stated summation //! range, three-valued: proved to vanish, proved nonzero (with the //! inhomogeneity `b(n)` explicit), or undecided — in which case nothing about -//! the sum may be claimed. [`boundary_term`] still returns `G(n,k)` for a caller -//! who would rather discharge the hypothesis by hand, and -//! [`boundary_side_condition`] states it in words. +//! the sum may be claimed. [`super::boundary::boundary_verdict`] returns the +//! same verdict with the integers `n` it is a theorem on, which a declared range +//! that is empty at small `n` makes a real restriction. [`boundary_term`] still +//! returns `G(n,k)` for a caller who would rather discharge the hypothesis by +//! hand, and [`boundary_side_condition`] states it in words. //! //! # Method //! diff --git a/alkahest-py/src/lib.rs b/alkahest-py/src/lib.rs index 49e65f17..3acc50c4 100644 --- a/alkahest-py/src/lib.rs +++ b/alkahest-py/src/lib.rs @@ -173,11 +173,11 @@ use alkahest_core::real::sos::{ }; // P1 item 7 — creative telescoping / holonomic (D-finite) machinery use alkahest_core::holonomic::{ - boundary_status as core_boundary_status, boundary_term as core_boundary_term, + boundary_term as core_boundary_term, boundary_verdict as core_boundary_verdict, natural_limits as core_natural_limits, zeilberger_search as core_zeilberger_search, - BoundaryStatus as CoreBoundaryStatus, HolonomicError as CoreHolonomicError, - OrderSearch as CoreOrderSearch, ZeilbergerOpts as CoreZeilbergerOpts, - ZeilbergerResult as CoreZeilbergerResult, + BoundaryStatus as CoreBoundaryStatus, BoundaryVerdict as CoreBoundaryVerdict, + HolonomicError as CoreHolonomicError, OrderSearch as CoreOrderSearch, + ZeilbergerOpts as CoreZeilbergerOpts, ZeilbergerResult as CoreZeilbergerResult, }; // M4(b) — q-analogue creative telescoping (q-Zeilberger) use alkahest_core::holonomic::qzeil::{ @@ -4773,6 +4773,13 @@ fn holonomic_modular_error_to_py(e: CoreHolonomicModularError) -> PyErr { /// the true relation is ``(n+2)·S(n+1) − (2n+2)·S(n) = 1``. Reading a /// recurrence off a certificate without this verdict is how a valid certificate /// becomes a false theorem. +/// +/// A verdict is also not a statement about every ``n``. The range in +/// :attr:`limits` need not *be* a range at every ``n`` — ``k = 3..n−3`` runs +/// backwards at ``n = 3, 4`` — and the telescoping needs ``G = R·F`` to be +/// finite at every integer ``k`` in it. :attr:`boundary_valid_from` and +/// :attr:`certificate_poles` carry those two bounds, rather than leaving a bare +/// ``b(n)`` with an implied "for every ``n``" attached to it. #[pyclass(name = "ZeilbergerCertificate")] struct PyZeilbergerCertificate { order: usize, @@ -4789,7 +4796,7 @@ struct PyZeilbergerCertificate { n_id: ExprId, k_id: ExprId, limits: (ExprId, ExprId), - status: CoreBoundaryStatus, + verdict: CoreBoundaryVerdict, } #[pymethods] @@ -4887,7 +4894,51 @@ impl PyZeilbergerCertificate { /// ``"unknown"`` means *no* statement about the sum may be made. #[getter] fn boundary(&self) -> &'static str { - self.status.tag() + self.verdict.tag() + } + + /// The smallest ``n`` this verdict is claimed for, or ``None``. + /// + /// A verdict is a statement about ``S(n)``, and the range in :attr:`limits` + /// is not a range at every ``n``: ``k = 3..n−3`` runs *backwards* at + /// ``n = 3`` and ``n = 4``, where a sum over it is ``0`` under one reading + /// and a signed sum under the other. The relation is false there, so those + /// ``n`` are excluded instead of being claimed — this attribute is where the + /// exclusion is recorded, and ``None`` means none was needed. + /// + /// It is a bound on ``n``, not a promise about it: the standing conditions + /// in :attr:`side_conditions` still apply above it, and it only says + /// anything next to a verdict — when :attr:`boundary` is ``"unknown"`` + /// nothing is claimed at any ``n`` regardless of what this holds. + #[getter] + fn boundary_valid_from(&self) -> Option { + self.verdict.valid_from + } + + /// Integer points ``k`` inside :attr:`limits` where the telescoping breaks, + /// as expressions in ``n``. + /// + /// ``G(n,k) = R(n,k)·F(n,k)`` has to be finite at every integer ``k`` in the + /// range for ``Σ_k (G(n,k+1) − G(n,k))`` to collapse to the two endpoints. + /// A pole of the certificate at an interior point — ``k = (n+3)/2`` for + /// ``C(n,k)/(n−2k+1)`` over ``k = 0..n``, an integer for every odd ``n`` — + /// breaks it in the *middle* of the sum, where no boundary value can see it. + /// Poles of the summand itself, which leave ``S(n)`` undefined, are listed + /// here too. + /// + /// Non-empty implies :attr:`boundary` is ``"unknown"``. Empty is not a + /// proof that there are none: locations with a denominator past ``4``, or + /// that only enter the range for large ``n``, are not searched for. + #[getter] + fn certificate_poles(&self, py: Python<'_>) -> Vec { + self.verdict + .certificate_poles + .iter() + .map(|&id| PyExpr { + id, + pool: self.pool.clone_ref(py), + }) + .collect() } /// ``b(n)`` in ``Σ_i a_i(n)·S(n+i) = b(n)``, or ``None``. @@ -4897,7 +4948,7 @@ impl PyZeilbergerCertificate { /// when it is ``"unknown"`` there is no recurrence for the sum to write down. #[getter] fn boundary_rhs(&self, py: Python<'_>) -> Option { - match &self.status { + match &self.verdict.status { CoreBoundaryStatus::Nonzero { rhs, .. } => Some(PyExpr { id: *rhs, pool: self.pool.clone_ref(py), @@ -4913,7 +4964,7 @@ impl PyZeilbergerCertificate { /// tells a caller whether to retry with a better range or close the branch. #[getter] fn boundary_reason(&self) -> String { - match &self.status { + match &self.verdict.status { CoreBoundaryStatus::Vanishes => { "the boundary difference was proved to vanish in exact arithmetic".to_string() } @@ -4931,15 +4982,16 @@ impl PyZeilbergerCertificate { /// (inhomogeneous, with :attr:`boundary_rhs`), ``False`` for ``"unknown"``. #[getter] fn implies_sum_recurrence(&self) -> bool { - self.status.implies_sum_recurrence() + self.verdict.implies_sum_recurrence() } /// Re-decide the boundary hypothesis over a different summation range. /// - /// Returns a ``dict`` with the same four keys as the attributes above — - /// ``boundary``, ``rhs``, ``reason``, ``side_conditions`` — without - /// re-running the search, which is the expensive half. Use it to ask what - /// the *same* certificate says about ``k = 0..n-1`` as well as ``k = 0..n``. + /// Returns a ``dict`` with the same keys as the attributes above — + /// ``boundary``, ``rhs``, ``reason``, ``valid_from``, ``certificate_poles``, + /// ``side_conditions`` — without re-running the search, which is the + /// expensive half. Use it to ask what the *same* certificate says about + /// ``k = 0..n-1`` as well as ``k = 0..n``. #[pyo3(signature = (k_lo, k_hi))] fn boundary_at( &self, @@ -4949,9 +5001,9 @@ impl PyZeilbergerCertificate { ) -> PyResult> { let lo = coerce_limit(py, &self.pool, k_lo, "k_lo")?; let hi = coerce_limit(py, &self.pool, k_hi, "k_hi")?; - let (status, range) = { + let (verdict, conditions) = { let pool = self.pool.borrow(py); - let status = core_boundary_status( + let verdict = core_boundary_verdict( &self.result, self.term_id, self.n_id, @@ -4960,11 +5012,12 @@ impl PyZeilbergerCertificate { &pool.inner, ); let range = format_range(&pool.inner, lo, hi); - (status, range) + let conditions = verdict.side_conditions(&range, &pool.inner); + (verdict, conditions) }; let out = PyDict::new_bound(py); - out.set_item("boundary", status.tag())?; - let rhs = match &status { + out.set_item("boundary", verdict.tag())?; + let rhs = match &verdict.status { CoreBoundaryStatus::Nonzero { rhs, .. } => Some(Py::new( py, PyExpr { @@ -4977,7 +5030,7 @@ impl PyZeilbergerCertificate { out.set_item("rhs", rhs)?; out.set_item( "reason", - match &status { + match &verdict.status { CoreBoundaryStatus::Vanishes => { "the boundary difference was proved to vanish in exact arithmetic".to_string() } @@ -4987,7 +5040,22 @@ impl PyZeilbergerCertificate { CoreBoundaryStatus::Unknown { reason } => reason.clone(), }, )?; - out.set_item("side_conditions", status.side_conditions(&range))?; + out.set_item("valid_from", verdict.valid_from)?; + let poles: Vec> = verdict + .certificate_poles + .iter() + .map(|&id| { + Py::new( + py, + PyExpr { + id, + pool: self.pool.clone_ref(py), + }, + ) + }) + .collect::>()?; + out.set_item("certificate_poles", poles)?; + out.set_item("side_conditions", conditions)?; Ok(out.unbind()) } @@ -5003,7 +5071,7 @@ impl PyZeilbergerCertificate { fn side_conditions(&self, py: Python<'_>) -> Vec { let pool = self.pool.borrow(py); let range = format_range(&pool.inner, self.limits.0, self.limits.1); - self.status.side_conditions(&range) + self.verdict.side_conditions(&range, &pool.inner) } /// Human-readable derivation log for the search that produced this. @@ -5027,7 +5095,7 @@ impl PyZeilbergerCertificate { } else { "" }, - self.status.tag(), + self.verdict.tag(), coeffs.join(", "), pool.inner.display(self.certificate_id) ) @@ -5163,7 +5231,7 @@ fn py_zeilberger( let derivation = derived.log.display_with(&pool.inner).to_string(); let report = derived.value; let boundary = core_boundary_term(&report.result, term.id, &pool.inner); - let status = core_boundary_status( + let status = core_boundary_verdict( &report.result, term.id, n.id, @@ -5195,7 +5263,7 @@ fn py_zeilberger( n_id: n.id, k_id: k.id, limits, - status, + verdict: status, }) } diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index 7df084ab..3c76a642 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -1421,7 +1421,7 @@ reg.coverage_report_markdown() # same, rendered as a Markdown table 20. **`zeilberger` does not claim its order is minimal** (since 3.9). The search visits `(order, degree)` cheapest-first, so it can reach a cheap order-2 probe before an expensive order-1 one; `cert.order_is_minimal` is `False` to say *not established*, never "a lower order exists". Pass `minimal=True` for an order-ascending search that does establish it — it costs the low-order sweep the default plan skips (Franel at `max_degree=16`: 0.23 s → 9.7 s), so claim minimality against the smallest `max_degree` you are willing to state. 21. **`guess_holonomic` returns `None` only for a swept grid** (since 3.9). It fits a P-recursive recurrence to exact `int`/`Fraction` terms, but only where the terms *over-determine* the ansatz — twice the unknowns by default — and reports `surplus_terms`, the equations that confirmed the fit without being needed. Too few terms to test the whole grid is `E-HOLO-005`, a refusal, not `None`; recording it as "not holonomic" closes a branch that was never explored. `float` terms are refused outright. -22. **A `zeilberger` certificate is about the *summand*; `cert.boundary` is what makes it about the *sum*** (since 3.9). `"vanishes"` licenses the homogeneous `Σ_i a_i(n)·S(n+i) = 0`; `"nonzero"` licenses the inhomogeneous `Σ_i a_i(n)·S(n+i) = b(n)` with `b(n)` in `cert.boundary_rhs` — a result, not a refusal; `"unknown"` licenses **nothing** about the sum, and recording the recurrence anyway is how a verified certificate becomes a false theorem (it did, on OEIS A279013). The verdict is about the range in `cert.limits`, which defaults to `k = 0..n` and is echoed back rather than inferred — pass `limits=(k_lo, k_hi)` when you are summing over anything else, because truncating a sum by one term generally flips `"vanishes"` to `"nonzero"`. `cert.boundary_at(k_lo, k_hi)` asks about another range without re-running the search. +22. **A `zeilberger` certificate is about the *summand*; `cert.boundary` is what makes it about the *sum*** (since 3.9). `"vanishes"` licenses the homogeneous `Σ_i a_i(n)·S(n+i) = 0`; `"nonzero"` licenses the inhomogeneous `Σ_i a_i(n)·S(n+i) = b(n)` with `b(n)` in `cert.boundary_rhs` — a result, not a refusal; `"unknown"` licenses **nothing** about the sum, and recording the recurrence anyway is how a verified certificate becomes a false theorem (it did, on OEIS A279013). The verdict is about the range in `cert.limits`, which defaults to `k = 0..n` and is echoed back rather than inferred — pass `limits=(k_lo, k_hi)` when you are summing over anything else, because truncating a sum by one term generally flips `"vanishes"` to `"nonzero"`. `cert.boundary_at(k_lo, k_hi)` asks about another range without re-running the search. A verdict is not a claim about every `n`: `cert.boundary_valid_from` is the smallest `n` it holds from, because a declared range can be *empty* there (`k = 3..n−3` runs backwards at `n = 3, 4`, and the returned `b(n)` is false at exactly those `n`; `k = 5..3`, backwards everywhere, is `"unknown"`), and `cert.certificate_poles` lists integer points inside the range where `G = R·F` or the summand is not finite — an interior pole breaks the telescoping in the middle of the sum, so a non-empty list means `"unknown"` no matter how clean the certificate looks. 23. **`asymptotics_from_recurrence` separates what the recurrence *proves* from what the terms *fitted*** (since 3.9). Hand it a `ZeilbergerCertificate`, a `GuessedRecurrence`, or a bare list of coefficient polynomials and it returns `growth_rate` / `polynomial_exponent` — derived by Poincaré–Perron, and **exact** as `growth_rate_exact` / `polynomial_exponent_exact` when the root is rational — plus `connection_constant`, which is **fitted** from the terms and is not implied by the recurrence at all. Quote the constant only with `connection_constant_converged`; `evidence()` returns the two halves under separate `derived` / `fitted` keys for exactly this reason. `verdict != "single_dominant_root"` means the hypotheses failed (`equal_modulus_roots`, `repeated_dominant_root`, `degenerate_leading_coefficient`, `eventually_zero`) and `growth_rate` is `None` — no root is reported as if it had won. `follows_dominant_root is False` is a real answer, not an error: the sequence's dominant component vanishes and it grows more slowly than the recurrence's generic solution. diff --git a/docs/features.md b/docs/features.md index 74fcb480..e4eafe2c 100644 --- a/docs/features.md +++ b/docs/features.md @@ -70,7 +70,7 @@ Current stable feature surface. - Difference equations / `rsolve`: constant-coefficient recurrences with polynomial RHS - Symbolic products: definite and indefinite via Γ-ratio telescoping (`product_definite`, `product_indefinite`, `Product`) - Creative telescoping / Zeilberger's algorithm (`zeilberger`): P-recursive recurrence for a proper hypergeometric term plus a rational certificate, re-checked as an exact `Q(n)(k)` identity before it is returned; refuses (`E-HOLO-*`) rather than guessing outside the class or beyond the search bounds. `order_is_minimal` reports whether the search established that no lower-order relation exists — the default cost-ordered search usually cannot, and says so; `minimal=True` searches order-ascending and can establish it, at a cost that grows with `max_degree` (free at `max_degree=4`, ~13 s versus 0.08 s at 16 on Apéry), so it is opt-in rather than the default -- Boundary verdict for creative telescoping (`ZeilbergerCertificate.boundary`): whether the certificate implies a recurrence for the **sum** over the range in `limits` (default `k = 0..n`, echoed back rather than inferred) — `"vanishes"` (homogeneous recurrence proved by exact order counting in `Q(n)`), `"nonzero"` (inhomogeneous recurrence proved, with `b(n)` in `boundary_rhs`) or `"unknown"` (nothing may be claimed). `boundary_at(k_lo, k_hi)` re-decides for another range without re-running the search +- Boundary verdict for creative telescoping (`ZeilbergerCertificate.boundary`): whether the certificate implies a recurrence for the **sum** over the range in `limits` (default `k = 0..n`, echoed back rather than inferred) — `"vanishes"` (homogeneous recurrence proved by exact order counting in `Q(n)`), `"nonzero"` (inhomogeneous recurrence proved, with `b(n)` in `boundary_rhs`) or `"unknown"` (nothing may be claimed). A verdict carries the `n` it is a theorem on rather than an implied "for every `n`": `boundary_valid_from` is the smallest `n` for which the declared range is a range at all (`k = 3..n−3` runs backwards at `n = 3, 4`, where the relation is false; a range that is backwards at every `n`, like `k = 5..3`, is `"unknown"`), and `certificate_poles` reports integer points inside the range where `G = R·F` — or the summand itself — is not finite, which breaks the telescoping in the *interior* rather than at an endpoint. `boundary_at(k_lo, k_hi)` re-decides for another range without re-running the search - `q`-analogue creative telescoping (`experimental.q_zeilberger`): `q`-Zeilberger for `q`-hypergeometric summands (Gaussian binomials `qbinomial(N, K)`, `q`-Pochhammer symbols `qpochhammer(u, d, v)`, powers of `q` with a degree-≤2 exponent in `n, k`), which the classical engine cannot express at all. The certificate is re-checked as an exact `Q(q)(q**n)(q**k)` identity before return; `sum_term(n0)` gives the exact `q`-series value from the definition of the `q`-Pochhammer symbol, so the returned recurrence can be checked independently of the machinery that produced it. The boundary verdict is two-valued — `"vanishes"` (proved for `S(n) = Σ_{k ∈ Z} F(n,k)`, with the proved support window in `support`) or `"unknown"` — and `q` is treated as transcendental throughout, so a verdict does not license specialising `q` to a root of unity. Refuses with `E-HOLO-020` (outside the class), `E-HOLO-021` (bounds exhausted), `E-HOLO-023` (malformed call) or `E-HOLO-024` (in the shape of the class but with a non-rational shift quotient, e.g. `(q; q**2)_k` shifted in `k`) - Recurrence guessing (`guess_holonomic`): fit a P-recursive recurrence to the first terms of a sequence in exact rational arithmetic, the guessing half of *guess then prove*. Only fits candidates the terms over-determine, reports how many surplus terms confirmed the fit, and refuses (`E-HOLO-005`) rather than returning an interpolation or reporting an untested grid as a negative - Modular / `p`-adic evaluation of a holonomic sequence (`ModularRecurrence`): `S(N) mod p^k` straight from `Σ_i a_i(n)·S(n+i) = b(n)`, in machine-word modular arithmetic and `O(1)` memory, without ever forming `S(N)` over `ℤ`. Indices where the leading coefficient `a_J(n)` is not a unit mod `p` are handled by a first pass that measures the total `p`-adic precision loss and a forward pass that runs at `p^(k+loss)`; a step that cannot be justified refuses (`E-HOLO-007`) and a working precision past the 64-bit modulus refuses (`E-HOLO-008`), so no path returns a residue that is silently short of the precision it claims. `supercongruence_sweep` drives it over a range of primes and reports counterexamples, the `v_p(LHS − RHS)` histogram and whether the claimed modulus is sharp diff --git a/docs/mdbook/src/telescoping.md b/docs/mdbook/src/telescoping.md index e4ff4ea8..4779ec25 100644 --- a/docs/mdbook/src/telescoping.md +++ b/docs/mdbook/src/telescoping.md @@ -115,6 +115,47 @@ echoed back*, not inferred from the summand, so a caller summing over something else can see the mismatch. And a range the analysis cannot place — endpoints that are not integer-affine in `n` — is `"unknown"`, never `"vanishes"`. +### The range is not a range at every `n` + +A declared range can be *empty*, and an empty range is where a verdict most +easily stops being true. `k = 3..n−3` runs backwards at `n = 3` and `n = 4`: a +sum over it is `0` under the "empty sum" reading and a signed sum under the +reversed-sum one, and a `b(n)` that is correct from `n = 5` on is simply wrong +below it. A loop that declares a range which happens to be empty for its first +few `n` is an ordinary thing to write. + +So a verdict carries the `n` it is claimed for: + +```python +cert = ak.zeilberger(F, n, k, limits=(3, n - pool.integer(3))) +cert.boundary # "nonzero" +cert.boundary_valid_from # 5 — the relation is FALSE at n = 3, 4 +``` + +`boundary_valid_from` is `None` when nothing is excluded on that ground, and the +range that is backwards at *every* `n` — `k = 5..3` — is `"unknown"`: there is +no domain left to claim it on. The one empty range that keeps a verdict is +`k_hi = k_lo − 1` (`k = n+1..n`), which both readings agree is `0`. + +### The telescoping has to survive the interior + +`Σ_k (G(n,k+1) − G(n,k))` collapses to the two endpoints only if `G = R·F` is +finite at every integer `k` in between. A pole of the *certificate* at an +interior point breaks it in the middle of the sum, where no boundary value can +see it. `cert.certificate_poles` reports those points, as expressions in `n`: + +```python +cert = ak.zeilberger(C(n, k) / (n - 2*k + one), n, k) # k = 0..n +cert.boundary # "unknown" +[str(p) for p in cert.certificate_poles] +# k = (n+1)/2 — a pole of the summand, so S(n) is undefined for odd n — and +# k = (n+3)/2, a pole of the certificate strictly inside the range +``` + +A non-empty `certificate_poles` always means `"unknown"`. An empty one is not a +proof that there are none: the search covers locations `k = (p·n + q)/c` with +`c ≤ 4` that are inside the range for large `n`. + ### What `"vanishes"` is worth It is a proof, not a numeric check. Each endpoint of `G` is evaluated by exact diff --git a/tests/test_holonomic_boundary.py b/tests/test_holonomic_boundary.py index 975825c7..846fff6f 100644 --- a/tests/test_holonomic_boundary.py +++ b/tests/test_holonomic_boundary.py @@ -32,7 +32,8 @@ def _exact(pool, n, expr, ni): the whole check stays in exact arithmetic — no float ever sees a value that a verdict depends on. """ - return Fraction(str(ak.simplify(ak.subs(expr, {n: pool.integer(ni)})).value)) + # `simplify` may hand back a parenthesised atom, e.g. "(20)". + return Fraction(str(ak.simplify(ak.subs(expr, {n: pool.integer(ni)})).value).strip("()")) def _residual(pool, n, cert, s, ni): @@ -335,5 +336,157 @@ def test_boundary_at_does_not_disturb_the_certificate(): "boundary", "rhs", "reason", + "valid_from", + "certificate_poles", "side_conditions", } + # The domain travels with the verdict here too, not only on the attribute. + assert cert.boundary_at(0, n)["valid_from"] == cert.boundary_valid_from + assert cert.boundary_at(3, n - pool.integer(3))["valid_from"] == 5 + + +# --------------------------------------------------------------------------- +# The domain the verdict is claimed on +# --------------------------------------------------------------------------- + + +def test_a_backwards_range_is_not_a_recurrence(): + """``k = 5..3`` is empty, so every ``S(n)`` is ``0``. + + The engine used to answer ``"nonzero"`` here with a degree-9 ``b(n)`` whose + residual ran ``4, 107, 800, 2725, …`` — a valid certificate implying a false + recurrence, which is the failure the whole verdict exists to prevent. There + is no ``n`` at which the relation holds, so there is nothing to claim. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + b = _binomial(pool, n, k) + + for lo, hi in [(5, 3), (3, 1), (2, 0), (4, 2)]: + cert = ak.zeilberger(b * b, n, k, limits=(lo, hi)) + assert cert.boundary == "unknown", f"k = {lo}..{hi}: {cert.boundary_reason}" + assert not cert.implies_sum_recurrence + assert cert.boundary_rhs is None + # Nothing is claimed at any n, so the bound has nothing to bound. + assert cert.boundary_valid_from is None + assert "backwards" in cert.boundary_reason + + +def test_an_n_dependent_range_that_starts_empty_carries_its_domain(): + """``k = 3..n−3`` is the realistic form: empty at ``n = 3, 4``, a range after. + + The verdict is a theorem for ``n ≥ 5`` and false below it, so it is returned + *with* that bound rather than discarded or over-claimed. Both halves are + checked against the actual sum. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + three = pool.integer(3) + cert = ak.zeilberger(_binomial(pool, n, k), n, k, limits=(three, n - three)) + + assert cert.boundary_valid_from == 5 + assert cert.boundary == "nonzero", cert.boundary_reason + assert any("n >= 5" in s for s in cert.side_conditions) + + def s(m): + return sum(math.comb(m, j) for j in range(3, m - 3 + 1)) + + # A theorem from n = 5 on ... + for ni in range(5, 10): + assert _residual(pool, n, cert, s, ni) == _exact(pool, n, cert.boundary_rhs, ni) + # ... and false below it, which is exactly what the bound says. + for ni in (3, 4): + assert s(ni) == 0 + assert _residual(pool, n, cert, s, ni) != _exact(pool, n, cert.boundary_rhs, ni) + + +def test_an_exactly_empty_range_is_still_a_proved_zero(): + """``k = 0..−1`` and ``k = n+1..n`` are empty too — and were always right. + + ``κ₁ = κ₀ − 1`` is the one empty range both readings agree is ``0``, and the + telescoping handles it, so it keeps its ``"vanishes"``. The old behaviour + was inconsistent in this pair's favour; the fix must not swap which half is + wrong. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + b = _binomial(pool, n, k) + one = pool.integer(1) + + for limits in [(0, -1), (n + one, n)]: + cert = ak.zeilberger(b * b, n, k, limits=limits) + assert cert.boundary == "vanishes", cert.boundary_reason + assert cert.implies_sum_recurrence + assert cert.boundary_valid_from is None + assert cert.certificate_poles == [] + + +# --------------------------------------------------------------------------- +# Poles inside the range +# --------------------------------------------------------------------------- + + +def test_an_interior_certificate_pole_is_reported(): + """``C(n,k)/(n−2k+1)`` over ``k = 0..n``. + + ``S(n)`` is undefined for every odd ``n`` — the summand has a pole at + ``k = (n+1)/2`` — *and* the certificate has one at ``k = (n+3)/2``, an + integer strictly inside the range. The telescoping breaks in the middle of + the sum, where no boundary value can see it; the verdict was ``"vanishes"``. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + one = pool.integer(1) + cert = ak.zeilberger(_binomial(pool, n, k) / (n - pool.integer(2) * k + one), n, k) + + assert cert.boundary == "unknown", cert.boundary_reason + assert not cert.implies_sum_recurrence + poles = [str(p) for p in cert.certificate_poles] + assert poles, "the poles must be reported, not just the refusal" + assert all("1/2" in p for p in poles), poles + assert any("interior" in s for s in cert.side_conditions) + + # The sum really is undefined at the odd n, which is what the pole says. + for m in (3, 5, 7): + assert (m - 2 * ((m + 1) // 2) + 1) == 0 + + +def test_a_summand_pole_inside_the_range_is_reported(): + """``C(n,k)/(k−3)`` over ``k = 0..n``: ``S(n)`` does not exist for ``n ≥ 3``. + + The old answer was ``"vanishes"`` with ``implies_sum_recurrence`` — the same + strings a genuine theorem gets. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + cert = ak.zeilberger(_binomial(pool, n, k) / (k - pool.integer(3)), n, k) + + assert cert.boundary == "unknown", cert.boundary_reason + assert not cert.implies_sum_recurrence + assert [str(p) for p in cert.certificate_poles] == ["3"] + + +def test_verdicts_that_were_already_right_are_left_alone(): + """Guards, so that refusing empty ranges and interior poles cannot spread. + + ``k = −n..n`` is honestly ``"unknown"``, ``k = −n..0`` is a real + ``"nonzero"``, and ``C(n,k)/(n−k+1)`` over ``k = 0..n`` has a certificate + pole *exactly* at ``k = k_hi+1`` that a zero of the summand cancels — a + ``0·∞`` endpoint the analysis resolves rather than refuses. + """ + pool = ak.ExprPool() + n, k = pool.symbol("n"), pool.symbol("k") + one = pool.integer(1) + b = _binomial(pool, n, k) + + assert ak.zeilberger(b * b, n, k, limits=(-n, n)).boundary == "unknown" + + half = ak.zeilberger(b, n, k, limits=(-n, 0)) + assert half.boundary == "nonzero", half.boundary_reason + + cont = ak.zeilberger(b / (n - k + one), n, k) + assert cont.boundary == "nonzero", cont.boundary_reason + assert cont.certificate_poles == [] + s = lambda m: sum(Fraction(math.comb(m, j), m - j + 1) for j in range(m + 1)) # noqa: E731 + for ni in range(2, 7): + assert _residual(pool, n, cont, s, ni) == _exact(pool, n, cont.boundary_rhs, ni) From 90f2498f34bb16534063e6e33fa9b10b82473d3e Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 21:32:12 +0000 Subject: [PATCH 06/11] fix(ball): tan and pow_f returned "enclosures" that exclude values in the box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two critical soundness bugs in `ArbBall`, both the same mistake: evaluate at the endpoints, take the hull, assume monotonicity. `tan` guarded its pole with an `f64` `v % π` test at the two endpoints plus "`tan(lo) > tan(hi)` means we crossed one". Neither sees a box that crosses a pole and comes back out in increasing order: `tan` over `[0.1, 3.3415926535897933]` was reported as `[0.1003…, 0.2027…]` while `tan(1.5) = 14.101…` and `1.5` is in the box. The pole test is now an enclosure of `cos` over the same box — the rule `validated::taylor::TaylorModel::tan` already applies one module over, and the one that actually proves the box lies inside a single branch. A box that fails it widens to `[-∞, ∞]`, which is what `tan` does there. `pow_f` hulled the four corners of `[lo_b, hi_b] × [lo_e, hi_e]`, which encloses the range only where `x ↦ x^y` is monotone in `x`. Its guard admitted a negative base with an integer-valued exponent, where it is not: `x**2.0` over `[-1, 3]` claimed `[1, 9]`, missing `x = 0`, and `x**-2.0` claimed the *finite* `[0.111…, 1]` for a function unbounded on the box. The corner hull is now kept only for a base that stays `≥ 0`; a base reaching 0 or below routes through `powi`, which assumes no monotonicity and returns `∞` when its reciprocal step straddles zero. Audited the rest of the module for the same pattern and fixed what it turned up: * `add_rounding_error` bumped by `|mid|·2^-prec`, covering the rounding of `mid` and nothing else. Every kernel that also rounds its `rad` — each endpoint hull, `cosh`, `erf`, `Mul` — could report a ball an ulp narrower than the truth, and where the radius dwarfs the midpoint that ulp is the whole error. It now bumps by `(|mid| + rad)·2^-(prec-3)`. * `lo()`/`hi()` rounded `mid ∓ rad` to *nearest*, handing every kernel a bound strictly inside the interval. They now round outward. * The monotone kernels (`exp`, `log`, `sqrt`, `sinh`, `cosh`, `tanh`, the inverse trig/hyperbolic family, `digamma`) and the `Div` corner hull now evaluate at `prec + 32` and reduce once, outward, through `from_endpoints`. * `erf`/`erfc` rounded their Lipschitz constant `2/√π` to nearest; a Lipschitz constant rounded down is not one. Rounded up. * `digamma` rejected its pole by walking *every* integer in the box — 10^18 iterations on `[1, 1e18]` — and evaluated at `f64`-reduced endpoints, which moves them inside the box. Now an O(1) test on the smallest integer in the box, evaluated at full precision. Regression tests, each verified to fail before the change: four in `ball::rounding_soundness_tests` (the two bugs, a tightness guard so the fix cannot be "return ∞ always", a 200-interval random sweep over 22 kernels sampling from the balls' own endpoints, and the digamma pole test), and three in `tests/test_ball_enclosure_soundness.py` — that file bound only *point* balls, which is why neither bug was reachable from it. Co-Authored-By: Claude Opus 5 --- alkahest-core/src/ball/mod.rs | 797 ++++++++++++++++--------- docs/mdbook/src/ball-arithmetic.md | 22 + tests/test_ball_enclosure_soundness.py | 217 +++++++ 3 files changed, 746 insertions(+), 290 deletions(-) diff --git a/alkahest-core/src/ball/mod.rs b/alkahest-core/src/ball/mod.rs index 618a615f..eb7f17de 100644 --- a/alkahest-core/src/ball/mod.rs +++ b/alkahest-core/src/ball/mod.rs @@ -65,7 +65,7 @@ use crate::kernel::expr::PredicateKind; use crate::kernel::{ExprData, ExprId, ExprPool}; use crate::primitive::PrimitiveRegistry; -use rug::{ops::Pow, Float}; +use rug::{float::Round, ops::Pow, Float}; use std::collections::HashMap; use std::fmt; use std::sync::OnceLock; @@ -162,14 +162,20 @@ impl ArbBall { v >= lo && v <= hi } - /// Lower bound of the interval. + /// Lower bound of the interval, rounded **down**. + /// + /// `mid - rad` does not generally fit in `prec` bits, and rounding it to + /// nearest would hand back a bound *above* the interval's true lower end — + /// which every kernel that starts from `lo()` then evaluates at, giving a + /// hull that is an ulp too narrow at the bottom. The same, mirrored, for + /// [`ArbBall::hi`]. pub fn lo(&self) -> Float { - Float::with_val(self.prec, &self.mid - &self.rad) + Float::with_val_round(self.prec, &self.mid - &self.rad, Round::Down).0 } - /// Upper bound of the interval. + /// Upper bound of the interval, rounded **up**. See [`ArbBall::lo`]. pub fn hi(&self) -> Float { - Float::with_val(self.prec, &self.mid + &self.rad) + Float::with_val_round(self.prec, &self.mid + &self.rad, Round::Up).0 } /// Midpoint as f64 (lossy). @@ -184,14 +190,27 @@ impl ArbBall { // ── arithmetic ─────────────────────────────────────────────────────── - /// Grow radius by a rounding-error term: `eps * |mid| * 2^{-prec}`. + /// Grow the radius to absorb the rounding of *both* fields: by + /// `(|mid| + rad) · 2^{-(prec-3)}`. + /// + /// This used to bump by `|mid| · 2^{-prec}`, which covers the rounding of + /// `mid` and nothing else. A kernel that also rounds its `rad` — every + /// endpoint hull, `cosh`, `erf`, `Mul` — can then report a ball an ulp + /// narrower than the truth, and on a ball whose radius dwarfs its midpoint + /// (`cosh([5.2, 9.2])`, `log([1/2, 2])`) the shortfall is the whole error. + /// The eight-ulp headroom covers the handful of roundings the widest of + /// those kernels performs (`Mul` rounds five times) and is still 10³⁴ + /// below `f64` resolution at the default 128 bits. fn add_rounding_error(&mut self) { if self.mid.is_infinite() || self.mid.is_nan() { self.rad = Float::with_val(self.prec, f64::INFINITY); return; } - let scale = Float::with_val(self.prec, &self.mid).abs() - * Float::with_val(self.prec, 2.0_f64.powi(-(self.prec as i32))); + let mut scale = Float::with_val( + self.prec, + Float::with_val(self.prec, self.mid.abs_ref()) + &self.rad, + ); + scale >>= self.prec.saturating_sub(3); self.rad += &scale; } } @@ -216,12 +235,10 @@ impl std::ops::Add for ArbBall { fn add(self, rhs: Self) -> Self { let prec = self.prec.max(rhs.prec); let mid = Float::with_val(prec, &self.mid + &rhs.mid); - let mut rad = Float::with_val(prec, &self.rad + &rhs.rad); - // Rounding error: 1 ulp - let eps = Float::with_val(prec, mid.abs_ref()) - * Float::with_val(prec, 2.0_f64.powi(-(prec as i32))); - rad += eps; - ArbBall { mid, rad, prec } + let rad = Float::with_val(prec, &self.rad + &rhs.rad); + let mut b = ArbBall { mid, rad, prec }; + b.add_rounding_error(); + b } } @@ -230,11 +247,10 @@ impl std::ops::Sub for ArbBall { fn sub(self, rhs: Self) -> Self { let prec = self.prec.max(rhs.prec); let mid = Float::with_val(prec, &self.mid - &rhs.mid); - let mut rad = Float::with_val(prec, &self.rad + &rhs.rad); - let eps = Float::with_val(prec, mid.abs_ref()) - * Float::with_val(prec, 2.0_f64.powi(-(prec as i32))); - rad += eps; - ArbBall { mid, rad, prec } + let rad = Float::with_val(prec, &self.rad + &rhs.rad); + let mut b = ArbBall { mid, rad, prec }; + b.add_rounding_error(); + b } } @@ -247,13 +263,12 @@ impl std::ops::Mul for ArbBall { let mid = Float::with_val(prec, &self.mid * &rhs.mid); let ma = Float::with_val(prec, self.mid.abs_ref()); let mb = Float::with_val(prec, rhs.mid.abs_ref()); - let mut rad = Float::with_val(prec, &ma * &rhs.rad) + let rad = Float::with_val(prec, &ma * &rhs.rad) + Float::with_val(prec, &mb * &self.rad) + Float::with_val(prec, &self.rad * &rhs.rad); - let eps = Float::with_val(prec, mid.abs_ref()) - * Float::with_val(prec, 2.0_f64.powi(-(prec as i32))); - rad += eps; - ArbBall { mid, rad, prec } + let mut b = ArbBall { mid, rad, prec }; + b.add_rounding_error(); + b } } @@ -278,12 +293,17 @@ impl std::ops::Div for ArbBall { // Monotone on positive/negative intervals let lo_rhs = rhs.lo(); let hi_rhs = rhs.hi(); - // Compute all 4 corners + // Compute all 4 corners, at working precision so the reduction to + // `prec` happens once, outward, in `from_endpoints` below. + // (`saturating_add`, not `+`: `clippy::suspicious_arithmetic_impl` + // flags a bare `+` anywhere inside a `Div` impl.) + let work = prec.saturating_add(32); + let quot = |a: &Float, b: &Float| Float::with_val(work, a) / Float::with_val(work, b); let corners = [ - Float::with_val(prec, self.lo() / lo_rhs.clone()), - Float::with_val(prec, self.lo() / hi_rhs.clone()), - Float::with_val(prec, self.hi() / lo_rhs.clone()), - Float::with_val(prec, self.hi() / hi_rhs.clone()), + quot(&self.lo(), &lo_rhs), + quot(&self.lo(), &hi_rhs), + quot(&self.hi(), &lo_rhs), + quot(&self.hi(), &hi_rhs), ]; // `∞/∞` is NaN, so an unbounded operand makes the corner ordering // partial and `partial_cmp(...).unwrap()` panics. `None` is the @@ -302,15 +322,9 @@ impl std::ops::Div for ArbBall { .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) .unwrap() .clone(); - let sum = Float::with_val(prec, &min + &max); - let diff = Float::with_val(prec, &max - &min); - let new_mid = sum / 2_f64; - let rad = diff / 2_f64; - Some(ArbBall { - mid: new_mid, - rad, - prec, - }) + // Outward rounding: the corners were computed at `work` bits and are + // being reported at `prec`. + Some(ArbBall::from_endpoints(&min, &max, prec)) } } @@ -340,52 +354,91 @@ impl ArbBall { result } + /// `self ^ exp` for a real exponent *ball*. + /// + /// # When the four-corner hull is legitimate, and when it is not + /// + /// This used to hull `base^exp` over the four corners of + /// `[lo_b, hi_b] × [lo_e, hi_e]` for every input a single guard let + /// through, `lo < 0 && !(exp.is_exact() && exp.lo().is_integer())`. A + /// corner hull encloses the range only where `x ↦ x^y` is monotone in `x`, + /// and that guard admits the two cases where it is not: + /// + /// * an **even** integer exponent on a base straddling `0` — `x²` over + /// `[-1, 3]` was reported as `[1, 9]`, missing `x = 0 ↦ 0`; + /// * a **negative** integer exponent on such a base — `x⁻²` over `[-1, 3]` + /// was reported as the *finite* `[0.111…, 1]` for a function that is + /// unbounded on the box (`0.001⁻² = 10⁶`). + /// + /// (`x ** 2` written with an `Integer` exponent node never reached here — + /// [`IntervalEval::eval_node`] routes it to [`ArbBall::powi`] — but the + /// same exponent arriving as a `Float`, a `Rational`, or a bound symbol + /// did.) + /// + /// So the corner hull is kept only on a base that stays `≥ 0`, where + /// `x^y = exp(y·ln x)` is monotone in `x` for fixed `y` *and* monotone in + /// `y` for fixed `x`; a function monotone in each variable separately + /// attains its extrema over a box at a corner. Everything else with a + /// real value — a base that reaches `0` or below, which forces an + /// integer-valued exponent — goes to [`ArbBall::powi`], which is repeated + /// ball multiplication and assumes no monotonicity at all (and whose + /// reciprocal step returns `∞` exactly when the denominator straddles 0). + /// What is left is complex-valued, and widens to `∞`. pub fn pow_f(&self, exp: &ArbBall) -> Self { - // [a,b]^[c,d] using interval exponentiation let prec = self.prec; let lo = self.lo(); let hi = self.hi(); - // A negative base only has a real power for an *integer* exponent. - // `is_exact` alone is not that test: `x^(3/2)` arrives here as an exact - // point ball at 1.5, `(-3.3)^1.5` is NaN, and the corner comparison - // below then unwrapped a `None` from `partial_cmp` and panicked — a - // Rust panic crossing the FFI boundary, which is a `BaseException` an - // `except Exception` handler does not catch. - if lo < 0 && !(exp.is_exact() && exp.lo().is_integer()) { - return ArbBall::infinity(prec); // complex result possible - } - // Conservative bound via corner evaluation - let corners = [ - Float::with_val(prec, lo.clone().pow(exp.lo())), - Float::with_val(prec, lo.clone().pow(exp.hi())), - Float::with_val(prec, hi.clone().pow(exp.lo())), - Float::with_val(prec, hi.clone().pow(exp.hi())), - ]; - // Defence in depth: any remaining NaN corner (an overflow, or a base - // interval straddling zero with a negative exponent) makes the ordering - // partial, and `partial_cmp(...).unwrap()` would panic on it. - if corners.iter().any(|c| c.is_nan()) { + let e_lo = exp.lo(); + let e_hi = exp.hi(); + if !(lo.is_finite() && hi.is_finite() && e_lo.is_finite() && e_hi.is_finite()) { return ArbBall::infinity(prec); } - let min = corners - .iter() - .min_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) - .unwrap() - .clone(); - let max = corners - .iter() - .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) - .unwrap() - .clone(); - let sum = Float::with_val(prec, &min + &max); - let diff = Float::with_val(prec, &max - &min); - let new_mid = sum / 2_f64; - let rad = diff / 2_f64; - ArbBall { - mid: new_mid, - rad, - prec, + // `lo == 0` joins the monotonicity argument as long as the exponent + // stays ≥ 0: `0^y` is 0 for `y > 0` and 1 for `y = 0`, still monotone + // in `y`. With a negative exponent in reach it is a pole instead. + if lo > 0 || (lo == 0 && e_lo >= 0) { + let work = prec + 32; + let corner = + |b: &Float, e: &Float| Float::with_val(work, b).pow(Float::with_val(work, e)); + let corners = [ + corner(&lo, &e_lo), + corner(&lo, &e_hi), + corner(&hi, &e_lo), + corner(&hi, &e_hi), + ]; + // Defence in depth: a NaN corner makes the ordering partial and + // `partial_cmp(...).unwrap()` would panic on it — a Rust panic + // crossing the FFI boundary is a `BaseException` that an + // `except Exception` handler does not catch. An infinite corner + // (overflow) is a true statement about an unbounded box, and `∞` + // is its enclosure. + if corners.iter().any(|c| !c.is_finite()) { + return ArbBall::infinity(prec); + } + let min = corners + .iter() + .min_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .unwrap() + .clone(); + let max = corners + .iter() + .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .unwrap() + .clone(); + // Outward rounding: the corners were computed at `work` bits and + // are being reported at `prec`. + return ArbBall::from_endpoints(&min, &max, prec); } + // The base reaches 0 or below. A real value then requires an exponent + // that is an exact integer; `x^(3/2)` arrives here as an exact point + // ball at 1.5 and `(-3.3)^1.5` is not real, so `is_exact` alone is not + // the test. + if exp.is_exact() && e_lo.is_integer() { + if let Some(n) = e_lo.to_integer().and_then(|n| n.to_i64()) { + return self.powi(n); + } + } + ArbBall::infinity(prec) // complex result possible, or unbounded } pub fn sin(&self) -> Self { @@ -411,20 +464,15 @@ impl ArbBall { pub fn exp(&self) -> Self { // e^[m-r, m+r] = [e^(m-r), e^(m+r)] let prec = self.prec; - let lo = Float::with_val(prec, self.lo().exp()); - let hi = Float::with_val(prec, self.hi().exp()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // `lo`/`hi` are themselves rounded to `prec`; without this the ball is + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).exp(); + let hi = Float::with_val(work, self.hi()).exp(); + // `from_endpoints` rounds outward. Without that the ball is // exact-looking (`rad == 0`) for an exact input, which is a false - // rigorous claim about a transcendental value. - b.add_rounding_error(); - b + // rigorous claim about a transcendental value; and on a ball whose + // radius dwarfs its midpoint the endpoints' own rounding is not + // covered by the `|mid|·2^-prec` bump `add_rounding_error` applies. + ArbBall::from_endpoints(&lo, &hi, prec) } pub fn log(&self) -> Option { @@ -432,20 +480,11 @@ impl ArbBall { return None; // log undefined for non-positive values } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().ln()); - let hi = Float::with_val(prec, self.hi().ln()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).ln(); + let hi = Float::with_val(work, self.hi()).ln(); + // Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&lo, &hi, prec)) } pub fn sqrt(&self) -> Option { @@ -453,84 +492,78 @@ impl ArbBall { return None; } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().sqrt()); - let hi = Float::with_val(prec, self.hi().sqrt()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).sqrt(); + let hi = Float::with_val(work, self.hi()).sqrt(); + // Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&lo, &hi, prec)) } - /// tan([m-r, m+r]) — Lipschitz constant: sec²(m+r) (may blow up near π/2). - /// Returns None if the interval contains a pole. + /// tan([m-r, m+r]). `None` only for a non-finite argument; a box that may + /// contain a pole widens to `[-∞, ∞]`, which is what `tan` does there. + /// + /// # Why the pole test is an enclosure of `cos`, not a test on the endpoints + /// + /// This used to guard the pole with `near_pole(lo) || near_pole(hi)` — an + /// `f64` test of `v % π` at the two endpoints — plus "`tan(lo) > tan(hi)` + /// means we crossed a pole". Neither sees a box that crosses a pole and + /// comes back out the other side in increasing order: on + /// `[0.1, 3.3415926535897933]` the endpoint hull is `[0.1003…, 0.2027…]` + /// while `tan(1.5) = 14.101…` and `1.5` is in the box. (`v % π` is not a + /// pole test at all once `|v|` is large enough that `v % π` loses the low + /// bits that decide the question.) + /// + /// `tan′ = sec² > 0`, so an endpoint hull *is* the exact range — but only + /// on an interval where `cos` does not vanish, because those intervals are + /// exactly the ones on which `tan` is continuous and increasing. That is + /// the rule the Taylor path already applies one module over + /// (`validated::taylor::TaylorModel::tan` refuses when the `cos` enclosure + /// contains zero, `E-VALIDATED-003`), and this is the same rule in ball + /// form: [`ArbBall::cos`] encloses the range of `cos` over the box, so + /// `0 ∉ cos([lo, hi])` *proves* the box lies inside a single branch. pub fn tan(&self) -> Option { let prec = self.prec; - let _pi_half = Float::with_val(prec, rug::float::Constant::Pi) / 2_f64; - // Check that neither bound is within ε of π/2 + k*π let lo = self.lo(); let hi = self.hi(); - // simple pole check: |lo mod π - π/2| > 0 and |hi mod π - π/2| > 0 - let lo_f = lo.to_f64(); - let hi_f = hi.to_f64(); - let pi_f: f64 = std::f64::consts::PI; - let near_pole = |v: f64| ((v % pi_f).abs() - pi_f / 2.0).abs() < 1e-9; - if near_pole(lo_f) || near_pole(hi_f) { + if !(lo.is_finite() && hi.is_finite()) { return None; } - let lo_tan = Float::with_val(prec, lo.tan()); - let hi_tan = Float::with_val(prec, hi.tan()); - // If lo_tan > hi_tan the interval crossed a pole — discard - if lo_tan > hi_tan { - return None; + // Pole test. A zero in the `cos` enclosure is the only way `tan` can + // fail to be continuous and increasing on `[lo, hi]`; it is also the + // only way it can be unbounded there, so `[-∞, ∞]` is the enclosure. + if self.cos().contains(0.0) { + return Some(ArbBall::infinity(prec)); } - let sum = Float::with_val(prec, &lo_tan + &hi_tan); - let diff = Float::with_val(prec, &hi_tan - &lo_tan); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo_tan = Float::with_val(work, &lo).tan(); + let hi_tan = Float::with_val(work, &hi).tan(); + // `from_endpoints` rounds outward, which both absorbs the reduction + // from `work` to `prec` and keeps an exact input from reporting + // `rad == 0` — a false claim that a transcendental value is exactly + // representable. + Some(ArbBall::from_endpoints(&lo_tan, &hi_tan, prec)) } pub fn sinh(&self) -> Self { let prec = self.prec; - let lo = Float::with_val(prec, self.lo().sinh()); - let hi = Float::with_val(prec, self.hi().sinh()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; see `exp`. - b.add_rounding_error(); - b + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).sinh(); + let hi = Float::with_val(work, self.hi()).sinh(); + // Rounded outward; see `exp`. + ArbBall::from_endpoints(&lo, &hi, prec) } pub fn cosh(&self) -> Self { let prec = self.prec; // cosh is even and has a minimum at 0; handle by evaluating at lo, hi, and 0 if in range - let lo = Float::with_val(prec, self.lo().cosh()); - let hi = Float::with_val(prec, self.hi().cosh()); + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).cosh(); + let hi = Float::with_val(work, self.hi()).cosh(); let (min_val, max_val) = if self.lo() <= 0 && self.hi() >= 0 { // minimum is cosh(0) = 1 let cosh_lo = lo.clone(); let cosh_hi = hi.clone(); - let min = Float::with_val(prec, 1_f64); + let min = Float::with_val(work, 1_f64); let max = if cosh_lo > cosh_hi { cosh_lo } else { cosh_hi }; (min, max) } else if lo < hi { @@ -538,33 +571,18 @@ impl ArbBall { } else { (hi, lo) }; - let sum = Float::with_val(prec, &min_val + &max_val); - let diff = Float::with_val(prec, &max_val - &min_val); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; see `exp`. - b.add_rounding_error(); - b + // Rounded outward; see `exp`. + ArbBall::from_endpoints(&min_val, &max_val, prec) } pub fn tanh(&self) -> Self { // tanh is monotone, maps ℝ → (-1, 1) let prec = self.prec; - let lo = Float::with_val(prec, self.lo().tanh()); - let hi = Float::with_val(prec, self.hi().tanh()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; see `exp`. - b.add_rounding_error(); - b + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).tanh(); + let hi = Float::with_val(work, self.hi()).tanh(); + // Rounded outward; see `exp`. + ArbBall::from_endpoints(&lo, &hi, prec) } pub fn asin(&self) -> Option { @@ -572,20 +590,11 @@ impl ArbBall { return None; } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().asin()); - let hi = Float::with_val(prec, self.hi().asin()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).asin(); + let hi = Float::with_val(work, self.hi()).asin(); + // Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&lo, &hi, prec)) } pub fn acos(&self) -> Option { @@ -593,54 +602,31 @@ impl ArbBall { return None; } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().acos()); - let hi = Float::with_val(prec, self.hi().acos()); - // acos is decreasing, so lo/hi swap - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &lo - &hi); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).acos(); + let hi = Float::with_val(work, self.hi()).acos(); + // acos is decreasing, so lo/hi swap; `from_endpoints` takes either + // order. Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&hi, &lo, prec)) } pub fn atan(&self) -> Self { let prec = self.prec; - let lo = Float::with_val(prec, self.lo().atan()); - let hi = Float::with_val(prec, self.hi().atan()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; see `exp`. - b.add_rounding_error(); - b + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).atan(); + let hi = Float::with_val(work, self.hi()).atan(); + // Rounded outward; see `exp`. + ArbBall::from_endpoints(&lo, &hi, prec) } /// asinh([m-r, m+r]) — monotone increasing on all of ℝ. pub fn asinh(&self) -> Self { let prec = self.prec; - let lo = Float::with_val(prec, self.lo().asinh()); - let hi = Float::with_val(prec, self.hi().asinh()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; see `exp`. - b.add_rounding_error(); - b + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).asinh(); + let hi = Float::with_val(work, self.hi()).asinh(); + // Rounded outward; see `exp`. + ArbBall::from_endpoints(&lo, &hi, prec) } /// acosh([m-r, m+r]) — monotone increasing on `[1, ∞)`. Returns `None` if @@ -650,20 +636,11 @@ impl ArbBall { return None; } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().acosh()); - let hi = Float::with_val(prec, self.hi().acosh()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).acosh(); + let hi = Float::with_val(work, self.hi()).acosh(); + // Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&lo, &hi, prec)) } /// atanh([m-r, m+r]) — monotone increasing on `(-1, 1)`. Returns `None` if @@ -673,28 +650,18 @@ impl ArbBall { return None; } let prec = self.prec; - let lo = Float::with_val(prec, self.lo().atanh()); - let hi = Float::with_val(prec, self.hi().atanh()); - let sum = Float::with_val(prec, &lo + &hi); - let diff = Float::with_val(prec, &hi - &lo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + let work = prec + 32; + let lo = Float::with_val(work, self.lo()).atanh(); + let hi = Float::with_val(work, self.hi()).atanh(); + // Rounded outward; see `exp`. + Some(ArbBall::from_endpoints(&lo, &hi, prec)) } pub fn erf(&self) -> Self { let prec = self.prec; // Use midpoint + Lipschitz: |erf'(x)| = 2/sqrt(π) * exp(-x²) ≤ 2/sqrt(π) ≈ 1.13 let mid = Float::with_val(prec, self.mid.clone().erf()); - let lipschitz = Float::with_val(prec, 2.0_f64 / std::f64::consts::PI.sqrt()); - let rad = Float::with_val(prec, &self.rad * &lipschitz); + let rad = Float::with_val(prec, &self.rad * &erf_lipschitz(prec)); let mut b = ArbBall { mid, rad, prec }; b.add_rounding_error(); b @@ -703,8 +670,7 @@ impl ArbBall { pub fn erfc(&self) -> Self { let prec = self.prec; let mid = Float::with_val(prec, self.mid.clone().erfc()); - let lipschitz = Float::with_val(prec, 2.0_f64 / std::f64::consts::PI.sqrt()); - let rad = Float::with_val(prec, &self.rad * &lipschitz); + let rad = Float::with_val(prec, &self.rad * &erf_lipschitz(prec)); let mut b = ArbBall { mid, rad, prec }; b.add_rounding_error(); b @@ -848,32 +814,30 @@ impl ArbBall { /// Digamma ψ(x). Returns `None` when the ball contains a non-positive /// integer pole. pub fn digamma(&self) -> Option { - let lo = self.lo().to_f64(); - let hi = self.hi().to_f64(); - let k_start = lo.ceil() as i64; - let k_end = hi.floor() as i64; - for k in k_start..=k_end { - if k <= 0 { - return None; - } - } let prec = self.prec; - let mut flo = Float::with_val(prec, lo); + let lo = self.lo(); + let hi = self.hi(); + if !(lo.is_finite() && hi.is_finite()) { + return None; + } + // ψ is increasing between consecutive poles (ψ′ = trigamma > 0), so the + // endpoint hull is the range as long as no pole is enclosed. The poles + // sit at the non-positive integers, so it is enough to look at the + // smallest integer in `[lo, hi]` — iterating over every integer in the + // box, as this used to, walks 10¹⁸ steps on a box like `[1, 1e18]`. + let k_min = Float::with_val(prec, lo.ceil_ref()); + if k_min <= hi && k_min <= 0 { + return None; + } + let work = prec + 32; + let mut flo = Float::with_val(work, &lo); flo.digamma_mut(); - let mut fhi = Float::with_val(prec, hi); + let mut fhi = Float::with_val(work, &hi); fhi.digamma_mut(); - let sum = Float::with_val(prec, &flo + &fhi); - let diff = Float::with_val(prec, &fhi - &flo); - let mut b = ArbBall { - mid: sum / 2_f64, - rad: diff / 2_f64, - prec, - }; - // Endpoints are rounded to `prec`; without this a ball built from an - // exact input reports `rad == 0`, falsely claiming an irrational result - // is exactly representable. - b.add_rounding_error(); - Some(b) + // Rounded outward; see `exp`. (The endpoints also used to be reduced + // to `f64` before evaluation, which moves them *inside* the box and can + // shrink the hull below the true range.) + Some(ArbBall::from_endpoints(&flo, &fhi, prec)) } /// Bessel function of the first kind Jₙ(x) for integer order `n`. @@ -962,6 +926,23 @@ impl ArbBall { } } +/// `2/√π ≈ 1.1283791…`, the Lipschitz constant of `erf` and `erfc`, rounded +/// **up**. `Float::with_val(prec, 2.0/π.sqrt())` rounds to nearest, and a +/// Lipschitz constant rounded down is not a Lipschitz constant. +fn erf_lipschitz(prec: u32) -> Float { + let work = prec + 32; + let two_over_sqrt_pi = Float::with_val( + work, + 2u32 / Float::with_val(work, rug::float::Constant::Pi).sqrt(), + ); + let mut v = Float::with_val(prec, &two_over_sqrt_pi); + // Round up: nudge by an ulp unless the reduction was exact. + if v < two_over_sqrt_pi { + v.next_up(); + } + v +} + /// A certified bracket `(low, high)` with `low ≤ W₀(x) ≤ high`, or `None` when /// `x` is outside the principal branch's domain `x ≥ −1/e`. /// @@ -1591,6 +1572,16 @@ mod rounding_soundness_tests { const PREC: u32 = 128; + /// Containment tested at ball precision. + /// + /// `ArbBall::contains` takes an `f64`, and rounding the true value to + /// `f64` moves it by ~2^-53 — enough to push a sample that sits exactly on + /// an endpoint outside an enclosure whose outward rounding is 2^-126 wide. + /// That is a defect of the *measurement*, not of the enclosure. + fn encloses(b: &ArbBall, v: &Float) -> bool { + *v >= b.lo() && *v <= b.hi() + } + /// Every transcendental op must carry a rounding term. /// /// `exp`/`log`/`sqrt`/`tan`/`asin`/`acos`/`atan`/`asinh`/`atanh` built their @@ -1682,6 +1673,232 @@ mod rounding_soundness_tests { } } + /// `tan` used to hull its two endpoints whenever they came out in + /// increasing order, which a box crossing a pole by nearly a whole period + /// does: `[0.1, 0.1 + π]` was reported as `[0.1003…, 0.2027…]`, and + /// `tan(1.5) = 14.101…` sits inside that box. + #[test] + fn tan_encloses_a_box_that_crosses_a_pole() { + let pi = std::f64::consts::PI; + for (lo, hi) in [ + (0.1, 0.2 + pi), + (0.1, 0.5 + pi), + (-1.0, 1.0 + pi), + (-1.4292036732051034, 4.570796326794897), + (1.5, 1.7), // straddles π/2 by less than a period + (-10.0, 10.0), // several periods + ] { + let b = ArbBall::from_midpoint_radius((lo + hi) / 2.0, (hi - lo) / 2.0, PREC); + let out = b.tan().expect("finite box"); + // Sample the true function across the box; every value must be in. + for k in 0..=200 { + let x = lo + (hi - lo) * (k as f64 / 200.0); + let v = Float::with_val(PREC + 32, x).tan(); + assert!( + encloses(&out, &v), + "tan({x}) = {v} escapes {out} for [{lo}, {hi}]" + ); + } + } + } + + /// …and the fix must not answer `∞` to everything: a pole-free box still + /// gets the exact endpoint hull, which is what `tan′ = sec² > 0` buys. + #[test] + fn tan_stays_tight_on_a_pole_free_box() { + let b = ArbBall::from_midpoint_radius(0.55, 0.45, PREC); // [0.1, 1.0] + let out = b.tan().expect("finite box"); + assert!(out.lo() > 0.10033, "lower bound {} is not tight", out.lo()); + assert!(out.hi() < 1.5575, "upper bound {} is not tight", out.hi()); + assert!(out.contains(0.5463024898437905)); // tan(0.5) + } + + /// `pow_f` hulled the four corners of `[lo_b, hi_b] × [lo_e, hi_e]`, which + /// encloses the range only where `x ↦ x^y` is monotone in `x`. A base + /// straddling zero with an even exponent is the counterexample: `x²` over + /// `[-1, 3]` came out as `[1, 9]`, missing `x = 0 ↦ 0`. With a *negative* + /// exponent the claim was worse than narrow — `[0.111…, 1]` for a function + /// unbounded on the box. + #[test] + fn pow_f_encloses_a_base_straddling_zero() { + let base = ArbBall::from_midpoint_radius(1.0, 2.0, PREC); // [-1, 3] + for e in [2.0_f64, 4.0, 2.0, 6.0] { + let out = base.pow_f(&ArbBall::from_f64(e, PREC)); + for k in 0..=100 { + let x = -1.0 + 4.0 * (k as f64 / 100.0); + let v = Float::with_val(PREC + 32, x).pow(e); + assert!(encloses(&out, &v), "{x}^{e} = {v} escapes {out}"); + } + } + // A negative exponent puts a pole at 0 inside the box. + let out = base.pow_f(&ArbBall::from_f64(-2.0, PREC)); + assert!( + out.contains(1e6), + "x^-2 over [-1, 3] claims the finite {out}" + ); + assert!( + out.contains(1e300), + "x^-2 over [-1, 3] claims the finite {out}" + ); + // A non-integer exponent on a negative base is not real at all. + let out = base.pow_f(&ArbBall::from_f64(1.5, PREC)); + assert!(out.rad.is_infinite(), "(-1)^1.5 is not real, got {out}"); + } + + /// The corner hull is kept where it is valid — a base that stays positive — + /// so this must not have widened into uselessness. + #[test] + fn pow_f_stays_tight_on_a_positive_base() { + let base = ArbBall::from_midpoint_radius(2.5, 0.5, PREC); // [2, 3] + let out = base.pow_f(&ArbBall::from_f64(0.5, PREC)); + let (sqrt2, sqrt3) = (std::f64::consts::SQRT_2, 1.7320508075688772_f64); + assert!( + out.lo() > sqrt2 - 1e-9, + "lower bound {} is not tight", + out.lo() + ); + assert!( + out.hi() < sqrt3 + 1e-9, + "upper bound {} is not tight", + out.hi() + ); + // Exponent intervals are allowed too: x^y over [2,3] × [-1, 2]. + let out = base.pow_f(&ArbBall::from_midpoint_radius(0.5, 1.5, PREC)); + for k in 0..=20 { + let x = 2.0 + (k as f64) / 20.0; + for j in 0..=20 { + let y = -1.0 + 3.0 * (j as f64) / 20.0; + let v = Float::with_val(PREC + 32, x).pow(y); + assert!(encloses(&out, &v), "{x}^{y} = {v} escapes {out}"); + } + } + assert!(out.hi() < 9.000001, "upper bound {} is not tight", out.hi()); + } + + /// Randomised enclosure check over *wide* balls for every kernel that + /// builds its answer from the two endpoints. This is the sweep that would + /// have caught `tan` and `pow_f`; a point ball reaches neither. + /// + /// Samples are taken from the ball's *own* endpoints in `Float` arithmetic: + /// recomputing `mid ± rad` in `f64` moves the sample by ~2^-53, which is + /// 10¹⁸ times the outward rounding an enclosure carries, and would put the + /// end samples outside the box being tested rather than on its boundary. + #[test] + fn endpoint_hull_kernels_enclose_random_wide_intervals() { + let work = PREC + 32; + let mut seed = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 11) as f64 / (1u64 << 53) as f64 + }; + for _ in 0..200 { + let centre = (next() - 0.5) * 20.0; + let radius = next() * 5.0; + let ball = ArbBall::from_midpoint_radius(centre, radius, PREC); + let lo = ball.lo(); + let width = Float::with_val(work, ball.hi() - &lo); + let kernels: Vec<(&str, Option)> = vec![ + ("exp", Some(ball.exp())), + ("sin", Some(ball.sin())), + ("cos", Some(ball.cos())), + ("tan", ball.tan()), + ("log", ball.log()), + ("sqrt", ball.sqrt()), + ("sinh", Some(ball.sinh())), + ("cosh", Some(ball.cosh())), + ("tanh", Some(ball.tanh())), + ("asin", ball.asin()), + ("acos", ball.acos()), + ("atan", Some(ball.atan())), + ("asinh", Some(ball.asinh())), + ("acosh", ball.acosh()), + ("atanh", ball.atanh()), + ("erf", Some(ball.erf())), + ("erfc", Some(ball.erfc())), + ("gamma", ball.gamma()), + ("digamma", ball.digamma()), + ("bessel_j0", Some(ball.bessel_jn(0))), + ("square", Some(ball.pow_f(&ArbBall::from_f64(2.0, PREC)))), + ("cube", Some(ball.pow_f(&ArbBall::from_f64(3.0, PREC)))), + ("recip_sq", Some(ball.pow_f(&ArbBall::from_f64(-2.0, PREC)))), + ("sqrt_pow", Some(ball.pow_f(&ArbBall::from_f64(0.5, PREC)))), + ]; + for (name, out) in kernels { + let Some(out) = out else { continue }; + for k in 0..=50 { + let t = Float::with_val(work, k as f64 / 50.0); + let x = Float::with_val(work, &lo + Float::with_val(work, &width * &t)); + let v = match name { + "exp" => x.clone().exp(), + "sin" => x.clone().sin(), + "cos" => x.clone().cos(), + "tan" => x.clone().tan(), + "log" => x.clone().ln(), + "sqrt" => x.clone().sqrt(), + "sinh" => x.clone().sinh(), + "cosh" => x.clone().cosh(), + "tanh" => x.clone().tanh(), + "asin" => x.clone().asin(), + "acos" => x.clone().acos(), + "atan" => x.clone().atan(), + "asinh" => x.clone().asinh(), + "acosh" => x.clone().acosh(), + "atanh" => x.clone().atanh(), + "erf" => x.clone().erf(), + "erfc" => x.clone().erfc(), + "gamma" => x.clone().gamma(), + "digamma" => { + let mut t = x.clone(); + t.digamma_mut(); + t + } + "bessel_j0" => { + let mut t = x.clone(); + t.jn_mut(0); + t + } + "square" => Float::with_val(work, &x * &x), + "cube" => Float::with_val(work, Float::with_val(work, &x * &x) * &x), + "recip_sq" => Float::with_val(work, 1u32 / Float::with_val(work, &x * &x)), + "sqrt_pow" => x.clone().sqrt(), + _ => unreachable!(), + }; + if !v.is_finite() { + continue; + } + assert!( + encloses(&out, &v), + "{name}({x}) = {v} escapes {out} on [{lo}, {}]", + ball.hi() + ); + } + } + } + } + + /// `digamma` used to reject its pole by walking *every* integer in the box, + /// which is 10¹⁸ iterations on a box this wide. + #[test] + fn digamma_does_not_walk_every_integer_in_the_box() { + // [1e16, 9.9e17] — an f64-exact box with 10^18 integers in it. + let b = ArbBall::from_midpoint_radius(5e17, 4.9e17, PREC); + let out = b.digamma().expect("no pole in [1e16, 9.9e17]"); + for x in [1e16_f64, 5e17, 9.9e17] { + let mut v = Float::with_val(PREC + 32, x); + v.digamma_mut(); + assert!(encloses(&out, &v), "ψ({x}) = {v} escapes {out}"); + } + // The pole test still fires when a non-positive integer is in reach. + assert!(ArbBall::from_midpoint_radius(0.0, 1.0, PREC) + .digamma() + .is_none()); + assert!(ArbBall::from_midpoint_radius(-2.5, 1.0, PREC) + .digamma() + .is_none()); + } + /// The radius must stay at the working-precision scale, not balloon. /// /// Soundness is trivially achievable by making every ball enormous; this diff --git a/docs/mdbook/src/ball-arithmetic.md b/docs/mdbook/src/ball-arithmetic.md index 0e42f948..b1616733 100644 --- a/docs/mdbook/src/ball-arithmetic.md +++ b/docs/mdbook/src/ball-arithmetic.md @@ -90,6 +90,28 @@ result = interval_eval(expr, { `interval_eval` guarantees that the output ball contains the true value for any input in the given input balls, accounting for all rounding in the intermediate computation. +### Boxes containing a singularity + +The guarantee is about the whole input box, not only its endpoints, so a box a +function is unbounded on cannot get a finite answer. Such a box widens to +`[-inf, inf]` — an honest, if useless, enclosure — rather than reporting the +hull of the values at the two ends: + +```python +import math +# [0.1, 0.1 + pi] contains pi/2, where tan has a pole +b = ArbBall((0.1 + 0.1 + math.pi) / 2, math.pi / 2) +interval_eval(tan(x), {x: b}) # ArbBall(0.000000 ± inf) + +# x**-2 over a box containing 0 +interval_eval(x**-2.0, {x: ArbBall(1.0, 2.0)}) # ArbBall(0.000000 ± inf) +``` + +A box on which the function is bounded but not monotone still gets a finite +enclosure — `x**2.0` over `[-1, 3]` covers `x = 0`, and `bessel_j0` over +`[-1, 1]` covers the peak at `x = 0` — it is only the unbounded case that +degenerates. + ## AcbBall Complex ball arithmetic for expressions over ℂ: diff --git a/tests/test_ball_enclosure_soundness.py b/tests/test_ball_enclosure_soundness.py index dc3738ef..b252ec41 100644 --- a/tests/test_ball_enclosure_soundness.py +++ b/tests/test_ball_enclosure_soundness.py @@ -23,6 +23,12 @@ ``lo <= v <= hi`` then rejects a value the ball genuinely encloses. They now round outward. +Two more, of a different shape, are covered by the wide-ball section at the +bottom of this file: ``tan`` accepted a box that crossed a pole, and ``pow`` +hulled four corners across a sign change. Neither is reachable from a *point* +ball, which is all this file used to bind — which is why both survived the +tests that exist to catch exactly this. + The constants below are correct to 40 significant digits and are compared with :mod:`decimal`, deliberately not ``mpmath`` — mpmath lives in the ``ci-extras`` group and is not installed for the Tier 1a run that must catch a regression @@ -136,3 +142,214 @@ def test_added_rounding_term_stays_at_precision_scale(pool, x): assert ball.rad < 1e-25 * max(1.0, abs(value)), ( f"radius {ball.rad} is far above the working-precision scale" ) + + +# --------------------------------------------------------------------------- +# Wide balls +# --------------------------------------------------------------------------- +# +# Everything above binds ``ak.ArbBall(at)`` — a *point* ball, ``rad = 0.0``. +# That is the case the two kernels below could not get wrong, and it is why +# they stayed wrong: an enclosure claim about a single point is a claim about +# one value, while the guarantee this file exists to defend is a claim about +# every value in a *box*. +# +# * ``tan`` guarded its pole by testing the two endpoints and by checking +# that ``tan(lo) <= tan(hi)``. A box that crosses a pole and comes out the +# far side satisfies both: ``tan`` over ``[0.1, 0.1 + pi]`` was reported as +# ``[0.1003, 0.2027]``, and ``tan(1.5) = 14.101…`` with ``1.5`` in the box. +# * ``pow`` hulled the four corners of ``base × exponent``, which encloses +# the range only while ``x ↦ x**y`` is monotone in ``x``. ``x**2.0`` over +# ``[-1, 3]`` was reported as ``[1, 9]``, missing ``x = 0``; ``x**-2.0`` +# over the same box was reported as the *finite* ``[0.111, 1.0]`` for a +# function unbounded there. +# +# Neither is reachable at ``rad = 0``. The two tests below are the wide-ball +# sweep that is. + +#: (function name, lo, hi, witness *strictly inside* the box, 40-digit truth). +#: +#: The witnesses avoid the two endpoints on purpose. ``ArbBall(mid, rad)`` +#: cannot represent an arbitrary ``[lo, hi]`` exactly — ``(0.1 + 1.0)/2`` and +#: ``(1.0 - 0.1)/2`` both round — so the box's real lower end sits an ulp or so +#: away from ``0.1``, and for a steep function that ulp exceeds the 40 digits +#: below. Endpoints are covered exactly, from the ball's own bounds, by +#: ``endpoint_hull_kernels_enclose_random_wide_intervals`` in +#: ``alkahest-core/src/ball/mod.rs``. +_WIDE_TRUTH = [ + ("tan", 0.1, 3.241592653589793, 1.5, "14.10141994717171938764608365198775644566"), + ("tan", -1.0, 4.141592653589793, 1.5707, "10381.32741756978658760268387828435979306"), + ("tan", 0.1, 1.0, 0.5, "0.5463024898437905132551794657802853832976"), + ("exp", -3.0, 5.0, 4.5, "90.01713130052181355011545674557436084793"), + ("exp", -3.0, 5.0, -2.5, "0.08208499862389879516952867446715980783780"), + ("log", 0.5, 2.0, 0.6, "-0.5108256237659907202129482504755473004409"), + ("log", 0.5, 2.0, 1.9, "0.6418538861723947292448033614074233770888"), + ("sqrt", 0.25, 9.0, 0.3, "0.5477225575051661033220665419872727524121"), + ("sin", 0.0, 3.0, 1.5707963267948966, "0.9999999999999999999999999999999981253003"), + ("cosh", -2.0, 3.0, 2.9, "9.114584294749733281202437702692431073838"), + ("cosh", -2.0, 3.0, 0.0, "1.0"), + ("tanh", -4.0, 4.0, 3.9, "0.9991808656700278991779684578147509577166"), + ("atan", -5.0, 5.0, 4.9, "1.369479218420255873378967790264769028244"), + ("erf", -2.0, 2.0, 1.9, "0.9927904292352574672372159671320949386714"), + ("gamma", 0.5, 5.0, 1.4616321449683622, "0.8856031944108887002788159005825926411111"), + ("digamma", 0.5, 3.0, 0.6, "-1.540619213893190495500737928815446515306"), + ("bessel_j0", -1.0, 1.0, 0.0, "1.0"), + ("bessel_j0", 2.0, 4.0, 3.0, "-0.2600519549019334376241546959773314368196"), + ("lambert_w", 0.0, 3.0, 1.0, "0.5671432904097838729999686622103555497538"), +] + + +def _wide(lo: float, hi: float) -> ak.ArbBall: + """The ball whose interval is (as closely as `f64` allows) ``[lo, hi]``.""" + return ak.ArbBall((lo + hi) / 2.0, (hi - lo) / 2.0, 128) + + +def _encloses(ball, truth: Decimal) -> bool: + """``lo <= truth <= hi``, with `inf` endpoints handled. + + An unbounded box is honestly reported as ``[-inf, inf]``; that contains + everything, which is exactly the claim being made. + """ + if ball.lo == float("-inf") and ball.hi == float("inf"): + return True + return Decimal(ball.lo) <= truth <= Decimal(ball.hi) + + +def test_wide_ball_enclosures_contain_externally_verified_values(pool, x): + """The guarantee, restated for a box: every value the box reaches is in. + + The witnesses are interior points and endpoints alike, and the truths are + 40-significant-digit constants (compared with :mod:`decimal`, so this runs + in the default tier without ``mpmath``). + """ + failures = [] + for name, lo, hi, witness, truth in _WIDE_TRUTH: + expr = getattr(ak, name)(x) + try: + ball = ak.interval_eval(expr, {x: _wide(lo, hi)}, prec=128) + except ValueError: + continue # a refusal is sound; it claims nothing + if not _encloses(ball, Decimal(truth)): + failures.append( + f"{name} over [{lo}, {hi}]: [{ball.lo}, {ball.hi}] " + f"excludes f({witness}) = {truth}" + ) + assert not failures, "wide ball excluded a value inside its box:\n" + "\n".join( + failures + ) + + +def test_pow_over_a_base_that_changes_sign(pool, x): + """``x**y`` for a base straddling 0, by every route the exponent can take. + + ``x ** p.integer(2)`` reaches a different kernel (``powi``) from + ``x ** 2.0``, ``x ** p.float(2.0)``, ``x ** p.rational(4, 2)`` and + ``x ** y`` with ``y`` bound to a point ball. Only the first was sound. + """ + box = _wide(-1.0, 3.0) + y = pool.symbol("y") + two = [ + ("float literal", x**2.0, {}), + ("p.float(2.0)", x ** pool.float(2.0), {}), + ("p.rational(4, 2)", x ** pool.rational(4, 2), {}), + ("p.integer(2)", x ** pool.integer(2), {}), + ("bound symbol", x**y, {y: ak.ArbBall(2.0, 0.0, 128)}), + ] + failures = [] + for label, expr, extra in two: + ball = ak.interval_eval(expr, {x: box, **extra}, prec=128) + # x = 0 is in the box and 0**2 = 0; the corner hull said [1, 9]. + if not _encloses(ball, Decimal(0)): + failures.append(f"x**2 via {label}: [{ball.lo}, {ball.hi}] excludes 0") + if not _encloses(ball, Decimal(9)): + failures.append(f"x**2 via {label}: [{ball.lo}, {ball.hi}] excludes 9") + + # A negative exponent puts a *pole* in the box, so no finite bound is true. + for label, expr in [ + ("float literal", x**-2.0), + ("p.float(-2.0)", x ** pool.float(-2.0)), + ("p.integer(-2)", x ** pool.integer(-2)), + ]: + ball = ak.interval_eval(expr, {x: box}, prec=128) + if not _encloses(ball, Decimal("999999.9999999999583666365765566310341165")): + failures.append( + f"x**-2 via {label}: [{ball.lo}, {ball.hi}] excludes " + f"f(0.001) = 1e6, and the box contains 0.001" + ) + assert not failures, "\n".join(failures) + + +def test_a_wide_ball_encloses_every_point_ball_inside_it(pool, x): + """Oracle-free sweep: the enclosure over a box contains the enclosure at + each point of that box. + + Both are rigorous claims about the same function, so the box's answer must + contain the point's — no external truth needed, which lets this cover + every kernel that has a ball implementation rather than only the ones with + a constant in the table above. Sampling stays off the two endpoints, where + an `f64` sample can fall a few ulps outside the box it was derived from. + """ + boxes = { + "exp": [(-4.0, 4.0), (0.0, 20.0)], + "log": [(0.25, 8.0), (1e-3, 1.0)], + "sqrt": [(0.0, 9.0)], + "sin": [(-7.0, 7.0), (0.0, 1.0)], + "cos": [(-7.0, 7.0), (0.0, 1.0)], + "tan": [(0.1, 3.3), (-1.0, 4.2), (0.1, 1.0), (1.6, 3.0), (-10.0, 10.0)], + "sinh": [(-3.0, 3.0)], + "cosh": [(-3.0, 3.0), (1.0, 5.0)], + "tanh": [(-4.0, 4.0)], + "asin": [(-0.9, 0.9)], + "acos": [(-0.9, 0.9)], + "atan": [(-5.0, 5.0)], + "asinh": [(-5.0, 5.0)], + "acosh": [(1.5, 6.0)], + "atanh": [(-0.9, 0.9)], + "erf": [(-3.0, 3.0)], + "erfc": [(-3.0, 3.0)], + "gamma": [(0.25, 5.0), (1.0, 2.0)], + "digamma": [(0.25, 5.0)], + "bessel_j0": [(-1.0, 1.0), (0.0, 12.0)], + "bessel_j1": [(0.0, 12.0)], + "lambert_w": [(0.0, 5.0)], + "abs": [(-2.0, 3.0)], + } + exprs = [(name, getattr(ak, name)(x), box) for name, bs in boxes.items() for box in bs] + exprs += [ + (label, expr, box) + for label, expr in [ + ("x**2.0", x**2.0), + ("x**3.0", x**3.0), + ("x**-2.0", x**-2.0), + ("x**0.5", x**0.5), + ("x**rational(4,2)", x ** pool.rational(4, 2)), + ] + for box in [(-1.0, 3.0), (0.5, 4.0), (-4.0, -0.5)] + ] + + failures = [] + for label, expr, (lo, hi) in exprs: + try: + wide = ak.interval_eval(expr, {x: _wide(lo, hi)}, prec=128) + except ValueError: + continue # a refusal is sound + if wide.lo == float("-inf") and wide.hi == float("inf"): + continue # unbounded: contains everything + for k in range(1, 40): + at = lo + (hi - lo) * (k / 40.0) + try: + point = ak.interval_eval(expr, {x: ak.ArbBall(at, 0.0, 128)}, prec=128) + except ValueError: + continue + # Slack absorbs the two enclosures' own outward rounding, which is + # at the 1e-38 scale; a non-monotonicity bug is of order 1. + slack = 1e-25 * max(1.0, abs(point.lo), abs(point.hi)) + if point.lo < wide.lo - slack or point.hi > wide.hi + slack: + failures.append( + f"{label} over [{lo}, {hi}] claims [{wide.lo}, {wide.hi}], " + f"but at x={at} the value is in [{point.lo}, {point.hi}]" + ) + break + assert not failures, "box enclosure does not contain a point inside it:\n" + "\n".join( + failures + ) From 8e68ee889989444caec5ae37e01826ca0ce301b7 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 21:42:06 +0000 Subject: [PATCH 07/11] fix: parenthesise a negative or fractional power base in all three printers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `str`, `latex` and `unicode_str` all rendered `(-1)^n` as `-1^n`, which under standard precedence — Python's, sympy's, LaTeX's, and this crate's own parser's (`BP_UNARY` < `BP_POW`) — means `-(1^n) = -1`. Nothing internal was wrong; only the exported text was, and it was wrong in the place that hurts most: an M1 boundary inhomogeneity handed to a caller for external checking. `b(n) = -16·(-2)^n` printed as `-16 * -2^n`, worth -64 at n=2 inside alkahest and +16 once re-parsed — enough to make an audit harness report a correct engine as unsound. The three printers now agree that a base binding looser than `^` needs parentheses: * `display.rs` gains `PREC_NEG` (25, mirroring the parser's `BP_UNARY`) and `literal_prec`, so a rendered literal carrying a leading `-` no longer claims `PREC_ATOM`. LaTeX/Unicode rationals drop to `PREC_MUL` as well — a fraction is a quotient, not an atom — except for the Unicode vulgar-fraction glyphs (`½`), which are. * `pool.rs` gains `fmt_pow_base`, which wraps anything `fmt_pow_atom` wraps plus any base rendering with a leading `-`. Deliberately unchanged: negative *exponents* (`x^-1`) stay bare, since `^` is right-associative and unary minus binds looser, so no ambiguity is possible; and `parse` itself, which was applying standard precedence correctly to the bad string it was handed. Regression tests: `tests/test_printer_roundtrip.py` checks `sympify(str(e).replace("^","**"))`, `parse(str(e))`, `latex` and `unicode_str` over negative, fractional, sum, product, negative-exponent and nested-power bases; `display.rs` and `pool.rs` gain unit tests for the same. Before the fix 4 of the 7 new Rust tests and 24 of the 58 new Python tests fail; after, all pass, as does `cargo test --workspace` and `pytest tests/`. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 19 +++ alkahest-core/src/kernel/display.rs | 126 ++++++++++++++++++-- alkahest-core/src/kernel/pool.rs | 54 ++++++++- tests/test_printer_roundtrip.py | 176 ++++++++++++++++++++++++++++ 4 files changed, 366 insertions(+), 9 deletions(-) create mode 100644 tests/test_printer_roundtrip.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..c499e822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +- **Every printer emitted `(-1)^n` as `-1^n`, which re-reads as `-(1^n)`.** A + negative power base was rendered without parentheses by all three exported + forms — `str`/`repr`, `latex` and `unicode_str` — so `str((-1)**n)` was + `'-1^n'`, which sympy, Python's own `eval`, LaTeX **and `alkahest.parse`** + all correctly read as `-(1^n) = -1`, not `(-1)^n`. Nothing internal was + wrong: `(-1)^4` evaluated to `1` all along; only the exported text lied, and + it lied in exactly the place it does the most damage — an `M1` boundary + result handed out for external checking. `b(n) = -16·(-2)^n` printed as + `-16 * -2^n`, worth `-64` at `n = 2` inside alkahest and `+16` once + re-parsed, which is enough to make an audit harness report a correct engine + as unsound. The printers now parenthesise any base that does not bind + tighter than `^`: a negative literal (unary minus binds looser than `^`, + matching `BP_UNARY` in the parser) and, in the LaTeX and Unicode renderers, + a fraction — `\left(\frac{1}{2}\right)^n`, `(3/7)^(n)`. Bases that were + already unambiguous are untouched (`2^n`, `x^n`, `½^(n)`), as are negative + *exponents* (`x^-1`), where a leading `-` cannot be misread because `^` is + right-associative. `alkahest.parse` itself was **not** changed: it was + applying standard precedence correctly to the bad string it was given. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-core/src/kernel/display.rs b/alkahest-core/src/kernel/display.rs index 02447158..d9641423 100644 --- a/alkahest-core/src/kernel/display.rs +++ b/alkahest-core/src/kernel/display.rs @@ -8,6 +8,11 @@ use crate::kernel::{ExprId, ExprPool}; const PREC_ADD: i32 = 10; const PREC_MUL: i32 = 20; +/// Unary minus: binds tighter than `*` but looser than `^`, matching `BP_UNARY` +/// in `parse.rs` (and Python, and sympy). A literal that renders with a leading +/// `-` therefore has to be parenthesised as a power base — `(-1)^n`, never +/// `-1^n`, which would re-read as `-(1^n)`. +const PREC_NEG: i32 = 25; const PREC_POW: i32 = 30; const PREC_ATOM: i32 = 100; @@ -158,6 +163,16 @@ fn to_superscript(s: &str) -> Option { Some(out) } +/// Precedence of a rendered numeric literal: [`PREC_NEG`] when it carries a +/// leading `-`, [`PREC_ATOM`] otherwise. +fn literal_prec(rendered: &str) -> i32 { + if rendered.starts_with('-') { + PREC_NEG + } else { + PREC_ATOM + } +} + fn unicode_frac(num: i64, den: i64) -> String { match (num, den) { (1, 2) => "½".into(), @@ -521,18 +536,24 @@ fn latex_piecewise(branches: &[(ExprId, ExprId)], default: ExprId, pool: &ExprPo fn latex_r(id: ExprId, pool: &ExprPool) -> (String, i32) { pool.with(id, |data| match data { ExprData::Symbol { name, .. } => (latex_symbol(name), PREC_ATOM), - ExprData::Integer(n) => (n.0.to_string(), PREC_ATOM), + ExprData::Integer(n) => (n.0.to_string(), literal_prec(&n.0.to_string())), ExprData::Rational(r) => { let num = r.0.numer(); let den = r.0.denom(); let s = latex_frac(num.to_string().trim_start_matches('-'), &den.to_string()); + // A fraction is a quotient, not an atom: it needs parentheses under + // `^` just like any other product/quotient does. if *num < 0 { - (format!("-{s}"), PREC_ATOM) + (format!("-{s}"), PREC_MUL) } else { - (s, PREC_ATOM) + (s, PREC_MUL) } } - ExprData::Float(f) => (f.inner.to_string(), PREC_ATOM), + ExprData::Float(f) => { + let s = f.inner.to_string(); + let prec = literal_prec(&s); + (s, prec) + } ExprData::Add(args) => (latex_add(args, pool), PREC_ADD), ExprData::Mul(args) => { let (sign, tex) = latex_signed_mul(args, pool); @@ -843,18 +864,26 @@ fn unicode_piecewise(branches: &[(ExprId, ExprId)], default: ExprId, pool: &Expr fn unicode_r(id: ExprId, pool: &ExprPool) -> (String, i32) { pool.with(id, |data| match data { ExprData::Symbol { name, .. } => (unicode_symbol(name), PREC_ATOM), - ExprData::Integer(n) => (n.0.to_string(), PREC_ATOM), + ExprData::Integer(n) => (n.0.to_string(), literal_prec(&n.0.to_string())), ExprData::Rational(r) => { let num = r.0.numer().to_i64().unwrap_or(0); let den = r.0.denom().to_i64().unwrap_or(1); let s = unicode_frac(num.abs(), den); + // `unicode_frac` returns a single vulgar-fraction glyph (`½`) for a + // handful of values and a `num/den` quotient otherwise; only the + // former is atomic under `^`. + let prec = if s.contains('/') { PREC_MUL } else { PREC_ATOM }; if num < 0 { - (format!("-{s}"), PREC_ATOM) + (format!("-{s}"), prec.min(PREC_NEG)) } else { - (s, PREC_ATOM) + (s, prec) } } - ExprData::Float(f) => (f.inner.to_string(), PREC_ATOM), + ExprData::Float(f) => { + let s = f.inner.to_string(); + let prec = literal_prec(&s); + (s, prec) + } ExprData::Add(args) => (unicode_add(args, pool), PREC_ADD), ExprData::Mul(args) => { let (sign, tex) = unicode_signed_mul(args, pool); @@ -889,3 +918,84 @@ fn unicode_r(id: ExprId, pool: &ExprPool) -> (String, i32) { } }) } + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernel::Domain; + + /// A negative power base must be parenthesised in every exported form: + /// `-1^n` means `-(1^n)` in LaTeX and in every parser that reads these + /// strings back, so `(-1)^n` is the only round-trippable rendering. + #[test] + fn negative_pow_base_is_parenthesised() { + let p = ExprPool::new(); + let n = p.symbol("n", Domain::Real); + let pow_m1 = p.pow(p.integer(-1_i32), n); + assert_eq!(render_latex(pow_m1, &p), r"\left(-1\right)^n"); + assert_eq!(render_unicode(pow_m1, &p), "(-1)^(n)"); + + let pow_m2 = p.pow(p.integer(-2_i32), n); + assert_eq!(render_latex(pow_m2, &p), r"\left(-2\right)^n"); + assert_eq!(render_unicode(pow_m2, &p), "(-2)^(n)"); + + let pow_mhalf = p.pow(p.rational(-1, 2), n); + assert_eq!(render_latex(pow_mhalf, &p), r"\left(-\frac{1}{2}\right)^n"); + assert_eq!(render_unicode(pow_mhalf, &p), "(-½)^(n)"); + } + + /// A fractional base is a quotient, not an atom, so it needs parentheses + /// too — except where the Unicode renderer has a single glyph for it. + #[test] + fn fractional_pow_base_is_parenthesised() { + let p = ExprPool::new(); + let n = p.symbol("n", Domain::Real); + let pow_half = p.pow(p.rational(1, 2), n); + assert_eq!(render_latex(pow_half, &p), r"\left(\frac{1}{2}\right)^n"); + assert_eq!(render_unicode(pow_half, &p), "½^(n)"); + + let pow_3_7 = p.pow(p.rational(3, 7), n); + assert_eq!(render_latex(pow_3_7, &p), r"\left(\frac{3}{7}\right)^n"); + assert_eq!(render_unicode(pow_3_7, &p), "(3/7)^(n)"); + } + + /// The negative base survives being embedded in a product — this is the + /// `b(n) = -16 * (-2)^n` inhomogeneity shape that surfaced the bug. + #[test] + fn negative_pow_base_inside_product() { + let p = ExprPool::new(); + let n = p.symbol("n", Domain::Real); + let prod = p.mul(vec![p.integer(-16_i32), p.pow(p.integer(-2_i32), n)]); + assert_eq!(render_latex(prod, &p), r"-16 \left(-2\right)^n"); + assert_eq!(render_unicode(prod, &p), "-16·(-2)^(n)"); + } + + /// Non-negative atoms stay bare — the fix must not add noise everywhere. + #[test] + fn positive_pow_base_is_bare() { + let p = ExprPool::new(); + let n = p.symbol("n", Domain::Real); + let x = p.symbol("x", Domain::Real); + let pow_2 = p.pow(p.integer(2_i32), n); + assert_eq!(render_latex(pow_2, &p), "2^n"); + assert_eq!(render_unicode(pow_2, &p), "2^(n)"); + let pow_x = p.pow(x, p.integer(2_i32)); + assert_eq!(render_latex(pow_x, &p), "x^2"); + assert_eq!(render_unicode(pow_x, &p), "x²"); + } + + /// A negative coefficient in a product is still printed bare (`-2 x`): + /// unary minus binds tighter than `*`, so parentheses are unnecessary. + #[test] + fn negative_coefficient_in_product_stays_bare() { + let p = ExprPool::new(); + let x = p.symbol("x", Domain::Real); + let prod = p.mul(vec![p.integer(-2_i32), x]); + assert_eq!(render_latex(prod, &p), "-2 x"); + assert_eq!(render_unicode(prod, &p), "-2·x"); + } +} diff --git a/alkahest-core/src/kernel/pool.rs b/alkahest-core/src/kernel/pool.rs index fd107225..fa921ca0 100644 --- a/alkahest-core/src/kernel/pool.rs +++ b/alkahest-core/src/kernel/pool.rs @@ -544,6 +544,25 @@ fn fmt_pow_atom(id: ExprId, pool: &ExprPool) -> String { } } +/// Format a power *base*. +/// +/// Everything [`fmt_pow_atom`] wraps needs wrapping here too, plus any literal +/// that renders with a leading `-`: unary minus binds looser than `^` in this +/// crate's own parser (`BP_UNARY` < `BP_POW` in `parse.rs`), in Python and in +/// sympy, so `-1^n` re-reads as `-(1^n)`. Only `(-1)^n` round-trips. +/// +/// The exponent side deliberately keeps the bare form (`x^-1`): `^` is +/// right-associative and unary minus binds looser than it, so a `-` there is +/// already unambiguous. +fn fmt_pow_base(id: ExprId, pool: &ExprPool) -> String { + let s = fmt_pow_atom(id, pool); + if s.starts_with('-') { + format!("({s})") + } else { + s + } +} + fn fmt_data(data: &ExprData, pool: &ExprPool, f: &mut fmt::Formatter<'_>) -> fmt::Result { match data { ExprData::Symbol { name, .. } => write!(f, "{}", name), @@ -573,7 +592,7 @@ fn fmt_data(data: &ExprData, pool: &ExprPool, f: &mut fmt::Formatter<'_>) -> fmt ExprData::Pow { base, exp } => { // Parenthesize compound bases/exponents so `x^(1/2)^3` cannot be // misread as `x^1 / 2^3`. Prefer `(x^(1/2))^3`. - let base_s = fmt_pow_atom(*base, pool); + let base_s = fmt_pow_base(*base, pool); let exp_s = fmt_pow_atom(*exp, pool); write!(f, "{base_s}^{exp_s}") } @@ -831,6 +850,39 @@ mod tests { assert_eq!(p.display(expr).to_string(), "(x^2 + 1)"); } + /// A negative power base must be parenthesised: `-1^n` re-reads as + /// `-(1^n)` in this crate's own parser, in Python and in sympy. + #[test] + fn display_negative_pow_base_is_parenthesised() { + let p = pool(); + let n = p.symbol("n", Domain::Real); + let m1 = p.integer(-1_i32); + assert_eq!(p.display(p.pow(m1, n)).to_string(), "(-1)^n"); + let m2 = p.integer(-2_i32); + assert_eq!(p.display(p.pow(m2, n)).to_string(), "(-2)^n"); + let half = p.rational(-1, 2); + assert_eq!(p.display(p.pow(half, n)).to_string(), "(-1/2)^n"); + // …including under a negative exponent, whose bare `-` is unambiguous. + let m3 = p.integer(-3_i32); + assert_eq!(p.display(p.pow(m2, m3)).to_string(), "(-2)^-3"); + // …and inside a product, the `b(n) = -16 * (-2)^n` boundary shape. + // `mul` orders its arguments canonically, hence the factor order here. + let m16 = p.integer(-16_i32); + let prod = p.mul(vec![m16, p.pow(m2, n)]); + assert_eq!(p.display(prod).to_string(), "((-2)^n * -16)"); + } + + /// A non-negative atom keeps the bare form — no gratuitous parentheses. + #[test] + fn display_positive_pow_base_is_bare() { + let p = pool(); + let n = p.symbol("n", Domain::Real); + let two = p.integer(2_i32); + assert_eq!(p.display(p.pow(two, n)).to_string(), "2^n"); + let x = p.symbol("x", Domain::Real); + assert_eq!(p.display(p.pow(x, n)).to_string(), "x^n"); + } + // --- send + sync: compile-time check --- fn assert_send_sync() {} diff --git a/tests/test_printer_roundtrip.py b/tests/test_printer_roundtrip.py new file mode 100644 index 00000000..9a5c616b --- /dev/null +++ b/tests/test_printer_roundtrip.py @@ -0,0 +1,176 @@ +"""Printed expressions must re-read as the expression that was printed. + +Regression guard for the `(-1)^n` → `-1^n` printer bug: a negative (or +otherwise non-atomic) power base was emitted without parentheses, so every +exported form meant `-(1^n)` under the standard precedence used by Python, +sympy, LaTeX — and by alkahest's own parser. +""" + +import alkahest as ak +import pytest +from alkahest import ExprPool, latex, parse, simplify, unicode_str + +sympy = pytest.importorskip("sympy") + + +@pytest.fixture +def pool(): + return ExprPool() + + +@pytest.fixture +def syms(pool): + return { + "n": pool.symbol("n"), + "x": pool.symbol("x"), + "a": pool.symbol("a"), + "b": pool.symbol("b"), + } + + +def _cases(pool, syms): + """`label -> (expr, python_source, latex, unicode)` for each printed form. + + `python_source` is an independent, unambiguous spelling of the same + mathematics; the round-trip test sympifies it and the printed form and + demands they agree. + """ + n, x, a, b = syms["n"], syms["x"], syms["a"], syms["b"] + one = pool.integer(1) + cases = [ + ("(-1)^n", pool.integer(-1) ** n, "(-1)**n", r"\left(-1\right)^n", "(-1)^(n)"), + ("(-2)^n", pool.integer(-2) ** n, "(-2)**n", r"\left(-2\right)^n", "(-2)^(n)"), + ( + "(-1/2)^n", + pool.rational(-1, 2) ** n, + "(sympy.Rational(-1, 2))**n", + r"\left(-\frac{1}{2}\right)^n", + "(-½)^(n)", + ), + ( + "(1/2)^n", + pool.rational(1, 2) ** n, + "(sympy.Rational(1, 2))**n", + r"\left(\frac{1}{2}\right)^n", + "½^(n)", + ), + ( + "(3/7)^n", + pool.rational(3, 7) ** n, + "(sympy.Rational(3, 7))**n", + r"\left(\frac{3}{7}\right)^n", + "(3/7)^(n)", + ), + ("(x + 1)^n", (x + one) ** n, "(x + 1)**n", r"\left(x + 1\right)^n", "(x + 1)^(n)"), + ("(-x)^n", (pool.integer(-1) * x) ** n, "(-x)**n", r"\left(-x\right)^n", "(-x)^(n)"), + ("(a*b)^n", (a * b) ** n, "(a*b)**n", r"\left(a b\right)^n", "(a·b)^(n)"), + # negative exponents + ("x^-2", x ** pool.integer(-2), "x**-2", "x^{-2}", "x⁻²"), + ( + "(-2)^-3", + pool.integer(-2) ** pool.integer(-3), + "(-2)**-3", + r"\left(-2\right)^{-3}", + "(-2)⁻³", + ), + ( + "(-x)^-1", + (pool.integer(-1) * x) ** pool.integer(-1), + "(-x)**-1", + r"\frac{1}{\left(-x\right)}", + "(-x)⁻¹", + ), + # nested powers + ( + "(x^2)^3", + (x ** pool.integer(2)) ** pool.integer(3), + "(x**2)**3", + r"\left(x^2\right)^3", + "(x²)³", + ), + ( + "((-1)^n)^2", + (pool.integer(-1) ** n) ** pool.integer(2), + "((-1)**n)**2", + r"\left(\left(-1\right)^n\right)^2", + "((-1)^(n))²", + ), + # a negative base inside a product — the M1 boundary shape `-16 * (-2)^n` + ( + "-16 * (-2)^n", + pool.integer(-16) * (pool.integer(-2) ** n), + "-16 * (-2)**n", + r"-16 \left(-2\right)^n", + "-16·(-2)^(n)", + ), + ] + return {c[0]: c[1:] for c in cases} + + +LABELS = [ + "(-1)^n", + "(-2)^n", + "(-1/2)^n", + "(1/2)^n", + "(3/7)^n", + "(x + 1)^n", + "(-x)^n", + "(a*b)^n", + "x^-2", + "(-2)^-3", + "(-x)^-1", + "(x^2)^3", + "((-1)^n)^2", + "-16 * (-2)^n", +] + + +def _sympify(src): + return sympy.sympify(src, locals={"sympy": sympy}) + + +@pytest.mark.parametrize("label", LABELS) +def test_str_round_trips_through_sympy(pool, syms, label): + """`sympify(str(e).replace("^", "**"))` is the expression that was printed.""" + expr, source, _tex, _uni = _cases(pool, syms)[label] + printed = _sympify(str(expr).replace("^", "**")) + expected = _sympify(source) + assert printed == expected, f"{label}: str={str(expr)!r} reads as {printed}, want {expected}" + + +@pytest.mark.parametrize("label", LABELS) +def test_str_round_trips_through_alkahest_parse(pool, syms, label): + """alkahest can re-read its own output.""" + expr, _source, _tex, _uni = _cases(pool, syms)[label] + reparsed = parse(str(expr), pool, syms) + assert simplify(reparsed).value == simplify(expr).value, ( + f"{label}: str={str(expr)!r} parses to {reparsed}" + ) + + +@pytest.mark.parametrize("label", LABELS) +def test_latex(pool, syms, label): + expr, _source, tex, _uni = _cases(pool, syms)[label] + assert latex(expr) == tex, label + + +@pytest.mark.parametrize("label", LABELS) +def test_unicode(pool, syms, label): + expr, _source, _tex, uni = _cases(pool, syms)[label] + assert unicode_str(expr) == uni, label + + +def test_case_table_is_complete(pool, syms): + """`LABELS` and the case table must not drift apart.""" + assert sorted(_cases(pool, syms)) == sorted(LABELS) + + +def test_negative_base_agrees_with_evaluation(pool, syms): + """The printed form of `(-1)^n` must not flip sign when re-read at `n = 4`.""" + n = syms["n"] + expr = pool.integer(-1) ** n + direct = simplify(ak.subs(expr, {n: pool.integer(4)})).value + reparsed = parse(str(expr), pool, syms) + via_text = simplify(ak.subs(reparsed, {n: pool.integer(4)})).value + assert str(direct) == "1" + assert str(via_text) == str(direct) From 80a70b2541b52a8113f0ba68e21ec41cc4e769d6 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 21:55:05 +0000 Subject: [PATCH 08/11] fix(sos): half-Newton-polytope reduction; retract the false N=2 Motzkin claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retracts a false mathematical claim recorded in five places and pinned by a passing test, and lands the search fix behind it. ## The false claim (issue #28) The previous round recorded that the homogeneous ternary Motzkin form requires multiplier power N=2, and called reaching it "a genuine, quantified numerical-hardness finding". The premise is false: (x²+y²+z²)(x⁴y²+x²y⁴−3x²y²z²+z⁶) = (½x³y+xy³−3⁄2xyz²)² + ¾(x³y−xyz²)² + (xy²z−xz³)² + (x²yz−yz³)² + (x²y²−z⁴)² This identity is precisely why Motzkin is the standard example of a PSD non-SOS form that becomes SOS after one factor of Σxᵢ². Re-verified here by exact rational expansion and independently in sympy before any edit. `psd_search_does_not_yet_reach_homogeneous_motzkin_times_sum_of_squares` asserted `is_none()`, *passed*, and its comment said the refusal was "expected to stay None permanently" — so CI defended the bug. It is replaced by its contrapositive, `psd_search_certifies_homogeneous_motzkin_times_sum_of_squares_at_n1`, which passes. The claim is corrected in positivity.md, alkahest-skill/alkahest.md and CHANGELOG.md (retraction kept alongside the original text so the reasoning is not re-derived); the planning doc is corrected in the temp-alkahest repo. ## The search fix (issue #29) `psd_search` did no half-Newton-polytope reduction. Reznick: if p = Σqᵢ² then Newton(qᵢ) ⊆ ½·Newton(p) for every i, so restricting the Gram basis to the lattice points of ½·Newton(p) is complete, not heuristic. `half_newton_reduce` decides membership exactly over Q with the module's own rational simplex. This is a *dimension* reduction and nothing else. On σ·Motzkin_hom it cuts the basis 15 → 9 and the family 75 → 18 parameters; on both bases the certificate is the unique PSD point, rank 5, λ_min exactly 0, so λ_min does not distinguish them — what changes is that the numeric solution lands *on* the certificate instead of ~0.96 away from it in parameter space. More Douglas-Rachford does not substitute. Robinson (15 → 15) is the guard case and is asserted unchanged. Measured, same host and load, psd_search alone: σ·Motzkin_hom N=1 123.0 s refuse -> 3.7 s certificate σ·Choi-Lam N=1 549.6 s refuse -> 21.1 s certificate σ·Robinson N=1 54.0 s cert -> 50.2 s certificate (unchanged) ## The silent ceilings (issues #31, 26h, 26g) Three budgets converted "we did not look" into a verdict that reads like "we looked and found nothing", all producing the same bare E-SOS-002: - MAX_FREE_PARAMETERS (200) dropped whole families with nothing logged. For the Horn/C₅ form that meant *no multiplier power was ever searched* (N=1 has 420 free parameters), so its refusal was not a search result. Now logged as NOT SEARCHED, and applied before `solve_affine` using pack_len(n) − rows as a dimension lower bound — C₇'s N=1 was paying for a 924x3570 exact Gauss-Jordan that was then discarded. - MAX_MULTIPLIER_BASIS_LEN (90) compared the *unreduced* monomial_basis count. C₇'s N=1 was rejected at 120 > 90 though its real basis is 84. Now uses psd::searched_basis_len, and logs both numbers. - The refusal text said "no Reznick multiplier up to N=4 made σ·p SOS within the search budget", implying four powers were tried. Now "that was actually searched", with the full trace appended and reachable via the new SosError::search_trace / hit_a_search_ceiling accessors (additive, semver-clean — no new enum variant). C₇ end to end: 1294.0 s -> 783.6 s, refusal unchanged but now fully traced. Also: SosOpts::basis_degree now reaches multiplier_search, which derived its basis degree from deg(σ·p) alone and never read the option — so the path was bit-identical at every setting while E-SOS-002 told callers to raise it. And a doc comment on symmetry_reduced_search claiming Douglas-Rachford "reliably closes the gap", citing a test that never existed, is corrected. ## Still open Horn/C₅ and C₇ at N=1. Their Newton polytopes are already full, so the reduction does not help, and their families (420 and 2646 free parameters) are above the numeric-search ceiling. Closing them needs an interior-point SDP solve, which must run on the reduced family. This is now reported rather than hidden. Verified: cargo fmt, clippy -D warnings, cargo test --workspace (2290 passed, 0 failed), pytest tests/test_sos.py (13 passed), tests/test_error_code_registry.py + tests/silent_errors/ (328 passed). Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 102 ++++++ alkahest-core/src/errors/codes.rs | 2 +- alkahest-core/src/real/sos/mod.rs | 285 ++++++++++++++++- alkahest-core/src/real/sos/psd.rs | 495 ++++++++++++++++++++++++------ alkahest-skill/alkahest.md | 2 +- docs/mdbook/src/errors.md | 2 +- docs/mdbook/src/positivity.md | 101 +++--- tests/test_sos.py | 79 +++++ 8 files changed, 922 insertions(+), 146 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..5b3ec8a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,83 @@ ## Unreleased +- **`sos_decompose` now certifies the homogeneous ternary Motzkin form and + Choi–Lam at multiplier power `N = 1`, via a half-Newton-polytope reduction + — and retracts the claim that it could not.** The previous round recorded, + as its closing M10 finding, that `(x²+y²+z²)·Motzkin_hom` is not SOS and + that the classical fact needs `N = 2`, and pinned that with a *passing* + test named `psd_search_does_not_yet_reach_…` whose comment said the `N = 1` + refusal was "expected to stay `None` permanently". **The premise was + false.** `(x²+y²+z²)(x⁴y²+x²y⁴−3x²y²z²+z⁶) = (½x³y+xy³−3⁄2xyz²)² + + ¾(x³y−xyz²)² + (xy²z−xz³)² + (x²yz−yz³)² + (x²y²−z⁴)²` — this identity is + the reason Motzkin is the standard example of a PSD non-SOS form that + becomes SOS after one multiplication by `Σxᵢ²`. The claim had propagated + into `docs/mdbook/src/positivity.md`, `alkahest-skill/alkahest.md` and this + file; all are corrected, and the test is now stated in the positive + direction (`psd_search_certifies_homogeneous_motzkin_times_sum_of_squares_at_n1`) + so a regression is a failure rather than a confirmation. + + **What was actually missing was a dimension reduction, not iterations.** + `psd_search` now restricts the Gram basis to the lattice points of + `½·Newton(p)` (`psd::half_newton_reduce`) before searching. Reznick's + theorem says the support of every square in every SOS decomposition of `p` + already lies there, so the restriction is *complete*, not heuristic — and + it cuts `σ·Motzkin_hom`'s degree-4 ternary basis from 15 monomials to 9, + its affine family from 75 free parameters to 18. That is the difference + between a numeric solution landing ≈ 0.96 away from the true certificate in + parameter space (so no rounding recovers it) and landing on it exactly. + Note what it is not: on both bases the certificate is the unique PSD point + of the family, rank 5, minimum eigenvalue exactly 0, so `λ_min` is a + misleading progress metric here and more Douglas–Rachford does not help + (4× the budget on the unreduced family still fails to round). Robinson's + form, whose Newton polytope is already full (15 → 15), is unaffected — + asserted as a guard case in + `psd::tests::half_newton_reduction_is_a_no_op_when_the_polytope_is_already_full`. + Measured on one host, same load, `psd_search` alone: `σ·Motzkin_hom` at + `N = 1` went from a 123.0 s refusal to a 3.7 s certificate, `σ·Choi–Lam` + from a 549.6 s refusal to a 21.1 s certificate; `σ·Robinson` is unchanged + at 54.0 s → 50.2 s, still a certificate. + +- **`E-SOS-002` now reports what the search actually did, so a budget that + fired is distinguishable from a search that came up empty.** Three separate + ceilings could previously convert "we did not look" into a verdict that + reads like "we looked and found nothing", all producing the same instant + `E-SOS-002` with nothing logged: + + - `psd::MAX_FREE_PARAMETERS` (200) dropped whole affine families silently. + For the Horn/C₅ copositivity form this meant *no multiplier power was + ever searched at all* (its `N = 1` family has 420 free parameters), so + its refusal was not a search result. The ceiling now logs a line marked + `NOT SEARCHED` naming the family size and the ceiling, and the refusal + message carries the whole trace. It is also applied *before* + `solve_affine` rather than after, using `pack_len(n) − rows` as a lower + bound on the family's dimension — C₇'s `N = 1` was paying for a + 924 × 3570 exact rational Gauss–Jordan whose result was thrown away. + End to end, C₇'s refusal went from 1294.0 s to 783.6 s and from a bare + message to a trace naming every power that ran and every one that did + not; the Horn form's went from 37.7 s to 43.9 s, likewise with a trace. + - `MAX_MULTIPLIER_BASIS_LEN` (90) was compared against the *unreduced* + `monomial_basis` count, which is not the basis that gets searched. C₇'s + `N = 1` was rejected at 120 > 90 though its real basis is 84. The + comparison now uses `psd::searched_basis_len`, and the log reports both + numbers. + - The refusal text said "no Reznick multiplier … up to `N = 4` made `σ·p` + SOS within the search budget", which reads as if four powers had been + tried. It now says "that was actually searched", and the appended trace + names each power and whether it ran. + +- **`SosOpts::basis_degree` now reaches the multiplier search.** + `multiplier_search` derived its basis degree from `deg(σ·p)` alone and + never read the option, so the multiplier path was bit-identical at every + setting — while `E-SOS-002`'s remediation told callers to raise it. The + option is now a floor on that path (a basis below `⌈deg(σ·p)/2⌉` cannot + reproduce `σ·p`'s top-degree terms, so it can only be raised, not lowered). + +- **Fixed a doc comment on `psd::symmetry_reduced_search`** that claimed + Douglas–Rachford "reliably closes the gap" on the homogeneous Motzkin + family and cited `psd::tests::psd_search_certifies_homogeneous_motzkin_at_multiplier_power_2`, + a test that has never existed. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search @@ -1467,6 +1544,26 @@ Both are detailed under *Behaviour changes to plan for*. `psd::tests::psd_search_certifies_robinsons_form_with_a_reznick_multiplier` check the identities by hand, independent of the search that proposed them. + > **RETRACTED 2026-08-20 — the premise of the block below is false.** It + > claims the homogeneous ternary Motzkin form needs multiplier power + > `N = 2` and that `N = 1` "is not classically expected to work … at all". + > `(x²+y²+z²)·(x⁴y²+x²y⁴−3x²y²z²+z⁶)` **is** a sum of squares: + > + > ```text + > = (½x³y+xy³−3⁄2xyz²)² + ¾(x³y−xyz²)² + (xy²z−xz³)² + (x²yz−yz³)² + (x²y²−z⁴)² + > ``` + > + > which is exactly why Motzkin is the standard example of a PSD non-SOS + > form that becomes SOS after one factor of `Σxᵢ²`. The measurements below + > are real; the diagnosis attached to them was not, and the round of + > engineering they motivated was aimed at a problem that does not exist. + > The real defect was a missing half-Newton-polytope reduction — see the + > Unreleased entry. The test cited below asserted the wrong mathematics and + > passed; it has been replaced by its contrapositive, + > `psd::tests::psd_search_certifies_homogeneous_motzkin_times_sum_of_squares_at_n1`. + > Kept here, struck through in spirit, because a retraction that deletes + > the claim leaves nothing to warn the next reader off re-deriving it. + **What's still open, now attempted to closure and precisely quantified (2026-08-17, round 3):** the homogeneous 3-variable form of Motzkin, `(x²+y²+z²)²·(x⁴y²+x²y⁴−3x²y²z²+z⁶)` — multiplier power `N = 2`, not `N = 1` @@ -1508,6 +1605,11 @@ Both are detailed under *Behaviour changes to plan for*. All three lines of evidence agree: this is now a genuine, quantified numerical-hardness finding (an unusually slowly-converging tangential intersection), not an under-tuned budget or an unexplored structural avenue. + *(2026-08-20: the measurements hold; the conclusion does not. The three + approaches were all applied to `N = 2`, which was never the right power, + and the tangential-intersection diagnosis does not explain the `N = 1` + refusal either — the cyclic AM-GM sextic has the identical geometry and + certifies on the unreduced family. See the retraction above.)* `symmetry_reduced_search` ships anyway as a real, general capability — it is wired into `psd_search` as a further fallback (only once the direct search and facial reduction have both already failed, so it adds no cost to any diff --git a/alkahest-core/src/errors/codes.rs b/alkahest-core/src/errors/codes.rs index 911e2edf..159a7c8a 100644 --- a/alkahest-core/src/errors/codes.rs +++ b/alkahest-core/src/errors/codes.rs @@ -241,7 +241,7 @@ pub const REGISTRY: &[ErrorSpec] = &[ // E-DOMAIN — reserved; DomainError is Python-only pending Rust implementation // E-SOS — SosError (P1 item 8: positivity certificates / Positivstellensatz) ErrorSpec { code: "E-SOS-001", class: "SosError", cause: Cause::UserInput, remediation: Some("positivity certificates are for polynomials in the listed variables; expand or clear denominators first, and pass every symbol that occurs as a variable") }, - ErrorSpec { code: "E-SOS-002", class: "SosError", cause: Cause::Unsupported, remediation: Some("record this as unknown, not as a closed branch: raise basis_degree (unconstrained) or level (constrained); the search covers the diagonally dominant subcone, so this is not a proof that no SOS decomposition exists, and still less that the inequality is false — alkahest.decide is the complete (and far more expensive) fallback") }, + ErrorSpec { code: "E-SOS-002", class: "SosError", cause: Cause::Unsupported, remediation: Some("record this as unknown, not as a closed branch: read the 'what the search actually did' trace in the message first — a line marked NOT SEARCHED is a size ceiling that fired, not a search that came up empty — then raise basis_degree (unconstrained) or level (constrained); this is not a proof that no SOS decomposition exists, and still less that the inequality is false — alkahest.decide is the complete (and far more expensive) fallback") }, ErrorSpec { code: "E-SOS-003", class: "SosError", cause: Cause::UserInput, remediation: Some("the witness point in the message satisfies the constraints and makes the target negative; the claim is false as stated") }, ErrorSpec { code: "E-SOS-004", class: "SosError", cause: Cause::UserInput, remediation: Some("pass at least one variable, and keep basis_degree/level within the supported range") }, ErrorSpec { code: "E-SOS-005", class: "SosError", cause: Cause::Internal, remediation: Some("internal: report the target and constraints as a minimal failing example") }, diff --git a/alkahest-core/src/real/sos/mod.rs b/alkahest-core/src/real/sos/mod.rs index 6aec0070..4f8df542 100644 --- a/alkahest-core/src/real/sos/mod.rs +++ b/alkahest-core/src/real/sos/mod.rs @@ -116,6 +116,45 @@ pub enum SosError { VerificationFailed(String), } +/// Separates the human-readable half of a [`SosError::NoCertificate`] +/// message from the trace of what the search actually did. Stable: callers +/// split on it via [`SosError::search_trace`]. +const SEARCH_TRACE_MARKER: &str = "what the search actually did:"; + +impl SosError { + /// The trace of what the certificate search actually did, one step per + /// line, or `None` for errors that carry no trace. + /// + /// `E-SOS-002` covers three materially different situations — a search + /// that ran and was exhausted, a search that ran up to an iteration or + /// rounding budget, and a basis or multiplier power that was **never + /// searched at all** because it was over a size ceiling — and the code + /// alone cannot distinguish them. Reading the trace can; so can + /// [`Self::hit_a_search_ceiling`], which is the single question most + /// callers actually have. + pub fn search_trace(&self) -> Option<&str> { + match self { + SosError::NoCertificate(msg) => msg + .split_once(SEARCH_TRACE_MARKER) + .map(|(_, trace)| trace.trim_start_matches('\n')), + _ => None, + } + } + + /// Did a size ceiling stop some part of the search from running at all? + /// + /// `true` means the refusal is *weaker* than an exhausted search: at + /// least one basis or multiplier power was skipped without being looked + /// at, so "no certificate was found" does not even mean "everything in + /// scope was tried". Never a reason to record a positive verdict either + /// way — `E-SOS-002` is `unknown` in all cases — but a caller escalating + /// to a more expensive method should know which kind of refusal it got. + pub fn hit_a_search_ceiling(&self) -> bool { + self.search_trace() + .is_some_and(|trace| trace.contains("NOT SEARCHED")) + } +} + impl fmt::Display for SosError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -150,11 +189,12 @@ impl crate::errors::AlkahestError for SosError { clear denominators first, and pass every symbol that occurs as a variable" } SosError::NoCertificate(_) => { - "record this as unknown, not as a closed branch: raise basis_degree \ - (unconstrained) or level (constrained); the search covers the diagonally \ - dominant subcone, so this is not a proof that no SOS decomposition exists, \ - and still less that the inequality is false — alkahest.decide is the \ - complete (and far more expensive) fallback" + "record this as unknown, not as a closed branch: read the 'what the search \ + actually did' trace in the message first — a line marked NOT SEARCHED is a \ + size ceiling that fired, not a search that came up empty — then raise \ + basis_degree (unconstrained) or level (constrained); this is not a proof \ + that no SOS decomposition exists, and still less that the inequality is \ + false — alkahest.decide is the complete (and far more expensive) fallback" } SosError::Negative(_) => { "the witness point in the message satisfies the constraints and makes the target \ @@ -267,11 +307,18 @@ const MAX_MULTIPLIER_BASIS_LEN: usize = 90; /// `max_power` and `max_basis_len` are explicit parameters rather than the /// module constants above so tests can exercise the "budget exhausted" /// path with a small, fast budget instead of the production one. +/// +/// `basis_degree` is the caller's `SosOpts::basis_degree`. `None` uses the +/// smallest basis that can possibly work for `σ·p` (`⌈deg(σ·p)/2⌉`); a +/// larger value genuinely widens the basis searched here. It used to be +/// ignored outright on this path, which made `E-SOS-002`'s advice to raise +/// `basis_degree` inert in exactly the case where that error fires most. fn multiplier_search( target: &RatPoly, nvars: usize, max_power: u32, max_basis_len: usize, + basis_degree: Option, log: &mut Vec, ) -> Option<(RatPoly, SosPoly, u32)> { for n in 1..=max_power { @@ -281,23 +328,41 @@ fn multiplier_search( if qdeg % 2 != 0 { continue; } - let basis_deg = qdeg.div_ceil(2); - let basis_len = gram::monomial_basis(nvars, basis_deg).len(); + let min_basis_deg = qdeg.div_ceil(2); + // A basis below ⌈deg(σ·p)/2⌉ cannot reproduce σ·p's top-degree + // terms at all, so the caller's request is a floor to raise to, not + // a ceiling to clamp at. + let basis_deg = basis_degree.unwrap_or(min_basis_deg).max(min_basis_deg); + // Budget against the basis actually searched, not the raw + // `monomial_basis` count: `psd_search` applies both the homogeneity + // restriction and the half-Newton-polytope reduction before it + // searches anything, and comparing the *unreduced* count against the + // budget rejects powers whose real basis is well inside it (C₇'s + // N=1, for instance: 120 raw, 84 real, against a budget of 90). + let raw_len = gram::monomial_basis(nvars, basis_deg).len(); + let basis_len = psd::searched_basis_len(&q, basis_deg); if basis_len > max_basis_len { log.push(format!( - "multiplier search: N={n} would need a degree-{basis_deg} basis \ - ({basis_len} monomials), over the search budget ({max_basis_len}); stopping" + "multiplier search: N={n} NOT SEARCHED — σ·p would need a degree-{basis_deg} \ + basis of {basis_len} monomials ({raw_len} before the homogeneity and \ + half-Newton reductions), over the search budget of {max_basis_len}; stopping \ + here, so no multiplier power from N={n} up was searched at all" )); break; } log.push(format!( "multiplier search: trying σ = (Σxᵢ²)^{n}, searching the full PSD Gram cone for \ - σ·p over the degree-{basis_deg} monomial basis ({basis_len} monomials)" + σ·p over the degree-{basis_deg} monomial basis ({basis_len} monomials, {raw_len} \ + before reduction)" )); - if let Some(sos) = psd::psd_search(&q, basis_deg) { + if let Some(sos) = psd::psd_search_logged(&q, basis_deg, log) { log.push(format!("multiplier search succeeded at N={n}")); return Some((sigma, sos, basis_deg)); } + log.push(format!( + "multiplier search: N={n} produced no certificate (see the lines above for what \ + ran at this power and what was skipped)" + )); } None } @@ -382,7 +447,7 @@ pub fn sos_decompose( log.push( "diagonally dominant search failed; trying the full PSD Gram cone directly".to_string(), ); - if let Some(sos) = psd::psd_search(&target, basis_deg) { + if let Some(sos) = psd::psd_search_logged(&target, basis_deg, &mut log) { log.push( "full PSD Gram search succeeded (p is SOS but its Gram matrix is not \ diagonally dominant)" @@ -407,6 +472,7 @@ pub fn sos_decompose( nvars, MAX_MULTIPLIER_POWER, MAX_MULTIPLIER_BASIS_LEN, + opts.basis_degree, &mut log, ) { return finish(PositivityCertificate { @@ -423,13 +489,28 @@ pub fn sos_decompose( log, }); } + // The trace is not decoration: `None` out of the searches above + // covers "searched and exhausted", "searched up to a budget" and + // "never searched at all" (a multiplier power skipped for basis + // size, or an affine family over `psd`'s free-parameter ceiling), + // and the bare error code cannot tell those apart. Reporting which + // multiplier powers actually ran — and which were skipped, with the + // reason — is the difference between "we looked and found nothing" + // and "we did not look". + let trace = log + .iter() + .map(|line| format!("\n - {line}")) + .collect::(); return Err(SosError::NoCertificate(format!( "undecided, not a refutation — no diagonally dominant or general PSD Gram matrix \ over the degree-{basis_deg} monomial basis reproduces p, and no Reznick multiplier \ - (Σxᵢ²)^N up to N={MAX_MULTIPLIER_POWER} made σ·p SOS within the search budget \ + (Σxᵢ²)^N up to N={MAX_MULTIPLIER_POWER} that was actually searched made σ·p SOS \ either. None of this is a proof that p is not SOS (with or without a multiplier), \ and still less that p is not non-negative — only that no certificate of these \ - shapes was found at this size. Raise basis_degree, or fall back to alkahest.decide" + shapes was found at this size. Read the trace below before treating this as \ + exhaustive: a line marked NOT SEARCHED is a budget that fired, not a search that \ + came up empty. Raise basis_degree, or fall back to alkahest.decide.\ + \n{SEARCH_TRACE_MARKER}{trace}" ))); }; @@ -694,7 +775,7 @@ mod tests { ]); let target = RatPoly::from_expr(p, &[x, y], &pool).unwrap(); let mut log = Vec::new(); - let out = multiplier_search(&target, 2, /* max_power */ 0, 90, &mut log); + let out = multiplier_search(&target, 2, /* max_power */ 0, 90, None, &mut log); assert!( out.is_none(), "a zero-power budget must not manufacture a certificate" @@ -718,6 +799,180 @@ mod tests { ); } + /// The C₇ copositivity form: `Σ_ij M_ij x_i² x_j²` with + /// `M = 3·(I + A(C₇)) − J`, `A(C₇)` the adjacency matrix of the 7-cycle. + /// A 7-variable quartic that is non-negative but not SOS. + fn c7_copositivity_form() -> RatPoly { + let n = 7usize; + let adj = |i: usize, j: usize| -> i64 { + let d = (i + n - j) % n; + i64::from(d == 1 || d == n - 1) + }; + let mut p = RatPoly::zero(n); + for i in 0..n { + for j in 0..n { + let m = 3 * (i64::from(i == j) + adj(i, j)) - 1; + if m == 0 { + continue; + } + let mut e = vec![0u32; n]; + e[i] += 2; + e[j] += 2; + p = p.add(&RatPoly::monomial(n, e, Rational::from(m))); + } + } + p + } + + #[test] + fn the_multiplier_budget_uses_the_basis_actually_searched() { + // `MAX_MULTIPLIER_BASIS_LEN` used to be compared against the *raw* + // `monomial_basis` count, which is not the basis `psd_search` + // searches: it applies a homogeneity restriction and a + // half-Newton-polytope reduction first. For C₇'s N=1 multiplier that + // is the difference between 120 (rejected, "over the search budget") + // and the 84 monomials really at stake (inside the budget of 90) — + // so the power was skipped on a number that never described the + // search, and the log reported a size the search never used. + let p = c7_copositivity_form(); + let q = p.mul(&RatPoly::sum_of_squares(7)); + assert_eq!(gram::monomial_basis(7, 3).len(), 120); + assert!(gram::monomial_basis(7, 3).len() > MAX_MULTIPLIER_BASIS_LEN); + assert_eq!(psd::searched_basis_len(&q, 3), 84); + assert!(psd::searched_basis_len(&q, 3) <= MAX_MULTIPLIER_BASIS_LEN); + + let mut log = Vec::new(); + let out = multiplier_search(&p, 7, 1, MAX_MULTIPLIER_BASIS_LEN, None, &mut log); + assert!(out.is_none(), "C₇'s N=1 certificate is still out of reach"); + let tried = log + .iter() + .find(|l| l.contains("trying σ")) + .unwrap_or_else(|| panic!("N=1 must no longer be rejected on basis size: {log:?}")); + assert!( + tried.contains("84 monomials") && tried.contains("120 before reduction"), + "the log must report the searched size, with the raw one for context: {tried}" + ); + } + + #[test] + fn a_refusal_reports_which_multiplier_powers_were_actually_searched() { + // The Horn form (copositivity of the Horn matrix, `Σ_ij H_ij x_i² + // x_j²`): non-negative, not SOS, and — as `E-SOS-002` used to present + // it — "no Reznick multiplier up to N=4 made σ·p SOS", which reads as + // if four multiplier powers had been tried. In fact *no* multiplier + // power is searched here at all: N=1's affine family has 420 free + // parameters, over `psd`'s ceiling, and the higher powers are over + // the basis-length budget. The refusal is the same either way; what + // must not be the same is what it claims to have done. + let (pool, _x, _y) = setup(); + let vars: Vec = (0..5) + .map(|i| pool.symbol(format!("h{i}"), Domain::Real)) + .collect(); + let h = [ + [1, -1, 1, 1, -1], + [-1, 1, -1, 1, 1], + [1, -1, 1, -1, 1], + [1, 1, -1, 1, -1], + [-1, 1, 1, -1, 1], + ]; + let mut terms = Vec::new(); + for i in 0..5 { + for j in 0..5 { + terms.push(pool.mul(vec![ + pool.integer(h[i][j]), + vars[i], + vars[i], + vars[j], + vars[j], + ])); + } + } + let p = pool.add(terms); + + let err = sos_decompose(p, &vars, &pool, &SosOpts::default()).expect_err("still refused"); + assert_eq!(err.code(), "E-SOS-002"); + let msg = err.to_string(); + assert!( + msg.contains("what the search actually did:"), + "the refusal must carry a trace of what ran: {msg}" + ); + assert!( + msg.contains("NOT SEARCHED"), + "a budget that fired must be reported as such, not folded into the same \ + undifferentiated refusal an exhausted search produces: {msg}" + ); + assert!( + msg.contains("that was actually searched"), + "the message must not imply multiplier powers were tried when they were not: {msg}" + ); + + // The same distinction, reachable without string-scraping. + let trace = err.search_trace().expect("NoCertificate carries a trace"); + assert!(trace.contains("NOT SEARCHED")); + assert!( + err.hit_a_search_ceiling(), + "this refusal is weaker than an exhausted search and must say so" + ); + + // …and the counterpart: an error with no trace does not pretend to + // have one, and a refusal is never silently upgraded to a refutation. + let neg = SosError::Negative("witness".into()); + assert!(neg.search_trace().is_none()); + assert!(!neg.hit_a_search_ceiling()); + } + + #[test] + fn basis_degree_is_not_ignored_on_the_multiplier_path() { + // `multiplier_search` derived its basis degree from `deg(σ·p)` alone + // and never read `opts.basis_degree`, so the multiplier path was + // bit-identical at every setting — while `E-SOS-002`'s remediation + // told callers to raise `basis_degree`. Raising it now genuinely + // widens the basis searched on this path. + let (pool, x, y) = setup(); + let p = pool.add(vec![ + pool.mul(vec![x, x, x, x, y, y]), + pool.mul(vec![x, x, y, y, y, y]), + pool.mul(vec![pool.integer(-3_i32), x, x, y, y]), + pool.integer(1_i32), + ]); + let target = RatPoly::from_expr(p, &[x, y], &pool).unwrap(); + + let mut default_log = Vec::new(); + multiplier_search( + &target, + 2, + 1, + MAX_MULTIPLIER_BASIS_LEN, + None, + &mut default_log, + ); + let mut raised_log = Vec::new(); + multiplier_search( + &target, + 2, + 1, + MAX_MULTIPLIER_BASIS_LEN, + Some(5), + &mut raised_log, + ); + let degree_of = |log: &[String]| -> String { + log.iter() + .find(|l| l.contains("trying σ")) + .expect("N=1 is searched either way") + .split("degree-") + .nth(1) + .and_then(|t| t.split(' ').next()) + .expect("the log names the basis degree") + .to_string() + }; + assert_eq!(degree_of(&default_log), "4"); + assert_eq!( + degree_of(&raised_log), + "5", + "basis_degree must reach the multiplier path, or E-SOS-002 must stop recommending it" + ); + } + #[test] fn non_polynomial_is_refused() { let (pool, x, y) = setup(); diff --git a/alkahest-core/src/real/sos/psd.rs b/alkahest-core/src/real/sos/psd.rs index 17a975d7..1dbb2f9f 100644 --- a/alkahest-core/src/real/sos/psd.rs +++ b/alkahest-core/src/real/sos/psd.rs @@ -45,6 +45,7 @@ use super::cert::SosPoly; use super::gram::monomial_basis; use super::linalg::{psd_decompose, solve_affine}; +use super::lp::{Lp, LpStatus, Rel}; use super::ratpoly::{Exponents, RatPoly}; use super::sdp::{min_eigenvalue, smallest_magnitude_eigenvectors, Family}; use rug::Rational; @@ -152,6 +153,92 @@ fn gram_system(target: &RatPoly, basis: &[Exponents]) -> (Vec>, Ve (out_rows, out_rhs) } +/// Above this much work — candidate basis monomials × support monomials of +/// the target — the half-Newton-polytope reduction below is skipped rather +/// than run: each candidate costs one exact-rational LP feasibility solve +/// whose column count is the support size, so this keeps the reduction's own +/// cost bounded no matter how large a basis the caller asks for. Skipping +/// only ever leaves the basis *wider* than necessary, so it can cost search +/// time but can never cost soundness or a certificate that would otherwise +/// have been found. +const MAX_NEWTON_REDUCTION_WORK: usize = 400_000; + +/// Is `point` in the convex hull of `support`? Decided exactly, over ℚ, by +/// the same rational simplex the DSOS search uses: the hull membership +/// `point = Σ λ_i·support[i]`, `λ ≥ 0`, `Σ λ_i = 1` is a linear feasibility +/// programme verbatim. +/// +/// A pivot-budget exhaustion (defensive; Bland's rule makes it unreachable) +/// is reported as `true` — "keep this monomial" — so an unexpected LP +/// outcome can only ever widen the basis, never narrow it wrongly. +fn in_convex_hull(support: &[Exponents], point: &[u32]) -> bool { + let mut lp = Lp::new(support.len()); + for k in 0..point.len() { + let row: Vec = support.iter().map(|s| Rational::from(s[k])).collect(); + lp.constrain(row, Rel::Eq, Rational::from(point[k])); + } + lp.constrain( + vec![Rational::from(1); support.len()], + Rel::Eq, + Rational::from(1), + ); + !matches!(lp.solve(), LpStatus::Infeasible) +} + +/// Restrict `basis` to the lattice points of `½·Newton(target)`. +/// +/// Reznick's theorem (1978): if `p = Σ_i q_i²` then `Newton(q_i) ⊆ +/// ½·Newton(p)` for **every** `i`. So every SOS decomposition of `p` is +/// already expressible over the monomials of `½·Newton(p)`, and dropping the +/// rest cannot lose a certificate — this is an exact, complete reduction, +/// not a heuristic narrowing, and that is what makes it safe to apply +/// unconditionally rather than as a fallback. +/// +/// It matters because the numeric search's cost and its *accuracy* both +/// scale with the free-parameter count, and the free-parameter count scales +/// quadratically with the basis size. On `(x²+y²+z²)·Motzkin_hom` this cuts +/// the degree-4 ternary basis from 15 monomials to 9, which is the +/// difference between a 75-parameter affine family whose numeric solution +/// lands ~0.96 away from the true certificate in parameter space (and so +/// never rounds onto it) and an 18-parameter family that lands on it +/// exactly. Note that this is a *dimension* reduction and nothing else: the +/// certificate is the unique PSD point of the affine family on both bases, +/// with the same rank and the same zero minimum eigenvalue, so `λ_min` alone +/// does not reveal the difference — distance in parameter space does. +/// +/// Forms whose Newton polytope already fills the simplex — Robinson's form +/// times `σ`, for instance, where 15 monomials reduce to 15 — come back +/// unchanged, which is exactly right: there is nothing to remove. +fn half_newton_reduce(target: &RatPoly, basis: Vec) -> Vec { + let support: Vec = target.terms().keys().cloned().collect(); + if support.is_empty() || basis.is_empty() { + return basis; + } + if basis.len().saturating_mul(support.len()) > MAX_NEWTON_REDUCTION_WORK { + return basis; + } + let nvars = support[0].len(); + // Cheap coordinate-wise bounding box of Newton(target): a point outside + // it cannot be in the hull, and this rejects most of the discarded + // monomials without an LP solve at all. + let mut hi = vec![0u32; nvars]; + for s in &support { + for k in 0..nvars { + hi[k] = hi[k].max(s[k]); + } + } + basis + .into_iter() + .filter(|e| { + let doubled: Vec = e.iter().map(|c| 2 * c).collect(); + if (0..nvars).any(|k| doubled[k] > hi[k]) { + return false; + } + in_convex_hull(&support, &doubled) + }) + .collect() +} + fn rat_to_f64(r: &Rational) -> f64 { r.to_f64() } @@ -322,6 +409,35 @@ const ROUNDING_CANDIDATES: usize = 6; /// budget", not "not SOS" — exactly like every other budget in this module. const MAX_FREE_PARAMETERS: usize = 200; +/// How [`MAX_FREE_PARAMETERS`] reports itself. Never silently: this ceiling +/// returns `None` *without searching at all*, which is a categorically +/// different thing from a search that ran and came up empty, and a caller +/// that cannot tell them apart will read "we did not look" as "we looked and +/// found nothing". `at_least` distinguishes the pre-solve estimate (a lower +/// bound on the dimension) from the exact post-solve count. +fn ceiling_note(basis_len: usize, free_params: usize, at_least: bool) -> String { + let qualifier = if at_least { "at least " } else { "" }; + format!( + "PSD Gram search: NOT SEARCHED — the affine Gram family over this {basis_len}-monomial \ + basis has {qualifier}{free_params} free parameters, above the numeric-search ceiling of \ + {MAX_FREE_PARAMETERS}; no search was attempted at this basis, so this is a budget that \ + fired, not a search that came up empty" + ) +} + +/// Number of rows [`gram_system`] would build for `target` over `basis` — +/// one per monomial occurring on either side — without building the (large, +/// exact-rational) system itself. +fn row_count(target: &RatPoly, basis: &[Exponents]) -> usize { + let mut exps: BTreeSet = target.terms().keys().cloned().collect(); + for i in 0..basis.len() { + for j in i..basis.len() { + exps.insert(add_exp(&basis[i], &basis[j])); + } + } + exps.len() +} + /// Over-relaxation parameters tried for the Douglas–Rachford polish, in /// increasing order of overshoot. `1.0` is plain (non-relaxed) /// Douglas–Rachford; the larger value is the standard mitigation for a @@ -483,6 +599,7 @@ fn search_rational_family( target: &RatPoly, base_rat: &[Vec], dirs_rat: &[Vec>], + log: &mut Vec, ) -> (Option, Vec>>) { let try_point = |t: &[Rational]| -> Option { let q = rat_family_at(base_rat, dirs_rat, t); @@ -516,8 +633,14 @@ fn search_rational_family( return (try_point(&[]), Vec::new()); } if dirs_rat.len() > MAX_FREE_PARAMETERS { + log.push(ceiling_note(basis.len(), dirs_rat.len(), false)); return (None, Vec::new()); } + log.push(format!( + "PSD Gram search: searching a {}-parameter affine family over a {}-monomial basis", + dirs_rat.len(), + basis.len() + )); let base: Vec> = base_rat .iter() @@ -691,6 +814,7 @@ fn facial_reduction_search( base_rat: &[Vec], dirs_rat: &[Vec>], candidates: &[Vec>], + log: &mut Vec, ) -> Option { let max_corank = *FACIAL_CORANK_GUESSES.iter().max().unwrap_or(&0); let mut budget = FACIAL_SEARCH_BUDGET; @@ -730,7 +854,7 @@ fn facial_reduction_search( } budget -= 1; let (found, _deeper) = - search_rational_family(nvars, basis, target, &new_base, &new_dirs); + search_rational_family(nvars, basis, target, &new_base, &new_dirs, log); if found.is_some() { return found; } @@ -986,19 +1110,25 @@ const MIN_PARAMS_FOR_SYMMETRY_SEARCH: usize = 100; /// parameter count, this simply returns `None` — a routine "nothing to /// exploit here", not a bug. /// -/// The motivating case is the *homogeneous* ternary Motzkin form at +/// The motivating case was the *homogeneous* ternary Motzkin form at /// multiplier power `N = 2`: 165 free parameters, all-even exponents in /// every monomial (invariant under any sign flip) and symmetric under -/// swapping `x, y` — an order-16 group — which collapses the family enough -/// for Douglas–Rachford to reliably close the gap that plain (unreduced) DR -/// left open even after ~600,000 iterations (see this function's test and -/// `psd::tests::psd_search_certifies_homogeneous_motzkin_at_multiplier_power_2`). +/// swapping `x, y` — an order-16 group — which collapses the family to 26 +/// parameters (see `tests::symmetry_group_and_zero_vector_shrink_n2_family`, +/// which asserts that shrink exactly). **That motivation was based on a +/// false premise** and is recorded here only so it is not re-derived: the +/// homogeneous ternary Motzkin form is SOS at `N = 1`, not `N = 2`, and it +/// is `half_newton_reduce` — a plain dimension reduction — that closes it, +/// not any amount of extra Douglas–Rachford. This fallback is *not* known to +/// close any case on its own; it is retained because it is cheap, sound, and +/// only ever runs after everything else has already given up. fn symmetry_reduced_search( nvars: usize, basis: &[Exponents], target: &RatPoly, base_rat: &[Vec], dirs_rat: &[Vec>], + log: &mut Vec, ) -> Option { if nvars == 0 || nvars > MAX_SYMMETRY_NVARS || dirs_rat.len() < MIN_PARAMS_FOR_SYMMETRY_SEARCH { return None; @@ -1028,35 +1158,47 @@ fn symmetry_reduced_search( let sym_dirs: Vec>> = reduced_packed.iter().map(|v| unpack(n, v)).collect(); let (found, sym_candidates) = - search_rational_family(nvars, basis, target, &sym_base, &sym_dirs); + search_rational_family(nvars, basis, target, &sym_base, &sym_dirs, log); if found.is_some() { return found; } if sym_candidates.is_empty() { return None; } - facial_reduction_search(nvars, basis, target, &sym_base, &sym_dirs, &sym_candidates) + facial_reduction_search( + nvars, + basis, + target, + &sym_base, + &sym_dirs, + &sym_candidates, + log, + ) } -pub fn psd_search(target: &RatPoly, basis_deg: u32) -> Option { +/// [`psd_search`], but also appending a human-readable trace of what the +/// search actually did to `log`. +/// +/// The trace exists because `None` out of this module covers three +/// materially different situations — the search ran and was exhausted, the +/// search ran and hit an iteration/rounding budget, and *the search never +/// ran at all* because the family was over [`MAX_FREE_PARAMETERS`] — and a +/// bare `None` (or the `E-SOS-002` it turns into) cannot distinguish them. +/// [`super::sos_decompose`] folds this trace into the error message so that +/// distinction reaches the caller. +pub fn psd_search_logged( + target: &RatPoly, + basis_deg: u32, + log: &mut Vec, +) -> Option { let nvars = target.nvars(); - // A homogeneous target of degree exactly `2·basis_deg` needs only the - // monomials of degree *exactly* `basis_deg` in its Gram basis — mixing in - // lower-degree monomials can only ever contribute to coefficients the - // target does not have, since every product of two basis monomials of - // unequal degree still sums to `2·basis_deg` only when *both* already - // have degree `basis_deg`. This is standard (Blekherman–Parrilo–Thomas, - // Prop. 3.29): a homogeneous SOS decomposition can always be taken with - // homogeneous summands. Restricting here is not just an optimisation — - // the search is numeric, and a smaller basis is the difference between - // "converges" and "not within budget" on cases like Motzkin. - let basis: Vec = match target.is_homogeneous() { - Some(d) if d == 2 * basis_deg => monomial_basis(nvars, basis_deg) - .into_iter() - .filter(|e| e.iter().sum::() == basis_deg) - .collect(), - _ => monomial_basis(nvars, basis_deg), - }; + let (basis, homogeneous_len) = restricted_basis(target, basis_deg); + if basis.len() < homogeneous_len { + log.push(format!( + "half-Newton-polytope reduction: {homogeneous_len} → {} monomials in the Gram basis", + basis.len() + )); + } let n = basis.len(); if n == 0 { return if target.is_zero() { @@ -1066,25 +1208,97 @@ pub fn psd_search(target: &RatPoly, basis_deg: u32) -> Option { }; } + // Apply the free-parameter ceiling *before* the exact nullspace solve, + // not after it: `solve_affine` is a rational Gauss–Jordan on a + // `rows × pack_len(n)` matrix and on a family this large costs far more + // than the search that is then skipped anyway (C₇'s N=1 multiplier is a + // 924 × 3570 exact solve, thrown away immediately). The rank of the + // coefficient-matching system is at most its row count, so + // `pack_len(n) − rows` is a *lower* bound on the family's dimension — + // when even that exceeds the ceiling the search provably cannot run. + let min_free = pack_len(n).saturating_sub(row_count(target, &basis)); + if min_free > MAX_FREE_PARAMETERS { + log.push(ceiling_note(n, min_free, true)); + return None; + } + let (rows, rhs) = gram_system(target, &basis); let sol = solve_affine(&rows, &rhs)?; let base_rat = unpack(n, &sol.particular); let dirs_rat: Vec>> = sol.nullspace.iter().map(|d| unpack(n, d)).collect(); - let (found, candidates) = search_rational_family(nvars, &basis, target, &base_rat, &dirs_rat); + let (found, candidates) = + search_rational_family(nvars, &basis, target, &base_rat, &dirs_rat, log); if found.is_some() { return found; } if candidates.is_empty() { return None; } - if let Some(found) = - facial_reduction_search(nvars, &basis, target, &base_rat, &dirs_rat, &candidates) - { + if let Some(found) = facial_reduction_search( + nvars, + &basis, + target, + &base_rat, + &dirs_rat, + &candidates, + log, + ) { return Some(found); } - symmetry_reduced_search(nvars, &basis, target, &base_rat, &dirs_rat) + symmetry_reduced_search(nvars, &basis, target, &base_rat, &dirs_rat, log) +} + +/// Search the full PSD-Gram cone for an exact rational sum-of-squares +/// decomposition of `target`, discarding the diagnostic trace. See +/// [`psd_search_logged`] for the variant that keeps it. +pub fn psd_search(target: &RatPoly, basis_deg: u32) -> Option { + let mut log = Vec::new(); + psd_search_logged(target, basis_deg, &mut log) +} + +/// The monomial basis [`psd_search_logged`] will actually search for +/// `target` at `basis_deg`, plus the size it had before the +/// half-Newton-polytope reduction. +/// +/// A homogeneous target of degree exactly `2·basis_deg` needs only the +/// monomials of degree *exactly* `basis_deg` in its Gram basis — mixing in +/// lower-degree monomials can only ever contribute to coefficients the +/// target does not have, since every product of two basis monomials of +/// unequal degree still sums to `2·basis_deg` only when *both* already have +/// degree `basis_deg`. This is standard (Blekherman–Parrilo–Thomas, +/// Prop. 3.29): a homogeneous SOS decomposition can always be taken with +/// homogeneous summands. Then [`half_newton_reduce`] drops whatever survives +/// that but still lies outside `½·Newton(target)`. Neither step is only an +/// optimisation — the search is numeric, and the parameter count (quadratic +/// in the basis size) is the difference between "rounds onto the exact +/// certificate" and "lands 0.96 away from it". +fn restricted_basis(target: &RatPoly, basis_deg: u32) -> (Vec, usize) { + let nvars = target.nvars(); + let basis: Vec = match target.is_homogeneous() { + Some(d) if d == 2 * basis_deg => monomial_basis(nvars, basis_deg) + .into_iter() + .filter(|e| e.iter().sum::() == basis_deg) + .collect(), + _ => monomial_basis(nvars, basis_deg), + }; + let homogeneous_len = basis.len(); + (half_newton_reduce(target, basis), homogeneous_len) +} + +/// Size of the monomial basis [`psd_search_logged`] will actually search for +/// `target` at `basis_deg`. +/// +/// Exposed so [`super::multiplier_search`] can budget against the size it is +/// really going to search rather than the raw `monomial_basis` count: those +/// two differ by nearly 4× on realistic targets (a degree-4 ternary basis is +/// 35 monomials raw, 15 after the homogeneity restriction, 9 after the +/// Newton reduction on `σ·Motzkin`), and budgeting against the raw number +/// rejects multiplier powers whose real basis is comfortably inside the +/// budget. +pub fn searched_basis_len(target: &RatPoly, basis_deg: u32) -> usize { + restricted_basis(target, basis_deg).0.len() } #[cfg(test)] @@ -1147,55 +1361,37 @@ mod tests { } #[test] - fn psd_search_does_not_yet_reach_homogeneous_motzkin_times_sum_of_squares() { + fn psd_search_certifies_homogeneous_motzkin_times_sum_of_squares_at_n1() { // The homogeneous Motzkin form, x^4y^2 + x^2y^4 - 3x^2y^2z^2 + z^6, is - // the textbook example of a PSD form that is not itself SOS. At - // multiplier power N=1, (x^2+y^2+z^2)*Motzkin is *not* SOS either — - // literature and the diagnostic evidence below agree the classical - // fact needs N=2, (x^2+y^2+z^2)^2*Motzkin, for this specific - // *homogeneous ternary* form (contrast the *affine* 2-variable - // Motzkin case, `real::sos::tests::motzkin_certifies_via_a_reznick_multiplier`, - // which genuinely does close at N=1). So this N=1 refusal is - // expected to stay `None` permanently, not just "not yet reached" — - // it is here mainly as a mechanism sanity check alongside the - // diagnostics below, not as the open item. + // the textbook example of a PSD form that is not itself SOS — and it + // is the textbook example precisely *because* multiplying it by + // σ = x^2+y^2+z^2 makes it SOS. The classical identity, exact over ℚ: + // + // σ·Motzkin = (½x³y + xy³ − 3⁄2xyz²)² + ¾(x³y − xyz²)² + // + (xy²z − xz³)² + (x²yz − yz³)² + (x²y² − z⁴)² // - // `diag::diag_step1_step2_trajectory_and_family_sanity` shows the - // annealed multi-start search converging monotonically toward the - // PSD cone's boundary (min eigenvalue from about -1.6 to about - // -0.0018 as the floor anneals to 0) without fully closing the gap — - // a tangential (non-transversal) intersection is the classic case - // where alternating projection's convergence rate degrades this way. - // `diag::diag_step3_planted_singular_example` confirms the search - // mechanism itself is sound: a synthetic boundary case of the same - // nullspace dimension *is* found and exactly re-verified. + // so `N = 1` suffices. This test used to be its own negation — + // `psd_search_does_not_yet_reach_...`, asserting `is_none()` and + // *passing*, with a comment claiming the classical fact needs N=2 and + // that "this N=1 refusal is expected to stay `None` permanently". That + // claim was simply false (the identity above expands to zero + // difference; re-check it in any CAS), and the green test pinned the + // search bug that produced the refusal. It is stated in the positive + // direction now so that a regression is a failure rather than a + // confirmation. // - // **N=2 was attempted directly (2026-08-17) and also does not close, - // for a reason now well quantified rather than merely "not yet - // reached":** at N=2 the raw affine family has 165 free parameters. - // `symmetry_group_and_zero_vector_shrink_n2_family` (below) shows the - // *new* `symmetry_reduced_search` machinery (added this round) - // exactly collapses that to 26 parameters via the polynomial's own - // order-16 signed-permutation symmetry (swap x,y; independently flip - // the sign of each variable — every exponent in this target is - // even), and a further *exact*, non-numeric restriction — imposing - // `Q·z(1,1,1) = 0`, forced because q(1,1,1)=0 and Q is PSD, using the - // literal all-ones vector as the null-vector candidate, no rounding - // involved — collapses it again to 16. That 16-parameter family is - // the smallest one reached, and it is still not found: escalating - // Douglas–Rachford there from 15,000 to 6,000,000 iterations moves - // the minimum eigenvalue from about -4·10⁻⁶ to about -1.4·10⁻⁸, a - // clearly *sublinear* (not exponential/finite-step) approach — the - // same tangential-intersection signature as N=1, just quantified — - // and rational rounding fails at every stage even with denominators - // tested up to roughly 10⁹. Also checked and ruled out: an - // additional tangent-direction null vector (`grad_x`, the exact - // gradient of the basis vector at (1,1,1)) is *inconsistent* with - // the family, confirming the corank contributed by this zero is - // exactly 1 (not a missed higher-corank guess). So this is now a - // genuine numerical-hardness finding specific to N=2, not an - // under-tuned budget or an unexplored structural avenue — see - // `CHANGELOG.md`'s M10 entry for the full writeup. + // What was actually missing was the half-Newton-polytope reduction + // (`half_newton_reduce`): σ·Motzkin's degree-4 ternary basis is 15 + // monomials, only 9 of which lie in ½·Newton(σ·Motzkin), and the + // 75-parameter family over the unreduced basis puts the numeric + // search ~0.96 away from the true point in parameter space — far too + // far to round onto it — while the 18-parameter family over the + // reduced basis lands on it exactly. Note this is a *dimension* + // effect, not a conditioning one: on both bases the certificate is + // the unique PSD point of the affine family, rank 5, with minimum + // eigenvalue exactly 0, so λ_min does not distinguish them and more + // Douglas–Rachford iterations do not help (4× the budget on the + // unreduced family still fails to round). let mut m = RatPoly::monomial(3, vec![4, 2, 0], Rational::from(1)); m = m.add(&RatPoly::monomial(3, vec![2, 4, 0], Rational::from(1))); m = m.add(&RatPoly::monomial(3, vec![2, 2, 2], Rational::from(-3))); @@ -1205,23 +1401,146 @@ mod tests { let q = m.mul(&sigma); assert_eq!(q.is_homogeneous(), Some(8)); + // The reduction itself, asserted rather than assumed: 15 → 9. + let (reduced, homogeneous_len) = restricted_basis(&q, 4); + assert_eq!(homogeneous_len, 15); + assert_eq!(reduced.len(), 9); + + let sos = psd_search(&q, 4).expect( + "(x^2+y^2+z^2)·Motzkin_hom is a sum of squares at N=1 — the classical fact that \ + makes Motzkin the standard PSD-not-SOS example — so the search must find a \ + certificate here", + ); + // Exact re-expansion over ℚ: the soundness argument, independent of + // whatever the numeric search converged to. + assert_eq!(sos.to_poly(3), q); + } + + #[test] + fn psd_search_certifies_choi_lam_times_sum_of_squares_at_n1() { + // The Choi–Lam form, x²y² + y²z² + z²x² + w⁴ − 4xyzw: a quaternary + // quartic that is PSD but not SOS, and (like Motzkin) becomes SOS + // after one multiplication by σ = Σxᵢ². Four variables, so the + // degree-3 basis is 20 monomials before the Newton reduction and 16 + // after; the corresponding drop in free parameters is what makes the + // search land on the certificate. + let mono = |ex: Vec, c: i64| RatPoly::monomial(4, ex, Rational::from(c)); + let mut cl = mono(vec![2, 2, 0, 0], 1); + cl = cl.add(&mono(vec![0, 2, 2, 0], 1)); + cl = cl.add(&mono(vec![2, 0, 2, 0], 1)); + cl = cl.add(&mono(vec![0, 0, 0, 4], 1)); + cl = cl.add(&mono(vec![1, 1, 1, 1], -4)); + assert_eq!(cl.is_homogeneous(), Some(4)); + + let q = cl.mul(&RatPoly::sum_of_squares(4)); + assert_eq!(q.is_homogeneous(), Some(6)); + let sos = psd_search(&q, 3) + .expect("(Σxᵢ²)·Choi–Lam is SOS at N=1; the search should find a certificate"); + assert_eq!(sos.to_poly(4), q); + } + + #[test] + fn half_newton_reduction_is_a_no_op_when_the_polytope_is_already_full() { + // Robinson's form times σ is the guard case for `half_newton_reduce`: + // its Newton polytope fills the degree-8 simplex, so *no* monomial of + // the degree-4 basis is outside ½·Newton, and the reduction must + // return all 15 — a reduction that trimmed anything here would be + // dropping monomials a real certificate needs. + let mono = |ex: Vec, c: i64| RatPoly::monomial(3, ex, Rational::from(c)); + let mut r = mono(vec![6, 0, 0], 1); + r = r.add(&mono(vec![0, 6, 0], 1)); + r = r.add(&mono(vec![0, 0, 6], 1)); + r = r.add(&mono(vec![4, 2, 0], -1)); + r = r.add(&mono(vec![2, 4, 0], -1)); + r = r.add(&mono(vec![0, 4, 2], -1)); + r = r.add(&mono(vec![0, 2, 4], -1)); + r = r.add(&mono(vec![4, 0, 2], -1)); + r = r.add(&mono(vec![2, 0, 4], -1)); + r = r.add(&mono(vec![2, 2, 2], 3)); + let q = r.mul(&RatPoly::sum_of_squares(3)); + let (reduced, homogeneous_len) = restricted_basis(&q, 4); + assert_eq!(homogeneous_len, 15); + assert_eq!(reduced.len(), 15, "Robinson admits no Newton reduction"); + + // Same check on the direct (unmultiplied) sextic, and on a target + // where the reduction genuinely bites, so this test pins both + // directions rather than only the no-op one. + let (reduced_direct, direct_len) = restricted_basis(&r, 3); + assert_eq!((direct_len, reduced_direct.len()), (10, 10)); + + // The cyclic AM-GM sextic x⁴y²+y⁴z²+z⁴x²−3x²y²z², times σ: the same + // 15 → 9 profile as Motzkin. + let mut c = mono(vec![4, 2, 0], 1); + c = c.add(&mono(vec![0, 4, 2], 1)); + c = c.add(&mono(vec![2, 0, 4], 1)); + c = c.add(&mono(vec![2, 2, 2], -3)); + let (reduced_cyclic, cyclic_len) = restricted_basis(&c.mul(&RatPoly::sum_of_squares(3)), 4); + assert_eq!((cyclic_len, reduced_cyclic.len()), (15, 9)); + } + + #[test] + fn the_free_parameter_ceiling_is_reported_not_silently_dropped() { + // x⁸ + y⁸ + z⁸ + 1 is a sum of squares by inspection. Searched over + // the degree-4 basis its affine Gram family still has 465 free + // parameters — above `MAX_FREE_PARAMETERS` — so `psd_search` gives up + // *without running any search at all*. That is a legitimate budget, + // but it used to be indistinguishable from an exhausted search: the + // same instant `None`, and nothing recorded. The point of this test + // is not the `None` (which is unchanged behaviour) but that the + // ceiling is now *reported*. + let mono = |ex: Vec, c: i64| RatPoly::monomial(3, ex, Rational::from(c)); + let mut p = mono(vec![8, 0, 0], 1); + p = p.add(&mono(vec![0, 8, 0], 1)); + p = p.add(&mono(vec![0, 0, 8], 1)); + p = p.add(&mono(vec![0, 0, 0], 1)); + + let mut log = Vec::new(); + assert!(psd_search_logged(&p, 4, &mut log).is_none()); + let ceiling = log + .iter() + .find(|l| l.contains("NOT SEARCHED")) + .unwrap_or_else(|| panic!("the free-parameter ceiling must be logged; got {log:?}")); + assert!( + ceiling.contains("465") && ceiling.contains(&MAX_FREE_PARAMETERS.to_string()), + "the ceiling report must name both the family size and the ceiling: {ceiling}" + ); + assert!( + !log.iter().any(|l| l.contains("searching a")), + "nothing was searched, so nothing may claim to have searched: {log:?}" + ); + } + + #[test] + fn a_search_that_really_runs_says_so() { + // The counterpart to the test above: when a search does run, the log + // says *searched*, so the two outcomes are distinguishable in the + // trace and not only in the (identical) `None`/`Some`. + let mut p = RatPoly::monomial(2, vec![4, 0], Rational::from(1)); + p = p.add(&RatPoly::monomial(2, vec![0, 4], Rational::from(1))); + p = p.add(&RatPoly::monomial(2, vec![2, 2], Rational::from(2))); + let mut log = Vec::new(); + assert!(psd_search_logged(&p, 2, &mut log).is_some()); assert!( - psd_search(&q, 4).is_none(), - "Motzkin's N=1 multiplier is not classically expected to be SOS for the \ - homogeneous ternary form (see this test's doc comment); if this starts passing \ - that would itself be worth investigating, not just promoting to a positive assertion" + log.iter().any(|l| l.contains("searching a")), + "a search that runs must record that it ran: {log:?}" ); + assert!(!log.iter().any(|l| l.contains("NOT SEARCHED"))); } - /// Fast, exact-arithmetic-only regression test for the `symmetry_reduced_search` - /// machinery added to close in on (though not fully close) the N=2 - /// homogeneous-ternary-Motzkin gap: documents the 165 → 26 → 16 + /// Fast, exact-arithmetic-only regression test for the + /// `symmetry_reduced_search` machinery: documents the 165 → 26 → 16 /// free-parameter shrink referenced in this module's doc comments, with /// real assertions rather than `eprintln!` probes. Deliberately does - /// *not* call the full `psd_search` (which, at N=2, spends several - /// minutes on Douglas–Rachford before giving up — see the doc comment on - /// `psd_search_does_not_yet_reach_homogeneous_motzkin_times_sum_of_squares` - /// for those numbers) so this stays fast enough to run on every `cargo test`. + /// *not* call the full `psd_search` (which at `N = 2` spends several + /// minutes on Douglas–Rachford before giving up) so this stays fast + /// enough to run on every `cargo test`. + /// + /// The `N = 2` family is exercised here purely because it is a large, + /// highly symmetric family that makes the reduction machinery easy to + /// assert on. It is **not** the multiplier power this target needs — the + /// homogeneous ternary Motzkin form is SOS at `N = 1`, and + /// `psd_search_certifies_homogeneous_motzkin_times_sum_of_squares_at_n1` + /// is where that is checked. #[test] fn symmetry_group_and_zero_vector_shrink_n2_family() { let mut m = RatPoly::monomial(3, vec![4, 2, 0], Rational::from(1)); diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index 7df084ab..3e29a349 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -1437,5 +1437,5 @@ reg.coverage_report_markdown() # same, rendered as a Markdown table 29. **`cert.specialize_at_root_of_unity(d, n)` is the decision that carries a `q_zeilberger` verdict to `q = ζ_d`, and it is three-valued** (since 3.9). A proved `Q(q)` recurrence does not by itself license setting `q` to a primitive `d`-th root of unity — a coefficient or a sum value can have a pole there, and specialising anyway is the `q`-analogue of the A279013 failure (item 22): a certificate that re-checks perfectly while the specialised claim is false. The hypotheses (no pole in any `a_i(qⁿ)` or `S(n+i)` at `ζ_d`) are decided **exactly**, by polynomial divisibility by `Φ_d(q)` over `Q` in the cyclotomic field `Q(ζ_d) = Q[q]/(Φ_d(q))` — never numerically — and `cyclotomic_polynomial(pool, d)` exposes `Φ_d(q)` itself so a caller can redo the check by hand. `status` is `"specializes"` (proved, and re-checked as an exact identity in `Q(ζ_d)` before being returned), `"obstructed"` (a pole was **exhibited** — `sum_value`/`coefficient` raise, but `sum_valuation(i)` is still available since the negative valuation *is* the obstruction — and this is not a claim the specialised identity is false, only that this route is blocked), or `"unknown"` (the generic boundary verdict was already `"unknown"`, so there is nothing to specialise). Three things a `"specializes"` verdict does **not** by itself mean, each with its own accessor: `is_vacuous` (every coefficient died — always true at `d = 1`, the `q → 1` limit — so the recurrence is `0 = 0`, still true, but empty), `leading_coefficient_survives` (`False` means the specialised recurrence no longer determines the last value from the earlier ones), and `support_shrinks` (`q`-Lucas killing terms — `[2;1]_q = 1 + q` is non-zero in `Q(q)` and zero at `ζ_2` — reported via `effective_support`, which can shrink but never grow). `sum_valuation(i)` is the `q`-supercongruence content itself: the exact integer `v` with `Φ_d(q)^v ∥ S(n+i)`, so `v ≥ r` is precisely `Φ_d(q)^r | S(n)`. -30. **`sos_decompose` tries the full PSD Gram cone and a Reznick multiplier search before refusing, and now certifies Motzkin and Robinson's form too** (since 3.9). Past diagonal dominance (`E-SOS-002` from DSOS alone) it searches the general PSD Gram cone, and past that — when `p` itself is not SOS — tries `(x_1²+…+x_n²)^N·p` for `N = 1..4` and searches *that* cone; a witness for `p < 0` still refuses separately with `E-SOS-003`, unaffected. Every certificate this returns is exact end to end: the numeric search only ever proposes a Gram matrix, which is rounded to nearby rationals and re-expanded to check it equals the target exactly before anything is returned — a `Some`/returned certificate is always sound regardless of what the float search converged to. Budget exhaustion is still `E-SOS-002`, undecided, never "not SOS" — say so, don't paraphrase it as a disproof. **The textbook PSD-not-SOS examples whose multiplier certificates are *singular* Gram matrices sitting exactly on the boundary of the PSD cone** — Motzkin's polynomial and Robinson's form — used to be out of reach for the original annealed alternating-projection search (a diagnosed convergence limitation at tangential PSD-cone intersections, not a soundness bug); the search now also tries Douglas–Rachford splitting with over-relaxation and a facial-reduction step, and with them both examples are found and exactly re-verified. **What's still open:** the homogeneous 3-variable form of Motzkin at multiplier power `N = 2` (`(x²+y²+z²)²·Motzkin_hom`, not `N = 1` — this specific homogeneous ternary form is not classically expected to be SOS at `N = 1` at all, unlike the affine 2-variable case) is still not reached, and this has now been attempted to closure rather than left as a budget skip: a new symmetry-reduction fallback (`real::sos::psd::symmetry_reduced_search`) exploits the target's own signed-permutation symmetry (order 16 here) to shrink its 165 free parameters to 26, and an exact algebraic zero-vector restriction (no numerics — Motzkin's known zero at `(1,1,1)` forces a specific null vector on any witnessing Gram matrix) shrinks that again to 16 — but deep Douglas–Rachford on that 16-parameter family still leaves the minimum eigenvalue at roughly `−1.4·10⁻⁸` after 6,000,000 iterations, a genuinely slow (not budget-limited) convergence. So a boundary-only certificate is still not guaranteed to be found in general — `E-SOS-002` still means "not found within this search", never "not SOS". Raise `basis_degree`, or fall back to `alkahest.decide`, exactly as for any other `E-SOS-002`. +30. **`sos_decompose` tries the full PSD Gram cone and a Reznick multiplier search before refusing, and now certifies Motzkin and Robinson's form too** (since 3.9). Past diagonal dominance (`E-SOS-002` from DSOS alone) it searches the general PSD Gram cone, and past that — when `p` itself is not SOS — tries `(x_1²+…+x_n²)^N·p` for `N = 1..4` and searches *that* cone; a witness for `p < 0` still refuses separately with `E-SOS-003`, unaffected. Every certificate this returns is exact end to end: the numeric search only ever proposes a Gram matrix, which is rounded to nearby rationals and re-expanded to check it equals the target exactly before anything is returned — a `Some`/returned certificate is always sound regardless of what the float search converged to. Budget exhaustion is still `E-SOS-002`, undecided, never "not SOS" — say so, don't paraphrase it as a disproof. **The textbook PSD-not-SOS examples whose multiplier certificates are *singular* Gram matrices sitting exactly on the boundary of the PSD cone** — Motzkin's polynomial and Robinson's form — used to be out of reach for the original annealed alternating-projection search (a diagnosed convergence limitation at tangential PSD-cone intersections, not a soundness bug); the search now also tries Douglas–Rachford splitting with over-relaxation and a facial-reduction step, and with them both examples are found and exactly re-verified. **Correction (2026-08-20):** earlier revisions of this entry said the homogeneous 3-variable Motzkin form "is not classically expected to be SOS at `N = 1` at all" and that reaching it required `N = 2`. That was **false**. `(x²+y²+z²)·(x⁴y²+x²y⁴−3x²y²z²+z⁶)` *is* a sum of squares — that identity is precisely why Motzkin is the standard example of a PSD non-SOS form made SOS by one factor of `Σxᵢ²` — and it now certifies at `N = 1`, together with Choi–Lam. What was missing was not iterations but a **half-Newton-polytope reduction**: `psd_search` now restricts the Gram basis to the lattice points of `½·Newton(p)` (Reznick: every square in every SOS decomposition already has its support there, so nothing is lost), which takes `σ·Motzkin_hom` from a 75-parameter family to an 18-parameter one — the difference between landing `0.96` away from the certificate in parameter space and landing on it exactly. **What's still open:** the Horn/C₅ and C₇ copositivity forms, whose Newton polytopes are already full and whose `N = 1` families (420 and 2646 free parameters) are above `psd_search`'s numeric-search ceiling of 200 — so for those *no multiplier power is searched at all*. `E-SOS-002` now carries a trace of what the search actually did, with `NOT SEARCHED` marking budgets that fired; read it before recording a refusal as exhaustive, because "we did not look" and "we looked and found nothing" share the error code. `E-SOS-002` still means "not found within this search", never "not SOS". `basis_degree` now does reach the multiplier path (it used to be ignored there), or fall back to `alkahest.decide`. 31. **Multi-sums need `experimental.telescope2d` (two bound indices) or `experimental.telescope_md` (any number `m >= 1`), not `zeilberger`** (since 3.9; `telescope_md` since 3.10). `zeilberger`/`q_zeilberger` reach a sum over *one* index; `telescope2d(term, n, j, k)` is the Apagodu–Zeilberger generalization to a proper hypergeometric `F(n,j,k)` with **two** bound indices `j`, `k`, returning `a_0(n), …, a_J(n)` and *two* certificates `cert1`, `cert2` with `Σ_i a_i(n)·F(n+i,j,k) = Δ_j(cert1·F) + Δ_k(cert2·F)`, re-checked exactly in `Q(n,j,k)`. `telescope_md(term, n, [x_1, ..., x_m])` is the same engine generalized to arbitrary `m` — `m = 1` degenerates to a single-sum-shaped search, `m = 2` behaves identically to `telescope2d` (which is now a thin wrapper over it), `m >= 3` is genuinely new — returning `cert.certs()` (a list of `m` certificates, a method not a property since it's a collection) instead of `cert1`/`cert2`. Four real, stated scope limits, not unfinished polish: (1) the certificate ansatz uses a *fixed* denominator built from `F`'s own shift-ratio denominators rather than a minimal Gosper normal form, so a search that finds nothing raises `E-HOLO-041` and does not prove no certificate exists; (2) `cert.boundary_status(j_lo, j_hi, k_lo, k_hi)` / `cert.boundary_status([(lo_1, hi_1), ..., (lo_m, hi_m)])` only accept **constant** (not `n`-dependent) boxes — for a natural range like `j = 0..n`, pick a fixed bound safely larger than any `n` you check and let `F`'s own combinatorial vanishing do the rest, exactly as the module's own worked examples do; (3) the boundary of a box is **`2m` `(m-1)`-dimensional face sums, not `2^m` corner-point evaluations** — a naive corner-evaluation formula is simply wrong — and this version only proves the sufficient (not necessary) condition that each face vanishes identically, so `boundary_status` can return `"unknown"` for a boundary that is genuinely `0` but not by that pointwise route; it never guesses `"vanishes"`. There is no inhomogeneous `"nonzero"` verdict yet — an unresolved face is always `"unknown"`; (4) `telescope_md`'s underlying exact linear solve is `O(rows · cols²)` and both grow fast with `m` and the certificate degree bound (measured: `m = 3` at certificate degree 2 already means a ~10,000-row, 245-unknown system taking ~47s to solve *per probe*), so two resource ceilings apply — a single probe above 400 unknowns is refused outright, and total work across every probe at or above 150 unknowns in one search call is capped to 300 — meaning `E-HOLO-041` can also mean "refused by a resource ceiling, not searched and found nothing," which the error message states explicitly; raising `m` or `max_cert_degree` further will not help once a ceiling is the reason. `E-HOLO-040` is the class refusal (not proper hypergeometric in the bound indices), `E-HOLO-042` a malformed call (indices not pairwise distinct, or `indices` empty for `telescope_md`). diff --git a/docs/mdbook/src/errors.md b/docs/mdbook/src/errors.md index 37722253..4f9e1e5f 100644 --- a/docs/mdbook/src/errors.md +++ b/docs/mdbook/src/errors.md @@ -111,7 +111,7 @@ loop must record as **undecided**, never as a negative result. | `E-LINALG-010` | `LinearAlgebraError` | An entry's vanishing could be proven neither zero nor non-zero, so `rank` / `rref` / `nullspace` / `eigenvects` / `jordan_form` declined to pick a branch | | `E-MAT-004` | `MatrixError` | Same, for a determinant: `inverse()` will not divide by something it cannot show is non-zero | | `E-CAD-001` | `CadError` | `decide` is outside its fragment, or the only candidate solutions lie at an irrational boundary point it cannot test exactly | -| `E-SOS-002` | `SosError` | No positivity certificate of this shape at this degree — a statement about the search, not a proof that none exists. **Record it as `unknown`, never as "not SOS" or "the inequality is false":** `p` may be SOS outside the LP subcone searched, SOS at a higher `basis_degree`, or non-negative without being SOS (Motzkin). `E-SOS-003`, which carries a witness point, is the only SOS *refutation*. See [Positivity certificates](./positivity.md#three-outcomes-deliberately-kept-apart) | +| `E-SOS-002` | `SosError` | No positivity certificate of this shape at this degree — a statement about the search, not a proof that none exists. **Record it as `unknown`, never as "not SOS" or "the inequality is false":** `p` may be SOS outside the LP subcone searched, SOS at a higher `basis_degree`, or non-negative without being SOS (Motzkin). `E-SOS-003`, which carries a witness point, is the only SOS *refutation*. The message carries a `what the search actually did:` trace; lines marked `NOT SEARCHED` are budgets that fired, not searches that came up empty, and mean the corresponding basis or multiplier power was never looked at. See [Positivity certificates](./positivity.md#three-outcomes-deliberately-kept-apart) | | `E-IDEAL-005` | `IdealRefusal` | `radical` cannot certify `√I` for this ideal. Only monomial, principal and zero-dimensional ideals — and anything whose primary decomposition is certified — are answered; the alternative is asserting `√I = I` with nothing behind it | | `E-IDEAL-006` | `IdealRefusal` | `primary_decomposition` reached a component it cannot show is primary, so it will not report the ideal itself with an unjustified `associated_prime` | | `E-SOLVE-004` | `TriangularizeRefusal` | `triangularize` extracted a chain that does not generate an ideal containing the input, i.e. one that cuts out a larger variety than the system. Splitting on the initials (Lazard–Kalkbrener) is not implemented | diff --git a/docs/mdbook/src/positivity.md b/docs/mdbook/src/positivity.md index c043511e..4677abf4 100644 --- a/docs/mdbook/src/positivity.md +++ b/docs/mdbook/src/positivity.md @@ -65,17 +65,27 @@ it does not report it as negative, and it does not invent a decomposition. `sos_decompose`'s full pipeline does not stop there, though (see "What the search actually covers" below) — it also tries multiplying by a power of `x²+y²` before giving up, and that succeeds for Motzkin, so the *end-to-end* -call returns a certificate, not a refusal. The homogeneous 3-variable form of -Motzkin still refuses even through the full pipeline (multiplier search -included), and — unlike the affine case above — this is not just "not yet -reached": the classical fact needs multiplier power `N = 2`, not `N = 1`, for -this specific homogeneous ternary form, and `N = 2` has now been attempted to -closure by a genuinely harder search (symmetry reduction plus an exact -algebraic zero-vector restriction, cutting 165 free parameters down to 16) -without succeeding — see "What's still open" below for the numbers. That is -the actual reachable illustration of a genuine `E-SOS-002` from this module -today: **the refusal is a property of the search, not of the polynomial**, -and which polynomials it applies to shifts as the search grows more complete. +call returns a certificate, not a refusal. The *homogeneous* 3-variable form +of Motzkin now certifies too, at multiplier power `N = 1` — which is the +classical fact: + +```text +(x²+y²+z²)(x⁴y²+x²y⁴−3x²y²z²+z⁶) + = (½x³y+xy³−3⁄2xyz²)² + ¾(x³y−xyz²)² + (xy²z−xz³)² + (x²yz−yz³)² + (x²y²−z⁴)² +``` + +> **Correction (2026-08-20).** Earlier releases of this page said the +> homogeneous ternary form "is still out of reach" and that the classical +> fact "needs `N = 2`, not `N = 1`". Both statements were wrong: the identity +> above is exactly why Motzkin is the standard example of a PSD non-SOS form +> that becomes SOS after one multiplication by `Σxᵢ²`. What was missing was a +> half-Newton-polytope reduction in the search — see "What the search +> actually covers" below. + +The general point stands regardless of which examples are currently +reachable: **an `E-SOS-002` refusal is a property of the search, not of the +polynomial**, and which polynomials it applies to shifts as the search grows +more complete. The three-way branch a loop should write: @@ -110,7 +120,12 @@ The search tries three things, in order, before refusing: perfect square as ordinary as `(x/2 + 1/3)²` has a Gram matrix — its only one — that is PSD but not DD. 2. **The full PSD Gram cone**, when DSOS fails (`real::sos::psd::psd_search`). - This subsumes DSOS but is not free: it leans on a floating-point search + The monomial basis is cut down first: to the monomials of degree exactly + `d/2` when `p` is homogeneous of degree `d`, and then to the lattice + points of `½·Newton(p)` — Reznick's theorem says the support of every + square in every SOS decomposition of `p` already lies there, so this loses + no certificate while removing free parameters quadratically. It then leans + on a floating-point search (Jacobi eigendecomposition, PSD-cone projection, an annealed schedule of shrinking eigenvalue floors with several random restarts — `real::sos::sdp`) to *propose* a Gram matrix, which is then rounded to @@ -141,30 +156,35 @@ the textbook behaviour of alternating projection at a tangential Douglas–Rachford splitting with over-relaxation and a facial-reduction step — both standard escapes for exactly this stall — and with them, both `(x²+y²)·Motzkin(x,y)` and `(x²+y²+z²)·Robinson(x,y,z)` are found and -exactly re-verified. **What's still open:** the *homogeneous 3-variable* -form of Motzkin at multiplier power `N = 2`, `(x²+y²+z²)²·(x⁴y²+x²y⁴−3x²y²z²+z⁶)` -(`N = 1` is not classically expected to work for this specific homogeneous -ternary form at all), is not — and this has now been pushed well past a -budget skip. A new fallback, symmetry reduction -(`real::sos::psd::symmetry_reduced_search`), restricts the search to the -subspace fixed by the target's own signed-permutation symmetry (here, order -16: swap `x, y`, and independently flip the sign of each variable, since -every exponent is even) whenever that subspace is genuinely smaller — cutting -this case's 165 free parameters to 26. An *exact*, non-numeric restriction on -top of that — Motzkin's known zero at `(1,1,1)` forces `Q·z(1,1,1) = 0` on -any witnessing Gram matrix `Q`, and `z(1,1,1)` is literally the all-ones -vector, no rounding involved — cuts it again to 16. Even on that -16-parameter family, though, Douglas–Rachford converges only very slowly: -6,000,000 iterations bring the minimum eigenvalue to roughly `−1.4·10⁻⁸`, -and rational rounding still fails at every stage even with denominators up to -roughly `10⁹`. This is now a genuine, quantified numerical-hardness finding, -not an unexplored avenue or an under-tuned budget — checked to be a real -search limitation and not a bug in the machinery the same way as before: an -independent sanity check confirms the affine Gram-matrix family (and each -reduced family) is constructed correctly, and a synthetic planted example -with a singular Gram matrix of the same size *is* found and exactly -re-verified. The remaining test for the 3-variable Motzkin form records -`undecided` rather than a false certificate. +exactly re-verified. + +**The half-Newton-polytope reduction is what closed the homogeneous cases.** +The *homogeneous* ternary Motzkin form at `N = 1` used to refuse, and the +refusal was misdiagnosed as numerical hardness: a great deal of extra search +machinery (symmetry reduction, an exact zero-vector restriction, 6,000,000 +Douglas–Rachford iterations) was spent on the wrong multiplier power. The +actual cause was dimension. `(x²+y²+z²)·Motzkin_hom` has a 15-monomial +degree-4 basis of which only 9 lie in `½·Newton`; the 75-parameter family +over the unreduced basis leaves the numeric search about `0.96` away from +the true certificate in parameter space — far too far to round onto it — +while the 18-parameter family over the reduced basis lands on it exactly. +Note what this is *not*: on both bases the certificate is the unique PSD +point of the affine family, rank 5, minimum eigenvalue exactly 0, so `λ_min` +does not distinguish them and more iterations do not help (4× the +Douglas–Rachford budget on the unreduced family still fails to round). With +the reduction, `(Σxᵢ²)·Motzkin_hom` and `(Σxᵢ²)·Choi–Lam` both certify at +`N = 1` in seconds. + +**What's still open:** larger copositivity forms — the Horn/C₅ form (5 +variables) and the C₇ form (7 variables) both admit `N = 1` certificates that +this search does not reach. Their Newton polytopes are already full, so the +reduction does not help, and their `N = 1` affine families have 420 and 2646 +free parameters respectively — above `psd_search`'s numeric-search ceiling of +200, so no search is attempted at those powers at all. That is now *reported* +in the `E-SOS-002` message (lines marked `NOT SEARCHED`) rather than being +indistinguishable from an exhausted search. Closing these needs a real +interior-point SDP solve on the reduced family, not more alternating +projection. ## Constrained certificates @@ -273,10 +293,11 @@ Motzkin's polynomial and Robinson's form), Handelman certificates on basic semialgebraic sets, exact verification, and Lean export. Not yet shipped: reliable certification of *every* boundary-case example — -the homogeneous 3-variable form of Motzkin specifically is still out of -reach (see above), so this is a real but narrower gap than "Motzkin doesn't -certify" was in the prior release — a proper interior-point solver that -would close it more systematically, and Putinar-style certificates with +the Horn/C₅ and C₇ copositivity forms are the ones currently out of reach, +and for a reason the error message now states outright (their affine families +are over the numeric-search ceiling, so no search runs) — a proper +interior-point solver that would close them more systematically, and +Putinar-style certificates with genuine SOS — rather than non-negative constant — multipliers on the *constraints*. `CertificateKind::Putinar` exists in the certificate type so those can be added without a shape change. diff --git a/tests/test_sos.py b/tests/test_sos.py index d2ce6269..6d718afc 100644 --- a/tests/test_sos.py +++ b/tests/test_sos.py @@ -136,6 +136,85 @@ def test_motzkin_certifies_via_a_multiplier(): assert lhs.count(")") >= 2 +def test_homogeneous_motzkin_certifies_at_multiplier_power_one(): + """The *homogeneous* ternary Motzkin form ``x⁴y² + x²y⁴ − 3x²y²z² + z⁶``. + + Multiplying by ``σ = x²+y²+z²`` makes it a sum of squares — that identity + is why Motzkin is the standard example of a PSD form that is not itself + SOS:: + + σ·M = (½x³y+xy³−3⁄2xyz²)² + ¾(x³y−xyz²)² + (xy²z−xz³)² + + (x²yz−yz³)² + (x²y²−z⁴)² + + Alkahest used to refuse this and, worse, recorded the refusal as a + mathematical fact ("not classically expected to be SOS at N = 1"). It was + a missing half-Newton-polytope reduction in the Gram-basis construction. + Asserted from Python as well as from Rust because the false claim reached + the user-facing documentation, and the user-facing entry point is here. + """ + pool = ak.ExprPool() + x, y, z = pool.symbol("x"), pool.symbol("y"), pool.symbol("z") + m = ( + _square(x) * _square(x) * _square(y) + + _square(x) * _square(y) * _square(y) + - pool.integer(3) * _square(x) * _square(y) * _square(z) + + _square(z) * _square(z) * _square(z) + ) + + cert = ak.sos_decompose(m, [x, y, z]) + + assert cert.kind == "sos" + # The exact re-expansion in Q is the soundness argument; the numeric + # search only ever proposed the Gram matrix. + assert cert.verify() is True + # A multiplier was needed (Motzkin is not itself SOS), which at this + # surface shows as a two-factor left-hand side in the identity. + lhs, _rhs = cert.identity.split("=", 1) + assert lhs.strip().startswith("(") + assert lhs.count(")") >= 2 + + +@pytest.mark.slow +def test_a_refusal_says_whether_it_actually_searched(): + """``E-SOS-002`` covers "searched and found nothing" *and* "never looked". + + Marked ``slow`` (~45 s: the Horn form's direct PSD search runs to + exhaustion before any multiplier power is considered). The same + assertion is made in the default CI tier by the Rust test + ``real::sos::tests::a_refusal_reports_which_multiplier_powers_were_actually_searched``; + this one exists because the false claim it guards against reached the + *user-facing* surface, and this is that surface. + + The Horn form (copositivity of the Horn matrix) is the case where the + difference bites: its ``N = 1`` multiplier family has 420 free + parameters, over the numeric search's ceiling, so no multiplier power is + searched at all. The refusal is legitimate; presenting it as an exhausted + search would not be. The message must carry the trace that distinguishes + them. + """ + pool = ak.ExprPool() + v = [pool.symbol(f"h{i}") for i in range(5)] + h = [ + [1, -1, 1, 1, -1], + [-1, 1, -1, 1, 1], + [1, -1, 1, -1, 1], + [1, 1, -1, 1, -1], + [-1, 1, 1, -1, 1], + ] + p = pool.integer(0) + for i in range(5): + for j in range(5): + p = p + pool.integer(h[i][j]) * _square(v[i]) * _square(v[j]) + + with pytest.raises(ak.SosError) as excinfo: + ak.sos_decompose(p, v) + + assert excinfo.value.code == "E-SOS-002" + msg = str(excinfo.value) + assert "what the search actually did:" in msg + assert "NOT SEARCHED" in msg + + def test_non_polynomial_is_refused(): pool = ak.ExprPool() x = pool.symbol("x") From 852529cd0e0d5efac5cbcc1fbe00d27df31edba6 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 22:00:38 +0000 Subject: [PATCH 09/11] fix(validated): enclose bounded integrands at a domain boundary; restore verified_sign monotonicity; make the searches interruptible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings from the 2026-08-19 autoresearch run, all in the validated-bounds surface. #11 (High) — `verified_integral` refused bounded, continuous integrands and called them "singular". `asin`/`acos`, `sqrt(1-x^2)`, `sqrt(x)`, `x^x`, `(1-cos x)/x^2` and `(x-1)/log x` on [0,1] all came back E-VALIDATED-003 "the integrand is singular at the right endpoint", which for `asin` — value pi/2 there — is not a conservative approximation but a false statement. The refusal was identical across a full order x prec x tol x max_subdivisions sweep and one ulp wide. Every Taylor-model rule needs a derivative bound, and asin/sqrt/log/recip have none at the end of their domain, so no model exists on the last panel however far it is bisected. Such a panel is now closed with a derivative-free `width x range` bound, the range from directed-rounding interval arithmetic over the extended reals (new private `validated::interval`). Endpoints stay exact — `1 - x^2` on [1-h,1] hits `[0, ...]` on the nose instead of dipping below sqrt's domain — and unboundedness is representable, so `log([0,h]) = [-inf, log h]` composes with the factor that tames it and `x^x = exp(x log x)` comes out in [0,1]. The same bound is also used to narrow a panel the Taylor model covers badly, which is what keeps the quadrature inside its subdivision budget near such a boundary; and the descent now stops at the existing width floor rather than running away into it. Separately, `RemovableQuotient::piece` now iterates Cauchy's mean value theorem, which is what `(1-cos x)/x^2` needs. Unbounded integrands have no bounded range and still refuse: -log x, (log x)^2, 1/sqrt(x), 1/sqrt(1-x^2), log(x) log(1-x), 1/x and sin(x)/x^2. The refusal message no longer calls a finite integrand singular; it names the rule that stopped and keeps the "no enclosure exists" vs "the integral does not exist" distinction. #12 (Medium) — `verified_sign` was not monotone in the box. The endpoint collar is only strong when the box endpoint *is* the tight point, so a dead band of left endpoints from ~1e-300 to 1e-9 answered `undecided` between two `true` regions: shrinking a box lost a proof. An `undecided` 1-D box stopping short of x = 0 is retried with the collar planted at 0; `true` on the superset implies `true` on the box, and only `true` is taken. Tightness elsewhere still shows the effect and is now documented. 26i (Medium) — the four validated entry points were uninterruptible from Python: a `signal.setitimer(60)` around a 109.9 s `verified_sign` fired at 182.8 s. Releasing the GIL is not enough, because CPython runs a main-thread signal handler only between bytecodes. The call now runs on a scoped worker thread while the calling thread polls `PyErr_CheckSignals` every 25 ms; a pending signal sets the cooperative cancellation flag, and the searches check it at every subdivision and wind down exactly as an exhausted `max_subdivisions` does — wider answer, sooner, never a wrong one. A 3 s timer now fires at 3 s. 26j (Medium, docs) — `bounds_supported`'s docstring and `docs/mdbook/src/validated-bounds.md` still listed bessel_j0, bessel_j1, digamma, lambert_w and gamma as outside Taylor-model coverage; all five were covered in 3.9.0. Verified against the build and corrected in both places, with the example moved to `floor`. A test asserts the live answer so the prose cannot drift again unnoticed. Verification: cargo test --workspace (2298 core tests), clippy -D warnings and cargo fmt clean; tests/test_validated_bounds.py 107 passed (23 of them fail against the pre-fix build, checked by stashing); the R4 fuzzers unchanged (fuzz_ball_trim 51788 samples / 5 failures, all the known ArbBall::tan one; fuzz_composition_notan 4000 trials / 0 failures); and 1209 random `verified_integral` enclosures checked against mpmath with no containment failure. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 75 +++ alkahest-core/src/validated/bounds.rs | 562 ++++++++++++++++--- alkahest-core/src/validated/interval.rs | 704 ++++++++++++++++++++++++ alkahest-core/src/validated/mod.rs | 3 + alkahest-py/src/lib.rs | 139 ++++- docs/mdbook/src/validated-bounds.md | 108 +++- tests/test_validated_bounds.py | 249 ++++++++- 7 files changed, 1708 insertions(+), 132 deletions(-) create mode 100644 alkahest-core/src/validated/interval.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f33e6f..a221c1a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,81 @@ ## Unreleased +- **`verified_integral` refused bounded, continuous integrands and called them + "singular".** `asin` and `acos` on `[0, 1]`, `sqrt(1 - x²)` on `[0, 1]`, + `sqrt(x)`, `xˣ`, `(1 - cos x)/x²` and `(x - 1)/log x` on `[0, 1]` all came + back `E-VALIDATED-003` — "the integrand is singular at the right endpoint" — + which for `asin`, whose value there is `π/2`, was not a conservative + approximation but a false statement. The refusal was a hard cliff, identical + across `order ∈ {2,…,24}` × `prec ∈ {64,…,512}` × `tol ∈ {1e-9,…,1}` × + `max_subdivisions ∈ {64,…,1e5}`, and one ulp wide: backing the endpoint off + by `1e-8` succeeded in every case. + + The cause is that every Taylor-model rule needs a derivative bound, and + `asin`, `sqrt`, `log` and the reciprocal have none at the end of their + domain, so no model exists on the last panel however far it is bisected. A + panel where the model fails but the integrand is *bounded* is now closed with + a derivative-free `width × range` bound instead. The range comes from + directed-rounding interval arithmetic over the extended reals + (`validated::interval`): endpoints stay exact, so `1 - x²` on `[1-h, 1]` + evaluates to `[0, …]` on the nose rather than dipping below `sqrt`'s domain, + and `log([0, h]) = [-∞, log h]` composes with the factor that tames it so + that `xˣ = exp(x·log x) ∈ [0, 1]` comes out bounded. Independently, the + removable-singularity argument now **iterates** Cauchy's mean value theorem, + which is what `(1 - cos x)/x²` needs (`D' = 2x` vanishes where `D = x²` does; + only `D'' = 2` is bounded away from zero). + + Genuinely unbounded integrands have no bounded range and still refuse: + `-log x`, `(log x)²`, `1/√x`, `1/√(1-x²)`, `log x · log(1-x)` on `[0, 1]`, + and `1/x` and `sin(x)/x²` across zero. The refusal message no longer calls a + finite integrand singular — it names the Taylor rule that stopped, says the + range fallback found nothing bounded either, and keeps the existing + distinction between "no enclosure of the integrand exists" and "the integral + does not exist". A 1209-enclosure containment fuzz against mpmath found no + soundness failure, and the classical-integral audit goes from 28 enclosed / + 12 refused to 33 enclosed / 7 refused with no enclosure missing its closed + form. + +- **`verified_sign` was not monotone in the box: a strictly smaller box could + lose a verdict the larger one had.** The collar the endpoint-series argument + plants is only strong when the box endpoint *is* the point the inequality is + tight at. Backing off by `δ` makes the leading coefficient `g(δ)`, which for + Cusa–Huygens (`g ~ x⁵/60`) cannot beat its own evaluation noise or the linear + term of the tail bound. The measured effect was a dead band of left endpoints + from about `1e-300` to `1e-9` answering `"undecided"`, sandwiched between + `"true"` at `0` and `"true"` from `1e-6` up — so shrinking a box could lose a + proof. An `"undecided"` one-dimensional box that stops short of `x = 0` is + now retried with the collar planted at `0`; `"true"` on the larger box + implies `"true"` on the box asked about, and only `"true"` is taken from the + retry. Cusa–Huygens, Mitrinović–Adamović, Wilker and Huygens are now `"true"` + across the whole band. Tightness at a point other than `0` still shows the + effect, and `verified_sign`'s documentation now says so and says what to do + about it. + +- **`verified_sign`, `verified_no_roots`, `verified_integral` and + `bound_on_box` were uninterruptible from Python.** A `verified_sign` on + `tan` with large integer coefficients ran 109.9 s, and a `signal.setitimer` + around it did not fire until the call had already returned (a 60 s timer + fired at 182.8 s). Releasing the GIL is not enough on its own: CPython runs a + Python-level signal handler only in the main thread and only between + bytecodes. The four entry points now run the native call on a scoped worker + thread while the calling thread polls `PyErr_CheckSignals` every 25 ms; a + pending signal sets the cooperative cancellation flag, and the searches check + it at every subdivision and wind down exactly as an exhausted + `max_subdivisions` does — wider answer, sooner, never a wrong one. A 3 s + timer now fires at 3 s. `alkahest_core::budget::request_cancel` and an + active `Budget` reach the same checkpoint for direct Rust callers. + +- **The validated-bounds docs still listed five functions as uncovered that + 3.9.0 had covered.** `bounds_supported`'s docstring example claimed + `bessel_j0` answers `(False, ['bessel_j0'])`; the real answer is + `(True, [])`, and `docs/mdbook/src/validated-bounds.md` likewise listed + `bessel_j0`, `bessel_j1`, `digamma`, `lambert_w` and `gamma` as outside + Taylor-model coverage. All five were verified against the build and both + places corrected, with the example moved to `floor`, which really has no + Taylor-model rule. A test now asserts the live answer for all seven covered + functions so the prose cannot drift again unnoticed. + - **`telescope2d` generalizes from two bound indices to an arbitrary `m ≥ 1`: `experimental.telescope_md`** (M4 extension). `telescope2d(term, n, j, k)` only ever reached exactly two bound indices; the underlying ansatz search diff --git a/alkahest-core/src/validated/bounds.rs b/alkahest-core/src/validated/bounds.rs index ae883141..f7094df2 100644 --- a/alkahest-core/src/validated/bounds.rs +++ b/alkahest-core/src/validated/bounds.rs @@ -18,15 +18,22 @@ //! enough to prove either case — it is never conflated with `False`. //! * [`verified_integral`] integrates the **continuous extension** of the //! integrand across a *removable* singularity `N(x)/D(x)` with -//! `N(p) = D(p) = 0` and `D'` non-vanishing — the enclosure there comes -//! from Cauchy's mean value theorem, not from ignoring the singular point. -//! Genuine (non-removable) singularities are still refused. +//! `N⁽ᵏ⁾(p) = D⁽ᵏ⁾(p) = 0` up to the order at which `D` stops vanishing — +//! the enclosure there comes from Cauchy's mean value theorem, iterated, +//! not from ignoring the singular point. Genuine (non-removable) +//! singularities are still refused. +//! * A sub-interval on which the Taylor model runs out of *domain* rather +//! than out of function — `asin` at `x = 1`, `sqrt` at `x = 0`, where the +//! integrand is finite but some derivative is not — is closed with a +//! derivative-free `width × range` bound (see [`super::interval`]). +//! Unbounded integrands have no bounded range and are still refused. //! * A sub-box that cannot be bounded rigorously (branch cut, pole, or //! domain violation persisting after the box has been bisected far below //! the scale of the original box) causes the whole call to **refuse** //! with the underlying [`super::ValidatedError`], rather than silently //! omitting that piece of the domain from the answer. +use super::interval::panel_integral; use super::taylor::{taylor_range, TaylorContext, MAX_ORDER}; use super::{ contains_zero, from_bounds, from_float, is_finite, lb, mag, ub, width, ValidatedError, @@ -159,6 +166,22 @@ fn split_widest(boxes: &[FBox], prec: u32) -> (Vec, Vec) { (b1, b2) } +/// Cooperative stop request: [`crate::budget::request_cancel`] on any thread, +/// or an active [`crate::budget::Budget`] that has run out. +/// +/// The searches here are the slowest entry points in the crate — a +/// `verified_sign` on a target with large integer coefficients can run for +/// minutes — and until this existed there was no way to stop one short of +/// killing the process. Every caller of this treats a stop exactly as it +/// treats an exhausted `max_subdivisions`: it stops *refining* and reports +/// what it already has, flagged `budget_exhausted`. So a cancelled search +/// never invents a tighter bound, never upgrades a verdict, and never drops a +/// piece of the domain from an integral — it only gives back a wider answer, +/// sooner. +fn stop_requested() -> bool { + crate::budget::check().is_err() +} + fn is_recoverable_domain_issue(e: &ValidatedError) -> bool { matches!( e, @@ -408,7 +431,7 @@ fn extremum_search( } } - if subdivisions >= opts.max_subdivisions { + if subdivisions >= opts.max_subdivisions || stop_requested() { exhausted = true; active.push((key, b)); break; @@ -738,14 +761,23 @@ fn point_value( r.mid.is_finite().then(|| r.mid.clone()) } -/// The ingredients of the L'Hôpital enclosure for an integrand written as a -/// quotient: `N`, `D`, `D'`, and the mean-value quotient `N'/D'`. +/// How many times the mean-value argument may be iterated before giving up. +/// +/// One step handles a simple `0/0` (`sin(x)/x`); `k` steps handle a zero of +/// order `k` in the denominator (`(1 - cos x)/x²` needs two, since `D' = 2x` +/// vanishes at the same point `D` does). Each step costs a symbolic derivative +/// of both halves and a `bound_on_fboxes` over the panel, and the derivatives +/// grow, so the ceiling is deliberately low. +const MAX_LHOPITAL_ORDER: usize = 4; + +/// The ingredients of the iterated L'Hôpital enclosure for an integrand +/// written as a quotient: the successive derivative pairs +/// `(N⁽ᵏ⁾, D⁽ᵏ⁾)` together with each mean-value quotient `N⁽ᵏ⁾/D⁽ᵏ⁾`. struct RemovableQuotient { num: ExprId, den: ExprId, - dden: ExprId, - /// `N' · (D')⁻¹`. - ratio: ExprId, + /// `(N⁽ᵏ⁾, D⁽ᵏ⁾, N⁽ᵏ⁾ · (D⁽ᵏ⁾)⁻¹)` for `k = 1 ..= MAX_LHOPITAL_ORDER`. + levels: Vec<(ExprId, ExprId, ExprId)>, } impl RemovableQuotient { @@ -754,15 +786,22 @@ impl RemovableQuotient { /// here asserts that a removable singularity exists. fn detect(expr: ExprId, pool: &ExprPool, var: ExprId) -> Option { let (num, den) = split_quotient(expr, pool)?; - let dnum = diff(num, var, pool).ok()?.value; - let dden = diff(den, var, pool).ok()?.value; - let ratio = pool.mul(vec![dnum, pool.pow(dden, pool.integer(-1_i32))]); - Some(RemovableQuotient { - num, - den, - dden, - ratio, - }) + let mut levels = Vec::with_capacity(MAX_LHOPITAL_ORDER); + let (mut n, mut d) = (num, den); + for _ in 0..MAX_LHOPITAL_ORDER { + // Simplified at each step, as `expand_at_endpoint` does, or the + // fourth derivative of a quotient is unusable in size. + n = simplify(diff(n, var, pool).ok()?.value, pool).value; + d = simplify(diff(d, var, pool).ok()?.value, pool).value; + let ratio = pool.mul(vec![n, pool.pow(d, pool.integer(-1_i32))]); + levels.push((n, d, ratio)); + } + Some(RemovableQuotient { num, den, levels }) + } + + /// `D'`, the first denominator derivative — the one Newton's method uses. + fn dden(&self) -> ExprId { + self.levels[0].1 } /// Newton iterates of `D` inside `[lo, hi]`, as *candidate* locations for @@ -790,7 +829,7 @@ impl RemovableQuotient { let Some(f) = point_value(self.den, pool, var, &z, order, prec) else { break; }; - let Some(df) = point_value(self.dden, pool, var, &z, order, prec) else { + let Some(df) = point_value(self.dden(), pool, var, &z, order, prec) else { break; }; if df.is_zero() || !df.is_finite() { @@ -823,17 +862,23 @@ impl RemovableQuotient { /// the analytic interior of the primitive's domain. Analytic implies /// differentiable, so the symbolic derivatives `N'`, `D'` really are the /// derivatives of `N`, `D` on `J`. - /// 2. `D'` has no zero on `J` (its enclosure excludes zero). Hence `D` is - /// strictly monotone on `J`, so `p` is its *only* zero there and - /// `D(x) ≠ 0` for every other `x ∈ J`. - /// 3. `R` is a rigorous enclosure of the range of `N'/D'` over `J`. + /// 2. The first `d` for which `D⁽ᵈ⁾` has no zero on `J` (its enclosure + /// excludes zero), with `N⁽ᵏ⁾(p) = D⁽ᵏ⁾(p) = 0` proven *exactly* for + /// every `k < d`, and `N⁽ᵏ⁾`, `D⁽ᵏ⁾` analytic on `J` for every `k ≤ d`. + /// 3. `R` is a rigorous enclosure of the range of `N⁽ᵈ⁾/D⁽ᵈ⁾` over `J`. + /// + /// Cauchy's mean value theorem, applied `d` times, then gives for every + /// `x ∈ J \ {p}` a point `ξ` strictly between `p` and `x` with + /// `N(x)/D(x) = N⁽ᵈ⁾(ξ)/D⁽ᵈ⁾(ξ) ∈ R`. For `d = 1` that is the one step + /// `(N(x) − N(p))·D'(ξ) = (D(x) − D(p))·N'(ξ)` with `N(p) = D(p) = 0`; each + /// further step needs the two facts the loop checks, namely that the level + /// above vanishes at `p` and that `D⁽ᵈ⁾ ≠ 0` on `J`. The latter also + /// supplies every non-vanishing hypothesis the chain uses: by Taylor with + /// Lagrange remainder, `D⁽ᵏ⁾(y) = D⁽ᵈ⁾(η)·(y − p)^{d−k}/(d − k)!` for some + /// `η ∈ J`, which is non-zero for every `y ∈ J \ {p}` and every `k < d`. /// - /// Cauchy's mean value theorem then gives, for every `x ∈ J \ {p}`, some - /// `ξ` strictly between `p` and `x` with - /// `(N(x) − N(p))·D'(ξ) = (D(x) − D(p))·N'(ξ)`; since `N(p) = D(p) = 0`, - /// `D(x) ≠ 0` and `D'(ξ) ≠ 0`, this is `N(x)/D(x) = N'(ξ)/D'(ξ) ∈ R`. - /// So the integrand is bounded by `R` on `J` minus a single point, and - /// `∫_J N/D dx ∈ (hi − lo)·R`. + /// Two steps are what `(1 − cos x)/x²` needs: `D' = 2x` vanishes at the + /// same point `D = x²` does, and only `D'' = 2` is bounded away from zero. /// /// Note what is being integrated: the integrand is *undefined* at `p`, and /// the value returned is the integral of its continuous extension (which @@ -862,40 +907,57 @@ impl RemovableQuotient { // the last of those is what reaches a singularity sitting at a point // the dyadic bisection grid never visits. A singularity at a point that // no candidate names exactly is simply refused. + let vanishes_at = |e: ExprId, p: &Float| { + vanishes_exactly(e, pool, var, p) + && enclosure_admits_zero(e, pool, var, p, opts.order, prec) + }; let mut candidates = vec![lo.clone(), hi.clone(), midpoint(lo, hi, prec)]; candidates.extend(self.newton_candidates(pool, var, lo, hi, opts)); - candidates.iter().find(|p| { - vanishes_exactly(self.den, pool, var, p) - && vanishes_exactly(self.num, pool, var, p) - && enclosure_admits_zero(self.den, pool, var, p, opts.order, prec) - && enclosure_admits_zero(self.num, pool, var, p, opts.order, prec) - })?; + let p = candidates + .iter() + .find(|p| vanishes_at(self.den, p) && vanishes_at(self.num, p))? + .clone(); let j = vec![(var, lo.clone(), hi.clone())]; // (1) N and D analytic on J. bound_on_fboxes(self.num, pool, &j, &bopts).ok()?; bound_on_fboxes(self.den, pool, &j, &bopts).ok()?; - // (2) D' non-vanishing on J. - let dd = bound_on_fboxes(self.dden, pool, &j, &bopts).ok()?; - if contains_zero(dd.enclosure()) { - return None; - } - // (3) the mean-value quotient. - let r = bound_on_fboxes(self.ratio, pool, &j, &bopts).ok()?; - let w = Float::with_val(prec, hi - lo); - let piece = from_float(&w, prec) * r.enclosure().clone(); - if !is_finite(&piece) { - return None; + // (2) Descend to the first level whose denominator is bounded away + // from zero on J, requiring both halves of every level passed over to + // vanish exactly at `p` — otherwise the chain of mean value theorems + // has no next link and the answer is a refusal, not a guess. + for &(nk, dk, ratio) in &self.levels { + bound_on_fboxes(nk, pool, &j, &bopts).ok()?; + let dd = bound_on_fboxes(dk, pool, &j, &bopts).ok()?; + if !contains_zero(dd.enclosure()) { + // (3) the mean-value quotient at this level. + let r = bound_on_fboxes(ratio, pool, &j, &bopts).ok()?; + let w = Float::with_val(prec, hi - lo); + let piece = from_float(&w, prec) * r.enclosure().clone(); + return is_finite(&piece).then_some(piece); + } + if !(vanishes_at(nk, &p) && vanishes_at(dk, &p)) { + return None; + } } - Some(piece) + None } } -/// Turn a refusal that survived bisection into a message that says *what* is -/// singular and *where*, and that distinguishes "no rigorous enclosure of the -/// integrand exists here" from "the integral does not exist". -fn describe_singularity( +/// Turn a refusal that survived bisection into a message that says which rule +/// gave up and *where*, without asserting anything about the integrand that has +/// not been established. +/// +/// The distinction matters and used to be got wrong: a Taylor-model rule +/// stopping at its domain boundary is a fact about the *rule*, and a message +/// that reports it as "the integrand is singular here" is simply false for +/// `asin` at `x = 1`, where the integrand takes the perfectly finite value +/// `π/2`. Those cases no longer reach this function at all — the `width × +/// range` fallback closes them — so what is left is genuinely a case where no +/// bounded enclosure of the integrand could be produced by any available rule, +/// which is still not the same claim as "the integral does not exist". +fn describe_refusal( cause: ValidatedError, lo: &Float, hi: &Float, @@ -915,12 +977,20 @@ fn describe_singularity( let at = midpoint(lo, hi, lo.prec()).to_f64(); ValidatedError::DomainViolation { what: format!( - "the integrand is singular at {where_} x ≈ {at:e} ({what}). \ - This is a statement about the *integrand*, not about the integral: \ - an integrable singularity still has a finite integral, which this \ - routine cannot certify. Removable singularities written as N(x)/D(x) \ - with N(p) = D(p) = 0 exactly and D'(p) ≠ 0 are handled automatically \ - (their continuous extension is integrated); this one is not of that form" + "no rigorous enclosure of the integrand could be built near {where_} \ + x ≈ {at:e}. The Taylor-model rule that stopped there was: {what}; \ + and the derivative-free fallback (width × range over the sub-interval, \ + by interval arithmetic) found no *bounded* range there either, which \ + is what a genuine pole or an unbounded integrable singularity looks \ + like. A bounded integrand whose Taylor model merely runs out of \ + domain — asin or sqrt at the end of its domain — is closed by that \ + fallback and never reaches this message. Quotients N(x)/D(x) whose \ + two halves vanish exactly at the same point, to the same order, with \ + some D⁽ᵈ⁾ non-vanishing, are handled by an iterated Cauchy mean value \ + theorem (their continuous extension is integrated); this one is not \ + of that form. None of this says the integral fails to exist: an \ + integrable singularity such as -log(x) on [0, 1] has a finite \ + integral that this routine still cannot certify" ), } } @@ -945,24 +1015,40 @@ fn describe_singularity( /// integral is singular. Such a sub-interval is enclosed through Cauchy's mean /// value theorem instead: `N(x)/D(x) = N'(ξ)/D'(ξ)` for some `ξ` in the /// sub-interval, so an enclosure `R` of `N'/D'` there — which is perfectly -/// regular — gives `∫_J N/D dx ∈ |J| · R`. The vanishing of `N` and `D` -/// at `p` is checked *symbolically* — a numeric enclosure cannot prove a value -/// is exactly zero — and `D'` must be certified non-vanishing on the -/// sub-interval, so a genuine pole is never mistaken for a removable one. The -/// number returned is the integral of the continuous extension. +/// regular — gives `∫_J N/D dx ∈ |J| · R`. When `D'` vanishes at `p` too the +/// step is repeated, up to a small fixed depth, which is what covers a +/// higher-order removable singularity such as `(1 - cos x)/x²`. The vanishing +/// of `N⁽ᵏ⁾` and `D⁽ᵏ⁾` at `p` is checked *symbolically* — a numeric enclosure +/// cannot prove a value is exactly zero — and some `D⁽ᵈ⁾` must be certified +/// non-vanishing on the sub-interval, so a genuine pole is never mistaken for a +/// removable one. The number returned is the integral of the continuous +/// extension. +/// +/// # Bounded integrands at a domain boundary +/// +/// `asin` on `[0, 1]`, `sqrt(1 - x²)` on `[0, 1]`, `sqrt(x)` on `[0, 1]`, +/// `xˣ` on `[0, 1]` are all bounded and continuous on the closed interval, but +/// the Taylor-model rule for `asin`/`sqrt`/`log` needs a derivative bound and +/// has none at the end of its domain, so no model exists on the last panel +/// however far it is bisected. Those panels are closed with a `width × range` +/// bound instead, the range computed by directed-rounding interval arithmetic +/// ([`super::interval`]) — the one enclosure that needs no derivative. The +/// panel is tiny by then, so the crude bound costs essentially nothing in +/// width. An integrand that is genuinely unbounded on the panel has no bounded +/// range, so this never converts a refusal into an unsound answer. /// /// Refuses (does not guess) when: /// - `a` or `b` is non-finite (infinite-limit improper integrals are not /// supported — there is no box to Taylor-expand over), /// - `a > b`, -/// - the integrand has a singularity in `[a, b]` that is not removable in the -/// above sense (e.g. `1/sqrt(x)` on `[0, 1]`, or the *integrable* endpoint -/// singularity of `-log(x)` on `[0, 1]`) — subdivision is tried first in -/// case the domain violation is only a boundary artefact of a coarse box, -/// but a persistent one refuses with a [`ValidatedError`] that names the -/// location and distinguishes "the integrand is singular here" from "the -/// integral does not exist", rather than silently skipping the offending -/// piece. +/// - the integrand has a singularity in `[a, b]` that is neither removable in +/// the above sense nor bounded (e.g. `1/sqrt(x)` on `[0, 1]`, or the +/// *integrable* endpoint singularity of `-log(x)` on `[0, 1]`) — subdivision +/// is tried first in case the domain violation is only a boundary artefact of +/// a coarse box, but a persistent one refuses with a [`ValidatedError`] that +/// names the location and the rule that stopped, and distinguishes "no +/// enclosure of the integrand could be built here" from "the integral does +/// not exist", rather than silently skipping the offending piece. pub fn verified_integral( expr: ExprId, pool: &ExprPool, @@ -1008,8 +1094,15 @@ pub fn verified_integral( // Structural `N/D` analysis of the integrand, built at most once and only // if a sub-interval actually refuses. let mut removable: Option> = None; + // Sticky, so the wind-down is uniform: once a stop has been seen every + // remaining panel is accepted as-is rather than refined, and the loop + // drains the stack instead of abandoning it. Abandoning it would leave + // part of `[a, b]` out of the sum, which is the one thing this routine + // must never do. + let mut stopped = false; while let Some((lo, hi)) = stack.pop() { + stopped = stopped || stop_requested(); let piece_w = Float::with_val(prec, &hi - &lo); let piece_tol = Float::with_val( prec, @@ -1027,16 +1120,55 @@ pub fn verified_integral( .as_ref(); match q.and_then(|q| q.piece(pool, var, &lo, &hi, opts)) { Some(piece) => Ok(piece), - None => Err(e), + // Failing that, close the panel the crude way: `width × + // range`, with the range computed by directed-rounding + // interval arithmetic (see [`super::interval`]). That needs + // no derivative, so it survives exactly the domain + // *boundaries* where the Taylor rules stop — `asin` at + // `x = 1`, `sqrt` at `x = 0` — while an integrand that is + // genuinely unbounded on the panel still has no bounded + // range and still refuses. The piece it returns is wide, so + // the loop below keeps bisecting it like any other; only + // the very last panel ends up carrying a crude bound. + None => match panel_integral(expr, pool, var, &lo, &hi, prec) { + Some(piece) => Ok(piece), + None => Err(e), + }, } } Err(e) => return Err(e), }; match outcome { - Ok(piece) => { - let w = width(&piece); - if w <= piece_tol || subdivisions >= opts.max_subdivisions { + Ok(mut piece) => { + let mut w = width(&piece); + if w > piece_tol { + // The Taylor model is about to be judged too wide here. + // A `width × range` bound needs no derivative, so next to a + // domain boundary — where the Taylor remainder blows up + // even though a model still exists — it is routinely the + // better of the two. They enclose the same integral, so + // keeping the narrower is sound, and it saves every + // bisection the wider one would otherwise have cost. + if let Some(alt) = panel_integral(expr, pool, var, &lo, &hi, prec) { + let aw = width(&alt); + if aw < w { + piece = alt; + w = aw; + } + } + } + // `floor` is the same depth limit the refusal path uses: a + // panel that has already been bisected 2^-60 of the way down + // contributes less than the accumulated rounding of the sum, so + // splitting it again cannot buy accuracy — it only spends + // budget that the rest of the interval still needs. Without + // this the depth-first descent into a boundary the crude + // fallback keeps closing (`xˣ` at `x = 0`) runs away, exhausts + // `max_subdivisions` before the search ever returns to the + // right-hand panels, and leaves *those* wide. + let at_floor = piece_w.to_f64_round(Round::Up) <= floor; + if w <= piece_tol || subdivisions >= opts.max_subdivisions || at_floor || stopped { if w > piece_tol { exhausted = true; } @@ -1053,8 +1185,8 @@ pub fn verified_integral( } Err(e) => { let w = piece_w.to_f64_round(Round::Up); - if subdivisions >= opts.max_subdivisions || w <= floor { - return Err(describe_singularity(e, &lo, &hi, &a_f, &b_f)); + if subdivisions >= opts.max_subdivisions || w <= floor || stopped { + return Err(describe_refusal(e, &lo, &hi, &a_f, &b_f)); } subdivisions += 1; let mid = midpoint(&lo, &hi, prec); @@ -1337,7 +1469,7 @@ fn root_exists_witness( let mut budget = opts.max_subdivisions; while let Some(b) = queue.pop_front() { - if budget == 0 { + if budget == 0 || stop_requested() { break; } budget -= 1; @@ -1468,6 +1600,20 @@ pub enum SignPredicate { /// predicate holds everywhere on the box, [`Verdict::False`] when the /// enclosure proves it is violated somewhere, and [`Verdict::Undecided`] /// when the enclosure straddles the boundary needed to decide either way. +/// +/// # `Undecided` is not monotone in the box +/// +/// `True` on a box logically implies `True` on every sub-box, but this is a +/// *search*, and the search does not have that property: an inequality tight at +/// a point `p` is decided by a series collar planted at `p`, and a box whose +/// endpoint sits a little way off `p` gets a much weaker collar (see +/// `widened_to_tight_point`). For `p = 0` — where every classical sharp +/// trigonometric inequality is tight — that gap is closed by retrying on the +/// box grown out to `0`, so `[δ, b]` now decides whenever `[0, b]` does. For a +/// tight point anywhere else the effect remains: if `[a, b]` comes back +/// `Undecided`, it is worth trying a *larger* box whose endpoint is the point +/// the inequality is tight at, rather than concluding the statement is out of +/// reach. pub fn verified_sign( expr: ExprId, pool: &ExprPool, @@ -1525,6 +1671,13 @@ pub fn verified_sign( return Ok(Verdict::True); } + // The two series attempts below each cost a run of symbolic derivatives, so + // a stop request is honoured before starting them rather than only inside + // their enclosures. + if stop_requested() { + return Ok(Verdict::Undecided); + } + // Still undecided, which is what a margin that *vanishes* at an endpoint // always looks like to a subdivision search. Try the series argument, which // is the only one of the two that can reach a tight endpoint at all. @@ -1532,9 +1685,71 @@ pub fn verified_sign( return Ok(v); } + // Last: the same series argument on a box *enlarged* to the tight point the + // caller stopped just short of. See `widened_to_tight_point` for why this + // is here and why only `True` may be taken from it. + if !stop_requested() { + if let Some(wider) = widened_to_tight_point(boxes) { + if let Some(Verdict::True) = + endpoint_series_verdict(expr, pool, &wider, predicate, opts) + { + return Ok(Verdict::True); + } + } + } + Ok(Verdict::Undecided) } +/// The box grown outward to `x = 0` on whichever side is nearest it, or `None` +/// when the box already reaches it (or is not one-dimensional). +/// +/// # Why +/// +/// The collar argument of [`endpoint_series_verdict`] expands `g` at the box's +/// own endpoint `p`, and its strength comes from the *leading* coefficient +/// `c_j` being provably signed while the ones below it are provably zero. When +/// `p` is the point at which the inequality is tight — `x = 0` for every +/// classical sharp trigonometric inequality — `simplify` proves `g(0) = g'(0) = +/// … = 0` symbolically, `j` is the true order of vanishing, and `c_j` is an +/// `O(1)` number. +/// +/// Back the box off that point by `δ` and the picture inverts. Nothing vanishes +/// exactly at `δ` any more, so `j = 0` and `c_0 = g(δ) ≈ δ^j·(leading)` — a +/// number that has to beat both the cancellation noise of evaluating `g` near a +/// zero of order `j` and the `|c_1|·δ` term of the tail bound. For +/// Cusa–Huygens, `g ~ x⁵/60`, and that fails for every `δ` between roughly +/// `10⁻³⁰⁰` and `10⁻⁹`, while succeeding both at `δ = 0` and from `10⁻⁶` up. +/// The measured consequence is a **dead band**: `[0, π/2]` is `True`, +/// `[10⁻¹², π/2]` — a strictly *smaller* box, so a strictly weaker claim — is +/// `Undecided`. +/// +/// A verdict that a smaller box can lose is a bad thing to hand a search loop, +/// which will read `Undecided` as "out of reach" and stop. Since `True` on a +/// superset implies `True` on the box, retrying on the enlarged box is sound +/// and restores monotonicity across the band. Only `True` may be taken from it: +/// `False` on a superset says nothing about the subset, and is discarded. +/// +/// `0` is the only candidate. It is where these inequalities are tight in +/// practice, and enlarging to an arbitrary nearby point would be a search +/// rather than a retry. A statement tight at some other point still shows the +/// non-monotonicity, which [`verified_sign`]'s documentation now records. +fn widened_to_tight_point(boxes: &[(ExprId, f64, f64)]) -> Option> { + let &[(var, lo, hi)] = boxes else { + return None; + }; + if !(lo.is_finite() && hi.is_finite()) || lo >= hi { + return None; + } + if lo > 0.0 { + Some(vec![(var, 0.0, hi)]) + } else if hi < 0.0 { + Some(vec![(var, lo, 0.0)]) + } else { + None + } +} + /// Rigorous one-sided bound computed with [`SearchGoal::DecideSign`]: a lower /// bound on `min f` for [`Extremum::Min`], an upper bound on `max f` for /// [`Extremum::Max`]. @@ -2303,6 +2518,199 @@ mod tests { assert_eq!(crate::errors::AlkahestError::code(&err), "E-VALIDATED-003"); } + // ── verified_integral: bounded integrands at a domain boundary ───── + + /// Every row of the table in the 2026-08-19 audit's issue #11, plus the + /// control that already worked. Each integrand is bounded and continuous on + /// the closed interval, and each was refused as "singular" — which for + /// `asin`, whose value at the offending endpoint is `π/2`, was simply + /// false. Reference values are mpmath at 40 dps. + #[test] + fn bounded_integrands_at_a_domain_boundary_are_enclosed() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let one = pool.integer(1_i32); + let log_x = pool.func("log", vec![x]); + let cases: Vec<(&str, ExprId, f64)> = vec![ + ( + "asin", + pool.func("asin", vec![x]), + std::f64::consts::FRAC_PI_2 - 1.0, + ), + ("acos", pool.func("acos", vec![x]), 1.0), + ( + "sqrt(1-x^2)", + pool.func("sqrt", vec![sub(&pool, one, pool.mul(vec![x, x]))]), + std::f64::consts::FRAC_PI_4, + ), + ("sqrt(x)", pool.func("sqrt", vec![x]), 2.0 / 3.0), + ( + "x^x", + pool.func("exp", vec![pool.mul(vec![x, log_x])]), + 0.783_430_510_712_134_4, + ), + ( + "(1-cos x)/x^2", + div( + &pool, + sub(&pool, one, pool.func("cos", vec![x])), + pool.pow(x, pool.integer(2_i32)), + ), + 0.486_385_376_235_322_7, + ), + ( + "(x-1)/log x", + div(&pool, sub(&pool, x, one), log_x), + std::f64::consts::LN_2, + ), + ( + "sin(x)/x", + div(&pool, pool.func("sin", vec![x]), x), + 0.946_083_070_367_183, + ), + ]; + for (label, e, truth) in cases { + let r = verified_integral(e, &pool, x, 0.0, 1.0, &iopts()) + .unwrap_or_else(|err| panic!("{label}: {err}")); + assert!( + r.lower() <= truth && truth <= r.upper(), + "{label}: [{}, {}] misses {truth}", + r.lower(), + r.upper() + ); + assert!( + r.upper() - r.lower() < 1e-3, + "{label}: enclosure too wide: {}", + r.upper() - r.lower() + ); + } + } + + /// The fallback fires exactly where the Taylor model runs out of domain, so + /// what has to keep working is the refusal for an *unbounded* integrand at + /// the very same boundary. `sqrt(1-x²)` and `1/sqrt(1-x²)` both stop the + /// `sqrt` rule at `x = 1`; only the first has a bounded range there. + #[test] + fn the_bounded_fallback_does_not_swallow_an_unbounded_integrand() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let one = pool.integer(1_i32); + let inner = sub(&pool, one, pool.mul(vec![x, x])); + let root = pool.func("sqrt", vec![inner]); + + assert!(verified_integral(root, &pool, x, 0.0, 1.0, &iopts()).is_ok()); + let err = + verified_integral(div(&pool, one, root), &pool, x, 0.0, 1.0, &iopts()).unwrap_err(); + assert_eq!(crate::errors::AlkahestError::code(&err), "E-VALIDATED-003"); + } + + /// The refusal message must not describe a Taylor rule running out of + /// domain as the *integrand* being singular, and must name the rule. + #[test] + fn the_refusal_names_the_rule_rather_than_calling_the_integrand_singular() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let e = pool.mul(vec![pool.integer(-1_i32), pool.func("log", vec![x])]); + let err = verified_integral(e, &pool, x, 0.0, 1.0, &iopts()).unwrap_err(); + let message = err.to_string(); + assert!(!message.contains("the integrand is singular"), "{message}"); + assert!( + message.contains("no rigorous enclosure of the integrand"), + "{message}" + ); + assert!(message.contains("log"), "{message}"); + } + + // ── verified_sign: monotone in the box ───────────────────────────── + + /// `[δ, b] ⊂ [0, b]` is a strictly weaker claim, so it must not be harder + /// to decide. Before the widened retry, every `δ` from about `1e-300` to + /// `1e-9` came back `Undecided` while `δ = 0` was `True`. + #[test] + fn shrinking_the_box_does_not_lose_a_tight_endpoint_verdict() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + // Cusa–Huygens, denominators cleared: x(2 + cos x) − 3 sin x ≥ 0. + let e = sub( + &pool, + pool.mul(vec![ + x, + pool.add(vec![pool.integer(2_i32), pool.func("cos", vec![x])]), + ]), + pool.mul(vec![pool.integer(3_i32), pool.func("sin", vec![x])]), + ); + for lo in [0.0, 1e-30, 1e-12] { + let v = verified_sign( + e, + &pool, + &[(x, lo, 1.5)], + SignPredicate::NonNegative, + &opts(), + ) + .unwrap(); + assert_eq!(v, Verdict::True, "lo = {lo:e}"); + } + } + + /// Only `True` may be taken from the enlarged box: a `False` there says + /// nothing about the sub-box that was actually asked about. + #[test] + fn the_widened_retry_never_manufactures_a_false() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let e = sub(&pool, x, pool.rational(1, 2)); + assert_eq!( + verified_sign( + e, + &pool, + &[(x, 0.0, 1.0)], + SignPredicate::NonNegative, + &opts() + ) + .unwrap(), + Verdict::False + ); + assert_eq!( + verified_sign( + e, + &pool, + &[(x, 0.6, 1.0)], + SignPredicate::NonNegative, + &opts() + ) + .unwrap(), + Verdict::True + ); + } + + /// A stop request must halt the search promptly and degrade to the same + /// wide-but-sound answer an exhausted subdivision budget gives — never to a + /// wrong one. + /// + /// Driven through a thread-local [`crate::budget::Budget`] rather than + /// [`crate::budget::request_cancel`]: cancellation is a process-wide flag, + /// and `cargo test` runs these in parallel, so setting it here would stop + /// every other search in the binary as a side effect. Both reach the same + /// checkpoint. + #[test] + fn a_stopped_search_halts_and_stays_sound() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let e = pool.mul(vec![x, sub(&pool, pool.integer(1_i32), x)]); + let tiny = BoundOptions { + tol: 1e-30, // unreachable, so only the stop can end the search + ..opts() + }; + let r = { + let _guard = crate::budget::enter(crate::budget::Budget::new().with_max_steps(3)); + bound_on_box(e, &pool, &[(x, 0.0, 1.0)], &tiny).unwrap() + }; + assert!(r.budget_exhausted); + assert!(r.subdivisions < 32, "did not stop early: {r:?}"); + // x(1-x) has true range [0, 1/4]; a stopped search still encloses it. + assert!(r.lower() <= 0.0 && r.upper() >= 0.25, "{r:?}"); + } + #[test] fn double_pole_with_a_simple_numerator_zero_is_refused() { // sin(x)/x² ~ 1/x near 0: the numerator vanishes to order 1 but the diff --git a/alkahest-core/src/validated/interval.rs b/alkahest-core/src/validated/interval.rs new file mode 100644 index 00000000..7b6aeda2 --- /dev/null +++ b/alkahest-core/src/validated/interval.rs @@ -0,0 +1,704 @@ +//! Directed-rounding interval arithmetic over the **extended** reals, used as +//! the last-resort range bound for one panel of [`super::bounds::verified_integral`]. +//! +//! # Why this exists next to the Taylor models +//! +//! [`super::taylor`] is the right tool almost everywhere: it tracks +//! correlations between subexpressions, so it does not lose `x - x` to the +//! dependency problem, and its remainder shrinks like `O(h^{p+1})` under +//! subdivision. What it cannot do is stand on the *boundary* of a primitive's +//! domain. Every rule there needs a derivative bound over the argument +//! enclosure, and `asin`, `acos`, `sqrt`, `log` and the reciprocal all have an +//! unbounded derivative where their domain ends. So a panel that touches +//! `x = 1` refuses for `asin`, and one that touches `x = 0` refuses for +//! `sqrt` — even though `asin` is *bounded* there (`asin(1) = π/2`) and +//! `sqrt(x) ∈ [0, √h]`. +//! +//! A bounded integrand on a panel of width `h` contributes at most `h · +//! range` to the integral, which is all that panel needs to be closed. This +//! module computes that `range` with the one tool that does not need a +//! derivative: plain interval arithmetic, evaluated with **directed rounding +//! on the endpoints** rather than midpoint/radius balls, and extended to +//! `±∞` so that a subexpression which really is unbounded (`log x` as +//! `x → 0⁺`) can still be composed with one that tames it (`exp(x·log x) ∈ +//! [0, 1]`). +//! +//! Two properties matter and are the reason this is not simply +//! [`crate::ball::IntervalEval`]: +//! +//! * **Endpoints stay exact.** The range of the integration variable over the +//! panel `[lo, hi]` is *exactly* `[lo, hi]`; there is no ball radius to +//! round outward. That is what makes `sqrt(x)` on `[0, h]` work at all — a +//! ball whose lower endpoint has been inflated to `-ε` is out of `sqrt`'s +//! domain and is (correctly) refused. Likewise `1 - x²` on `[1-h, 1]` +//! evaluates to `[0, …]` on the nose, because `1 - 1·1` is exact. +//! * **Unboundedness is representable, not an error.** `log([0, h])` is +//! `[-∞, log h]`, and `[0, h] · [-∞, log h]` is `[-∞, 0]`. Only the *final* +//! range has to be finite; refusing at the first infinity would lose every +//! integrand whose boundedness comes from a cancellation between an +//! unbounded factor and a vanishing one. +//! +//! # Soundness +//! +//! Every rule below returns a superset of `{ f(x) : x ∈ box }`: +//! +//! * Endpoint arithmetic uses [`rug::float::Round::Down`] for lower bounds and +//! [`rug::float::Round::Up`] for upper bounds, so no rounding step can ever +//! narrow an interval. +//! * A NaN endpoint (`∞ - ∞`, `0/0`) makes the interval meaningless and +//! returns `None`, which propagates to a refusal. +//! * `0 · ±∞` is taken to be `0` in the multiplication corners. This is the +//! IEEE 1788 convention and it is exactly right here: the product set of a +//! bounded interval containing `0` with an unbounded one has `0` among its +//! attainable values (take the finite factor to be `0`), and `±∞` as its +//! limit (take the finite factor away from `0`) — which is what the corner +//! rule produces. +//! * A primitive whose domain the argument interval genuinely leaves — `log` +//! or `sqrt` of an interval with a *strictly negative* lower endpoint, +//! `asin` outside `[-1, 1]` — returns `None`. A domain *endpoint* (`log` of +//! `[0, h]`, `sqrt` of `[0, h]`) is not a violation: the value set over the +//! half-open panel is enclosed, and the single point where the integrand is +//! undefined is a null set, exactly as for the removable singularities +//! [`super::bounds::verified_integral`] already integrates through. +//! * Anything with no rule here returns `None`. There is no fallback that +//! guesses. + +use super::from_bounds; +use crate::ball::ArbBall; +use crate::kernel::{ExprData, ExprId, ExprPool}; +use rug::float::Round; +use rug::ops::Pow; +use rug::Float; + +/// Recursion ceiling, so a pathological expression cannot blow the stack in +/// what is only ever a best-effort fallback. +const MAX_DEPTH: usize = 256; + +/// A closed interval of the extended reals `[-∞, +∞]`. +/// +/// Invariant: `lo <= hi`, neither endpoint is NaN. Both endpoints are held at +/// the working precision and every operation rounds them outward. +#[derive(Clone, Debug)] +pub(super) struct XInterval { + lo: Float, + hi: Float, + prec: u32, +} + +/// `a op b` rounded in `dir`, or `None` when the result is NaN. +fn rounded(v: Float, dir: Round, prec: u32) -> Option { + let out = Float::with_val_round(prec, v, dir).0; + (!out.is_nan()).then_some(out) +} + +/// `a · b` with the IEEE 1788 convention `0 · ±∞ = 0`; `None` on NaN. +fn xmul(a: &Float, b: &Float, dir: Round, prec: u32) -> Option { + if (a.is_zero() && b.is_infinite()) || (b.is_zero() && a.is_infinite()) { + return Some(Float::new(prec)); + } + rounded(Float::with_val(prec + 32, a * b), dir, prec) +} + +fn fmin(a: Float, b: Float) -> Float { + if a <= b { + a + } else { + b + } +} + +fn fmax(a: Float, b: Float) -> Float { + if a >= b { + a + } else { + b + } +} + +impl XInterval { + fn new(lo: Float, hi: Float, prec: u32) -> Option { + if lo.is_nan() || hi.is_nan() || lo > hi { + return None; + } + Some(XInterval { lo, hi, prec }) + } + + fn constant(v: f64, prec: u32) -> Option { + XInterval::new(Float::with_val(prec, v), Float::with_val(prec, v), prec) + } + + fn is_finite(&self) -> bool { + self.lo.is_finite() && self.hi.is_finite() + } + + /// Midpoint and an upward-rounded radius, both finite. `None` for an + /// unbounded interval. + fn mid_rad(&self) -> Option<(Float, Float)> { + if !self.is_finite() { + return None; + } + let p = self.prec; + let mid = Float::with_val(p, Float::with_val(p + 32, &self.lo + &self.hi) / 2u32); + let a = Float::with_val_round(p, Float::with_val(p + 32, &mid - &self.lo), Round::Up).0; + let b = Float::with_val_round(p, Float::with_val(p + 32, &self.hi - &mid), Round::Up).0; + Some((mid, fmax(a, b))) + } + + fn add(&self, other: &Self) -> Option { + let p = self.prec; + let lo = rounded( + Float::with_val(p + 32, &self.lo + &other.lo), + Round::Down, + p, + )?; + let hi = rounded(Float::with_val(p + 32, &self.hi + &other.hi), Round::Up, p)?; + XInterval::new(lo, hi, p) + } + + fn mul(&self, other: &Self) -> Option { + let p = self.prec; + let ends = [ + (&self.lo, &other.lo), + (&self.lo, &other.hi), + (&self.hi, &other.lo), + (&self.hi, &other.hi), + ]; + let mut lo: Option = None; + let mut hi: Option = None; + for (a, b) in ends { + let l = xmul(a, b, Round::Down, p)?; + let h = xmul(a, b, Round::Up, p)?; + lo = Some(match lo { + Some(c) => fmin(c, l), + None => l, + }); + hi = Some(match hi { + Some(c) => fmax(c, h), + None => h, + }); + } + XInterval::new(lo?, hi?, p) + } + + /// `1/self`. An interval with `0` in its *interior* becomes `[-∞, +∞]`; + /// one whose closed end is `0` keeps the one-sided bound the values on the + /// other side of it actually have. + fn recip(&self) -> Option { + let p = self.prec; + let one = Float::with_val(p, 1); + let inv = |v: &Float, dir: Round| -> Option { + if v.is_zero() { + return None; + } + rounded(Float::with_val(p + 32, &one / v), dir, p) + }; + let zlo = self.lo.is_zero(); + let zhi = self.hi.is_zero(); + if zlo && zhi { + return None; + } + if self.lo < 0 && self.hi > 0 { + return XInterval::new( + Float::with_val(p, f64::NEG_INFINITY), + Float::with_val(p, f64::INFINITY), + p, + ); + } + if zlo { + // values in (0, hi] ⇒ [1/hi, +∞] + return XInterval::new( + inv(&self.hi, Round::Down)?, + Float::with_val(p, f64::INFINITY), + p, + ); + } + if zhi { + // values in [lo, 0) ⇒ [-∞, 1/lo] + return XInterval::new( + Float::with_val(p, f64::NEG_INFINITY), + inv(&self.lo, Round::Up)?, + p, + ); + } + // 0 is outside the closed interval, so `1/·` is monotone decreasing on it. + XInterval::new(inv(&self.hi, Round::Down)?, inv(&self.lo, Round::Up)?, p) + } + + fn neg(&self) -> Option { + let p = self.prec; + XInterval::new( + Float::with_val(p, -&self.hi), + Float::with_val(p, -&self.lo), + p, + ) + } + + fn abs(&self) -> Option { + let p = self.prec; + if self.lo >= 0 { + return Some(self.clone()); + } + if self.hi <= 0 { + return self.neg(); + } + let m = fmax( + Float::with_val(p, self.lo.abs_ref()), + Float::with_val(p, self.hi.abs_ref()), + ); + XInterval::new(Float::new(p), m, p) + } + + fn powi(&self, n: i64) -> Option { + let p = self.prec; + if n == 0 { + return XInterval::constant(1.0, p); + } + if n < 0 { + return self.powi(-n)?.recip(); + } + let pw = |v: &Float, dir: Round| -> Option { + let e = u32::try_from(n).ok()?; + rounded(Float::with_val(p + 32, v.pow(e)), dir, p) + }; + if n % 2 == 1 { + // Odd powers are increasing on the whole line. + return XInterval::new(pw(&self.lo, Round::Down)?, pw(&self.hi, Round::Up)?, p); + } + if self.lo >= 0 { + return XInterval::new(pw(&self.lo, Round::Down)?, pw(&self.hi, Round::Up)?, p); + } + if self.hi <= 0 { + return XInterval::new(pw(&self.hi, Round::Down)?, pw(&self.lo, Round::Up)?, p); + } + // Straddles zero: the minimum of an even power is 0, the maximum is at + // whichever endpoint is farther from it. + let a = pw(&self.lo, Round::Up)?; + let b = pw(&self.hi, Round::Up)?; + XInterval::new(Float::new(p), fmax(a, b), p) + } + + /// Image under a function that is non-decreasing on the whole of `self`. + fn increasing(&self, f: impl Fn(&Float, Round) -> Option) -> Option { + XInterval::new( + f(&self.lo, Round::Down)?, + f(&self.hi, Round::Up)?, + self.prec, + ) + } + + /// Image under a function that is non-increasing on the whole of `self`. + fn decreasing(&self, f: impl Fn(&Float, Round) -> Option) -> Option { + XInterval::new( + f(&self.hi, Round::Down)?, + f(&self.lo, Round::Up)?, + self.prec, + ) + } + + fn exp(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.exp_ref()), d, p)) + } + + /// `log`, defined on `(0, ∞)`. A *closed* left end at `0` is not a + /// violation — it contributes `-∞`, which the caller may still tame. + fn log(&self) -> Option { + let p = self.prec; + if self.lo < 0 { + return None; + } + if self.lo.is_zero() { + let hi = if self.hi.is_zero() { + Float::with_val(p, f64::NEG_INFINITY) + } else { + rounded(Float::with_val(p + 32, self.hi.ln_ref()), Round::Up, p)? + }; + return XInterval::new(Float::with_val(p, f64::NEG_INFINITY), hi, p); + } + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.ln_ref()), d, p)) + } + + fn sqrt(&self) -> Option { + let p = self.prec; + if self.lo < 0 { + return None; + } + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.sqrt_ref()), d, p)) + } + + fn asin(&self) -> Option { + let p = self.prec; + if self.lo < -1 || self.hi > 1 { + return None; + } + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.asin_ref()), d, p)) + } + + fn acos(&self) -> Option { + let p = self.prec; + if self.lo < -1 || self.hi > 1 { + return None; + } + self.decreasing(|v, d| rounded(Float::with_val(p + 32, v.acos_ref()), d, p)) + } + + fn atan(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.atan_ref()), d, p)) + } + + fn atanh(&self) -> Option { + let p = self.prec; + if self.lo < -1 || self.hi > 1 { + return None; + } + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.atanh_ref()), d, p)) + } + + fn asinh(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.asinh_ref()), d, p)) + } + + fn acosh(&self) -> Option { + let p = self.prec; + if self.lo < 1 { + return None; + } + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.acosh_ref()), d, p)) + } + + fn sinh(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.sinh_ref()), d, p)) + } + + fn tanh(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.tanh_ref()), d, p)) + } + + fn erf(&self) -> Option { + let p = self.prec; + self.increasing(|v, d| rounded(Float::with_val(p + 32, v.erf_ref()), d, p)) + } + + fn erfc(&self) -> Option { + let p = self.prec; + self.decreasing(|v, d| rounded(Float::with_val(p + 32, v.erfc_ref()), d, p)) + } + + fn cosh(&self) -> Option { + let p = self.prec; + let f = |v: &Float, d: Round| rounded(Float::with_val(p + 32, v.cosh_ref()), d, p); + if self.lo >= 0 { + return self.increasing(f); + } + if self.hi <= 0 { + return self.decreasing(f); + } + let a = f(&self.lo, Round::Up)?; + let b = f(&self.hi, Round::Up)?; + XInterval::new(Float::with_val(p, 1), fmax(a, b), p) + } + + /// `sin`/`cos` through the Lipschitz bound `|f'| ≤ 1` around the midpoint, + /// intersected with the global range `[-1, 1]`. + /// + /// This is deliberately not a monotonicity analysis: locating the extrema + /// means deciding whether a multiple of `π/2` lies in the interval, and + /// getting that wrong by an ulp would be unsound. `|f(m + t) - f(m)| ≤ |t|` + /// needs nothing but the mean value theorem, and on the floor-width panels + /// this fallback runs on it is tight to the last bit. + fn trig(&self, cosine: bool) -> Option { + let p = self.prec; + let (mid, rad) = self.mid_rad()?; + let centre = if cosine { + Float::with_val(p + 32, mid.cos_ref()) + } else { + Float::with_val(p + 32, mid.sin_ref()) + }; + let lo = rounded(Float::with_val(p + 32, ¢re - &rad), Round::Down, p)?; + let hi = rounded(Float::with_val(p + 32, ¢re + &rad), Round::Up, p)?; + XInterval::new( + fmax(lo, Float::with_val(p, -1)), + fmin(hi, Float::with_val(p, 1)), + p, + ) + } +} + +/// Rigorous enclosure of the range of `expr` over the panel `[lo, hi]`, or +/// `None` when no rule applies or the range is not bounded. +/// +/// The returned ball is a superset of `{ f(x) : x ∈ [lo, hi], f defined }`. +pub(super) fn panel_range( + expr: ExprId, + pool: &ExprPool, + var: ExprId, + lo: &Float, + hi: &Float, + prec: u32, +) -> Option { + let x = XInterval::new( + Float::with_val_round(prec, lo, Round::Down).0, + Float::with_val_round(prec, hi, Round::Up).0, + prec, + )?; + let r = eval(expr, pool, var, &x, prec, 0)?; + r.is_finite() + .then(|| from_bounds(&r.lo, &r.hi, prec)) + .filter(super::is_finite) +} + +/// Same as [`panel_range`], phrased as the `width × range` contribution of one +/// panel to an integral: `∫_lo^hi f dx ∈ (hi − lo) · range(f)`. +/// +/// The width factor is the interval `[⌊hi − lo⌋, ⌈hi − lo⌉]`, i.e. the exact +/// width bracketed by directed rounding — *not* `[0, hi − lo]`. The difference +/// is not cosmetic: for a range that does not contain zero (`asin` near +/// `x = 1`, whose values are all close to `π/2`) the loose factor multiplies +/// the panel's contribution by the whole of `π/2` instead of by the tiny +/// variation of `asin` across the panel, and the quadrature loop then bisects +/// the panel until it runs out of budget trying to recover. +pub(super) fn panel_integral( + expr: ExprId, + pool: &ExprPool, + var: ExprId, + lo: &Float, + hi: &Float, + prec: u32, +) -> Option { + let range = panel_range(expr, pool, var, lo, hi, prec)?; + let wide = Float::with_val(prec + 32, hi - lo); + let w_lo = Float::with_val_round(prec, &wide, Round::Down).0; + let w_hi = Float::with_val_round(prec, &wide, Round::Up).0; + let piece = from_bounds(&w_lo, &w_hi, prec) * range; + super::is_finite(&piece).then_some(piece) +} + +fn eval( + expr: ExprId, + pool: &ExprPool, + var: ExprId, + x: &XInterval, + prec: u32, + depth: usize, +) -> Option { + if depth > MAX_DEPTH { + return None; + } + if expr == var { + return Some(x.clone()); + } + match pool.get(expr) { + // Bracketed straight from the exact value, at the working precision. + // Rounding to an intermediate `Float` (or to `f64`) first and bracketing + // *that* would enclose the rounded number rather than the literal, which + // for a constant wider than `prec` bits is not an enclosure at all. + ExprData::Integer(n) => XInterval::new( + Float::with_val_round(prec, &n.0, Round::Down).0, + Float::with_val_round(prec, &n.0, Round::Up).0, + prec, + ), + ExprData::Rational(r) => XInterval::new( + Float::with_val_round(prec, &r.0, Round::Down).0, + Float::with_val_round(prec, &r.0, Round::Up).0, + prec, + ), + ExprData::Float(f) => XInterval::new( + Float::with_val_round(prec, &f.inner, Round::Down).0, + Float::with_val_round(prec, &f.inner, Round::Up).0, + prec, + ), + // A free symbol other than the integration variable has no interval. + ExprData::Symbol { .. } => None, + ExprData::Add(args) => { + let mut acc = XInterval::constant(0.0, prec)?; + for a in args { + acc = acc.add(&eval(a, pool, var, x, prec, depth + 1)?)?; + } + Some(acc) + } + ExprData::Mul(args) => { + let mut acc = XInterval::constant(1.0, prec)?; + for a in args { + acc = acc.mul(&eval(a, pool, var, x, prec, depth + 1)?)?; + } + Some(acc) + } + ExprData::Pow { base, exp } => { + let b = eval(base, pool, var, x, prec, depth + 1)?; + if let ExprData::Integer(n) = pool.get(exp) { + return b.powi(n.0.to_i64()?); + } + // Anything else goes through `exp(e · log b)`, which is the real + // branch only for a non-negative base — and `log` refuses when the + // base interval reaches below zero. + let e = eval(exp, pool, var, x, prec, depth + 1)?; + b.log()?.mul(&e)?.exp() + } + ExprData::Func { name, args } if args.len() == 1 => { + let a = eval(args[0], pool, var, x, prec, depth + 1)?; + match name.as_str() { + "exp" => a.exp(), + "log" | "ln" => a.log(), + "sqrt" => a.sqrt(), + "sin" => a.trig(false), + "cos" => a.trig(true), + "asin" => a.asin(), + "acos" => a.acos(), + "atan" => a.atan(), + "sinh" => a.sinh(), + "cosh" => a.cosh(), + "tanh" => a.tanh(), + "asinh" => a.asinh(), + "acosh" => a.acosh(), + "atanh" => a.atanh(), + "erf" => a.erf(), + "erfc" => a.erfc(), + "abs" => a.abs(), + // `tan`, the Bessel functions, `gamma`, `digamma` and + // `lambert_w` have no interval rule here. They are not refused + // outright anywhere else — the Taylor models handle them — and + // guessing a monotone rule for a function with poles is exactly + // the mistake this module exists to avoid. + _ => None, + } + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernel::{Domain, ExprPool}; + + const P: u32 = 128; + + fn f(v: f64) -> Float { + Float::with_val(P, v) + } + + #[test] + fn variable_range_is_exact_at_the_panel_endpoints() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let r = panel_range(x, &pool, x, &f(0.0), &f(0.25), P).unwrap(); + assert!(r.lo() <= 0.0 && r.hi() >= 0.25); + // The panel's own endpoints must not be inflated past the domain of a + // `sqrt` sitting on top of them. + let s = pool.func("sqrt", vec![x]); + let rs = panel_range(s, &pool, x, &f(0.0), &f(0.25), P).unwrap(); + assert!(rs.lo() <= 0.0); + assert!(rs.hi() >= 0.5 - 1e-30); + } + + #[test] + fn asin_is_bounded_at_the_end_of_its_domain() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let a = pool.func("asin", vec![x]); + let r = panel_range(a, &pool, x, &f(1.0 - 1e-9), &f(1.0), P).unwrap(); + let half_pi = std::f64::consts::FRAC_PI_2; + assert!(r.hi() >= half_pi); + assert!(r.lo() <= half_pi); + } + + #[test] + fn one_minus_x_squared_reaches_exactly_zero() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let one = pool.integer(1_i32); + let inner = pool.add(vec![ + one, + pool.mul(vec![pool.integer(-1_i32), pool.mul(vec![x, x])]), + ]); + let s = pool.func("sqrt", vec![inner]); + // The Taylor model refuses here; interval arithmetic with exact + // endpoints does not, because `1 - 1·1` is exact. + let r = panel_range(s, &pool, x, &f(1.0 - 1e-9), &f(1.0), P).unwrap(); + assert!(r.lo() <= 0.0); + assert!(r.hi() >= 4.4e-5); + } + + #[test] + fn x_to_the_x_is_bounded_at_zero() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + // exp(x · log x) — unbounded factor times a vanishing one. + let e = pool.func("exp", vec![pool.mul(vec![x, pool.func("log", vec![x])])]); + let r = panel_range(e, &pool, x, &f(0.0), &f(1e-9), P).unwrap(); + assert!(r.lo() <= 0.0); + assert!(r.hi() >= 1.0); + } + + #[test] + fn genuine_poles_stay_unbounded() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + // 1/x on [0, h] — the integral does not exist, and no range does either. + let recip = pool.pow(x, pool.integer(-1_i32)); + assert!(panel_range(recip, &pool, x, &f(0.0), &f(1e-9), P).is_none()); + // -log(x) on [0, h] — integrable but unbounded; still refused. + let nlog = pool.mul(vec![pool.integer(-1_i32), pool.func("log", vec![x])]); + assert!(panel_range(nlog, &pool, x, &f(0.0), &f(1e-9), P).is_none()); + // 1/sqrt(x) on [0, h]. + let isq = pool.pow(pool.func("sqrt", vec![x]), pool.integer(-1_i32)); + assert!(panel_range(isq, &pool, x, &f(0.0), &f(1e-9), P).is_none()); + } + + #[test] + fn out_of_domain_on_a_set_of_positive_measure_is_refused() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let s = pool.func("sqrt", vec![x]); + assert!(panel_range(s, &pool, x, &f(-1.0), &f(1.0), P).is_none()); + let l = pool.func("log", vec![x]); + assert!(panel_range(l, &pool, x, &f(-1.0), &f(1.0), P).is_none()); + let a = pool.func("asin", vec![x]); + assert!(panel_range(a, &pool, x, &f(0.5), &f(2.0), P).is_none()); + } + + #[test] + fn trig_bound_contains_the_true_range() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let s = pool.func("sin", vec![x]); + let r = panel_range(s, &pool, x, &f(0.0), &f(3.2), P).unwrap(); + // sin reaches 1 on [0, 3.2]; the Lipschitz bound must not miss it. + assert!(r.hi() >= 1.0); + assert!(r.lo() <= 0.0); + let c = pool.func("cos", vec![x]); + let rc = panel_range(c, &pool, x, &f(0.0), &f(6.5), P).unwrap(); + assert!(rc.lo() <= -1.0 && rc.hi() >= 1.0); + } + + #[test] + fn a_constant_wider_than_the_working_precision_is_still_bracketed() { + // 2^200 + 1 is not representable in 128 bits: rounding it to a `Float` + // and bracketing *that* would enclose the rounded value, not the + // literal. The interval has to straddle the true integer. + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let big: rug::Integer = (rug::Integer::from(1) << 200) + 1; + let e = pool.add(vec![pool.integer(big.clone()), pool.mul(vec![x, x])]); + let r = panel_range(e, &pool, x, &f(0.0), &f(0.0), P).unwrap(); + let truth = Float::with_val(400, &big); + assert!(r.lo() <= truth, "lo {} > {}", r.lo(), truth); + assert!(r.hi() >= truth, "hi {} < {}", r.hi(), truth); + assert!( + r.lo() < r.hi(), + "a 201-bit integer cannot be exact at 128 bits" + ); + } + + #[test] + fn no_rule_means_no_bound() { + let pool = ExprPool::new(); + let x = pool.symbol("x", Domain::Real); + let t = pool.func("tan", vec![x]); + assert!(panel_range(t, &pool, x, &f(0.0), &f(0.1), P).is_none()); + } +} diff --git a/alkahest-core/src/validated/mod.rs b/alkahest-core/src/validated/mod.rs index 3997aa29..1a4230a8 100644 --- a/alkahest-core/src/validated/mod.rs +++ b/alkahest-core/src/validated/mod.rs @@ -69,6 +69,9 @@ //! ``` pub mod bounds; +/// Directed-rounding interval arithmetic over the extended reals — the +/// last-resort range bound for one panel of `bounds::verified_integral`. +mod interval; pub mod taylor; use crate::ball::ArbBall; diff --git a/alkahest-py/src/lib.rs b/alkahest-py/src/lib.rs index 49e65f17..6bf6ebb4 100644 --- a/alkahest-py/src/lib.rs +++ b/alkahest-py/src/lib.rs @@ -11267,9 +11267,11 @@ impl PyBoundsSupport { /// >>> p = ak.ExprPool(); x = p.symbol("x") /// >>> bool(ak.bounds_supported(ak.sin(x) * ak.exp(x))) /// True -/// >>> answer = ak.bounds_supported(ak.bessel_j0(x)) +/// >>> bool(ak.bounds_supported(ak.bessel_j0(x))) # covered since 3.9.0 +/// True +/// >>> answer = ak.bounds_supported(ak.floor(x)) /// >>> bool(answer), answer.functions -/// (False, ['bessel_j0']) +/// (False, ['floor']) #[pyfunction] #[pyo3(name = "bounds_supported")] fn py_bounds_supported(py: Python<'_>, expr: PyRef) -> PyResult { @@ -11296,6 +11298,77 @@ fn py_bounds_supported(py: Python<'_>, expr: PyRef) -> PyResult(py: Python<'_>, f: F) -> PyResult +where + F: FnOnce() -> T + Send, + T: Send, +{ + // Preserve a cancellation the caller had already requested: clearing one we + // did not set would silently resurrect somebody else's abandoned search. + let preset = alkahest_core::budget::is_cancelled(); + let mut interrupt: Option = None; + let value = py.allow_threads(|| { + std::thread::scope(|scope| { + let handle = scope.spawn(f); + while !handle.is_finished() { + std::thread::sleep(INTERRUPT_POLL); + if interrupt.is_some() { + continue; + } + if let Err(e) = Python::with_gil(|py| py.check_signals()) { + interrupt = Some(e); + alkahest_core::budget::request_cancel(); + } + } + handle.join() + }) + }); + if interrupt.is_some() && !preset { + alkahest_core::budget::clear_cancel(); + } + let value = value.map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("the validated-bounds worker thread panicked") + })?; + match interrupt { + Some(e) => Err(e), + None => Ok(value), + } +} + /// `alkahest.bound_on_box(expr, box, *, order=6, prec=128, tol=1e-9, max_subdivisions=2048)` /// /// Rigorous enclosure of the **range** of `expr` over an axis-aligned box, @@ -11330,8 +11403,9 @@ fn py_bound_on_box( max_subdivisions, }; let pool = pool_py.borrow(py); - let r = - core_bound_on_box(expr.id, &pool.inner, &boxes, &opts).map_err(validated_error_to_py)?; + let (id, inner) = (expr.id, &pool.inner); + let r = run_interruptible(py, || core_bound_on_box(id, inner, &boxes, &opts))? + .map_err(validated_error_to_py)?; Ok(PyEnclosure { lower: r.lower(), upper: r.upper(), @@ -11350,16 +11424,30 @@ fn py_bound_on_box( /// singular or improper integrands rather than guessing. /// /// A **removable** singularity is not a refusal: an integrand written as -/// ``N(x)/D(x)`` with ``N(p) = D(p) = 0`` and ``D'(p) != 0`` — ``log(1+x)/x`` -/// on ``[0, 1]``, ``sin(x)/x`` on ``[-1, 1]`` — is enclosed via Cauchy's mean -/// value theorem, and the value returned is the integral of the continuous -/// extension. The two zeros are checked *symbolically*, so a genuine pole is -/// never mistaken for a removable one. -/// -/// A singularity that is integrable but not removable (``-log(x)`` on -/// ``[0, 1]``, ``1/sqrt(1-x*x)`` on ``[0, 1]``) is still refused: the integral -/// exists, but no rigorous enclosure of the *integrand* does. The -/// :class:`ValidatedError` message says which of the two situations it is. +/// ``N(x)/D(x)`` whose two halves vanish at the same point ``p``, to the same +/// order, with some ``D**(d)(p) != 0`` — ``log(1+x)/x`` on ``[0, 1]``, +/// ``sin(x)/x`` on ``[-1, 1]``, ``(1-cos(x))/(x*x)`` on ``[0, 1]`` — is +/// enclosed via Cauchy's mean value theorem, iterated as many times as the +/// order of the zero requires, and the value returned is the integral of the +/// continuous extension. The zeros are checked *symbolically*, so a genuine +/// pole is never mistaken for a removable one. +/// +/// A **bounded** integrand whose Taylor model runs out of *domain* rather than +/// out of function is not a refusal either. ``asin`` and ``acos`` on +/// ``[0, 1]``, ``sqrt(1 - x*x)`` on ``[0, 1]``, ``sqrt(x)`` and ``x**x`` on +/// ``[0, 1]`` are all finite and continuous on the closed interval, but the +/// Taylor rule needs a derivative bound it does not have at the end of the +/// domain, so no model exists on the last panel however far it is bisected. +/// That panel is closed with a ``width * range`` bound computed by interval +/// arithmetic, which needs no derivative. The panel is of order ``2**-60`` of +/// the interval by then, so the crude bound costs essentially nothing. +/// +/// An **unbounded** integrand is still refused, whether or not the integral +/// converges (``-log(x)`` on ``[0, 1]``, ``1/sqrt(1-x*x)`` on ``[0, 1]``): the +/// integral exists, but no rigorous enclosure of the *integrand* does, and no +/// bounded range exists either. The :exc:`ValidatedError` message names the +/// Taylor rule that stopped and where, and does not claim the integral fails +/// to exist. #[allow(clippy::too_many_arguments)] #[pyfunction] #[pyo3(name = "verified_integral", signature = (expr, var, a, b, *, order = 8, prec = 128, tol = 1e-9, max_subdivisions = 4096))] @@ -11384,8 +11472,11 @@ fn py_verified_integral( max_subdivisions, }; let pool = pool_py.borrow(py); - let r = core_verified_integral(expr.id, &pool.inner, var.id, a, b, &opts) - .map_err(validated_error_to_py)?; + let (id, var_id, inner) = (expr.id, var.id, &pool.inner); + let r = run_interruptible(py, || { + core_verified_integral(id, inner, var_id, a, b, &opts) + })? + .map_err(validated_error_to_py)?; Ok(PyEnclosure { lower: r.lower(), upper: r.upper(), @@ -11439,7 +11530,8 @@ fn py_verified_no_roots( max_subdivisions, }; let pool = pool_py.borrow(py); - let v = core_verified_no_roots(expr.id, &pool.inner, &boxes, &opts) + let (id, inner) = (expr.id, &pool.inner); + let v = run_interruptible(py, || core_verified_no_roots(id, inner, &boxes, &opts))? .map_err(validated_error_to_py)?; Ok(verdict_str(v).to_string()) } @@ -11460,6 +11552,16 @@ fn py_verified_no_roots( /// boxes reaching ``x = 0``. Tightness in the *interior* is not covered and /// stays ``"undecided"``. /// +/// ``"undecided"`` is **not monotone in the box**: it is a search verdict, not +/// a logical one, and a box that stops just short of the point an inequality is +/// tight at gets a much weaker collar than one that reaches it. A box that +/// stops short of ``x = 0`` is retried with the collar planted at ``0`` — a +/// ``"true"`` on the larger box implies ``"true"`` on the box asked about — so +/// the classical inequalities decide on ``[1e-12, pi/2]`` as well as on +/// ``[0, pi/2]``. For a statement tight somewhere else the effect remains: on +/// ``"undecided"``, try the *larger* box whose endpoint is the tight point +/// before concluding the statement is out of reach. +/// /// ``tol`` sets the tolerance of the *enclosure*, not of the verdict: it is an /// absolute width, so it does not bound how close to zero the answer may be. /// Once the enclosure has been computed, the search is re-run with the sign @@ -11505,7 +11607,8 @@ fn py_verified_sign( max_subdivisions, }; let pool = pool_py.borrow(py); - let v = core_verified_sign(expr.id, &pool.inner, &boxes, pred, &opts) + let (id, inner) = (expr.id, &pool.inner); + let v = run_interruptible(py, || core_verified_sign(id, inner, &boxes, pred, &opts))? .map_err(validated_error_to_py)?; Ok(verdict_str(v).to_string()) } diff --git a/docs/mdbook/src/validated-bounds.md b/docs/mdbook/src/validated-bounds.md index 913ca182..beb91ec3 100644 --- a/docs/mdbook/src/validated-bounds.md +++ b/docs/mdbook/src/validated-bounds.md @@ -159,13 +159,34 @@ The limits are worth knowing: middle; it stays `"undecided"` rather than being upgraded on the strength of an enclosure that merely touches zero. +### `"undecided"` is not monotone in the box + +`"true"` on a box *logically* implies `"true"` on every sub-box, but this is a +search and the search does not have that property. The collar is planted at the +box's own endpoint, and its strength comes from `simplify` proving the low-order +Taylor coefficients there exactly zero. Back the box off the tight point by a +whisker and nothing vanishes exactly any more: the leading coefficient becomes +`g(δ)`, a number that has to beat both the cancellation noise of evaluating `g` +near a zero of order `j` and the linear term of the tail bound. For Cusa–Huygens +(`g ~ x⁵/60`) that failed for every left endpoint between about `10⁻³⁰⁰` and +`10⁻⁹`, while `[0, π/2]` and `[10⁻⁶, π/2]` both came back `"true"` — a dead band +in the middle of two `"true"` regions. + +An `"undecided"` box that stops short of `x = 0` is therefore retried with the +collar planted at `0` — `"true"` on the larger box implies `"true"` on the +box asked about, so the retry is sound, and only `"true"` is taken from it. That +closes the band for the classical inequalities, all of which are tight at `0`. +A statement tight at some *other* point still shows the effect: if `[a, b]` +comes back `"undecided"`, try the larger box whose endpoint is the point the +inequality is tight at before concluding it is out of reach. + ## Which functions are covered — ask before you build the workload Taylor models reach the **elementary fragment**: `exp`, `log`, `sqrt`, `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `sinh`, `cosh`, `tanh`, `asinh`, `acosh`, `atanh`, `abs`, plus arithmetic and integer/rational powers — and, -since 3.9.0, `erf` and `erfc`. Outside it are `bessel_j0`, `bessel_j1`, -`digamma`, `lambert_w`, `gamma`, the elliptic integrals, `floor` and `ceil`, +since 3.9.0, `erf`, `erfc`, `bessel_j0`, `bessel_j1`, `digamma`, `gamma` and +`lambert_w`. Outside it are the elliptic integrals, `floor`, `ceil` and `sign`, and so is any two-argument function such as `atan2`. The three inverse hyperbolics carry the domain restriction their branch has: @@ -180,20 +201,21 @@ instead of discovering it by hitting `E-VALIDATED-001`: ```python ak.bounds_supported(ak.sin(x) * ak.exp(x)) # truthy -answer = ak.bounds_supported(ak.bessel_j0(x)) -bool(answer), answer.functions # (False, ['bessel_j0']) -answer.blocker # "function `bessel_j0`" +ak.bounds_supported(ak.bessel_j0(x)) # truthy since 3.9.0 +answer = ak.bounds_supported(ak.floor(x)) +bool(answer), answer.functions # (False, ['floor']) +answer.blocker # "function `floor`" # Per primitive, in the agent contract: {row["name"] for row in ak.capabilities()["primitives"] if row["taylor_model"]} ``` **`numeric_ball` is not this flag.** It reports pointwise ball arithmetic, -which `bessel_j0`, `digamma`, `lambert_w` and `floor` all have; a Taylor model -additionally needs a rule with a rigorous Lagrange remainder, which they do -not. Both bits are honest — they answer different questions. `taylor_model` -and `bounds_supported` are derived by *running* the Taylor evaluator, not from -a maintained list, so neither can drift from what `bound_on_box` accepts. +which `floor` and `ceil` have; a Taylor model additionally needs a rule with a +rigorous Lagrange remainder, which a step function cannot have. Both bits are +honest — they answer different questions. `taylor_model` and +`bounds_supported` are derived by *running* the Taylor evaluator, not from a +maintained list, so neither can drift from what `bound_on_box` accepts. A `True` answer means "will not be refused with `E-VALIDATED-001`". It is not a promise of success: a covered function can still hit a domain violation or @@ -234,15 +256,24 @@ N(p) = D(p) = 0, D' ≠ 0 on J ⟹ ∀ x ∈ J\{p} : N(x)/D(x) = N'(ξ)/D' so the piece is bounded by an enclosure of `N'/D'`, which is perfectly regular. The number returned is the integral of the continuous extension. +When `D'` vanishes at `p` as well, the step is **repeated** — up to a small +fixed depth — provided `N'` vanishes there too. That is what covers a +higher-order removable singularity such as `(1 − cos x)/x²`, whose value at +`x = 0` is `1/2` but whose denominator has a double zero: only `D'' = 2` is +bounded away from zero, and `N''/D'' = cos(x)/2` is the quotient that gets +enclosed. + Three guards keep this from swallowing a genuine pole: -- `N(p) = 0` and `D(p) = 0` are established **symbolically** (substitute the - exact rational `p`, simplify, require a literal zero). No numeric enclosure - can prove a value is exactly zero, so none is asked to. -- `D'` must be *certified non-vanishing* on the sub-interval. That is what fails - for `sin(x)/x²`, where the denominator has a double zero and the integral does - not converge. -- `N` and `D` must each have a successful enclosure over the whole +- `N⁽ᵏ⁾(p) = 0` and `D⁽ᵏ⁾(p) = 0` are established **symbolically** at every + level the descent passes (substitute the exact rational `p`, simplify, + require a literal zero). No numeric enclosure can prove a value is exactly + zero, so none is asked to. +- Some `D⁽ᵈ⁾` must be *certified non-vanishing* on the sub-interval. That is + what fails for `sin(x)/x²`: `D' = 2x` vanishes at `0` but `N' = cos x` does + not, so the descent stops with nothing proven — as it must, since the + integral does not converge. +- `N⁽ᵏ⁾` and `D⁽ᵏ⁾` must each have a successful enclosure over the whole sub-interval, which certifies they are analytic — and hence that the symbolic derivatives really are their derivatives. @@ -252,20 +283,51 @@ ak.verified_integral(ak.sin(x) / x, x, -1.0, 1.0) # ≈ 1.8921 ak.verified_integral(pool.integer(1) / x, x, -1.0, 1.0) # refuses: N(0) ≠ 0 ``` +## Bounded integrands whose Taylor model runs out of domain + +`asin` on `[0, 1]` is bounded and continuous on the closed interval — +`asin(1) = π/2` — but the Taylor rule for `asin` needs a bound on +`1/√(1−x²)`, and there is none at `x = 1`. Bisecting does not help: every +sub-interval that still touches `1` refuses, however narrow. The same happens +for `√x` and `xˣ` at `x = 0`. + +Those panels are closed with a **`width × range`** bound instead. A range needs +no derivative, so it survives the domain boundary, and it is computed by +directed-rounding interval arithmetic over the extended reals — which is what +lets `log([0, h]) = [−∞, log h]` be composed with the factor that tames it, so +`xˣ = exp(x·log x) ∈ [0, 1]` comes out bounded. By the time a panel needs this +it has been bisected down to a width of order `2⁻⁶⁰` of the interval, so the +crude bound costs essentially nothing in accuracy. + +The soundness argument is that an integrand which is genuinely unbounded on the +panel has no bounded range either, so the fallback still refuses on it. Nothing +is clamped into a domain it is not proven to be in: `√` and `log` of an +interval whose lower endpoint is *strictly* negative are refused, and only a +closed endpoint sitting exactly on the domain boundary is accepted. + +| Integral | Value | Status | +|---|---|---| +| `∫₀¹ asin(x) dx` | `π/2 − 1` | enclosed (`width × range` on the last panel) | +| `∫₀¹ √(1−x²) dx` | `π/4` | enclosed, same | +| `∫₀¹ √x dx` | `2/3` | enclosed, same | +| `∫₀¹ xˣ dx` | 0.78343… | enclosed, same | +| `∫₀¹ (x−1)/ln x dx` | `ln 2` | enclosed — removable at `1`, bounded at `0` | +| `∫₀¹ (1−cos x)/x² dx` | 0.48638… | enclosed (order-2 removable) | + ### What is still refused -An **integrable but non-removable** singularity is refused, and the message says -so rather than implying the integral does not exist: +An **unbounded** integrand is refused, whether or not the integral converges, +and the message names the rule that stopped rather than implying the integral +does not exist: | Integral | Value | Status | |---|---|---| | `∫₀¹ ln(1+x)/x dx` | `π²/12` | enclosed (removable) | | `∫_{-1}^{1} sin(x)/x dx` | `2·Si(1)` | enclosed (removable) | -| `∫₀¹ −ln x dx` | 1 | refused — `log` enclosure reaches 0, not a `0/0` quotient | +| `∫₀¹ −ln x dx` | 1 | refused — unbounded at `0`, and not a `0/0` quotient | | `∫₀¹ (ln x)² dx` | 2 | refused, same reason | -| `∫₀¹ dx/√(1−x²) dx` | `π/2` | refused — endpoint singularity, numerator does not vanish | -| `∫₀¹ xˣ dx` | 0.78343… | refused — `log` enclosure reaches 0 | -| `∫₀¹ ln(x)·ln(1−x) dx` | `2 − π²/6` | refused — singular at both ends | +| `∫₀¹ dx/√(1−x²) dx` | `π/2` | refused — unbounded at `1`, numerator does not vanish | +| `∫₀¹ ln(x)·ln(1−x) dx` | `2 − π²/6` | refused — unbounded at both ends | These need an integrable-tail bound or a singularity-removing substitution, neither of which can be derived rigorously from the expression alone today. The diff --git a/tests/test_validated_bounds.py b/tests/test_validated_bounds.py index bacd32dd..05ba0ef2 100644 --- a/tests/test_validated_bounds.py +++ b/tests/test_validated_bounds.py @@ -192,16 +192,20 @@ def test_a_double_pole_with_a_simple_numerator_zero_is_still_refused(): ak.verified_integral(ak.sin(x) / (x * x), x, -1.0, 1.0) -def test_a_second_order_removable_singularity_is_refused_not_guessed(): - """`(1-cos x)/x**2` really is removable (it tends to 1/2), but the proof - needs a *second*-order argument: `D' = 2x` vanishes at 0, so Cauchy's mean - value theorem does not apply and the enclosure is declined rather than - stretched to fit.""" +def test_a_second_order_removable_singularity_is_enclosed_by_iterating(): + """`(1-cos x)/x**2` is removable (it tends to 1/2), but the proof needs a + *second*-order argument: `D' = 2x` vanishes at 0 too, so one application of + Cauchy's mean value theorem is not enough and the step has to be repeated + until `D'' = 2`, which is bounded away from zero. + + Reference value from mpmath at 40 dps. + """ pool = ak.ExprPool() x = pool.symbol("x") - with pytest.raises(ak.ValidatedError): - ak.verified_integral((pool.integer(1) - ak.cos(x)) / (x * x), x, 0.0, 1.0) + r = ak.verified_integral((pool.integer(1) - ak.cos(x)) / (x * x), x, 0.0, 1.0) + + assert r.lower <= 0.486385376235322732342288196293 <= r.upper @pytest.mark.parametrize( @@ -209,15 +213,26 @@ def test_a_second_order_removable_singularity_is_refused_not_guessed(): [ (lambda pool, x: -ak.log(x), 0.0, 1.0, "-log x"), (lambda pool, x: ak.log(x) * ak.log(x), 0.0, 1.0, "(log x)^2"), - (lambda pool, x: ak.exp(x * ak.log(x)), 0.0, 1.0, "x^x"), + (lambda pool, x: pool.integer(1) / ak.sqrt(x), 0.0, 1.0, "1/sqrt(x)"), + ( + lambda pool, x: ak.log(x) * ak.log(pool.integer(1) - x), + 0.0, + 1.0, + "log(x) log(1-x)", + ), ], ) -def test_integrable_but_not_removable_singularities_refuse_with_an_honest_message( - build, a, b, label -): +def test_unbounded_integrands_refuse_with_an_honest_message(build, a, b, label): """These integrals all exist. What does not exist is a rigorous enclosure - of the *integrand*, and the error text must say which of the two it means - rather than implying the integral is undefined.""" + of the *integrand* — it is genuinely unbounded on the last panel, so the + `width x range` fallback has no bounded range either — and the error text + must say which of the two it means rather than implying the integral is + undefined. + + It must also not call the integrand "singular" on the strength of a Taylor + rule refusing: it names the rule that stopped instead. `asin(1) = pi/2` is + finite, and the old wording called it singular. + """ pool = ak.ExprPool() x = pool.symbol("x") @@ -225,11 +240,77 @@ def test_integrable_but_not_removable_singularities_refuse_with_an_honest_messag ak.verified_integral(build(pool, x), x, a, b) message = str(excinfo.value) - assert "integrand is singular" in message, f"{label}: {message}" + assert "no rigorous enclosure of the integrand" in message, f"{label}: {message}" assert "integrable singularity" in message, f"{label}: {message}" + assert "the integrand is singular" not in message, f"{label}: {message}" assert excinfo.value.remediation +# Bounded, continuous integrands whose Taylor model runs out of *domain* rather +# than out of function. Every one of these was refused with "the integrand is +# singular at the right endpoint", which is false: `asin(1) = pi/2`. Reference +# values are mpmath at 40 dps. +_BOUNDED_TABLE = [ + ("asin(x)", lambda pool, x: ak.asin(x), 0.57079632679489661923132169164), + ("acos(x)", lambda pool, x: ak.acos(x), 1.0), + ( + "sqrt(1-x^2)", + lambda pool, x: ak.sqrt(pool.integer(1) - x * x), + 0.78539816339744830961566084582, + ), + ("sqrt(x)", lambda pool, x: ak.sqrt(x), 0.66666666666666666666666666667), + ( + "x**x", + lambda pool, x: ak.exp(x * ak.log(x)), + 0.783430510712134407059264386527, + ), + ( + "(1-cos x)/x^2", + lambda pool, x: (pool.integer(1) - ak.cos(x)) / (x * x), + 0.486385376235322732342288196293, + ), + ( + "(x-1)/log(x)", + lambda pool, x: (x - pool.integer(1)) / ak.log(x), + 0.693147180559945309417232121458, + ), + ("sin(x)/x", lambda pool, x: ak.sin(x) / x, 0.946083070367183014941353313823), +] + + +@pytest.mark.parametrize( + ("label", "build", "truth"), _BOUNDED_TABLE, ids=[r[0] for r in _BOUNDED_TABLE] +) +def test_bounded_integrands_at_a_domain_boundary_are_enclosed(label, build, truth): + """Each of these is bounded and continuous on the closed interval `[0, 1]`, + so `width x range` closes the last panel even though no Taylor model exists + on it. The enclosure must contain the true value: soundness is the point, + and an enclosure that succeeds but misses is far worse than a refusal. + """ + pool = ak.ExprPool() + x = pool.symbol("x") + + r = ak.verified_integral(build(pool, x), x, 0.0, 1.0) + + assert r.lower <= truth <= r.upper, f"{label}: [{r.lower!r}, {r.upper!r}]" + assert r.width < 1e-3, f"{label}: enclosure width {r.width}" + + +def test_the_bounded_fallback_does_not_swallow_a_pole_at_the_same_endpoint(): + """The fallback fires exactly where the Taylor model runs out of domain, so + the guard that matters is that an *unbounded* integrand at that same + endpoint still refuses. `1/sqrt(1-x*x)` on [0,1] and `sqrt(1-x*x)` on [0,1] + both hit `sqrt` at zero; only the second is bounded. + """ + pool = ak.ExprPool() + x = pool.symbol("x") + inner = pool.integer(1) - x * x + + assert ak.verified_integral(ak.sqrt(inner), x, 0.0, 1.0).contains(math.pi / 4) + with pytest.raises(ak.ValidatedError): + ak.verified_integral(pool.integer(1) / ak.sqrt(inner), x, 0.0, 1.0) + + # --------------------------------------------------------------------------- # Three-valued predicates — the third value is never collapsed # --------------------------------------------------------------------------- @@ -662,3 +743,143 @@ def test_bound_on_box_terminates_when_the_tolerance_is_unreachable(): assert elapsed < 60.0, f"took {elapsed:.1f}s" assert r.lower <= 0.0 <= r.upper + + +# --------------------------------------------------------------------------- +# `verified_sign` is monotone in the box across the endpoint dead band +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", _CLASSICAL_NAMES) +@pytest.mark.parametrize("lo", [1e-30, 1e-12]) +def test_a_smaller_box_never_loses_a_verdict_the_larger_one_had(name, lo): + """`[lo, 1.5]` is a strict subset of `[0, 1.5]` — which the test above + certifies `true` — so it is a strictly weaker claim and must not be harder + to decide. + + It used to be. The collar the endpoint-series argument plants is only strong + when the box endpoint *is* the point the inequality is tight at: back off by + `1e-12` and the leading coefficient becomes `g(1e-12)`, far too small to + beat its own evaluation noise and the linear term of the tail bound. The + measured result was a dead band of left endpoints from about `1e-300` to + `1e-9` answering `undecided`, sandwiched between two `true` regions at `0` + and from `1e-6` up. + """ + pool = ak.ExprPool() + x = pool.symbol("x") + + assert ak.verified_sign(_classical(pool, x)[name], _box(x, lo, 1.5), "nonnegative") == "true" + + +def test_the_dead_band_is_closed_at_its_far_end_too(): + """`1e-300` is where the tail bound, not the noise, is the binding + constraint: the halving sequence for the collar bottoms out long before it + reaches a width that small.""" + pool = ak.ExprPool() + x = pool.symbol("x") + + f = _classical(pool, x)["cusa_huygens"] + + assert ak.verified_sign(f, _box(x, 1e-300, 1.5), "nonnegative") == "true" + + +def test_the_widened_retry_cannot_manufacture_a_false(): + """Only `true` may be taken from the enlarged box. `false` on a superset + says nothing about the subset, and taking it would be a false negative: + `x - 1/2` is `false` on `[0, 1]` and genuinely `true` on `[0.6, 1]`, which + is exactly the box the retry enlarges back to `[0, 1]`. + """ + pool = ak.ExprPool() + x = pool.symbol("x") + f = x - pool.rational(1, 2) + + assert ak.verified_sign(f, _box(x, 0.0, 1.0), "nonnegative") == "false" + assert ak.verified_sign(f, _box(x, 0.6, 1.0), "nonnegative") == "true" + + +# --------------------------------------------------------------------------- +# The long-running native calls are interruptible +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif( + not hasattr(__import__("signal"), "setitimer"), + reason="needs POSIX interval timers", +) +def test_a_python_timer_can_bound_a_long_verified_sign(): + """`P*x - tan(x)*(P - 4e18*x*x) >= 0` on `[0, pi/2]` ran for 109.9 s before + answering `undecided`, and a `signal.setitimer(60)` around it fired at + 182.8 s — after the call had already returned. A Python-level signal handler + only runs in the main thread and only between bytecodes, so releasing the + GIL is not by itself enough: the call has to come *back* for the GIL + periodically and ask whether a signal is pending. + """ + import signal + import time + + class _Fired(Exception): + pass + + def _raise(signum, frame): + raise _Fired() + + pool = ak.ExprPool() + x = pool.symbol("x") + big = pool.integer(9869604401089358619) # ~ 1e18 * pi**2 + f = big * x - ak.tan(x) * (big - pool.integer(4 * 10**18) * x * x) + + previous = signal.signal(signal.SIGALRM, _raise) + start = time.monotonic() + try: + signal.setitimer(signal.ITIMER_REAL, 3.0) + try: + outcome = ak.verified_sign(f, _box(x, 0.0, math.pi / 2), "nonnegative") + except _Fired: + outcome = "interrupted" + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + finally: + signal.signal(signal.SIGALRM, previous) + elapsed = time.monotonic() - start + + assert elapsed < 45.0, f"the timer did not preempt the call: {elapsed:.1f}s" + assert outcome in ("interrupted", "true", "false", "undecided") + # The interpreter must be left in a usable state, with no cancellation + # request lingering for the next caller. + assert ak.verified_sign(ak.sin(x), _box(x, 0.1, 1.0), "positive") == "true" + + +# --------------------------------------------------------------------------- +# Advertised coverage matches what the evaluator actually does +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name", ["bessel_j0", "bessel_j1", "digamma", "gamma", "lambert_w", "erf", "erfc"] +) +def test_documented_taylor_model_coverage_is_real(name): + """The docs and the `bounds_supported` docstring both listed these as + *outside* Taylor-model coverage after the release that added them. Assert + the live answer so the text cannot drift again without a red test. + """ + pool = ak.ExprPool() + x = pool.symbol("x") + f = getattr(ak, name)(x) + + answer = ak.bounds_supported(f) + + assert bool(answer) is True, f"{name}: {answer.functions}" + assert answer.functions == [] + ak.bound_on_box(f, _box(x, 1.2, 1.4)) + + +def test_a_primitive_with_no_taylor_rule_is_still_reported(): + """The counterexample the docstring example now uses.""" + pool = ak.ExprPool() + x = pool.symbol("x") + + answer = ak.bounds_supported(ak.floor(x)) + + assert bool(answer) is False + assert answer.functions == ["floor"] + assert "floor" in answer.blocker From d23a2d3b3b17134e785f8f72e970ed2cd7899c99 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Thu, 20 Aug 2026 23:08:54 +0000 Subject: [PATCH 10/11] fix(m9): accept Q(params) input, verify specialisation refusals, and connect parametric elimination to rosenfeld_groebner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five defects from the 2026-08-19 autoresearch run (#8, #13, #14, #16, #17), plus three prolongation bugs found while wiring #16 — without which the composition #16 asks for returns relations the system does not imply. #8 — `ParametricGroebnerBasis.contains` / `.reduce` rejected their own `to_exprs()` output. The basis lives in `Q(params)[vars]`, so its generators carry `den**-1` factors in the parameters by construction, and the membership entry points routed them through the denominator-free `Expr -> GbPoly` conversion. New `expr_to_param_gbpoly` converts straight to `ParamGbPoly`, treating a negative power whose base is free of the ring variables as what it is: an element of the coefficient field. A denominator in a *ring variable* is still refused. `equals_ideal` / `contains_ideal` answer the question the issue says a loop actually needs — exactly, over the fraction field, with neither basis's `conditions()` entering the answer. #14 — `specialize(values, verify=True)` re-solves the specialised system over Q and compares, instead of refusing on the recorded conditions alone. `conditions()` is sufficient but not necessary, so a quarter to a half of refusals on small-integer grids are unnecessary. Reproduced the run's oracle exactly on its own 8-system {-2..2} sweep: 646 refused, 334 now returned, 312 still refused (239 genuine poles + 73 necessary). Completeness only — the refusals were separately verified sound, and the default path is unchanged. #16 — `rosenfeld_groebner(dae, params=[...])` raised `TypeError`. `rosenfeld_groebner_parametric` runs the prolongation loop over `Q(params)` and returns a `ParametricRosenfeldGroebnerResult`. #13 — `eliminate=[...]` plus `minimal=True`, and a `UserWarning` when an earlier round would have done. On SIR: 0.03s and one 4-term relation at the first informative round, against no result in ten minutes one round later. `minimal_prolongation_rounds` is documented as a cost signal, not a certificate — it is known to be wrong for multi-output models. #17 — one sentence, with its hypothesis: an IO-elimination route answers multi-experiment identifiability, by Ovchinnikov-Pillay-Pogudin-Scanlon (arXiv:2004.07774) Theorem 19, which requires the IO equations to be the characteristic presentation of `I_Sigma ∩ C(theta){y,u}` — something a lex elimination at a hand-picked jet order does not guarantee. Prolongation, found on the way to #16 and fixed because #16 is meaningless otherwise: * `differentiate_equation` counted a promoted derivative twice, once from its own state pair and once from the pair the previous round added; * a jet that was never promoted had its contribution dropped, because the next differentiation treated it as a constant. On `R' = I, I' = -I` the two together force `I = 0`; * the jet ranking was recomputed each round and is not append-only that way, so a jet reachable only through the previous round's equations could drop out and shift every later exponent slot under already-padded polynomials. On the issue log's own `x' = a*x` repro this dropped the input equation from the basis and replaced it with `x*x'' - x'`. Regression tests fail before and pass after: 16 new Python tests in tests/test_parametric_groebner.py, 9 new Rust tests in parametric.rs and diffalg/mod.rs. `cargo test --workspace --features "parallel egraph cranelift groebner"` 2292 passed / 0 failed; `pytest tests/` 3439 passed / 61 skipped; clippy clean; `cargo semver-checks` reports no semver update required. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 63 ++ alkahest-core/src/dae/mod.rs | 36 +- alkahest-core/src/diffalg/mod.rs | 549 +++++++++++++++++- alkahest-core/src/lib.rs | 14 +- alkahest-core/src/poly/groebner/parametric.rs | 351 ++++++++++- alkahest-core/src/solver/mod.rs | 157 +++++ alkahest-py/src/lib.rs | 400 +++++++++++-- docs/mdbook/src/ode-dae.md | 4 + docs/mdbook/src/solving.md | 71 ++- python/alkahest/experimental/__init__.py | 16 +- tests/test_parametric_groebner.py | 341 +++++++++++ 11 files changed, 1936 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acc32e50..4a05a30e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,69 @@ ## Unreleased +- **The parametric Gröbner surface (M9) accepts its own output, checks a + refusal before making it, and composes with differential elimination.** + Five defects from the 2026-08-19 autoresearch run, plus two prolongation + bugs found while wiring the fourth: + + - **`ParametricGroebnerBasis.contains` / `.reduce` accept input that is + rational in the parameters.** The basis lives in `Q(params)[vars]`, so its + generators carry `den**-1` factors by construction — and the membership + entry points routed them through the denominator-free `Expr → GbPoly` + conversion and refused with *"negative exponent -1 in polynomial"*. The + trivially-true `gb.contains(gb.to_exprs()[i])` could not be run, and + neither could the question a loop actually needs. A denominator in a *ring + variable* is still refused. New `equals_ideal` / `contains_ideal` answer + "do these two parametric bases generate the same ideal?" — exactly, over + the fraction field, with neither basis's `conditions()` entering the + answer. + - **`specialize(values, verify=True)`** re-solves the specialised system + over ℚ and compares, instead of refusing on the recorded conditions alone. + `conditions()` is sufficient but not necessary — most of it is leading + coefficients that were inverted inside the Buchberger loop and then + cancelled — so **a quarter to a half of refusals on small-integer grids + are unnecessary**, dominated by parameters equal to exactly 0 (`{-2..2}` + 52 %, `{-2,-1,1,2}` 25 %, `{-3..3}` 58 %). Over eight systems on + `{-2..2}`: 646 refusals, 334 now returned, 239 genuine poles and 73 + necessary still refused. The refusals were separately verified *sound* + (1,930 regular points, zero disagreements), so this closes a completeness + gap, not a correctness one; the default stays `False` because verifying + costs a second Gröbner basis over ℚ. + - **`rosenfeld_groebner(dae, params=[...])`** runs the prolongation loop + over `Q(params)` and returns a `ParametricRosenfeldGroebnerResult` whose + `final_basis()` is a `ParametricGroebnerBasis`. It previously raised + `TypeError`: the differential-elimination surface did not compose with M9 + at all, so every model had to be prolonged by hand. + - **`eliminate=[...]` and `minimal=True`**, and a `UserWarning` when neither + was used but an earlier round would have done. One prolongation too many + is expensive out of all proportion — on SIR, stopping at the first + informative round is 0.03 s and one 4-term relation, one round further + does not finish in ten minutes — and the over-supplied answer is + *correct*, so nothing else signalled it. `minimal_prolongation_rounds` + reports the first informative round; its scope is documented, and it is + **known to be wrong for multi-output models**, so it is a cost signal, not + a certificate. + - **Prolongation no longer emits relations the system does not imply.** Two + causes, both reachable from `rosenfeld_groebner` on any system where one + equation mentions another's derivative: a promoted derivative had its + contribution counted twice (once from its own state pair, once from the + pair the previous round added), and a jet that was never promoted had its + contribution dropped entirely, because the next differentiation treated it + as a constant. On `R' = I, I' = -I` the two together forced `I = 0`. The + jet ranking is also append-only across rounds now: it was recomputed each + round, so a jet reachable only through the previous round's equations + could drop out and shift every later exponent slot under polynomials that + had already been padded. + - **The docs say which structural identifiability this decides.** An + IO-elimination route answers **multi-experiment** identifiability, by + Ovchinnikov–Pillay–Pogudin–Scanlon, *Computing all identifiable functions + of parameters for ODE models* (arXiv:2004.07774), Theorem 19 — which is + why it can call a model globally identifiable where a single-experiment + tool such as SIAN reports "locally, not globally". Stated with the + hypothesis it needs: Theorem 19 requires the IO equations to be the + characteristic presentation of `I_Σ ∩ C(θ){y,u}`, and an algebraic lex + elimination at a hand-picked finite jet order is not guaranteed to be one. + <<<<<<< HEAD - **The M11 novelty filter reads more of what OEIS actually writes, pages its searches, can represent a `q`-recurrence, and cross-checks the terms it is diff --git a/alkahest-core/src/dae/mod.rs b/alkahest-core/src/dae/mod.rs index ac02d812..1d22659e 100644 --- a/alkahest-core/src/dae/mod.rs +++ b/alkahest-core/src/dae/mod.rs @@ -34,12 +34,21 @@ pub fn extend_derivative_state_vectors( ) { for (j, _) in variables.clone().iter().enumerate() { let deriv = derivatives[j]; - if structurally_depends(new_eq, deriv, pool) && !variables.contains(&deriv) { - let d2_name = pool.with(deriv, |d| match d { - ExprData::Symbol { name, .. } => format!("d{name}/dt"), - _ => "d?/dt".to_string(), - }); - let d2 = pool.symbol(&d2_name, Domain::Real); + if variables.contains(&deriv) { + continue; + } + let d2_name = pool.with(deriv, |d| match d { + ExprData::Symbol { name, .. } => format!("d{name}/dt"), + _ => "d?/dt".to_string(), + }); + let d2 = pool.symbol(&d2_name, Domain::Real); + // `deriv` becomes a state when the equation mentions it — or when it + // mentions `d(deriv)/dt`, which means `deriv` has *already* been + // differentiated once. Without the second test the chain stops: the + // jet is in the equation but has no successor, so the next + // differentiation treats it as a constant and drops its contribution, + // asserting a relation the system does not imply. + if structurally_depends(new_eq, deriv, pool) || structurally_depends(new_eq, d2, pool) { variables.push(deriv); derivatives.push(d2); } @@ -342,8 +351,19 @@ pub(crate) fn differentiate_equation( let term = pool.mul(vec![dg_dyi, deriv]); terms.push(term); } - // Also differentiate w.r.t. the derivative (for higher-index terms) - let dg_ddyi = diff(equation, deriv, pool)?.value; + // Also differentiate w.r.t. the derivative (for higher-index terms) — + // but only when `deriv` has not itself been promoted to the state + // vector. Once it has, its own `(deriv, d²y_i)` pair contributes + // exactly this term via the branch above, and emitting it here too + // doubles it. Prolongation promotes derivatives as it goes, so this + // fires from the very first round on any system where one equation + // mentions another's derivative. + let already_a_state = variables.contains(&deriv); + let dg_ddyi = if already_a_state { + pool.integer(0_i32) + } else { + diff(equation, deriv, pool)?.value + }; if dg_ddyi != pool.integer(0_i32) { // d(dy_i/dt)/dt is a new symbol — use the naming convention let d2_name = pool.with(deriv, |d| match d { diff --git a/alkahest-core/src/diffalg/mod.rs b/alkahest-core/src/diffalg/mod.rs index b152ae55..4365c433 100644 --- a/alkahest-core/src/diffalg/mod.rs +++ b/alkahest-core/src/diffalg/mod.rs @@ -22,9 +22,11 @@ use crate::dae::{ }; use crate::errors::AlkahestError; use crate::kernel::{ExprData, ExprId, ExprPool}; -use crate::poly::groebner::{GbPoly, GroebnerBasis, MonomialOrder}; -use crate::solver::expr_to_gbpoly; +use crate::poly::groebner::{ + GbPoly, GroebnerBasis, MonomialOrder, ParamGbPoly, ParamGroebnerBasis, ParamGroebnerError, +}; use crate::solver::SolverError; +use crate::solver::{expr_to_gbpoly, expr_to_param_gbpoly}; use std::collections::{BTreeMap, HashSet}; use std::fmt; @@ -203,6 +205,24 @@ fn vars_for_dae(dae: &DAE, scratch: &[ExprId], pool: &ExprPool) -> Vec { out } +/// Append every symbol of `fresh` that `vars` does not already name. +/// +/// Prolongation pads existing polynomials by extending their exponent vectors, +/// which is only meaningful if slot `i` keeps naming the same symbol from round +/// to round. Recomputing the ranking from scratch does **not** guarantee that: +/// a jet reached only through the previous round's equations drops out of the +/// next round's scratch set, every later slot shifts down by one, and the +/// padded polynomials silently start asserting relations about other variables. +/// Merging instead of replacing keeps the ranking append-only, which is what +/// [`pad_gbpoly`] assumes. +fn merge_vars(vars: &mut Vec, fresh: Vec) { + for v in fresh { + if !vars.contains(&v) { + vars.push(v); + } + } +} + fn polys_from_equations( eqs: &[ExprId], vars: &[ExprId], @@ -299,7 +319,7 @@ pub fn rosenfeld_groebner_ranked( .copied() .chain(prolong_exprs.iter().copied()) .collect(); - vars = vars_for_dae(&work, &scratch, pool); + merge_vars(&mut vars, vars_for_dae(&work, &scratch, pool)); let n = vars.len(); for p in &mut active { *p = pad_gbpoly(p, n); @@ -385,6 +405,380 @@ pub fn rosenfeld_groebner_ranked( )) } +// --------------------------------------------------------------------------- +// M9 × V2-13 — differential elimination with the parameters in Q(params) +// --------------------------------------------------------------------------- + +/// Knobs for [`rosenfeld_groebner_parametric`]. +#[derive(Clone, Copy, Debug)] +pub struct ParametricProlongOpts<'a> { + /// Monomial order for each round's basis. `Lex` with the variables to + /// eliminate ordered first is what elimination needs. + pub order: MonomialOrder, + /// Prolongation budget — the number of formal time derivatives taken. + pub max_prolong_rounds: usize, + /// The variables the caller intends to eliminate, e.g. the unobserved + /// states of an ODE model. Their whole jet chain is eliminated with them: + /// naming `x` also names `dx/dt`, `d2x/dt2`, … as they appear. + /// + /// Empty disables both the informativeness check and [`Self::minimal`]. + pub eliminate: &'a [ExprId], + /// Stop at the **first** prolongation round whose elimination ideal is + /// non-trivial, instead of prolonging to the budget. + /// + /// See [`ParametricRosenfeldResult::minimal_prolongation_rounds`] for the + /// scope of "first informative" — it is not a guarantee of minimality. + pub minimal: bool, +} + +impl Default for ParametricProlongOpts<'_> { + fn default() -> Self { + ParametricProlongOpts { + order: MonomialOrder::Lex, + max_prolong_rounds: DEFAULT_MAX_PROLONG_ROUNDS, + eliminate: &[], + minimal: false, + } + } +} + +/// Result of [`rosenfeld_groebner_parametric`]. +#[derive(Clone, Debug)] +pub struct ParametricRosenfeldResult { + /// `false` iff the unit ideal was reached over `Q(params)`. + pub consistent: bool, + /// `true` if prolongation stopped on the budget, or on `minimal`, rather + /// than because differentiating stopped adding relations. A truncated + /// basis is a sound set of consequences but need not be complete. + pub truncated: bool, + /// Number of prolongation rounds that contributed new relations. + pub prolongation_rounds: usize, + /// The prolonged [`DAE`]: the input plus every jet introduced. + pub working_dae: DAE, + /// The saturated basis over `Q(params)`, or `None` when inconsistent. + pub final_basis: Option, + /// The lowest round count at which the elimination ideal with respect to + /// [`ParametricProlongOpts::eliminate`] was non-empty, or `None` when no + /// round was informative or no `eliminate` list was given. + /// + /// **Scope.** This is "the first round at which eliminating those variables + /// leaves a generator", not a theorem about the differential ideal. For a + /// single-output model it coincides with the jet order the input–output + /// relation needs; **for multi-output models the criterion is known to be + /// wrong**, because one output can become informative several rounds before + /// the others and the truncated basis then misses their relations. Treat + /// it as a cost signal, not a certificate. + /// + /// It matters because the cost is not gentle: on the SIR model one extra + /// prolongation past the informative round takes the elimination from a + /// single 4-term generator to thirteen generators of up to 233 terms with + /// 30-digit rational-function coefficients — four orders of magnitude of + /// time, for the same relation. + pub minimal_prolongation_rounds: Option, +} + +fn param_err_to_diffalg(e: ParamGroebnerError) -> DiffAlgError { + DiffAlgError::NotPolynomial(e.to_string()) +} + +fn is_unit_ideal_param(gb: &ParamGroebnerBasis) -> bool { + gb.generators().iter().any(|g| { + g.terms.len() == 1 + && g.terms + .keys() + .next() + .is_some_and(|e| e.iter().all(|&x| x == 0)) + }) +} + +/// The exponent slots of `vars` naming `roots` or any jet descended from one. +/// +/// `dae.variables[i]` differentiates to `dae.derivatives[i]`, so following that +/// map to a fixed point turns "eliminate `x`" into "eliminate `x`, `dx/dt`, +/// `d2x/dt2`, …" — which is what a caller eliminating a state means. +fn jet_closure_slots(dae: &DAE, vars: &[ExprId], roots: &[ExprId]) -> Vec { + let mut closed: HashSet = roots.iter().copied().collect(); + loop { + let mut grew = false; + for (i, v) in dae.variables.iter().enumerate() { + if closed.contains(v) { + if let Some(&d) = dae.derivatives.get(i) { + grew |= closed.insert(d); + } + } + } + if !grew { + break; + } + } + vars.iter() + .enumerate() + .filter(|(_, v)| closed.contains(v)) + .map(|(i, _)| i) + .collect() +} + +/// True when at least one generator is free of every slot in `slots` — i.e. the +/// elimination ideal is non-trivial. +fn param_elimination_is_informative(gb: &ParamGroebnerBasis, slots: &[usize]) -> bool { + gb.generators().iter().any(|g| { + !g.terms + .keys() + .any(|e| slots.iter().any(|&i| e.get(i).copied().unwrap_or(0) > 0)) + }) +} + +/// The jet chain `v, dv, d²v, …` of a state, `depth` derivatives deep, using +/// the same `d{name}/dt` naming convention prolongation itself uses. +fn jet_chain(v: ExprId, dv: ExprId, depth: usize, pool: &ExprPool) -> Vec { + let mut out = vec![v, dv]; + let mut cur = dv; + for _ in 0..depth { + let name = pool.with(cur, |d| match d { + ExprData::Symbol { name, .. } => format!("d{name}/dt"), + _ => "d?/dt".to_string(), + }); + cur = pool.symbol(&name, crate::kernel::Domain::Real); + out.push(cur); + } + out +} + +/// A ranking with the whole jet tower laid out up front, eliminated states +/// first. +/// +/// Two things need this. Elimination by generator filtering is only valid +/// under a lex order that ranks the eliminated variables *above* the rest, and +/// [`vars_for_dae`] interleaves states with outputs instead. And the ranking +/// has to be append-only across rounds for [`pad_param_gbpoly`] to mean +/// anything, which it cannot be if new jets of an eliminated state keep having +/// to be inserted in front of the outputs. Laying the tower out to the +/// prolongation depth settles both; the jets that never get used are unused +/// variables in the ring, which cost nothing but a slot. +fn ranked_jet_vars( + dae: &DAE, + eliminate: &[ExprId], + params: &[ExprId], + depth: usize, + pool: &ExprPool, +) -> Vec { + let mut elim_first: Vec = Vec::new(); + let mut rest: Vec = Vec::new(); + for (i, &v) in dae.variables.iter().enumerate() { + let Some(&dv) = dae.derivatives.get(i) else { + continue; + }; + let chain = jet_chain(v, dv, depth, pool); + if eliminate.contains(&v) { + elim_first.extend(chain); + } else { + rest.extend(chain); + } + } + let mut out: Vec = vec![dae.time_var]; + for v in elim_first.into_iter().chain(rest) { + if !params.contains(&v) && !out.contains(&v) { + out.push(v); + } + } + out +} + +fn param_vars_for_dae( + dae: &DAE, + scratch: &[ExprId], + params: &[ExprId], + pool: &ExprPool, +) -> Vec { + vars_for_dae(dae, scratch, pool) + .into_iter() + .filter(|v| !params.contains(v)) + .collect() +} + +fn pad_param_gbpoly(p: &ParamGbPoly, new_n: usize) -> ParamGbPoly { + if new_n == p.n_vars { + return p.clone(); + } + assert!(new_n > p.n_vars); + let pad = new_n - p.n_vars; + ParamGbPoly { + terms: p + .terms + .iter() + .map(|(e, c)| { + let mut e = e.clone(); + e.extend(std::iter::repeat(0u32).take(pad)); + (e, c.clone()) + }) + .collect(), + n_vars: new_n, + n_params: p.n_params, + } +} + +/// Rosenfeld-style prolongation with `params` in the **coefficient field** +/// `Q(params)` rather than as extra ring variables (M9 × V2-13). +/// +/// [`rosenfeld_groebner_ranked`] puts every free symbol in the ring, so a model +/// parameter enlarges the monomial order, the pair schedule and the staircase. +/// Here the parameters are moved into the coefficients, which is the difference +/// between eliminating states from `Q[states, jets, params]` and from +/// `Q(params)[states, jets]` — the computation the input–output relations of an +/// ODE model actually need. +/// +/// The returned basis is **generic** in the parameters, exactly as +/// [`ParamGroebnerBasis`] describes: read +/// [`ParamGroebnerBasis::conditions`] for the hypotheses it used. +/// +/// `params` must be disjoint from the DAE's variables, derivatives and time +/// variable; a parameter listed there is dropped from the ring, so it must not +/// be one of the unknowns. +pub fn rosenfeld_groebner_parametric( + dae: &DAE, + pool: &ExprPool, + params: &[ExprId], + opts: ParametricProlongOpts<'_>, +) -> Result<(ParametricRosenfeldResult, DifferentialRanking), DiffAlgError> { + if dae.equations.is_empty() { + return Err(DiffAlgError::EmptySystem); + } + + let source_eqs = dae.equations.clone(); + let mut work = dae.clone(); + let mut scratch: Vec = source_eqs.clone(); + let mut vars = if opts.eliminate.is_empty() { + param_vars_for_dae(&work, &scratch, params, pool) + } else { + let mut v = ranked_jet_vars( + &work, + opts.eliminate, + params, + opts.max_prolong_rounds + 1, + pool, + ); + merge_vars(&mut v, param_vars_for_dae(&work, &scratch, params, pool)); + v + }; + let mut active: Vec = work + .equations + .iter() + .map(|&eq| expr_to_param_gbpoly(eq, &vars, params, pool).map_err(solver_err_to_diffalg)) + .collect::>()?; + + let mut prolong_exprs = source_eqs.clone(); + let mut minimal_prolongation_rounds: Option = None; + + // The budget counts prolongations, so `max + 1` bases get computed: one for + // the unprolonged system and one after each round. + for round in 0..=opts.max_prolong_rounds { + let gb = ParamGroebnerBasis::compute(active.clone(), opts.order) + .map_err(param_err_to_diffalg)?; + + if is_unit_ideal_param(&gb) { + return Ok(( + ParametricRosenfeldResult { + consistent: false, + truncated: false, + prolongation_rounds: round, + working_dae: work, + final_basis: None, + minimal_prolongation_rounds, + }, + DifferentialRanking { vars }, + )); + } + + if !opts.eliminate.is_empty() { + let slots = jet_closure_slots(&work, &vars, opts.eliminate); + if minimal_prolongation_rounds.is_none() + && param_elimination_is_informative(&gb, &slots) + { + minimal_prolongation_rounds = Some(round); + if opts.minimal { + return Ok(( + ParametricRosenfeldResult { + consistent: true, + truncated: true, + prolongation_rounds: round, + working_dae: work, + final_basis: Some(gb), + minimal_prolongation_rounds, + }, + DifferentialRanking { vars }, + )); + } + } + } + + if round == opts.max_prolong_rounds { + return Ok(( + ParametricRosenfeldResult { + consistent: true, + truncated: true, + prolongation_rounds: round, + working_dae: work, + final_basis: Some(gb), + minimal_prolongation_rounds, + }, + DifferentialRanking { vars }, + )); + } + + let mut next_prolong = Vec::with_capacity(prolong_exprs.len()); + for &eq in &prolong_exprs { + let d_eq = + differentiate_equation(eq, &work.variables, &work.derivatives, work.time_var, pool) + .map_err(|e| DiffAlgError::DiffError(e.to_string()))?; + extend_dae_for_derivative_symbols(&mut work, d_eq, pool); + next_prolong.push(d_eq); + } + prolong_exprs = next_prolong; + scratch = source_eqs + .iter() + .copied() + .chain(prolong_exprs.iter().copied()) + .collect(); + let old_n = vars.len(); + merge_vars(&mut vars, param_vars_for_dae(&work, &scratch, params, pool)); + let n = vars.len(); + for p in &mut active { + *p = pad_param_gbpoly(p, n); + } + // The previous round's basis is still a basis over the wider ring, so + // the new relations can be tested against it without recomputing. + let gb_check = gb.extend_vars(n - old_n); + + let mut to_add: Vec = Vec::new(); + for &d_eq in &prolong_exprs { + let p = + expr_to_param_gbpoly(d_eq, &vars, params, pool).map_err(solver_err_to_diffalg)?; + if !gb_check.contains(&p) { + to_add.push(p); + } + } + + if to_add.is_empty() { + // Saturated: differentiating adds nothing the ideal did not have. + return Ok(( + ParametricRosenfeldResult { + consistent: true, + truncated: false, + prolongation_rounds: round, + working_dae: work, + final_basis: Some(gb_check), + minimal_prolongation_rounds, + }, + DifferentialRanking { vars }, + )); + } + + active.extend(to_add); + } + + unreachable!("the `round == max_prolong_rounds` arm returns") +} + /// Calls [`rosenfeld_groebner_with_options`] with the default maximum prolongation rounds. pub fn rosenfeld_groebner( dae: &DAE, @@ -481,6 +875,155 @@ mod tests { assert!(!r.consistent); } + /// Prolongation used to emit relations the system does not imply. + /// + /// Two independent causes, both visible on `R' = I, I' = -I` after two + /// rounds: + /// + /// * a state whose derivative had already been promoted got its + /// contribution counted twice — once from its own `(I, I')` pair and once + /// from the `(I', I'')` pair the previous round added; and + /// * a jet that was *not* promoted got its contribution dropped entirely, + /// because the next differentiation treated it as a constant. `I''` + /// reaches the equations through `R`'s chain without `R'` ever appearing + /// in one, so `R'` never became a state and the chain stopped there. + /// + /// The first inflates a coefficient, the second deletes a term; together + /// they collapsed the second prolongation of `R' = I` into `-2·I'' = 0`, + /// which forces `I = 0` — a relation about a decaying exponential that is + /// simply false. + #[test] + fn prolongation_does_not_invent_relations() { + let p = pool(); + let t = p.symbol("t", Domain::Real); + let i = p.symbol("I", Domain::Real); + let r_ = p.symbol("R", Domain::Real); + let di = p.symbol("dI/dt", Domain::Real); + let dr = p.symbol("dR/dt", Domain::Real); + // R' = I, I' = -I. + let eq1 = p.add(vec![dr, p.mul(vec![p.integer(-1), i])]); + let eq2 = p.add(vec![di, i]); + let dae = DAE::new(vec![eq1, eq2], vec![r_, i], vec![dr, di], t); + + let (res, ranking) = rosenfeld_groebner_ranked(&dae, &p, MonomialOrder::Lex, 2).unwrap(); + assert!(res.consistent); + let gb = res.final_basis.expect("consistent"); + + // The genuine consequence. + let d2i = p.symbol("ddI/dt/dt", Domain::Real); + let truth = p.add(vec![d2i, p.mul(vec![p.integer(-1), i])]); + let truth_poly = expr_to_gbpoly(truth, &ranking.vars, &p).unwrap(); + assert!(gb.contains(&truth_poly), "I'' - I is a consequence"); + + // Nothing forces the state to vanish identically. + for false_claim in [i, di, d2i] { + let q = expr_to_gbpoly(false_claim, &ranking.vars, &p).unwrap(); + assert!( + !gb.contains(&q), + "prolongation asserted a jet of the state vanishes identically" + ); + } + } + + /// The jet ranking has to be append-only across prolongation rounds. + /// + /// It used to be recomputed from scratch each round, and + /// [`vars_for_dae`] appends the symbols it scrapes out of the equations + /// *after* the declared jets — so introducing `d²x/dt²` pushed the trailing + /// parameter `a` one slot to the right, under polynomials that had already + /// been padded on the assumption that slot `i` still meant what it meant + /// last round. On `x' = a·x`, one prolongation, that silently turned the + /// input equation into a relation about `d²x/dt²` and lost it from the + /// basis entirely. + #[test] + fn the_jet_ranking_is_append_only() { + let p = pool(); + let t = p.symbol("t", Domain::Real); + let x = p.symbol("x", Domain::Real); + let dx = p.symbol("dx/dt", Domain::Real); + let a = p.symbol("a", Domain::Real); + // x' = a·x, with `a` an ordinary ring variable (no params here). + let eq = p.add(vec![dx, p.mul(vec![p.integer(-1), a, x])]); + let dae = DAE::new(vec![eq], vec![x], vec![dx], t); + + let (res, ranking) = + rosenfeld_groebner_ranked(&dae, &p, MonomialOrder::GRevLex, 1).unwrap(); + let gb = res.final_basis.expect("consistent"); + + // The system's own equation is a consequence of itself. + let src = expr_to_gbpoly(eq, &ranking.vars, &p).unwrap(); + assert!(gb.contains(&src), "the input equation left the ideal"); + + // ...and x·x'' - x' is not: it would need a·x = 1. + let d2x = p.symbol("ddx/dt/dt", Domain::Real); + let bogus = p.add(vec![p.mul(vec![x, d2x]), p.mul(vec![p.integer(-1), dx])]); + let q = expr_to_gbpoly(bogus, &ranking.vars, &p).unwrap(); + assert!( + !gb.contains(&q), + "a shifted exponent slot invented a relation" + ); + } + + /// The parametric route reads the input–output relation of an ODE model + /// straight out of the DAE, with the rate constant in `Q(a)` rather than as + /// a fourth ring variable (2026-08-19 issue #16). + #[test] + fn parametric_prolongation_yields_the_io_relation() { + let p = pool(); + let t = p.symbol("t", Domain::Real); + let x = p.symbol("x", Domain::Real); + let y = p.symbol("y", Domain::Real); + let dx = p.symbol("dx/dt", Domain::Real); + let dy = p.symbol("dy/dt", Domain::Real); + let a = p.symbol("a", Domain::Real); + // x' = -a·x, y = x => y' + a·y = 0. + let eq1 = p.add(vec![dx, p.mul(vec![a, x])]); + let eq2 = p.add(vec![y, p.mul(vec![p.integer(-1), x])]); + let dae = DAE::new(vec![eq1, eq2], vec![x, y], vec![dx, dy], t); + + let (r, ranking) = rosenfeld_groebner_parametric( + &dae, + &p, + &[a], + ParametricProlongOpts { + order: MonomialOrder::Lex, + max_prolong_rounds: 3, + eliminate: &[x], + minimal: true, + }, + ) + .unwrap(); + + // One derivative of the output is enough, not three. + assert_eq!(r.minimal_prolongation_rounds, Some(1)); + assert_eq!(r.prolongation_rounds, 1); + assert!( + !ranking.vars.contains(&a), + "`a` must not be a ring variable" + ); + + let gb = r.final_basis.expect("consistent"); + let state_slots: Vec = ranking + .vars + .iter() + .enumerate() + .filter(|(_, &v)| { + let name = p.with(v, |d| match d { + ExprData::Symbol { name, .. } => name.clone(), + _ => String::new(), + }); + name.trim_start_matches('d').split('/').next() == Some("x") + }) + .map(|(i, _)| i) + .collect(); + let io = gb.eliminate(&state_slots); + assert_eq!(io.len(), 1); + + let relation = p.add(vec![dy, p.mul(vec![a, y])]); + let q = expr_to_param_gbpoly(relation, &ranking.vars, &[a], &p).unwrap(); + assert!(io.contains(&q), "y' + a·y = 0 is the input–output relation"); + } + #[test] fn textbook_library_runs() { // Ten tiny polynomial DAE snapshots (autonomous, explicit first derivatives). diff --git a/alkahest-core/src/lib.rs b/alkahest-core/src/lib.rs index f7fb29cf..3f60a79d 100644 --- a/alkahest-core/src/lib.rs +++ b/alkahest-core/src/lib.rs @@ -208,8 +208,9 @@ pub use lean::{ #[cfg(feature = "groebner")] pub use diffalg::{ dae_index_reduce, dae_index_reduce_ranked, rosenfeld_groebner, rosenfeld_groebner_algebraic, - rosenfeld_groebner_ranked, rosenfeld_groebner_with_options, DaeIndexReduction, DiffAlgError, - DifferentialIdeal, DifferentialRanking, DifferentialRing, RegularDifferentialChain, + rosenfeld_groebner_parametric, rosenfeld_groebner_ranked, rosenfeld_groebner_with_options, + DaeIndexReduction, DiffAlgError, DifferentialIdeal, DifferentialRanking, DifferentialRing, + ParametricProlongOpts, ParametricRosenfeldResult, RegularDifferentialChain, RosenfeldGroebnerResult, }; #[cfg(feature = "groebner")] @@ -231,10 +232,11 @@ pub use primitive::{ }; #[cfg(feature = "groebner")] pub use solver::{ - diophantine, expr_to_gbpoly, extract_regular_chain_from_basis, gbpoly_to_expr, - main_variable_recursive, solve_numerical, solve_polynomial_system, solve_transcendental, - triangularize, CertifiedPoint, DiophantineError, DiophantineSolution, HomotopyError, - HomotopyOpts, RegularChain, Solution, SolutionSet, SolverError, TranscendentalOutcome, + diophantine, expr_to_gbpoly, expr_to_param_gbpoly, extract_regular_chain_from_basis, + gbpoly_to_expr, main_variable_recursive, solve_numerical, solve_polynomial_system, + solve_transcendental, triangularize, CertifiedPoint, DiophantineError, DiophantineSolution, + HomotopyError, HomotopyOpts, RegularChain, Solution, SolutionSet, SolverError, + TranscendentalOutcome, }; pub fn version() -> &'static str { diff --git a/alkahest-core/src/poly/groebner/parametric.rs b/alkahest-core/src/poly/groebner/parametric.rs index 859a8be6..c53750ec 100644 --- a/alkahest-core/src/poly/groebner/parametric.rs +++ b/alkahest-core/src/poly/groebner/parametric.rs @@ -54,6 +54,7 @@ use crate::poly::groebner::ideal::GbPoly; use crate::poly::groebner::monomial_order::MonomialOrder; use crate::poly::groebner::pairs::{lcm_exp, update_pairs, CriticalPair}; use crate::poly::groebner::paramfield::{ParamPoly, QParam}; +use crate::poly::groebner::GroebnerBasis; // --------------------------------------------------------------------------- // Errors @@ -291,6 +292,40 @@ impl ParamGbPoly { } } + /// `self · other`. + pub fn mul(&self, other: &Self) -> Self { + let mut out = ParamGbPoly::zero(self.n_vars, self.n_params); + for (ea, ca) in &self.terms { + for (eb, cb) in &other.terms { + let e: Vec = ea.iter().zip(eb.iter()).map(|(a, b)| a + b).collect(); + let slot = out + .terms + .entry(e) + .or_insert_with(|| QParam::zero(self.n_params)); + *slot = slot.add(&ca.mul(cb)); + } + } + out.terms.retain(|_, c| !c.is_zero()); + out + } + + /// The polynomial as a single element of `Q(params)`, or `None` when it + /// mentions a ring variable. + /// + /// The zero polynomial is `Some(0)`. This is what decides whether a + /// negative power is a coefficient-field denominator (fine — that is where + /// the parameters live) or a genuine non-polynomial (not fine). + pub fn as_coeff(&self) -> Option { + match self.terms.len() { + 0 => Some(QParam::zero(self.n_params)), + 1 => { + let (e, c) = self.terms.iter().next()?; + e.iter().all(|&k| k == 0).then(|| c.clone()) + } + _ => None, + } + } + /// `self · c · x^shift`. pub fn mul_monomial(&self, shift: &[u32], c: &QParam) -> Self { if c.is_zero() { @@ -522,6 +557,13 @@ pub struct ParamGroebnerBasis { n_vars: usize, n_params: usize, conditions: Vec, + /// The generators this basis was computed from, kept so that + /// [`Self::specialize_verified`] can rebuild the oracle at a parameter + /// point instead of refusing on [`Self::conditions`] alone. + input_generators: Vec, + /// Variable slots dropped by [`Self::eliminate`], in call order. The + /// oracle has to eliminate them too, or it is not comparing like with like. + eliminated: Vec, } impl ParamGroebnerBasis { @@ -550,6 +592,7 @@ impl ParamGroebnerBasis { } } + let input_generators = gens.clone(); let initial: Vec = gens .into_iter() .filter(|g| !g.is_zero()) @@ -563,6 +606,8 @@ impl ParamGroebnerBasis { n_vars, n_params, conditions: conds.finish(), + input_generators, + eliminated: vec![], }); } @@ -613,6 +658,8 @@ impl ParamGroebnerBasis { n_vars, n_params, conditions: conds.finish(), + input_generators, + eliminated: vec![], }) } @@ -715,6 +762,107 @@ impl ParamGroebnerBasis { Ok(out) } + /// Specialise at `values`, **checking** the parameter point rather than + /// refusing on [`Self::conditions`] alone. + /// + /// [`Self::conditions`] is sufficient, not necessary: most of what it lists + /// are leading coefficients that were inverted somewhere inside the + /// Buchberger loop and then cancelled, so a large fraction of the points it + /// excludes are perfectly ordinary. (On small-integer parameter grids a + /// quarter to a half of the refusals are of that kind, dominated by + /// parameters equal to exactly 0.) + /// + /// This does the work instead of guessing: + /// + /// 1. off the locus, it is exactly [`Self::specialize`]; + /// 2. on it, `σ(F)` is re-Gröbner-ised over ℚ from scratch and compared with + /// `σ(G)`. They agree ⇒ the refusal was unnecessary and `σ(G)` is + /// returned. They disagree, or `σ(G)` has a genuine pole ⇒ + /// [`ParamGroebnerError::Degenerate`], as before. + /// + /// So this is strictly more complete and never less sound — but on the + /// locus it pays for a second Gröbner basis over ℚ, which is why + /// [`Self::specialize`] is still the default path. + /// + /// If [`Self::eliminate`] was applied, the oracle eliminates the same + /// variable slots before comparing; otherwise it would not be comparing + /// like with like. + pub fn specialize_verified( + &self, + values: &[Rational], + ) -> Result, ParamGroebnerError> { + if values.len() != self.n_params { + return Err(ParamGroebnerError::WrongArity { + expected: self.n_params, + got: values.len(), + }); + } + if self.is_regular_at(values) { + return self.specialize(values); + } + + let vanishing = self.vanishing_conditions(values); + let degenerate = || ParamGroebnerError::Degenerate { + vanishing: vanishing.clone(), + }; + + // σ(G): the generic basis at the point. A pole here is a genuine + // degeneration — the basis has no value at all there. + let mut spec = Vec::with_capacity(self.generators.len()); + for g in &self.generators { + spec.push(g.specialize(values).ok_or_else(degenerate)?); + } + + // σ(F): the input system at the point, re-solved over ℚ. + let mut direct = Vec::with_capacity(self.input_generators.len()); + for g in &self.input_generators { + let p = g.specialize(values).ok_or_else(degenerate)?; + if !p.terms.is_empty() { + direct.push(p); + } + } + if direct.is_empty() { + // Nothing left to generate the ideal with; `σ(G)` cannot be checked + // against anything, so keep the refusal. + return Err(degenerate()); + } + let mut oracle = GroebnerBasis::compute(direct, self.order); + if !self.eliminated.is_empty() { + oracle = GroebnerBasis::from_generators( + oracle + .generators() + .iter() + .filter(|g| { + !g.terms.keys().any(|e| { + self.eliminated + .iter() + .any(|&i| e.get(i).copied().unwrap_or(0) > 0) + }) + }) + .cloned() + .collect(), + self.order, + ); + } + + // Equal ideals, checked both ways: `σ(G)` is then a Gröbner basis of the + // specialised ideal, which is all `specialize` ever promised. + if !spec.iter().all(|g| oracle.contains(g)) { + return Err(degenerate()); + } + let spec_gb = GroebnerBasis::compute( + spec.iter() + .filter(|g| !g.terms.is_empty()) + .cloned() + .collect(), + self.order, + ); + if !oracle.generators().iter().all(|g| spec_gb.contains(g)) { + return Err(degenerate()); + } + Ok(spec) + } + /// Reduce a polynomial modulo this basis and return the remainder. pub fn reduce(&self, p: &ParamGbPoly) -> ParamGbPoly { let mut sink = ConditionLog::default(); @@ -726,6 +874,65 @@ impl ParamGroebnerBasis { self.reduce(p).is_zero() } + /// True when every generator of `other` lies in this ideal, i.e. + /// `⟨other⟩ ⊆ ⟨self⟩` as ideals of `Q(params)[vars]`. + /// + /// `false` when the two are written over different numbers of variables or + /// parameters — there is no shared ring in which to compare them. + pub fn contains_ideal(&self, other: &ParamGroebnerBasis) -> bool { + if (self.n_vars, self.n_params) != (other.n_vars, other.n_params) { + return false; + } + other.generators.iter().all(|g| self.contains(g)) + } + + /// True when the two bases generate the same ideal of `Q(params)[vars]`. + /// + /// This is an *exact* statement about the fraction field: reduction over + /// `Q(params)` only ever divides by non-zero field elements, so no + /// genericity hypothesis is involved and neither basis's + /// [`Self::conditions`] enters the answer. Those conditions still bound + /// what each basis says about a *specialised* parameter point. + pub fn equals_ideal(&self, other: &ParamGroebnerBasis) -> bool { + self.contains_ideal(other) && other.contains_ideal(self) + } + + /// This basis re-read in a ring with `extra` further variables, appended + /// after the existing ones. + /// + /// A Gröbner basis stays one when the ring gains variables none of its + /// generators mention: every leading monomial is unchanged and every S-pair + /// reduces exactly as before. Prolongation needs this — each round + /// introduces new jets, and recomputing the previous round's basis over the + /// wider ring just to test membership would double the work. + pub fn extend_vars(&self, extra: usize) -> ParamGroebnerBasis { + if extra == 0 { + return self.clone(); + } + let pad = |p: &ParamGbPoly| ParamGbPoly { + terms: p + .terms + .iter() + .map(|(e, c)| { + let mut e = e.clone(); + e.extend(std::iter::repeat(0u32).take(extra)); + (e, c.clone()) + }) + .collect(), + n_vars: p.n_vars + extra, + n_params: p.n_params, + }; + ParamGroebnerBasis { + generators: self.generators.iter().map(pad).collect(), + order: self.order, + n_vars: self.n_vars + extra, + n_params: self.n_params, + conditions: self.conditions.clone(), + input_generators: self.input_generators.iter().map(pad).collect(), + eliminated: self.eliminated.clone(), + } + } + /// The elimination ideal `I ∩ Q(params)[remaining vars]`. /// /// Drops every generator whose support mentions one of `vars`, exactly as @@ -743,12 +950,16 @@ impl ParamGroebnerBasis { }) .cloned() .collect(); + let mut eliminated = self.eliminated.clone(); + eliminated.extend_from_slice(vars); ParamGroebnerBasis { generators, order: self.order, n_vars: self.n_vars, n_params: self.n_params, conditions: self.conditions.clone(), + input_generators: self.input_generators.clone(), + eliminated, } } } @@ -763,7 +974,7 @@ mod tests { use crate::poly::groebner::GroebnerBasis; /// `c · x^var_exp · p^par_exp` as a one-term parametric polynomial. - fn term( + pub(super) fn term( n_vars: usize, n_params: usize, var_exp: &[u32], @@ -781,19 +992,19 @@ mod tests { p } - fn sum(parts: Vec) -> ParamGbPoly { + pub(super) fn sum(parts: Vec) -> ParamGbPoly { let mut it = parts.into_iter(); let first = it.next().expect("non-empty"); it.fold(first, |a, b| a.add(&b)) } - fn rat(v: i64) -> Rational { + pub(super) fn rat(v: i64) -> Rational { Rational::from(v) } /// The same system written over ℚ with the parameters substituted, as a /// plain `GbPoly` — the oracle for the specialisation tests. - fn gb_over_q(polys: &[Vec<(Vec, Rational)>], n_vars: usize) -> GroebnerBasis { + pub(super) fn gb_over_q(polys: &[Vec<(Vec, Rational)>], n_vars: usize) -> GroebnerBasis { let gens: Vec = polys .iter() .map(|terms| GbPoly { @@ -980,3 +1191,135 @@ mod tests { assert_eq!(err.code(), "E-PARAMGB-002"); } } + +#[cfg(test)] +mod ideal_and_verify_tests { + use super::tests::*; + use super::*; + + /// `{a·x + b·y - 1, c·x + d·y - 1}` over `Q(a, b, c, d)[x, y]`. + fn cramer() -> ParamGroebnerBasis { + let f = sum(vec![ + term(2, 4, &[1, 0], &[1, 0, 0, 0], 1), + term(2, 4, &[0, 1], &[0, 1, 0, 0], 1), + term(2, 4, &[0, 0], &[0, 0, 0, 0], -1), + ]); + let g = sum(vec![ + term(2, 4, &[1, 0], &[0, 0, 1, 0], 1), + term(2, 4, &[0, 1], &[0, 0, 0, 1], 1), + term(2, 4, &[0, 0], &[0, 0, 0, 0], -1), + ]); + ParamGroebnerBasis::compute(vec![f, g], MonomialOrder::Lex).unwrap() + } + + #[test] + fn a_basis_contains_its_own_generators() { + let gb = cramer(); + assert!(!gb.is_empty()); + for g in gb.generators() { + assert!(gb.contains(g), "a basis must contain its own generators"); + } + } + + #[test] + fn equals_ideal_is_mutual_containment() { + // ⟨a·x - 1⟩ and ⟨a²·x - a⟩ are the same ideal of Q(a)[x]: over the + // fraction field `a` is a unit. + let f = term(1, 1, &[1], &[1], 1).sub(&term(1, 1, &[0], &[0], 1)); + let g = term(1, 1, &[1], &[2], 1).sub(&term(1, 1, &[0], &[1], 1)); + let gf = ParamGroebnerBasis::compute(vec![f], MonomialOrder::Lex).unwrap(); + let gg = ParamGroebnerBasis::compute(vec![g], MonomialOrder::Lex).unwrap(); + assert!(gf.equals_ideal(&gg)); + assert!(gg.equals_ideal(&gf)); + + // ⟨x⟩ is a different ideal. + let h = term(1, 1, &[1], &[0], 1); + let gh = ParamGroebnerBasis::compute(vec![h], MonomialOrder::Lex).unwrap(); + assert!(!gf.equals_ideal(&gh)); + assert!(!gh.contains_ideal(&gf)); + } + + #[test] + fn equals_ideal_refuses_a_shape_mismatch() { + let f = term(1, 1, &[1], &[1], 1); + let g = term(1, 2, &[1], &[1, 0], 1); + let gf = ParamGroebnerBasis::compute(vec![f], MonomialOrder::Lex).unwrap(); + let gg = ParamGroebnerBasis::compute(vec![g], MonomialOrder::Lex).unwrap(); + assert!(!gf.equals_ideal(&gg)); + } + + #[test] + fn specialize_verified_accepts_an_unnecessary_refusal() { + let gb = cramer(); + // a = 0, b = c = d = 1: `a` is a recorded condition, but the only + // denominator in the basis is a·d - b·c = -1. + let pt = [rat(0), rat(1), rat(1), rat(1)]; + assert!(!gb.is_regular_at(&pt)); + assert!(matches!( + gb.specialize(&pt), + Err(ParamGroebnerError::Degenerate { .. }) + )); + + let spec = gb + .specialize_verified(&pt) + .expect("refusal was unnecessary"); + // y = 1, x = 0. + let direct = gb_over_q( + &[ + vec![(vec![0, 1], rat(1)), (vec![0, 0], rat(-1))], + vec![ + (vec![1, 0], rat(1)), + (vec![0, 1], rat(1)), + (vec![0, 0], rat(-1)), + ], + ], + 2, + ); + let spec_gb = GroebnerBasis::compute(spec, MonomialOrder::Lex); + for g in direct.generators() { + assert!(spec_gb.contains(g)); + } + for g in spec_gb.generators() { + assert!(direct.contains(g)); + } + } + + #[test] + fn specialize_verified_still_refuses_a_genuine_pole() { + let gb = cramer(); + // a·d - b·c = 0: the coefficients have no value there at all. + for pt in [ + [rat(1), rat(1), rat(1), rat(1)], + [rat(1), rat(1), rat(2), rat(2)], + ] { + assert!(matches!( + gb.specialize_verified(&pt), + Err(ParamGroebnerError::Degenerate { .. }) + )); + } + } + + #[test] + fn specialize_verified_matches_specialize_at_regular_points() { + let gb = cramer(); + let pt = [rat(1), rat(2), rat(3), rat(4)]; + assert!(gb.is_regular_at(&pt)); + let a = gb.specialize(&pt).unwrap(); + let b = gb.specialize_verified(&pt).unwrap(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(b.iter()) { + assert_eq!(x.terms, y.terms); + } + } + + #[test] + fn extend_vars_keeps_membership() { + let gb = cramer(); + let wide = gb.extend_vars(3); + assert_eq!(wide.n_vars(), gb.n_vars() + 3); + for (narrow, padded) in gb.generators().iter().zip(wide.generators()) { + assert_eq!(narrow.n_terms(), padded.n_terms()); + assert!(wide.contains(padded)); + } + } +} diff --git a/alkahest-core/src/solver/mod.rs b/alkahest-core/src/solver/mod.rs index 200015d2..713d2ef5 100644 --- a/alkahest-core/src/solver/mod.rs +++ b/alkahest-core/src/solver/mod.rs @@ -52,6 +52,7 @@ use crate::errors::AlkahestError; use crate::kernel::{ExprData, ExprId, ExprPool}; use crate::poly::collect_free_vars; use crate::poly::groebner::{GbPoly, GroebnerBasis, MonomialOrder}; +use crate::poly::groebner::{ParamGbPoly, ParamPoly, QParam}; use rug::ops::Pow; use rug::Rational; use std::collections::{BTreeMap, BTreeSet}; @@ -277,6 +278,162 @@ fn expr_to_gbpoly_rec( } } +// --------------------------------------------------------------------------- +// Expr → ParamGbPoly conversion (M9) +// --------------------------------------------------------------------------- + +/// Convert an `Expr` to a [`ParamGbPoly`] over `Q(params)[vars]`. +/// +/// The expression must be *polynomial in `vars`* and *rational in `params`*. +/// That second half is the difference from [`expr_to_gbpoly`], which refuses +/// any negative exponent: here a negative power whose base is free of `vars` is +/// just a denominator in the coefficient field, which is exactly where the +/// parameters live. Without it a parametric basis cannot be fed its own +/// [`crate::poly::groebner::ParamGroebnerBasis::generators`] back — those carry +/// `den^-1` factors by construction. +/// +/// Exponent slot `i` names `vars[i]`; parameter slot `j` names `params[j]`. +/// `vars` and `params` must be disjoint; a symbol in neither list is an error, +/// as it is for [`expr_to_gbpoly`]. +pub fn expr_to_param_gbpoly( + expr: ExprId, + vars: &[ExprId], + params: &[ExprId], + pool: &ExprPool, +) -> Result { + expr_to_param_gbpoly_rec(expr, vars, params, pool) +} + +fn param_constant(c: QParam, n_vars: usize, n_params: usize) -> ParamGbPoly { + let mut p = ParamGbPoly::zero(n_vars, n_params); + if !c.is_zero() { + p.terms.insert(vec![0u32; n_vars], c); + } + p +} + +fn expr_to_param_gbpoly_rec( + expr: ExprId, + vars: &[ExprId], + params: &[ExprId], + pool: &ExprPool, +) -> Result { + let (n_vars, n_params) = (vars.len(), params.len()); + if let Some(idx) = vars.iter().position(|&v| v == expr) { + let mut exp = vec![0u32; n_vars]; + exp[idx] = 1; + let mut p = ParamGbPoly::zero(n_vars, n_params); + p.terms.insert(exp, QParam::one(n_params)); + return Ok(p); + } + if let Some(idx) = params.iter().position(|&p| p == expr) { + let c = QParam::from_poly(ParamPoly::var(idx, n_params)); + return Ok(param_constant(c, n_vars, n_params)); + } + + enum Node { + IntConst(rug::Integer), + RatConst(Rational), + FloatConst(f64), + FreeSymbol(String), + Add(Vec), + Mul(Vec), + Pow(ExprId, ExprId), + Func(String), + Other, + } + + let node = pool.with(expr, |data| match data { + ExprData::Integer(n) => Node::IntConst(n.0.clone()), + ExprData::Rational(r) => Node::RatConst(r.0.clone()), + ExprData::Float(f) => Node::FloatConst(f.inner.to_f64()), + ExprData::Symbol { name, .. } => Node::FreeSymbol(name.clone()), + ExprData::Add(args) => Node::Add(args.clone()), + ExprData::Mul(args) => Node::Mul(args.clone()), + ExprData::Pow { base, exp } => Node::Pow(*base, *exp), + ExprData::Func { name, .. } => Node::Func(name.clone()), + _ => Node::Other, + }); + + let rational_const = |r: Rational| { + Ok(param_constant( + QParam::from_rational(&r, n_params), + n_vars, + n_params, + )) + }; + + match node { + Node::IntConst(n) => rational_const(Rational::from(n)), + Node::RatConst(r) => rational_const(r), + Node::FloatConst(v) => rational_const(Rational::from_f64(v).unwrap_or_else(|| 0.into())), + Node::FreeSymbol(name) => Err(SolverError::NotPolynomial(format!( + "free symbol '{name}' is neither a ring variable nor a parameter" + ))), + Node::Add(args) => { + let mut acc = ParamGbPoly::zero(n_vars, n_params); + for a in args { + acc = acc.add(&expr_to_param_gbpoly_rec(a, vars, params, pool)?); + } + Ok(acc) + } + Node::Mul(args) => { + let mut acc = param_constant(QParam::one(n_params), n_vars, n_params); + for a in args { + acc = acc.mul(&expr_to_param_gbpoly_rec(a, vars, params, pool)?); + } + Ok(acc) + } + Node::Pow(base, exp_id) => { + let exp_node = pool.with(exp_id, |d| match d { + ExprData::Integer(n) => n.0.to_i64(), + _ => None, + }); + let Some(n_val) = exp_node else { + return Err(SolverError::NotPolynomial( + "symbolic or non-integer exponent".to_string(), + )); + }; + let base_poly = expr_to_param_gbpoly_rec(base, vars, params, pool)?; + let (base_poly, k) = if n_val < 0 { + // A denominator is only admissible in the coefficient field. + let Some(c) = base_poly.as_coeff() else { + return Err(SolverError::NotPolynomial(format!( + "negative exponent {n_val} on a ring variable; only the \ + coefficient field Q(params) admits denominators" + ))); + }; + let Some(inv) = c.inv() else { + return Err(SolverError::NotPolynomial( + "negative exponent on zero".to_string(), + )); + }; + (param_constant(inv, n_vars, n_params), n_val.unsigned_abs()) + } else { + (base_poly, n_val as u64) + }; + let mut result = param_constant(QParam::one(n_params), n_vars, n_params); + let mut cur = base_poly; + let mut rem = k; + while rem > 0 { + if rem & 1 == 1 { + result = result.mul(&cur); + } + let cur2 = cur.clone(); + cur = cur.mul(&cur2); + rem >>= 1; + } + Ok(result) + } + Node::Func(name) => Err(SolverError::NotPolynomial(format!( + "function '{name}' is not a polynomial" + ))), + Node::Other => Err(SolverError::NotPolynomial( + "unsupported expression node".to_string(), + )), + } +} + // --------------------------------------------------------------------------- // GbPoly → Expr conversion // --------------------------------------------------------------------------- diff --git a/alkahest-py/src/lib.rs b/alkahest-py/src/lib.rs index 5a697f8b..49bfb567 100644 --- a/alkahest-py/src/lib.rs +++ b/alkahest-py/src/lib.rs @@ -12029,10 +12029,10 @@ fn py_cuda_device_count() -> usize { #[cfg(feature = "groebner")] use alkahest_core::{ - dae_index_reduce_ranked, expr_to_gbpoly, gbpoly_to_expr, primary_decomposition, - radical as core_ideal_radical, rosenfeld_groebner_ranked, DaeIndexReduction, GbPoly, - GroebnerBasis, MonomialOrder, ParamGbPoly, ParamGroebnerBasis, ParamGroebnerError, ParamPoly, - QParam, + dae_index_reduce_ranked, expr_to_gbpoly, expr_to_param_gbpoly, gbpoly_to_expr, + primary_decomposition, radical as core_ideal_radical, rosenfeld_groebner_parametric, + rosenfeld_groebner_ranked, DaeIndexReduction, GbPoly, GroebnerBasis, MonomialOrder, + ParamGbPoly, ParamGroebnerBasis, ParamGroebnerError, ParamPoly, ParametricProlongOpts, QParam, }; /// A sparse multivariate polynomial over ℚ, as used by the Gröbner machinery. @@ -12997,22 +12997,15 @@ impl PyParamGroebnerBasis { "a symbol cannot be both a ring variable and a coefficient-field parameter", )); } - let mut all_ids = var_ids.clone(); - all_ids.extend_from_slice(¶m_ids); - let pool_py = polys[0].pool.clone_ref(py); let mut gens = Vec::with_capacity(polys.len()); { let pool = pool_py.borrow(py); for p in &polys { - let gbp = expr_to_gbpoly(p.id, &all_ids, &pool.inner) + // Rational in the parameters is fine — that is the coefficient + // field. Only a denominator in a *ring variable* is refused. + let pg = expr_to_param_gbpoly(p.id, &var_ids, ¶m_ids, &pool.inner) .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; - let pg = ParamGbPoly::from_gbpoly(&gbp, var_ids.len(), param_ids.len()) - .ok_or_else(|| { - pyo3::exceptions::PyValueError::new_err( - "internal: polynomial arity does not match vars + params", - ) - })?; gens.push(pg); } } @@ -13180,16 +13173,44 @@ impl PyParamGroebnerBasis { /// "E-PARAMGB-004"`` rather than handing back something that is not a /// basis — check first with :meth:`is_regular_at` if that is a normal /// outcome for your caller. + /// + /// Parameters + /// ---------- + /// values : list + /// One rational value per parameter, in :meth:`parameters` order. + /// verify : bool, optional + /// When the point is on the locus, re-solve the specialised system over + /// ℚ and compare, instead of refusing on :meth:`conditions` alone. + /// :meth:`conditions` is sufficient but not necessary — most of it is + /// leading coefficients that were inverted inside the Buchberger loop + /// and then cancelled — so on small-integer parameter grids **a quarter + /// to a half of the refusals are unnecessary**, dominated by parameters + /// equal to exactly 0. With ``verify=True`` those points return a + /// basis; genuinely degenerate points still raise ``E-PARAMGB-004``. + /// The default is ``False`` because verification costs a second Gröbner + /// basis over ℚ. It changes only *completeness* — the refusals were + /// never unsound. + /// + /// Example:: + /// + /// gb = alkahest.GroebnerBasis.compute([a*x - y, c*x + d*y - one], + /// [x, y], params=[a, c, d]) + /// gb.specialize([0, 1, 1]) # E-PARAMGB-004 + /// gb.specialize([0, 1, 1], verify=True) # a GroebnerBasis + #[pyo3(signature = (values, verify=false))] fn specialize( &self, py: Python<'_>, values: Vec>, + verify: bool, ) -> PyResult { let vals = self.rational_values(&values)?; - let gens = self - .inner - .specialize(&vals) - .map_err(param_groebner_error_to_py)?; + let gens = if verify { + self.inner.specialize_verified(&vals) + } else { + self.inner.specialize(&vals) + } + .map_err(param_groebner_error_to_py)?; Ok(PyGroebnerBasis { inner: GroebnerBasis::from_generators(gens, self.inner.order()), pool: Some(self.pool.clone_ref(py)), @@ -13264,18 +13285,57 @@ impl PyParamGroebnerBasis { /// Reduce a polynomial modulo this basis; the remainder is a /// :class:`ParametricGbPoly`. /// - /// Accepts a :class:`ParametricGbPoly` or an :class:`Expr`. + /// Accepts a :class:`ParametricGbPoly` or an :class:`Expr`. The `Expr` may + /// be **rational in the parameters** — a ``den**-1`` factor is an ordinary + /// element of the coefficient field ``Q(params)``, not a non-polynomial — + /// so this basis's own :meth:`to_exprs` output is valid input. Only a + /// denominator in a *ring variable* is refused. fn reduce(&self, py: Python<'_>, p: &Bound<'_, PyAny>) -> PyResult { let poly = self.coerce(py, p)?; Ok(self.wrap(py, self.inner.reduce(&poly))) } /// Ideal membership: true exactly when :meth:`reduce` gives zero. + /// + /// Same input contract as :meth:`reduce`; in particular + /// ``gb.contains(gb.to_exprs()[i])`` is true for every ``i``. + /// + /// Example:: + /// + /// gb = alkahest.GroebnerBasis.compute([a*x - one], [x], params=[a]) + /// gb.to_exprs() # ['(x + (-1 * a^-1))'] + /// gb.contains(gb.to_exprs()[0]) # True fn contains(&self, py: Python<'_>, p: &Bound<'_, PyAny>) -> PyResult { let poly = self.coerce(py, p)?; Ok(self.inner.contains(&poly)) } + /// True when every generator of *other* lies in this ideal, i.e. + /// `` ⊆ `` in ``Q(params)[vars]``. + /// + /// Returns ``False`` when the two bases are written over different numbers + /// of variables or parameters — there is no shared ring to compare them in. + fn contains_ideal(&self, other: PyRef) -> bool { + self.inner.contains_ideal(&other.inner) + } + + /// True when *other* generates the **same ideal** of ``Q(params)[vars]``. + /// + /// This is exact, not generic: reduction over ``Q(params)`` only ever + /// divides by non-zero field elements, so neither basis's + /// :meth:`conditions` enters the answer. Those conditions still bound what + /// each basis says about a *specialised* parameter point — equal ideals + /// over the fraction field can specialise differently on the locus. + /// + /// Example:: + /// + /// g1 = alkahest.GroebnerBasis.compute([a*x - one], [x], params=[a]) + /// g2 = alkahest.GroebnerBasis.compute([a*a*x - a], [x], params=[a]) + /// g1.equals_ideal(g2) # True + fn equals_ideal(&self, other: PyRef) -> bool { + self.inner.equals_ideal(&other.inner) + } + fn __len__(&self) -> usize { self.inner.len() } @@ -13320,17 +13380,17 @@ impl PyParamGroebnerBasis { return Ok(pg.borrow().inner.clone()); } if let Ok(expr) = p.downcast::() { - let mut all_ids = self.var_ids.clone(); - all_ids.extend_from_slice(&self.param_ids); let pool = self.pool.borrow(py); - let gbp = expr_to_gbpoly(expr.borrow().id, &all_ids, &pool.inner) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; - return ParamGbPoly::from_gbpoly(&gbp, self.var_ids.len(), self.param_ids.len()) - .ok_or_else(|| { - pyo3::exceptions::PyValueError::new_err( - "internal: polynomial arity does not match vars + params", - ) - }); + // Over `Q(params)` a `den**-1` factor in the parameters is an + // ordinary coefficient, not a non-polynomial — so this accepts the + // basis's own `to_exprs()` output, which always carries them. + return expr_to_param_gbpoly( + expr.borrow().id, + &self.var_ids, + &self.param_ids, + &pool.inner, + ) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string())); } Err(pyo3::exceptions::PyTypeError::new_err( "expected a ParametricGbPoly or an Expr", @@ -13503,6 +13563,123 @@ impl PyDaeIndexReduction { } } +/// M9 × V2-13 — result of a **parametric** differential elimination. +/// +/// Returned by :func:`rosenfeld_groebner` when ``params`` is given. Same shape +/// as :class:`RosenfeldGroebnerResult`, except that :meth:`final_basis` is a +/// :class:`ParametricGroebnerBasis` over ``Q(params)`` — so it carries +/// :meth:`~ParametricGroebnerBasis.conditions` and +/// :meth:`~ParametricGroebnerBasis.specialize`, and the parameters never enter +/// the monomial order. +/// +/// Attributes +/// ---------- +/// consistent : bool +/// ``False`` iff the unit ideal was reached over ``Q(params)``. +/// truncated : bool +/// ``True`` if prolongation stopped on the budget or on ``minimal=True`` +/// rather than because differentiating stopped adding relations. A +/// truncated basis is a *sound* set of consequences of the system but need +/// not be complete. +/// prolongation_rounds : int +/// Number of prolongation rounds that contributed new relations. +/// minimal_prolongation_rounds : int or None +/// The lowest round count at which the elimination ideal with respect to +/// ``eliminate`` was non-empty; ``None`` when no round was informative or +/// no ``eliminate`` list was given. **Scope:** this is "the first round +/// that leaves a generator after elimination", not a theorem — for +/// multi-output models the criterion is known to be wrong, because one +/// output can become informative several rounds before the others. Treat +/// it as a cost signal, not a certificate. +#[cfg(feature = "groebner")] +#[pyclass(name = "ParametricRosenfeldGroebnerResult")] +struct PyParametricRosenfeldResult { + #[pyo3(get)] + consistent: bool, + #[pyo3(get)] + truncated: bool, + #[pyo3(get)] + prolongation_rounds: usize, + #[pyo3(get)] + minimal_prolongation_rounds: Option, + working_dae: DAE, + final_basis: Option, + pool: Py, + var_ids: Vec, + param_ids: Vec, +} + +#[cfg(feature = "groebner")] +#[pymethods] +impl PyParametricRosenfeldResult { + /// The prolonged :class:`DAE`: the input system plus every derivative jet + /// introduced while differentiating it. + fn working_dae(&self, py: Python<'_>) -> PyDAE { + PyDAE { + inner: self.working_dae.clone(), + pool: self.pool.clone_ref(py), + } + } + + /// The jet variables indexing the basis, in exponent-slot order. + /// + /// The parameters are **not** here — they are in the coefficient field. + fn variables(&self, py: Python<'_>) -> Vec { + self.var_ids + .iter() + .map(|&id| PyExpr { + id, + pool: self.pool.clone_ref(py), + }) + .collect() + } + + /// The parameters of the coefficient field, in order. + fn parameters(&self, py: Python<'_>) -> Vec { + self.param_ids + .iter() + .map(|&id| PyExpr { + id, + pool: self.pool.clone_ref(py), + }) + .collect() + } + + /// The saturated :class:`ParametricGroebnerBasis`, or ``None`` when the + /// system is inconsistent. + /// + /// Example:: + /// + /// r = alkahest.rosenfeld_groebner(dae, params=[a], eliminate=[x]) + /// io = r.final_basis().eliminate([x, dx]) + /// [str(e) for e in io.to_exprs()] + fn final_basis(&self, py: Python<'_>) -> PyResult>> { + match &self.final_basis { + None => Ok(None), + Some(gb) => Ok(Some(Py::new( + py, + PyParamGroebnerBasis { + inner: gb.clone(), + pool: self.pool.clone_ref(py), + var_ids: self.var_ids.clone(), + param_ids: self.param_ids.clone(), + }, + )?)), + } + } + + fn __repr__(&self) -> String { + format!( + "ParametricRosenfeldGroebnerResult(consistent={}, truncated={}, \ + prolongation_rounds={}, minimal_prolongation_rounds={:?})", + self.consistent, + self.truncated, + self.prolongation_rounds, + self.minimal_prolongation_rounds + ) + } +} + /// `alkahest.rosenfeld_groebner(dae, order=None, max_prolong_rounds=None)` — /// Rosenfeld–Gröbner-style differential elimination. /// @@ -13523,12 +13700,48 @@ impl PyDaeIndexReduction { /// Prolongation budget (default 8). Nonlinear jets often do not saturate /// in finitely many algebraic steps, so hitting the budget is normal and /// sets :attr:`RosenfeldGroebnerResult.truncated`. +/// params : list[Expr], optional +/// Symbols to put in the **coefficient field** ``Q(params)`` rather than in +/// the ring (M9). Without this every free symbol is a ring variable, so a +/// model parameter enlarges the monomial order, the pair schedule and the +/// staircase — which is the difference between eliminating states from +/// ``Q[states, jets, params]`` and from ``Q(params)[states, jets]``. With +/// ``params`` the return type is +/// :class:`ParametricRosenfeldGroebnerResult` and the basis is +/// :class:`ParametricGroebnerBasis`; without it nothing changes. +/// +/// The parameters must not be DAE variables, derivatives or the time +/// variable. +/// eliminate : list[Expr], optional +/// Variables the caller intends to eliminate — typically the unobserved +/// states. Their whole jet chain goes with them: naming ``x`` also names +/// ``dx/dt``, ``d2x/dt2``, … as prolongation introduces them. Supplying +/// this makes :attr:`ParametricRosenfeldGroebnerResult.minimal_prolongation_rounds` +/// available and enables the over-prolongation warning. Requires +/// ``params``. +/// minimal : bool, optional +/// Stop at the first prolongation round whose elimination ideal with +/// respect to ``eliminate`` is non-empty, instead of prolonging to the +/// budget. One prolongation too many is expensive out of all proportion — +/// on SIR it is 0.002 s and one 4-term relation against 20.9 s and thirteen +/// generators of up to 233 terms — and the over-supplied result is correct, +/// so nothing else signals it. Requires ``eliminate``. +/// +/// Not a minimality guarantee: see +/// :attr:`ParametricRosenfeldGroebnerResult.minimal_prolongation_rounds` +/// for the scope, in particular for multi-output models. /// /// Returns /// ------- -/// RosenfeldGroebnerResult +/// RosenfeldGroebnerResult or ParametricRosenfeldGroebnerResult /// Read the relations with ``result.final_basis().to_exprs()``. /// +/// Warns +/// ----- +/// UserWarning +/// When ``eliminate`` is given, the elimination ideal was already non-empty +/// at an earlier round, and ``minimal`` was not set. +/// /// Example:: /// /// t, x, dx = p.symbol("t"), p.symbol("x"), p.symbol("dx/dt") @@ -13536,15 +13749,115 @@ impl PyDaeIndexReduction { /// r = alkahest.rosenfeld_groebner(dae, max_prolong_rounds=1) /// r.consistent # True /// r.final_basis().to_exprs() # the eliminated relations, as Expr +/// +/// # a is a coefficient, not a fourth ring variable +/// dae = alkahest.DAE.new([dx - a*x], [x], [dx], t) +/// r = alkahest.rosenfeld_groebner(dae, params=[a], max_prolong_rounds=1) +/// r.final_basis().conditions() #[cfg(feature = "groebner")] #[pyfunction] -#[pyo3(name = "rosenfeld_groebner", signature = (dae, order=None, max_prolong_rounds=None))] +#[pyo3( + name = "rosenfeld_groebner", + signature = (dae, order=None, max_prolong_rounds=None, params=None, eliminate=None, minimal=false) +)] fn py_rosenfeld_groebner( py: Python<'_>, dae: PyRef, order: Option<&str>, max_prolong_rounds: Option, -) -> PyResult { + params: Option>>, + eliminate: Option>>, + minimal: bool, +) -> PyResult { + let param_ids: Vec = params + .as_ref() + .map(|ps| ps.iter().map(|p| p.id).collect()) + .unwrap_or_default(); + let eliminate_ids: Vec = eliminate + .as_ref() + .map(|es| es.iter().map(|e| e.id).collect()) + .unwrap_or_default(); + + if param_ids.is_empty() { + if !eliminate_ids.is_empty() || minimal { + return Err(pyo3::exceptions::PyValueError::new_err( + "eliminate= and minimal= are only available on the parametric path; \ + pass params=[...] as well", + )); + } + return py_rosenfeld_groebner_plain(py, dae, order, max_prolong_rounds); + } + if minimal && eliminate_ids.is_empty() { + return Err(pyo3::exceptions::PyValueError::new_err( + "minimal=True needs eliminate=[...]: there is no notion of an informative \ + prolongation round without knowing which variables are being eliminated", + )); + } + + let pool_py = dae.pool.clone_ref(py); + let requested_rounds = max_prolong_rounds.unwrap_or(8); + let out = { + let pool = pool_py.borrow(py); + rosenfeld_groebner_parametric( + &dae.inner, + &pool.inner, + ¶m_ids, + ParametricProlongOpts { + order: py_monomial_order_for_dae(order), + max_prolong_rounds: requested_rounds, + eliminate: &eliminate_ids, + minimal, + }, + ) + }; + let (r, ranking) = out.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + + // The blow-up from one prolongation too many is silent otherwise: the + // answer is correct, just enormously more expensive. + if !minimal { + if let Some(m) = r.minimal_prolongation_rounds { + if m < r.prolongation_rounds { + let msg = format!( + "rosenfeld_groebner prolonged {} rounds, but the elimination ideal was \ + already non-empty after {m}; the extra rounds are correct but can cost \ + orders of magnitude. Pass minimal=True to stop at the first informative \ + round (see minimal_prolongation_rounds for its scope).", + r.prolongation_rounds + ); + pyo3::PyErr::warn_bound( + py, + &py.get_type_bound::(), + &msg, + 1, + )?; + } + } + } + + Ok(Py::new( + py, + PyParametricRosenfeldResult { + consistent: r.consistent, + truncated: r.truncated, + prolongation_rounds: r.prolongation_rounds, + minimal_prolongation_rounds: r.minimal_prolongation_rounds, + working_dae: r.working_dae, + final_basis: r.final_basis, + pool: pool_py, + var_ids: ranking.vars, + param_ids, + }, + )? + .into_py(py)) +} + +#[cfg(feature = "groebner")] +fn py_rosenfeld_groebner_plain( + py: Python<'_>, + dae: PyRef, + order: Option<&str>, + max_prolong_rounds: Option, +) -> PyResult { let pool_py = dae.pool.clone_ref(py); let r = { let pool = pool_py.borrow(py); @@ -13556,15 +13869,19 @@ fn py_rosenfeld_groebner( ) }; let (r, ranking) = r.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; - Ok(PyRosenfeldGroebnerResult { - consistent: r.consistent, - truncated: r.truncated, - prolongation_rounds: r.prolongation_rounds, - working_dae: r.working_dae, - final_basis: r.final_basis, - pool: pool_py, - var_ids: ranking.vars, - }) + Ok(Py::new( + py, + PyRosenfeldGroebnerResult { + consistent: r.consistent, + truncated: r.truncated, + prolongation_rounds: r.prolongation_rounds, + working_dae: r.working_dae, + final_basis: r.final_basis, + pool: pool_py, + var_ids: ranking.vars, + }, + )? + .into_py(py)) } #[cfg(feature = "groebner")] @@ -15473,6 +15790,7 @@ fn alkahest(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/docs/mdbook/src/ode-dae.md b/docs/mdbook/src/ode-dae.md index 248e05b3..1c4fd2c8 100644 --- a/docs/mdbook/src/ode-dae.md +++ b/docs/mdbook/src/ode-dae.md @@ -133,6 +133,10 @@ result.variables() # [t, x, dx/dt, ddx/dt/dt] — jets, in exponent-slot order `final_basis()` returns `None` when `consistent` is `False`. +### Model parameters in the coefficient field + +By default every free symbol is a ring variable, so a model parameter enlarges the monomial order, the pair schedule and the staircase. `rosenfeld_groebner(dae, params=[...])` moves the listed symbols into `Q(params)` instead and returns a `ParametricRosenfeldGroebnerResult` whose `final_basis()` is a `ParametricGroebnerBasis`. With `eliminate=[...]` (the unobserved states, jet chains included) it also reports `minimal_prolongation_rounds` and warns when it prolonged past it, because one prolongation too many costs orders of magnitude for the same relation. See [Coefficient fields](./solving.md#differential-elimination-with-the-parameters-in-qparams) for the full surface, the scope of "minimal", and — where this is used for structural identifiability — which identifiability it decides (multi-experiment, by Ovchinnikov–Pillay–Pogudin–Scanlon Theorem 19). + ## Sensitivity analysis Sensitivity analysis computes how solutions depend on parameters. `sensitivity_system(ode, params)` augments the state with `∂x/∂p`; `adjoint_system(ode, objective_grad)` takes the gradient of the objective with respect to the state, as a list parallel to `ode.state_vars()`: diff --git a/docs/mdbook/src/solving.md b/docs/mdbook/src/solving.md index a4949a1c..cbc49098 100644 --- a/docs/mdbook/src/solving.md +++ b/docs/mdbook/src/solving.md @@ -160,9 +160,76 @@ gb.specialize([-1]) # raises ParamGroebnerError, code "E-PARAMGB-004" `conditions()` lists the hypersurfaces the computation assumed non-vanishing — every leading-coefficient inversion contributes its numerator and denominator, every input coefficient contributes its denominator — factored into irreducible, primitive pieces so the report is a list of conditions rather than one opaque polynomial in many parameters. The list is **sufficient, not necessary**: it can flag a point that turns out fine (a removable coincidence the bookkeeping cannot see), but it never misses a point where the generic basis is genuinely wrong. `specialize` refuses on the flagged locus with `ParamGroebnerError` (`E-PARAMGB-004`) rather than returning something that is not a basis; check `is_regular_at` first if a degenerate point is a normal outcome for your caller. -The read path matches `GroebnerBasis`: the object is a sequence of `ParametricGbPoly`, each with `to_expr()` / `terms()`, and the basis itself has `to_exprs()`, `eliminate(vars)` (same `Lex`-with-eliminated-variables-first contract, refuses to eliminate a coefficient-field parameter since there is nothing to eliminate), `reduce`, and `contains`. `GroebnerBasis.compute(..., params=None)` or `params=[]` is the unmodified `Q[vars]` path; `ParametricGroebnerBasis.compute(polys, vars, params, order=None)` is the equivalent direct constructor in `alkahest.experimental`. +### `specialize(values, verify=True)` -This surface is experimental (`alkahest.experimental.ParametricGroebnerBasis` / `ParametricGbPoly`) and requires `--features groebner`. +Most of what `conditions()` lists are leading coefficients that were inverted somewhere inside the Buchberger loop and then cancelled, so a large share of the points it excludes are ordinary. Measured over eight parametric systems on the small-integer box `{-2..2}`: 646 refusals, of which 239 are genuine poles, 73 are necessary, and **334 are unnecessary**. The fraction is box-dependent and dominated by parameters equal to exactly 0 — `{-2..2}` gives 52%, `{-2,-1,1,2}` gives 25%, `{-3..3}` gives 58% — so the honest headline is **a quarter to a half of refusals on small-integer grids**, not a single number. + +```python +gb = GroebnerBasis.compute([a*x + b*y - one, c*x + d*y - one], [x, y], + params=[a, b, c, d]) +gb.conditions() # [c, a*d - b*c, a] + +gb.specialize([0, 1, 1, 1]) # E-PARAMGB-004: a = 0 is on the locus +gb.specialize([0, 1, 1, 1], verify=True) # [x, y - 1] — the refusal was unnecessary +gb.specialize([1, 1, 2, 2], verify=True) # still E-PARAMGB-004: a*d - b*c = 0 +``` + +`verify=True` re-solves the specialised system over ℚ and compares it with the specialised generic basis, instead of deciding on the recorded conditions alone. It is strictly more complete and never less sound: the refusals were separately checked to be *sound* (1,930 regular points across three sweeps, zero disagreements between `specialize` and the basis computed directly at the point), so this closes a completeness gap, not a correctness one. It is off by default because on the locus it pays for a second Gröbner basis over ℚ. + +### Feeding a parametric basis back in + +`reduce` and `contains` accept input that is **rational in the parameters**. A `den**-1` factor is an ordinary element of `Q(params)`, not a non-polynomial, so a basis accepts its own `to_exprs()` output — only a denominator in a *ring variable* is refused: + +```python +gb = GroebnerBasis.compute([a*x - one], [x], params=[a]) +gb.to_exprs() # ['(x + (-1 * a^-1))'] +gb.contains(gb.to_exprs()[0]) # True +``` + +`equals_ideal` answers the question that needs: do two parametric bases generate the same ideal of `Q(params)[vars]`? + +```python +g1 = GroebnerBasis.compute([a*x - one], [x], params=[a]) +g2 = GroebnerBasis.compute([a*a*x - a], [x], params=[a]) +g1.equals_ideal(g2) # True +g1.contains_ideal(g2) # True — one direction only +``` + +This is exact, not generic: reduction over `Q(params)` only ever divides by non-zero field elements, so neither basis's `conditions()` enters the answer. Those conditions still bound what each basis says about a *specialised* parameter point — equal ideals over the fraction field can specialise differently on the locus. Bases over different numbers of variables or parameters compare `False`; there is no shared ring. + +The read path matches `GroebnerBasis`: the object is a sequence of `ParametricGbPoly`, each with `to_expr()` / `terms()`, and the basis itself has `to_exprs()`, `eliminate(vars)` (same `Lex`-with-eliminated-variables-first contract, refuses to eliminate a coefficient-field parameter since there is nothing to eliminate), `reduce`, `contains`, `contains_ideal` and `equals_ideal`. `GroebnerBasis.compute(..., params=None)` or `params=[]` is the unmodified `Q[vars]` path; `ParametricGroebnerBasis.compute(polys, vars, params, order=None)` is the equivalent direct constructor in `alkahest.experimental`. + +### Differential elimination with the parameters in `Q(params)` + +`rosenfeld_groebner(dae, params=[...])` runs the prolongation loop of [Rosenfeld–Gröbner](./ode-dae.md) over `Q(params)` instead of putting the model parameters in the ring, and returns a `ParametricRosenfeldGroebnerResult` whose `final_basis()` is a `ParametricGroebnerBasis`. That is the input–output relation of an ODE model computed end to end, rather than prolonged by hand: + +```python +# SIR: S' = -b*S*I, I' = b*S*I - g*I, R' = g*I, y0 = I +dae = DAE.new([dS + b*S*I, dI - b*S*I + g*I, dR - g*I, y0 - I], + [S, I, R, y0], [dS, dI, dR, dy0], t) + +r = rosenfeld_groebner(dae, params=[b, g], eliminate=[S, I, R], + minimal=True, order="lex") +r.minimal_prolongation_rounds # 2 +states = ("S", "I", "R") +io = r.final_basis().eliminate([v for v in r.variables() + if str(v).lstrip("d").split("/")[0] in states]) +io.to_exprs() # y0*y0'' - y0'^2 + b*y0^2*y0' + b*g*y0^3, over Q(b,g) +``` + +`eliminate=[...]` names the variables the caller intends to eliminate — typically the unobserved states. Their whole jet chain goes with them (`x` also names `dx/dt`, `d2x/dt2`, …), and the ranking is built with those jets first so the `Lex` elimination contract holds. + +**One prolongation too many is expensive out of all proportion.** On the SIR model above, stopping at the first informative round costs 0.03 s and gives one 4-term relation; prolonging one round further does not finish in ten minutes. The over-supplied answer is *correct*, just enormously more expensive, so nothing else signals it. `minimal=True` stops at the first informative round; without it, a run that prolonged past one emits a `UserWarning` naming the round it could have stopped at, and `minimal_prolongation_rounds` records it either way. + +**Scope of "minimal".** `minimal_prolongation_rounds` is "the first round at which eliminating those variables leaves a generator", not a theorem about the differential ideal. For a single-output model it coincides with the jet order the input–output relation needs; **for multi-output models the criterion is known to be wrong**, because one output can become informative several rounds before the others and the truncated basis then misses their relations. Treat it as a cost signal, not a certificate — and note that a `minimal=True` result is always flagged `truncated`. + +### Which structural identifiability this decides + +Where this pipeline is used for structural identifiability, it answers **multi-experiment** identifiability, not single-experiment: by Ovchinnikov–Pillay–Pogudin–Scanlon, [*Computing all identifiable functions of parameters for ODE models*](https://arxiv.org/abs/2004.07774), Theorem 19, a function of the parameters is multi-experiment identifiable iff it is input–output identifiable, for general rational systems. That is why an IO-elimination route can call a model globally identifiable where a single-experiment tool such as SIAN reports "locally, not globally" — both verdicts are right, about different questions. + +**Hypothesis, not a guarantee.** Theorem 19 requires the input–output equations to be the *characteristic presentation* of `I_Σ ∩ C(θ){y,u}`. An algebraic lex elimination at a hand-picked finite jet order — what the surface above does — is not guaranteed to produce one. So "this is a multi-experiment tool" holds under that standard; where it fails, the computed field can be a proper subfield of the identifiable one. + +This surface is experimental (`alkahest.experimental.ParametricGroebnerBasis` / `ParametricGbPoly` / `ParametricRosenfeldGroebnerResult`) and requires `--features groebner`. ## Performance diff --git a/python/alkahest/experimental/__init__.py b/python/alkahest/experimental/__init__.py index 49504287..30c0218c 100644 --- a/python/alkahest/experimental/__init__.py +++ b/python/alkahest/experimental/__init__.py @@ -114,7 +114,14 @@ ``GroebnerBasis.compute(polys, vars, params=[...])``. The basis is generic, so it reports the hypersurfaces its leading coefficients assumed non-zero (``conditions()``) and refuses to ``specialize()`` on them instead of - returning something that is not a basis + returning something that is not a basis — or, with ``specialize(pt, + verify=True)``, re-solves at the point and refuses only if the refusal was + really necessary +- :class:`ParametricRosenfeldGroebnerResult` — differential elimination with + the parameters in the coefficient field, from + ``rosenfeld_groebner(dae, params=[...])``; with ``eliminate=[...]`` it also + reports the first prolongation round that was informative, and warns when + more rounds were taken than that Numeric ODE integrators (Phase 16b): - :func:`ode_integrate_rk4` — fixed-step 4th-order Runge–Kutta integrator @@ -215,7 +222,11 @@ # M9 — Gröbner bases over the coefficient field Q(params). Registered by the # extension only on `groebner` builds, hence the suppressed import. with contextlib.suppress(ImportError): - from alkahest.alkahest import ParametricGbPoly, ParametricGroebnerBasis + from alkahest.alkahest import ( + ParametricGbPoly, + ParametricGroebnerBasis, + ParametricRosenfeldGroebnerResult, + ) with contextlib.suppress(ImportError): from alkahest.alkahest import CudaCompiledFn, compile_cuda @@ -240,6 +251,7 @@ # M9 — coefficient fields for elimination "ParametricGbPoly", "ParametricGroebnerBasis", + "ParametricRosenfeldGroebnerResult", # M11 — novelty filtering "QRecurrenceClaim", # M4 — root-of-unity specialisation diff --git a/tests/test_parametric_groebner.py b/tests/test_parametric_groebner.py index 9d030e98..7c9b5c96 100644 --- a/tests/test_parametric_groebner.py +++ b/tests/test_parametric_groebner.py @@ -416,3 +416,344 @@ def test_empty_params_list_is_the_ordinary_engine(pool): gb_plain = ak.GroebnerBasis.compute([x * x - one], [x]) assert isinstance(gb_param, ak.GroebnerBasis) assert [str(g) for g in gb_param.to_exprs()] == [str(g) for g in gb_plain.to_exprs()] + + +# --------------------------------------------------------------------------- +# 8. Feeding a parametric basis its own output back in (2026-08-19 issue #8) +# --------------------------------------------------------------------------- +# +# The basis lives in Q(params)[vars], so its generators carry `den**-1` factors +# in the parameters by construction. `contains` / `reduce` used to route those +# through the denominator-free Expr -> GbPoly conversion and refuse with +# "negative exponent -1 in polynomial", which made the trivially-true +# `gb.contains(gb.to_exprs()[i])` unrunnable -- and with it the question a loop +# actually needs: do these two parametric bases generate the same ideal? + + +def test_contains_accepts_the_basis_own_generators(pool): + a, x = pool.symbol("a"), pool.symbol("x") + one = pool.integer(1) + + gb = ak.GroebnerBasis.compute([a * x - one], [x], params=[a]) + + # The generator really does carry a parameter denominator. + assert "a^-1" in str(gb.to_exprs()[0]) + + for i, e in enumerate(gb.to_exprs()): + assert gb.contains(e) is True, f"generator {i} not recognised: {e}" + assert gb.reduce(e).is_zero is True + # Denominator-free input keeps working. + assert gb.contains(a * x - one) is True + # ...as does the ParametricGbPoly read path. + assert all(gb.contains(p.to_expr()) for p in gb.polynomials()) + + +def test_contains_accepts_own_generators_multivariate(pool): + a, x, y = pool.symbol("a"), pool.symbol("x"), pool.symbol("y") + one = pool.integer(1) + + gb = ak.GroebnerBasis.compute([a * x - y, x + y - one], [x, y], params=[a]) + for i, e in enumerate(gb.to_exprs()): + assert gb.contains(e) is True, f"generator {i} not recognised: {e}" + + +def test_nonparametric_control_still_accepts_its_own_generators(pool): + x = pool.symbol("x") + one = pool.integer(1) + + gb = ak.GroebnerBasis.compute([2 * x - one], [x]) + for e in gb.to_exprs(): + assert gb.contains(e) is True + + +def test_a_denominator_in_a_ring_variable_is_still_refused(pool): + a, x = pool.symbol("a"), pool.symbol("x") + one = pool.integer(1) + + gb = ak.GroebnerBasis.compute([a * x - one], [x], params=[a]) + with pytest.raises(ValueError, match="negative exponent"): + gb.contains(x ** -1) + + +def test_parametric_input_may_be_rational_in_the_parameters(pool): + a, x = pool.symbol("a"), pool.symbol("x") + one = pool.integer(1) + + # x - 1/a is the same ideal as a*x - 1, written with a denominator. + gb = ak.GroebnerBasis.compute([x - a ** -1], [x], params=[a]) + assert gb.contains(a * x - one) is True + + +def test_equals_ideal_answers_the_two_bases_question(pool): + a, x = pool.symbol("a"), pool.symbol("x") + one = pool.integer(1) + + g1 = ak.GroebnerBasis.compute([a * x - one], [x], params=[a]) + g2 = ak.GroebnerBasis.compute([a * a * x - a], [x], params=[a]) + g3 = ak.GroebnerBasis.compute([x - one], [x], params=[a]) + + assert g1.equals_ideal(g2) is True + assert g2.equals_ideal(g1) is True + assert g1.equals_ideal(g3) is False + assert g1.contains_ideal(g1) is True + + # One-directional containment is reported as such: is not , + # but == <1> contains both. + big = ak.GroebnerBasis.compute([a * x - one, x], [x], params=[a]) + assert big.contains_ideal(g1) is True + assert g1.contains_ideal(big) is False + + +def test_equals_ideal_refuses_to_compare_across_rings(pool): + a, b, x = pool.symbol("a"), pool.symbol("b"), pool.symbol("x") + one = pool.integer(1) + + g1 = ak.GroebnerBasis.compute([a * x - one], [x], params=[a]) + g2 = ak.GroebnerBasis.compute([b * x - one], [x], params=[a, b]) + assert g1.equals_ideal(g2) is False + + +# --------------------------------------------------------------------------- +# 9. specialize(values, verify=True) (2026-08-19 issue #14) +# --------------------------------------------------------------------------- +# +# `conditions()` is sufficient but not necessary, and on small-integer grids a +# quarter to a half of the refusals are unnecessary -- dominated by parameters +# equal to exactly 0, which are intermediate leading-coefficient inversions +# that cancelled. `verify=True` recomputes and compares instead of refusing on +# the recorded conditions alone. The refusals were separately verified *sound*, +# so this is a completeness fix; the sound path must not weaken. + + +@pytest.fixture +def cramer(pool): + """a*x + b*y = 1, c*x + d*y = 1 over Q(a, b, c, d).""" + a, b, c, d = (pool.symbol(s) for s in "abcd") + x, y = pool.symbol("x"), pool.symbol("y") + one = pool.integer(1) + gb = ak.GroebnerBasis.compute( + [a * x + b * y - one, c * x + d * y - one], [x, y], params=[a, b, c, d] + ) + return gb + + +def test_verify_accepts_a_point_the_recorded_conditions_refuse(cramer): + # a = 0 is listed in conditions() although the only denominator in the + # returned basis is a*d - b*c, which is -1 here. + assert cramer.is_regular_at([0, 1, 1, 1]) is False + assert "a" in [str(c) for c in cramer.conditions()] + + with pytest.raises(ak.ParamGroebnerError) as exc: + cramer.specialize([0, 1, 1, 1]) + assert exc.value.code == "E-PARAMGB-004" + + verified = cramer.specialize([0, 1, 1, 1], verify=True) + assert isinstance(verified, ak.GroebnerBasis) + # b*y = 1 and x + y = 1 => y = 1, x = 0. + assert sorted(str(e).replace(" ", "") for e in verified.to_exprs()) == [ + "(y+-1)", + "x", + ] + + +def test_verify_still_refuses_a_genuinely_singular_point(cramer): + # a*d - b*c = 0: the basis really does have a pole, and the specialised + # system is a different (dependent) system. + for pt in ([1, 1, 1, 1], [1, 1, 2, 2]): + with pytest.raises(ak.ParamGroebnerError) as exc: + cramer.specialize(pt, verify=True) + assert exc.value.code == "E-PARAMGB-004" + + +def test_verify_agrees_with_the_direct_basis_where_it_accepts(pool, cramer): + a, b, c, d = (pool.symbol(s) for s in "abcd") + x, y = pool.symbol("x"), pool.symbol("y") + one = pool.integer(1) + + pt = [0, 1, 1, 1] + verified = cramer.specialize(pt, verify=True) + direct = ak.GroebnerBasis.compute( + [pool.integer(0) * x + one * y - one, one * x + one * y - one], [x, y] + ) + assert sorted(str(e).replace(" ", "") for e in verified.to_exprs()) == sorted( + str(e).replace(" ", "") for e in direct.to_exprs() + ) + + +def test_verify_defaults_off_and_leaves_the_sound_path_alone(pool, cramer): + # Regular points are unaffected either way. + for verify in (False, True): + gb = cramer.specialize([1, 2, 3, 4], verify=verify) + assert isinstance(gb, ak.GroebnerBasis) + with pytest.raises(ak.ParamGroebnerError): + cramer.specialize([1, 1, 1, 1]) + + +def test_verify_recovers_a_measurable_share_of_a_small_integer_grid(cramer): + """A quarter to a half of refusals on small-integer grids are unnecessary. + + Kept to the 4-parameter `cramer` system on {-1, 0, 1} so it stays a + CI-tier test; the full 8-system {-2..2} sweep gives 646 refused / 334 + recovered / 239 genuine poles. + """ + import itertools + + refused = recovered = 0 + for pt in itertools.product([-1, 0, 1], repeat=4): + pt = list(pt) + if cramer.is_regular_at(pt): + continue + refused += 1 + try: + cramer.specialize(pt, verify=True) + recovered += 1 + except ak.ParamGroebnerError: + pass + assert refused > 0 + # Every recovered point is a real recovery; the interesting claim is that + # the share is substantial rather than a rounding error. + assert 0.25 <= recovered / refused <= 0.5, (refused, recovered) + + +# --------------------------------------------------------------------------- +# 10. rosenfeld_groebner(dae, params=[...]) (2026-08-19 issues #16 and #13) +# --------------------------------------------------------------------------- + + +def _decay_dae(pool): + """x' = -a*x with output y = x, as a DAE. IO relation: y' + a*y = 0.""" + t, x, y = pool.symbol("t"), pool.symbol("x"), pool.symbol("y") + dx, dy = pool.symbol("dx/dt"), pool.symbol("dy/dt") + a = pool.symbol("a") + dae = ak.DAE.new([dx + a * x, y - x], [x, y], [dx, dy], t) + return dae, x, y, a + + +def test_rosenfeld_groebner_accepts_params(pool): + """Issue #16: this raised TypeError -- M9 did not compose with V2-13.""" + dae, x, _y, a = _decay_dae(pool) + + r = ak.rosenfeld_groebner(dae, params=[a], max_prolong_rounds=1, order="lex") + + assert isinstance(r.final_basis(), ake.ParametricGroebnerBasis) + assert r.consistent is True + # `a` is a coefficient, not a ring variable. + assert "a" not in [str(v) for v in r.variables()] + assert [str(p) for p in r.parameters()] == ["a"] + assert r.final_basis().n_params == 1 + + +def test_rosenfeld_groebner_parametric_gives_the_io_relation(pool): + dae, x, y, a = _decay_dae(pool) + + r = ak.rosenfeld_groebner( + dae, params=[a], eliminate=[x], minimal=True, order="lex" + ) + basis = r.final_basis() + state_jets = [ + v for v in r.variables() if str(v).lstrip("d").split("/")[0] == "x" + ] + io = basis.eliminate(state_jets) + + assert len(io) == 1 + # y + y'/a = 0, i.e. y' = -a*y. + dy = pool.symbol("dy/dt") + assert io.contains(a * y + dy) is True + + +def test_rosenfeld_groebner_minimal_stops_at_the_first_informative_round(pool): + dae, x, _y, a = _decay_dae(pool) + + minimal = ak.rosenfeld_groebner( + dae, params=[a], eliminate=[x], minimal=True, max_prolong_rounds=3, + order="lex", + ) + assert minimal.minimal_prolongation_rounds == 1 + assert minimal.prolongation_rounds == 1 + assert minimal.truncated is True + + with pytest.warns(UserWarning, match="already non-empty after 1"): + over = ak.rosenfeld_groebner( + dae, params=[a], eliminate=[x], max_prolong_rounds=3, order="lex" + ) + assert over.prolongation_rounds == 3 + assert over.minimal_prolongation_rounds == 1 + + # Over-supplying is *correct*, just more expensive -- that is exactly why + # nothing else signals it. + state_jets = [ + v for v in over.variables() if str(v).lstrip("d").split("/")[0] == "x" + ] + assert len(over.final_basis().eliminate(state_jets)) > len( + minimal.final_basis().eliminate( + [v for v in minimal.variables() + if str(v).lstrip("d").split("/")[0] == "x"] + ) + ) + + +def test_rosenfeld_groebner_sir_minimal_matches_the_hand_relation(pool): + """SIR at the minimal informative jet order (2026-08-19 issue #13). + + S' = -b*S*I, I' = b*S*I - g*I, R' = g*I, y0 = I. By hand, + I*I'' - I'^2 + b*I^2*I' + b*g*I^3 = 0. The state decouples -- R never + enters the relation -- so "as many derivatives as there are states" is one + too many, and one too many is four orders of magnitude here. + """ + t = pool.symbol("t") + S, I, R, y0 = (pool.symbol(s) for s in ("S", "I", "R", "y0")) + dS, dI, dR, dy0 = ( + pool.symbol(s) for s in ("dS/dt", "dI/dt", "dR/dt", "dy0/dt") + ) + b, g = pool.symbol("b"), pool.symbol("g") + dae = ak.DAE.new( + [dS + b * S * I, dI - b * S * I + g * I, dR - g * I, y0 - I], + [S, I, R, y0], + [dS, dI, dR, dy0], + t, + ) + + r = ak.rosenfeld_groebner( + dae, params=[b, g], eliminate=[S, I, R], minimal=True, + max_prolong_rounds=4, order="lex", + ) + # Two derivatives of the output, not three. + assert r.minimal_prolongation_rounds == 2 + + state_jets = [ + v for v in r.variables() + if str(v).lstrip("d").split("/")[0] in ("S", "I", "R") + ] + io = r.final_basis().eliminate(state_jets) + assert len(io) == 1 + + y1, y2 = pool.symbol("dy0/dt"), pool.symbol("ddy0/dt/dt") + hand = y0 * y2 - y1 * y1 + b * y0 * y0 * y1 + b * g * y0 * y0 * y0 + assert io.contains(hand) is True + # ...and the relation is the *whole* content: 4 terms, not 233. + assert io.polynomials()[0].n_terms == 4 + + +def test_eliminate_and_minimal_require_params(pool): + dae, x, _y, _a = _decay_dae(pool) + + with pytest.raises(ValueError, match="params"): + ak.rosenfeld_groebner(dae, eliminate=[x]) + with pytest.raises(ValueError, match="params"): + ak.rosenfeld_groebner(dae, minimal=True) + + +def test_minimal_requires_eliminate(pool): + dae, _x, _y, a = _decay_dae(pool) + + with pytest.raises(ValueError, match="eliminate"): + ak.rosenfeld_groebner(dae, params=[a], minimal=True) + + +def test_rosenfeld_groebner_without_params_is_unchanged(pool): + t, x, dx = pool.symbol("t"), pool.symbol("x"), pool.symbol("dx/dt") + dae = ak.DAE.new([dx - x], [x], [dx], t) + + r = ak.rosenfeld_groebner(dae, max_prolong_rounds=1) + assert isinstance(r.final_basis(), ak.GroebnerBasis) + assert r.consistent is True From 28b2fc25fcadaa0f835090ab93bb408116a467c2 Mon Sep 17 00:00:00 2001 From: Areg Gevorgyan Date: Fri, 21 Aug 2026 02:29:39 +0000 Subject: [PATCH 11/11] fix(budget): an out-of-memory exact solve refused, not an uncatchable abort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-08-19 issues #5, #10 and 26d — three resource gaps, one theme: the engines could not say "no". #5 (High). `telescope_md` on the m = 4 multinomial, **at its own default arguments**, ran ~20 minutes and then died with `GNU MP: Cannot allocate memory (size=8)` and a core dump. Two more subsystems reached the same end by different routes and with a second allocator. There was no exception, no `BudgetExceededError`, no `try`/`except` that could help, and no memory analogue of `Budget.wall_ms` — so an unattended loop lost the whole interpreter and every result it was holding, not just the offending call. This cannot be fixed inside the allocator, and the commit does not pretend otherwise. GMP's contract for a replacement allocation function forbids returning NULL — the library has no failure path to take — and a Rust `panic!` may not cross a C frame. What a replacement *can* do soundly is count. `budget::memory` wraps GMP's own allocation functions (delegating to them, never allocating, never unwinding, never returning NULL where they would not) and maintains a live-byte total; the refusal then happens in ordinary Rust at Alkahest's own cooperative checkpoints, *before* the allocation that would have died. Two ceilings feed those checkpoints: * `Budget(max_bytes=...)` / `budget::enter_with_memory` (`E-BUDGET-004`) — the size budget the engines were missing. Their existing ceilings bound the *shape* of a linear system (how many unknowns); it is the *size of its numbers* that exhausts memory. * the address-space guard (`E-BUDGET-005`), which needs **no budget at all**: under a finite `RLIMIT_AS` (`ulimit -v`, a container limit) the process refuses within a reserve of its own limit. The operator already said how much the process may have, and stopping inside that number is strictly better than dying at it — which is what makes the default-arguments case survivable. What remains, stated plainly: the guard is checkpoint-granular, so a single allocation large enough to cross the whole reserve in one step still aborts; address-space usage is only observable on Linux; and `max_bytes` counts GMP memory, not Rust-side allocations. #10 (Medium/High). `q_zeilberger` consulted no budget and had no ceiling — `Σ_k [2n;k]_q` ran 8+ minutes at the documented defaults and had to be killed, while `Σ_k [n;k]_q` decides in half a second. It now has ceilings of the same shape `telescope_md` has (per-probe and cumulative system size) plus the two the shape ceilings are blind to: a size ceiling on `Q(q)(x)` elements, and a bit-length/work ceiling on the `Z[q][x]` subresultant gcd underneath every field operation, where a *small* system spends its minutes. The gcd ceilings are opt-in via `qfield::GcdWorkScope`, so `RatK` arithmetic in engines that have not asked for them is untouched. 26d (Medium). `prove_nonneg` ran 418.5 s inside `Budget(wall_ms=3000)` and ended in `E-SOS-002`. Budget checks at the search-loop boundaries, and — the part that matters more than the timing — the trip is recorded out of band so the bindings raise `BudgetExceededError`. `E-SOS-002` already conflates exhausted / budget-limited / never-attempted, and a loop that reads a timeout as "not SOS" records a false negative. Every engine error enum here is public and exhaustive, so none of them grows a `Budget` variant; the real cause travels through `budget::record_trip` / `take_trip` and the bindings re-raise it, the pattern `calculus::limits::last_budget_trip` established. `Budget` itself is unchanged for the same reason — `max_bytes` rides beside it via `enter_with_memory`, and `cargo semver-checks` reports no semver update required. Also replaces the wall-clock assertion in `chained_product_at_original_bounds_refuses_fast_via_resource_ceiling` with the deterministic quantity the ceiling actually bounds (unknowns spent on large probes). That assertion was measuring the machine — ~76 s idle against 1100-1300 s loaded — not the ceiling. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + alkahest-core/Cargo.toml | 4 + alkahest-core/src/budget/memory.rs | 322 +++++++++++++ alkahest-core/src/budget/mod.rs | 450 ++++++++++++++++++ alkahest-core/src/errors/codes.rs | 6 + alkahest-core/src/holonomic/qfield.rs | 227 +++++++++ alkahest-core/src/holonomic/qzeil/field.rs | 100 ++++ alkahest-core/src/holonomic/qzeil/mod.rs | 60 +++ alkahest-core/src/holonomic/qzeil/search.rs | 234 ++++++++- .../src/holonomic/telescoping2d/mod.rs | 72 ++- .../src/holonomic/telescoping2d/search.rs | 86 +++- alkahest-core/src/real/sos/mod.rs | 64 +++ alkahest-core/src/real/sos/psd.rs | 12 + alkahest-py/src/lib.rs | 75 ++- alkahest-skill/alkahest.md | 11 +- docs/mdbook/src/budgets.md | 67 ++- docs/mdbook/src/errors.md | 2 +- python/alkahest/_budget.py | 44 +- python/alkahest/_context.py | 7 +- tests/test_resource_budgets.py | 267 +++++++++++ 20 files changed, 2065 insertions(+), 46 deletions(-) create mode 100644 alkahest-core/src/budget/memory.rs create mode 100644 tests/test_resource_budgets.py diff --git a/Cargo.lock b/Cargo.lock index 336f098c..23b8e987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,7 @@ dependencies = [ "egglog", "gmp-mpfr-sys", "inkwell", + "libc", "libm", "num-integer", "proptest", diff --git a/alkahest-core/Cargo.toml b/alkahest-core/Cargo.toml index 659e8489..4ea570e9 100644 --- a/alkahest-core/Cargo.toml +++ b/alkahest-core/Cargo.toml @@ -65,6 +65,10 @@ libm = "0.2" # force-cross lets gmp-mpfr-sys build when the Rust host and target triples # differ (e.g. x86_64-pc-windows-msvc host → x86_64-pc-windows-gnu target). gmp-mpfr-sys = { version = "1", features = ["force-cross"] } +# getrlimit(RLIMIT_AS)/sysconf for the address-space guard in `budget::memory`, +# which refuses before GMP's uncatchable out-of-memory abort. Already in the +# lock file transitively via gmp-mpfr-sys. +libc = "0.2" egglog = { version = "0.4", optional = true } inkwell = { version = "0.9", features = ["llvm15-0-prefer-dynamic"], optional = true } cranelift-jit = { version = "0.132", optional = true } diff --git a/alkahest-core/src/budget/memory.rs b/alkahest-core/src/budget/memory.rs new file mode 100644 index 00000000..9b236faa --- /dev/null +++ b/alkahest-core/src/budget/memory.rs @@ -0,0 +1,322 @@ +//! Memory accounting for the exact-arithmetic paths, and the address-space +//! guard that turns an imminent out-of-memory `abort()` into a refusal. +//! +//! # The failure this exists to remove +//! +//! `rug`/GMP's default reaction to a failed allocation is to print +//! `GNU MP: Cannot allocate memory (size=N)` and call `abort()`. The Rust +//! allocator's is `memory allocation of N bytes failed` followed by the same +//! `abort()`. Neither is catchable: an exact-rational solve that outgrows the +//! machine takes the whole interpreter with it, and an unattended research +//! loop loses every result it was holding, not just the offending call. +//! +//! # What is enforceable, and what is not +//! +//! GMP's contract for a replacement allocation function is that it *must not +//! return `NULL`* — the library has no failure path to take. Nor may a +//! replacement unwind: a Rust `panic!` crossing a C frame is undefined +//! behaviour (and, since Rust 1.81, aborts anyway). So a custom allocator +//! cannot itself convert an out-of-memory condition into an error. +//! +//! What it *can* do soundly is **count**. The functions installed by +//! [`install`] delegate to whatever GMP was already using and maintain a +//! process-wide live-byte total. Nothing in them allocates, unwinds, or +//! returns `NULL`, so they are safe to run inside GMP frames. The refusal +//! then happens at Alkahest's own cooperative checkpoints +//! ([`crate::budget::check_all`]), which are ordinary Rust code returning an +//! ordinary `Err` — a *pre-flight* refusal, before the allocation that would +//! have died is attempted. +//! +//! Two ceilings feed those checkpoints: +//! +//! * **`Budget::max_bytes`** — an explicit, caller-supplied ceiling on how +//! much GMP memory one guarded block may hold live. See +//! [`crate::budget::enter_with_memory`]. +//! * **The address-space guard** — active with no budget at all. When the +//! process runs under a finite `RLIMIT_AS` (`ulimit -v`, a container limit, +//! a batch scheduler), [`headroom_exhausted`] reports when the process has +//! climbed to within [`reserve_bytes`] of that limit, and the checkpoint +//! refuses. This is what makes the *default-arguments* case survivable: +//! the operator already said how much the process may have, so Alkahest +//! stops inside that number instead of dying at it. With no limit set +//! (`ulimit -v unlimited`), the guard is inert and behaviour is unchanged. +//! +//! # What remains +//! +//! The guard is checkpoint-granular. A single allocation large enough to jump +//! the whole reserve in one step, between two consecutive checkpoints, still +//! aborts — nothing short of a fallible allocator can fix that, and GMP does +//! not have one. `reserve_bytes` is sized to make that unlikely rather than +//! impossible. Address-space *usage* is only observable on Linux +//! (`/proc/self/statm`); on other platforms the guard degrades to the +//! `Budget::max_bytes` ceiling alone. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Once, OnceLock}; + +use gmp_mpfr_sys::gmp; + +// --------------------------------------------------------------------------- +// GMP allocation accounting +// --------------------------------------------------------------------------- + +/// Live bytes currently held by GMP allocations, process-wide. +static LIVE_BYTES: AtomicU64 = AtomicU64::new(0); + +/// The allocation functions GMP was using before [`install`] wrapped them, +/// stored as raw addresses because a function pointer is not a `const`- +/// initialisable atomic. Written once, before `INSTALLED` is set. +static ORIG_ALLOC: AtomicUsize = AtomicUsize::new(0); +static ORIG_REALLOC: AtomicUsize = AtomicUsize::new(0); +static ORIG_FREE: AtomicUsize = AtomicUsize::new(0); + +static INSTALLED: AtomicBool = AtomicBool::new(false); +static INSTALL_ONCE: Once = Once::new(); + +type AllocFn = extern "C" fn(usize) -> *mut c_void; +type ReallocFn = extern "C" fn(*mut c_void, usize, usize) -> *mut c_void; +type FreeFn = unsafe extern "C" fn(*mut c_void, usize); + +fn add_live(n: usize) { + LIVE_BYTES.fetch_add(n as u64, Ordering::Relaxed); +} + +fn sub_live(n: usize) { + // Saturating, not wrapping: blocks allocated *before* `install` ran are + // freed through our wrapper without ever having been counted, so the + // total can legitimately try to go negative. Under-counting only ever + // makes the ceiling fire late, never spuriously. + let _ = LIVE_BYTES.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| { + Some(v.saturating_sub(n as u64)) + }); +} + +extern "C" fn wrap_alloc(size: usize) -> *mut c_void { + let orig: AllocFn = unsafe { std::mem::transmute(ORIG_ALLOC.load(Ordering::Acquire)) }; + let p = orig(size); + add_live(size); + p +} + +extern "C" fn wrap_realloc(ptr: *mut c_void, old: usize, new: usize) -> *mut c_void { + let orig: ReallocFn = unsafe { std::mem::transmute(ORIG_REALLOC.load(Ordering::Acquire)) }; + let p = orig(ptr, old, new); + sub_live(old); + add_live(new); + p +} + +unsafe extern "C" fn wrap_free(ptr: *mut c_void, size: usize) { + let orig: FreeFn = std::mem::transmute(ORIG_FREE.load(Ordering::Acquire)); + orig(ptr, size); + sub_live(size); +} + +/// Install the counting wrappers around GMP's current allocation functions. +/// +/// Idempotent and thread-safe (guarded by a [`Once`]); returns `true` if +/// accounting is active. Call it as early as the embedding allows — the +/// Python extension does it from its module initialiser — but it is *not* +/// required to run before GMP's first allocation: the wrappers delegate to +/// the functions that were installed at the time, so a block allocated +/// before installation is still freed by the allocator that produced it, and +/// the only consequence of installing late is that the live total starts from +/// a baseline it never saw allocated (handled by [`sub_live`]'s saturation). +/// +/// GMP's own documentation warns that changing the allocation functions while +/// other threads are inside GMP is unsafe; the [`Once`] makes the swap happen +/// exactly once, and the intended call site is process start-up. +pub fn install() -> bool { + INSTALL_ONCE.call_once(|| { + let mut alloc: gmp::allocate_function = None; + let mut realloc: gmp::reallocate_function = None; + let mut free: gmp::free_function = None; + // SAFETY: three out-pointers to live locals, which is exactly what + // `mp_get_memory_functions` expects. + unsafe { gmp::get_memory_functions(&mut alloc, &mut realloc, &mut free) }; + let (Some(alloc), Some(realloc), Some(free)) = (alloc, realloc, free) else { + // GMP always reports concrete functions (the defaults if nothing + // was installed); if it somehow does not, leave it alone. + return; + }; + ORIG_ALLOC.store(alloc as usize, Ordering::Release); + ORIG_REALLOC.store(realloc as usize, Ordering::Release); + ORIG_FREE.store(free as usize, Ordering::Release); + // SAFETY: the wrappers delegate to the functions just captured, never + // return NULL where the original did not, never unwind, and never + // allocate — the three things GMP requires of a replacement. + unsafe { + gmp::set_memory_functions(Some(wrap_alloc), Some(wrap_realloc), Some(wrap_free)); + } + INSTALLED.store(true, Ordering::Release); + }); + INSTALLED.load(Ordering::Acquire) +} + +/// `true` if [`install`] has swapped in the counting wrappers. +pub fn is_installed() -> bool { + INSTALLED.load(Ordering::Acquire) +} + +/// Bytes currently held live by GMP allocations, process-wide. +/// +/// Zero when [`install`] has not run. Process-wide rather than per-thread +/// because GMP's allocation hooks are global: a block allocated on one thread +/// may be freed on another, so a thread-local total could not stay balanced. +pub fn gmp_live_bytes() -> u64 { + LIVE_BYTES.load(Ordering::Relaxed) +} + +// --------------------------------------------------------------------------- +// Address-space guard +// --------------------------------------------------------------------------- + +/// The soft `RLIMIT_AS` of this process in bytes, or `None` when it is +/// unlimited (or cannot be read). +/// +/// Read once and cached: a process that raises its own limit mid-run is not a +/// case worth a syscall on every checkpoint, and caching can only make the +/// guard *more* conservative. +pub fn address_space_limit() -> Option { + static LIMIT: OnceLock> = OnceLock::new(); + *LIMIT.get_or_init(read_address_space_limit) +} + +#[cfg(unix)] +fn read_address_space_limit() -> Option { + let mut rl = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `getrlimit` writes a `struct rlimit` through the out-pointer. + if unsafe { libc::getrlimit(libc::RLIMIT_AS, &mut rl) } != 0 { + return None; + } + if rl.rlim_cur == libc::RLIM_INFINITY { + None + } else { + // The cast is a no-op where `rlim_t` is already `u64` (Linux, macOS) + // and load-bearing where it is not, so it stays. + #[allow(clippy::unnecessary_cast)] + Some(rl.rlim_cur as u64) + } +} + +#[cfg(not(unix))] +fn read_address_space_limit() -> Option { + None +} + +/// Virtual address space currently mapped by this process, in bytes. +/// +/// Linux only (`/proc/self/statm`); `None` elsewhere, which disables the +/// address-space guard rather than guessing. +#[cfg(target_os = "linux")] +pub fn address_space_used() -> Option { + use std::io::Read; + let mut buf = [0u8; 64]; + let mut f = std::fs::File::open("/proc/self/statm").ok()?; + let n = f.read(&mut buf).ok()?; + let text = std::str::from_utf8(buf.get(..n)?).ok()?; + let pages: u64 = text.split_whitespace().next()?.parse().ok()?; + Some(pages.saturating_mul(page_size())) +} + +#[cfg(not(target_os = "linux"))] +pub fn address_space_used() -> Option { + None +} + +#[cfg(target_os = "linux")] +fn page_size() -> u64 { + static PAGE: OnceLock = OnceLock::new(); + *PAGE.get_or_init(|| { + // SAFETY: `sysconf` takes an int and returns a long; no pointers. + let n = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if n > 0 { + n as u64 + } else { + 4096 + } + }) +} + +/// Headroom the address-space guard keeps in reserve below `RLIMIT_AS`. +/// +/// Sized to be crossed by *many* checkpoint intervals, not by one: the +/// allocations between two consecutive checkpoints in an exact-rational +/// elimination are limb arrays measured in kilobytes, so 32 MiB is hundreds of +/// them. +/// +/// It is deliberately **not** a large fraction of the limit. `RLIMIT_AS` caps +/// virtual address space, and importing the extension already maps ~600 MB of +/// it (arena reservations and thread stacks, ~46 MB of which is resident), so +/// a reserve of "an eighth of the limit" would refuse every call under a +/// 900 MB `ulimit -v` — including the ones that fit comfortably today. A flat +/// floor with a gentle fraction for large limits keeps the guard out of the +/// way until the process is genuinely at the edge. +pub fn reserve_bytes(limit: u64) -> u64 { + const FLOOR: u64 = 32 * 1024 * 1024; + const CEILING: u64 = 256 * 1024 * 1024; + (limit / 64).clamp(FLOOR, CEILING) +} + +/// `Some((used, limit))` when the process has climbed to within +/// [`reserve_bytes`] of its address-space limit, else `None`. +/// +/// `None` — the guard is inert — when no finite `RLIMIT_AS` is set, or when +/// address-space usage is not observable on this platform. +pub fn headroom_exhausted() -> Option<(u64, u64)> { + let limit = address_space_limit()?; + let used = address_space_used()?; + if used.saturating_add(reserve_bytes(limit)) >= limit { + Some((used, limit)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gmp_accounting_tracks_a_big_rational() { + assert!(install(), "GMP accounting must install"); + let before = gmp_live_bytes(); + let big = { + let mut z = rug::Integer::from(1); + z <<= 8_000_000; // ~1 MB of limbs + z + }; + let during = gmp_live_bytes(); + assert!( + during > before + 500_000, + "expected the shift to be counted: {before} -> {during}" + ); + drop(big); + let after = gmp_live_bytes(); + assert!( + after < during, + "freeing must decrement the live total: {during} -> {after}" + ); + } + + #[test] + fn reserve_is_clamped_to_the_floor_and_ceiling() { + assert_eq!(reserve_bytes(1024), 32 * 1024 * 1024); + assert_eq!(reserve_bytes(900_000_000), 32 * 1024 * 1024); + assert_eq!(reserve_bytes(64 * 1024 * 1024 * 1024), 256 * 1024 * 1024); + } + + #[test] + fn headroom_guard_is_inert_without_a_limit() { + // The test binary itself runs with no `ulimit -v` in CI, so the guard + // must not fire. (When a limit *is* set the subprocess regression + // test in tests/ exercises the firing path.) + if address_space_limit().is_none() { + assert_eq!(headroom_exhausted(), None); + } + } +} diff --git a/alkahest-core/src/budget/mod.rs b/alkahest-core/src/budget/mod.rs index df502601..f9eebe48 100644 --- a/alkahest-core/src/budget/mod.rs +++ b/alkahest-core/src/budget/mod.rs @@ -51,6 +51,21 @@ //! | `E-BUDGET-001` | [`BudgetError::WallClock`] | wall-clock deadline elapsed | //! | `E-BUDGET-002` | [`BudgetError::Steps`] | step counter exceeded `max_steps` | //! | `E-BUDGET-003` | [`BudgetError::Cancelled`] | [`request_cancel`] was called | +//! +//! Memory ceilings are reported through [`BudgetTrip`] rather than +//! [`BudgetError`]: `BudgetError` is a public *exhaustive* enum, so growing it +//! a `Memory` variant is a major semver break. [`check_all`] returns +//! [`BudgetTrip`], which wraps a [`BudgetError`] or carries one of the two +//! memory refusals: +//! +//! | Code | Variant | Cause | +//! |----------------|-------------------------------|-------------------| +//! | `E-BUDGET-004` | [`BudgetTrip::Memory`] | the active budget's `max_bytes` ceiling | +//! | `E-BUDGET-005` | [`BudgetTrip::AddressSpace`] | the process is about to exhaust `RLIMIT_AS` | +//! +//! See [`mod@memory`] for why the memory ceiling is enforced at these +//! checkpoints rather than inside a fallible allocator (GMP does not have +//! one, and a Rust `panic!` may not cross a C frame). use crate::errors::AlkahestError; use std::cell::{Cell, RefCell}; @@ -58,6 +73,10 @@ use std::fmt; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; +pub mod memory; + +pub use memory::{gmp_live_bytes, install as install_memory_accounting}; + // --------------------------------------------------------------------------- // Budget // --------------------------------------------------------------------------- @@ -112,6 +131,13 @@ struct Frame { max_steps: Option, steps: Cell, seed: Option, + /// Ceiling on GMP bytes held live by this block, from + /// [`enter_with_memory`]. + max_bytes: Option, + /// GMP live-byte total when this frame was pushed, so `max_bytes` measures + /// *this block's* appetite rather than whatever the process was already + /// holding. + bytes_at_entry: u64, } thread_local! { @@ -148,12 +174,33 @@ impl Drop for BudgetGuard { /// frame (matching the non-merging nesting semantics of /// `alkahest.context(...)` on the Python side). pub fn enter(budget: Budget) -> BudgetGuard { + enter_with_memory(budget, None) +} + +/// [`enter`], plus a ceiling on the GMP memory the guarded block may hold +/// live (see [`mod@memory`]); `None` imposes no ceiling. +/// +/// # Why this is not a `Budget` field +/// +/// [`Budget`] has only public fields and is not `#[non_exhaustive]`, so every +/// caller may build one with a struct literal — and adding a field to such a +/// struct is a major semver break (`cargo semver-checks`' +/// `constructible_struct_adds_field`). A free function is additive, so the +/// memory ceiling travels beside the budget instead of inside it. The Python +/// binding still spells it `Budget(max_bytes=...)`, because a `@dataclass` +/// with a defaulted trailing field *is* additive there. +pub fn enter_with_memory(budget: Budget, max_bytes: Option) -> BudgetGuard { + if max_bytes.is_some() { + memory::install(); + } let frame = Frame { start: Instant::now(), wall: budget.wall, max_steps: budget.max_steps, steps: Cell::new(0), seed: budget.seed, + max_bytes, + bytes_at_entry: memory::gmp_live_bytes(), }; STACK.with(|s| s.borrow_mut().push(frame)); BudgetGuard { @@ -172,6 +219,25 @@ pub fn seed() -> Option { STACK.with(|s| s.borrow().last().and_then(|f| f.seed)) } +/// The `max_bytes` ceiling of the innermost active budget frame, or `None`. +pub fn max_bytes() -> Option { + STACK.with(|s| s.borrow().last().and_then(|f| f.max_bytes)) +} + +/// GMP bytes held live *by the innermost active budget frame* — the total now, +/// less the total when that frame was entered. +/// +/// Zero when no budget is active. Saturating: a block allocated before the +/// frame was entered and freed inside it would otherwise underflow. +pub fn bytes_used() -> u64 { + STACK.with(|s| { + s.borrow() + .last() + .map(|f| memory::gmp_live_bytes().saturating_sub(f.bytes_at_entry)) + .unwrap_or(0) + }) +} + // --------------------------------------------------------------------------- // Cancellation — process-wide, not scoped to a thread or Budget frame // --------------------------------------------------------------------------- @@ -253,6 +319,175 @@ pub fn check() -> Result<(), BudgetError> { }) } +// --------------------------------------------------------------------------- +// Memory checkpoints +// --------------------------------------------------------------------------- + +/// How often [`check_memory`] pays for a `/proc/self/statm` read. +/// +/// The per-frame `max_bytes` ceiling is a pair of atomic loads and is checked +/// every time; the address-space probe is a file read (a few microseconds), so +/// it runs on every `PROBE_INTERVAL`-th call — often enough that a pivot loop +/// cannot climb a whole [`memory::reserve_bytes`] between probes, rarely +/// enough not to show up in a profile. +const PROBE_INTERVAL: u32 = 16; + +/// How many probe intervals' worth of *observed* growth the address-space +/// guard keeps in reserve, on top of [`memory::reserve_bytes`]. +/// +/// A flat reserve alone is a bet that no probe interval can cross it, and that +/// bet is wrong for a solve whose matrix entries double: under a 900 MB +/// `ulimit -v` the m = 4 multinomial was seen to add 26 MB between two +/// consecutive probes. Scaling the reserve by the growth actually observed +/// makes the guard tighten exactly when the workload accelerates, and stay out +/// of the way when it does not — which matters, because the import alone maps +/// ~600 MB of address space, so a large flat reserve would refuse work that +/// fits. +const GROWTH_RESERVE_FACTOR: u64 = 4; + +thread_local! { + static PROBE_TICK: Cell = const { Cell::new(0) }; + /// Address space mapped at the previous probe, for the growth term above. + /// `0` means "no history yet on this thread". + static LAST_VSZ: Cell = const { Cell::new(0) }; + /// The trip behind the engine-specific error the current thread is about + /// to return — see [`record_trip`]. + static LAST_TRIP: Cell> = const { Cell::new(None) }; +} + +/// [`check`] *and* the memory ceilings — the checkpoint a heavy exact- +/// arithmetic loop should call. +/// +/// Returns [`BudgetTrip::Budget`] for the wall/step/cancel cases (so an +/// existing `check` call site can be upgraded without changing what it +/// reports) and [`BudgetTrip::Memory`] / [`BudgetTrip::AddressSpace`] for the +/// memory ones. +pub fn check_all() -> Result<(), BudgetTrip> { + check()?; + check_memory() +} + +/// The memory half of [`check_all`], without the wall/step/cancel checks. +pub fn check_memory() -> Result<(), BudgetTrip> { + let framed = STACK.with(|s| { + let stack = s.borrow(); + stack.last().and_then(|f| { + f.max_bytes.map(|limit| { + ( + limit, + memory::gmp_live_bytes().saturating_sub(f.bytes_at_entry), + ) + }) + }) + }); + if let Some((limit, used)) = framed { + if used > limit { + return Err(BudgetTrip::Memory { limit, used }); + } + } + if memory::address_space_limit().is_some() { + let probe = PROBE_TICK.with(|t| { + let n = t.get().wrapping_add(1); + t.set(n); + n % PROBE_INTERVAL == 1 + }); + if probe { + if let (Some(limit), Some(used)) = + (memory::address_space_limit(), memory::address_space_used()) + { + let prev = LAST_VSZ.with(|c| c.replace(used)); + let growth = if prev == 0 { + 0 + } else { + used.saturating_sub(prev) + }; + let reserve = + memory::reserve_bytes(limit).max(growth.saturating_mul(GROWTH_RESERVE_FACTOR)); + if used.saturating_add(reserve) >= limit { + return Err(BudgetTrip::AddressSpace { + limit, + used, + reserve, + }); + } + } + } + } + Ok(()) +} + +/// Pre-flight check for a caller that is about to allocate `bytes` in one go. +/// +/// Unlike [`check_memory`] this always pays for the address-space probe: a +/// site that can name its allocation size up front is exactly the site where +/// one step can cross the whole reserve, so it is worth a syscall to refuse +/// *before* the allocation rather than after the next checkpoint. +pub fn check_alloc(bytes: u64) -> Result<(), BudgetTrip> { + check()?; + let framed = STACK.with(|s| { + let stack = s.borrow(); + stack.last().and_then(|f| { + f.max_bytes.map(|limit| { + ( + limit, + memory::gmp_live_bytes().saturating_sub(f.bytes_at_entry), + ) + }) + }) + }); + if let Some((limit, used)) = framed { + if used.saturating_add(bytes) > limit { + return Err(BudgetTrip::Memory { + limit, + used: used.saturating_add(bytes), + }); + } + } + if let (Some(limit), Some(used)) = (memory::address_space_limit(), memory::address_space_used()) + { + let projected = used.saturating_add(bytes); + let reserve = memory::reserve_bytes(limit); + if projected.saturating_add(reserve) >= limit { + return Err(BudgetTrip::AddressSpace { + limit, + used: projected, + reserve, + }); + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Out-of-band trip reporting +// --------------------------------------------------------------------------- + +/// Record the trip that is about to be reported as an engine-specific error. +/// +/// Engines whose error enums are public and exhaustive (`HolonomicError`, +/// `Telescoping2dError`, `SosError`, …) cannot grow a `Budget` variant without +/// a major semver break, so they return their own "gave up" variant and leave +/// the real cause here for the bindings to pick up with [`take_trip`] — the +/// pattern [`crate::calculus::limits::last_budget_trip`] established for +/// wall-clock trips inside `LimitError::DepthExceeded`. +/// +/// Call [`clear_trip`] at the outermost entry of such an engine so a stale +/// trip from an earlier call can never be attributed to this one. +pub fn record_trip(trip: BudgetTrip) { + LAST_TRIP.with(|c| c.set(Some(trip))); +} + +/// Take (and clear) the trip recorded by [`record_trip`] on this thread. +pub fn take_trip() -> Option { + LAST_TRIP.with(|c| c.take()) +} + +/// Clear any recorded trip. Call at the outermost entry of an engine that +/// reports trips out of band. +pub fn clear_trip() { + LAST_TRIP.with(|c| c.set(None)); +} + // --------------------------------------------------------------------------- // Error type // --------------------------------------------------------------------------- @@ -317,6 +552,116 @@ impl AlkahestError for BudgetError { } } +/// Why a [`check_all`] checkpoint stopped a call. +/// +/// # Why this is not a `BudgetError` variant +/// +/// [`BudgetError`] is a public *exhaustive* enum: adding `Memory` to it is a +/// major semver break, and so is marking it `#[non_exhaustive]` to allow one +/// later. This enum is new, so it can be `#[non_exhaustive]` from birth and +/// grow without breaking anyone; [`From`] keeps the two in one +/// `?`-chain. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum BudgetTrip { + /// A wall-clock, step, or cancellation trip — see [`BudgetError`]. + Budget(BudgetError), + /// The active budget's `max_bytes` ceiling was reached + /// ([`enter_with_memory`], `Budget(max_bytes=...)` in Python). + Memory { + /// The ceiling, in bytes. + limit: u64, + /// GMP bytes held live by the guarded block when it tripped. + used: u64, + }, + /// The process is within [`memory::reserve_bytes`] of its `RLIMIT_AS`, so + /// the next large allocation would `abort()` inside GMP or the Rust + /// allocator rather than fail. + /// + /// Unlike [`BudgetTrip::Memory`] this fires with **no budget active**: the + /// operator who set `ulimit -v` (or a container memory limit) already said + /// how much the process may have, and refusing inside that number is + /// strictly better than dying at it. + AddressSpace { + /// The process's soft `RLIMIT_AS`, in bytes. + limit: u64, + /// Address space mapped by the process when it tripped, in bytes. + used: u64, + /// Headroom the guard was keeping below `limit` — the flat reserve of + /// [`memory::reserve_bytes`], widened by the growth observed between + /// the last two probes. + reserve: u64, + }, +} + +impl From for BudgetTrip { + fn from(e: BudgetError) -> Self { + BudgetTrip::Budget(e) + } +} + +impl BudgetTrip { + /// The [`BudgetError`] behind a wall/step/cancel trip, or `None` for a + /// memory trip. + pub fn budget_error(&self) -> Option { + match self { + BudgetTrip::Budget(e) => Some(*e), + _ => None, + } + } +} + +impl fmt::Display for BudgetTrip { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BudgetTrip::Budget(e) => e.fmt(f), + BudgetTrip::Memory { limit, used } => write!( + f, + "budget exceeded: exact-arithmetic memory limit {limit} bytes reached \ + ({used} bytes held live)" + ), + BudgetTrip::AddressSpace { + limit, + used, + reserve, + } => write!( + f, + "budget exceeded: refusing before the process address-space limit \ + ({used} of {limit} bytes mapped, reserve {reserve} bytes) — the allocation \ + that would follow cannot fail safely, GMP and the Rust allocator both abort" + ), + } + } +} + +impl std::error::Error for BudgetTrip {} + +impl AlkahestError for BudgetTrip { + fn code(&self) -> &'static str { + match self { + BudgetTrip::Budget(e) => e.code(), + BudgetTrip::Memory { .. } => "E-BUDGET-004", + BudgetTrip::AddressSpace { .. } => "E-BUDGET-005", + } + } + + fn remediation(&self) -> Option<&'static str> { + match self { + BudgetTrip::Budget(e) => e.remediation(), + BudgetTrip::Memory { .. } => Some( + "raise Budget(max_bytes=...), or ask for a smaller problem (fewer unknowns, \ + a lower order/degree) — the exact coefficients, not the shape of the system, \ + are what grew", + ), + BudgetTrip::AddressSpace { .. } => Some( + "raise the process address-space limit (ulimit -v, or the container/cgroup \ + memory limit), or ask for a smaller problem — this refusal replaces the \ + uncatchable abort that would otherwise follow", + ), + } + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -454,6 +799,111 @@ mod tests { assert_eq!(check().unwrap_err(), BudgetError::Cancelled); } + /// ~8 MB of GMP limbs, well clear of the 1 MiB ceiling the memory tests + /// set and of any noise from tests running in parallel (the GMP live-byte + /// total is process-wide by necessity — see `memory::gmp_live_bytes`). + fn big_gmp_integer() -> rug::Integer { + let mut z = rug::Integer::from(1); + z <<= 64_000_000; + z + } + + #[test] + fn memory_budget_trips_when_gmp_memory_passes_max_bytes() { + let _serial = serial(); + let _guard = enter_with_memory(Budget::new(), Some(1 << 20)); + assert!( + check_all().is_ok(), + "an empty frame must not trip before anything is allocated" + ); + let z = big_gmp_integer(); + let err = check_all().unwrap_err(); + assert_eq!(err.code(), "E-BUDGET-004"); + assert!( + matches!(err, BudgetTrip::Memory { limit, used } if limit == 1 << 20 && used > limit), + "{err:?}" + ); + drop(z); + } + + #[test] + fn a_generous_memory_budget_does_not_trip() { + let _serial = serial(); + // 1 TiB: nothing this process can allocate reaches it, so this pins + // that the ceiling is a ceiling and not an unconditional refusal. + let _guard = enter_with_memory(Budget::new(), Some(1 << 40)); + let z = big_gmp_integer(); + assert!(check_all().is_ok()); + drop(z); + } + + #[test] + fn check_alloc_refuses_before_the_allocation_is_made() { + let _serial = serial(); + let _guard = enter_with_memory(Budget::new(), Some(1 << 20)); + // The refusal is entirely pre-flight — this frame has allocated + // nothing — which is the whole point: GMP has no failure path to take + // once it has been asked for the memory. + let err = check_alloc(4 << 20).unwrap_err(); + assert_eq!(err.code(), "E-BUDGET-004"); + // The reported `used` is the *projected* total, i.e. it accounts for + // the allocation that has not happened. Not asserted exactly: GMP's + // live-byte total is process-wide (its allocation hooks are), so a + // test running in parallel can contribute to this frame's delta. + assert!( + matches!(err, BudgetTrip::Memory { used, .. } if used >= 4 << 20), + "{err:?}" + ); + } + + #[test] + fn max_bytes_is_visible_and_scoped_to_its_frame() { + let _serial = serial(); + assert_eq!(max_bytes(), None); + { + let _guard = enter_with_memory(Budget::new(), Some(4096)); + assert_eq!(max_bytes(), Some(4096)); + // A plain `enter` shadows it, matching the non-merging nesting of + // every other budget field. + let _inner = enter(Budget::new()); + assert_eq!(max_bytes(), None); + } + assert_eq!(max_bytes(), None); + } + + #[test] + fn trips_round_trip_through_the_out_of_band_carrier() { + let _serial = serial(); + clear_trip(); + assert_eq!(take_trip(), None); + record_trip(BudgetTrip::Memory { limit: 1, used: 2 }); + let taken = take_trip().expect("recorded"); + assert_eq!(taken.code(), "E-BUDGET-004"); + // Taking clears, so a later unrelated error cannot inherit it. + assert_eq!(take_trip(), None); + } + + #[test] + fn budget_trip_codes_have_remediation() { + for trip in [ + BudgetTrip::Budget(BudgetError::Cancelled), + BudgetTrip::Memory { limit: 1, used: 2 }, + BudgetTrip::AddressSpace { + limit: 3, + used: 2, + reserve: 1, + }, + ] { + assert!(trip.code().starts_with("E-BUDGET-")); + assert!(trip.remediation().is_some()); + assert!(!trip.to_string().is_empty()); + } + assert_eq!( + BudgetTrip::from(BudgetError::Cancelled).budget_error(), + Some(BudgetError::Cancelled) + ); + } + #[test] fn error_codes_have_remediation() { for err in [ diff --git a/alkahest-core/src/errors/codes.rs b/alkahest-core/src/errors/codes.rs index 911e2edf..8703b8f0 100644 --- a/alkahest-core/src/errors/codes.rs +++ b/alkahest-core/src/errors/codes.rs @@ -234,6 +234,12 @@ pub const REGISTRY: &[ErrorSpec] = &[ ErrorSpec { code: "E-BUDGET-001", class: "BudgetError", cause: Cause::Resource, remediation: Some("raise Budget(wall_ms=...), or accept a heuristic/numeric result for this candidate instead of an exact one") }, ErrorSpec { code: "E-BUDGET-002", class: "BudgetError", cause: Cause::Resource, remediation: Some("raise Budget(max_steps=...), or accept a partial/heuristic result for this candidate instead of an exact one") }, ErrorSpec { code: "E-BUDGET-003", class: "BudgetError", cause: Cause::Resource, remediation: Some("call alkahest.clear_cancel() (Python) or budget::clear_cancel() (Rust) before starting the next candidate") }, + // E-BUDGET-004/005 — BudgetTrip. Carried on `BudgetTrip`, not `BudgetError`: + // the latter is an exhaustive public enum, so a `Memory` variant would be a + // major semver break. 005 fires with no budget active — it is the guard that + // turns GMP's `abort()` under a `ulimit -v` into a catchable refusal. + ErrorSpec { code: "E-BUDGET-004", class: "BudgetError", cause: Cause::Resource, remediation: Some("raise Budget(max_bytes=...), or ask for a smaller problem (fewer unknowns, a lower order/degree) — the exact coefficients, not the shape of the system, are what grew") }, + ErrorSpec { code: "E-BUDGET-005", class: "BudgetError", cause: Cause::Resource, remediation: Some("raise the process address-space limit (ulimit -v, or the container/cgroup memory limit), or ask for a smaller problem — this refusal replaces the uncatchable abort that would otherwise follow") }, // E-DEPTH — DepthLimitError (expression nesting ceiling; see kernel::depth). // Resource, not UserInput: the expression is well-formed, we decline to // recurse over it because a native stack overflow would kill the process. diff --git a/alkahest-core/src/holonomic/qfield.rs b/alkahest-core/src/holonomic/qfield.rs index 8f842255..7891e645 100644 --- a/alkahest-core/src/holonomic/qfield.rs +++ b/alkahest-core/src/holonomic/qfield.rs @@ -468,6 +468,14 @@ impl IPoly { fn pow_usize(&self, e: usize) -> Self { let mut acc = IPoly::one(); for _ in 0..e { + // Subresultant PRS raises the leading coefficient to `delta`, and + // `delta` grows with the degree gap, so one call here can be the + // whole runtime of a gcd. Bailing leaves a *wrong* power, which + // `BiPoly::gcd` never uses: its own loop-top check fires on the + // next iteration and returns `1`. + if gcd_should_stop(ipoly_bits(&acc)) { + return acc; + } acc = acc.mul(self); } acc @@ -568,6 +576,13 @@ impl IPoly { let lb = b.lc(); let mut e = (r.degree() - db + 1) as usize; while !r.is_zero() && r.degree() >= db { + // `r` is left partially reduced when this fires, which would be a + // wrong remainder — but the only caller is `BiPoly::gcd`, which + // re-checks the flag before using it and returns `1` instead. See + // `GcdStop`. + if gcd_should_stop(ipoly_bits(&r)) { + return r; + } let shift = (r.degree() - db) as usize; let lr = r.lc(); r = r.scale_int(&lb).sub(&b.shift_pow(shift).scale_int(&lr)); @@ -582,6 +597,9 @@ impl IPoly { /// gcd in `Z[n]` by subresultant PRS, normalized to a positive leading /// coefficient (`0` only when both inputs are `0`). fn gcd(a: &Self, b: &Self) -> Self { + if gcd_should_stop(0) { + return IPoly::one(); + } let mut x = a.clone().trim(); let mut y = b.clone().trim(); if x.is_zero() { @@ -604,8 +622,16 @@ impl IPoly { let mut g = Integer::from(1); let mut h = Integer::from(1); loop { + // Same contract as `BiPoly::gcd`: `1` is the safe degenerate + // answer, so a stopped gcd costs a cancellation, never soundness. + if gcd_should_stop(ipoly_bits(&x).saturating_add(ipoly_bits(&y))) { + return IPoly::one(); + } let delta = (x.degree() - y.degree()) as usize; let r = IPoly::pseudo_rem(&x, &y); + if gcd_stopped() { + return IPoly::one(); + } if r.is_zero() { break; } @@ -795,6 +821,11 @@ impl BiPoly { let lb = b.lc(); let mut e = (r.degree() - db + 1) as usize; while !r.is_zero() && r.degree() >= db { + // See `IPoly::pseudo_rem`: a stopped remainder is wrong, and + // `BiPoly::gcd` re-checks the flag before it is used. + if gcd_should_stop(bipoly_bits(&r)) { + return r; + } let shift = (r.degree() - db) as usize; let lr = r.lc(); r = r.scale(&lb).sub(&b.shift_pow(shift).scale(&lr)); @@ -809,6 +840,9 @@ impl BiPoly { /// gcd in `Z[n][k]` by Brown's subresultant PRS (content handled /// separately, as the algorithm requires primitive inputs). fn gcd(a: &Self, b: &Self) -> Self { + if gcd_should_stop(0) { + return BiPoly::one(); + } let mut x = a.clone().trim(); let mut y = b.clone().trim(); if x.is_zero() { @@ -831,8 +865,18 @@ impl BiPoly { let mut g = IPoly::one(); let mut h = IPoly::one(); loop { + // `1` is the safe degenerate answer: `RatK::normalize`'s caller + // treats a unit gcd as "nothing to cancel" and leaves the + // representation alone, so a stopped gcd costs readability, never + // correctness. + if gcd_should_stop(bipoly_bits(&x).saturating_add(bipoly_bits(&y))) { + return BiPoly::one(); + } let delta = (x.degree() - y.degree()) as usize; let r = BiPoly::pseudo_rem(&x, &y); + if gcd_stopped() { + return BiPoly::one(); + } if r.is_zero() { break; } @@ -859,6 +903,189 @@ impl BiPoly { } } +// --------------------------------------------------------------------------- +// Cooperative stop for the Z[n][k] gcd +// --------------------------------------------------------------------------- + +/// Total bit-length one operand of [`BiPoly::gcd`] may reach before the gcd +/// gives up. +/// +/// The subresultant PRS is where a *small* system spends minutes: degrees fall +/// on every step while the integer coefficients grow, so every ceiling phrased +/// in degrees or unknown-counts — which is every ceiling the search layers +/// above have — is blind to it. 2^20 bits (128 KB of integers across one gcd +/// operand pair) is far above anything the working cases in this crate's tests +/// reach and still bounds the pathological ones: `Σ_k [2n;k]_q` crosses it +/// while its *degrees* are still in the twenties, where `Σ_k [n;k]_q` decides +/// in half a second without coming near it. +const MAX_GCD_BITS: u64 = 1 << 20; + +/// Why the `Z[n][k]` gcd stopped early, if it did. +#[derive(Clone, Copy, Debug)] +pub enum GcdStop { + /// An active [`crate::budget`] stopped it. + Budget(crate::budget::BudgetTrip), + /// An operand passed [`MAX_GCD_BITS`]; the payload is the size it reached. + Size(u64), + /// The active [`GcdWorkScope`] passed [`MAX_GCD_WORK`] units of work. + Work(u64), +} + +thread_local! { + static GCD_STOP: std::cell::Cell> = const { std::cell::Cell::new(None) }; + static GCD_WORK: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Subresultant-PRS *work* — gcd invocations plus PRS steps — that one +/// [`GcdWorkScope`] may spend in this module. +/// +/// The bit-length ceiling above catches *one* object growing without bound; +/// this catches the other shape of the same problem, which is the one +/// `Σ_k [2n;k]_q` has: thousands of individually unremarkable gcds, each +/// triggered by one `Q(q)(x)` multiplication, adding up to minutes with no +/// single object ever looking large. The same idea as `modular.rs`'s +/// `BINOMIAL_WORK_BUDGET`. +const MAX_GCD_WORK: u64 = 2_000_000; + +/// An opt-in scope for the [`MAX_GCD_WORK`] and [`MAX_GCD_BITS`] ceilings. +/// +/// Outside every scope both are **inert**, so an engine that has not asked for +/// them is bit-for-bit unaffected: `RatK` is shared by the classical +/// Zeilberger search, the boundary analysis and the hypergeometric-term +/// parser, and silently changing what those normalise is not a change this +/// module gets to make on their behalf. +/// +/// Scopes nest by save/restore rather than by combining, matching +/// [`crate::budget`]'s frames. +pub struct GcdWorkScope { + prev: Option, +} + +impl Drop for GcdWorkScope { + fn drop(&mut self) { + let prev = self.prev; + GCD_WORK.with(|c| c.set(prev)); + // A stop is meaningful only inside the scope that produced it. Leaving + // one set would disable cancellation in `RatK` for every *later* call + // on this thread — including engines that never opted in — and an + // un-normalised `Q(n)(k)` grows without bound, so the leak would show + // up as the next unrelated call hanging. + clear_gcd_stop(); + } +} + +/// Enter a [`GcdWorkScope`]; the ceilings apply until the guard is dropped. +pub fn enter_gcd_work_scope() -> GcdWorkScope { + clear_gcd_stop(); + let prev = GCD_WORK.with(|c| c.replace(Some(0))); + GcdWorkScope { prev } +} + +/// Work spent in the innermost active [`GcdWorkScope`], or `0` outside one. +pub fn gcd_work() -> u64 { + GCD_WORK.with(|c| c.get().unwrap_or(0)) +} + +/// Clear any recorded gcd stop. Call before a unit of work whose result you +/// intend to check with [`take_gcd_stop`]. +pub fn clear_gcd_stop() { + GCD_STOP.with(|c| c.set(None)); +} + +/// Take (and clear) the gcd stop recorded on this thread, if any. +pub fn take_gcd_stop() -> Option { + GCD_STOP.with(|c| c.take()) +} + +/// `true` when a gcd on this thread has already given up — see [`GcdStop`]. +pub fn gcd_stop_pending() -> bool { + gcd_stopped() +} + +fn gcd_stopped() -> bool { + GCD_STOP.with(|c| { + let v = c.get(); + c.set(v); + v.is_some() + }) +} + +fn note_gcd_stop(s: GcdStop) { + GCD_STOP.with(|c| c.set(Some(s))); +} + +/// `true` when a [`GcdWorkScope`] is active, i.e. when the ceilings apply. +fn ceilings_active() -> bool { + GCD_WORK.with(|c| c.get().is_some()) +} + +/// Count one unit of gcd work; `true` when the ceiling has been passed. +/// Always `false` outside a [`GcdWorkScope`]. +fn spend_gcd_work() -> bool { + GCD_WORK.with(|c| match c.get() { + None => false, + Some(n) => { + let n = n.saturating_add(1); + c.set(Some(n)); + n > MAX_GCD_WORK + } + }) +} + +/// `true` when this gcd should give up, recording why. +/// +/// Everything here is gated on an active [`GcdWorkScope`]: outside one the +/// function is a single thread-local load and returns `false`, so `RatK` +/// arithmetic in engines that have not opted in is untouched — including the +/// budget check, whose caller has its own checkpoints and does not need this +/// one to bail on its behalf. +/// +/// `bits` is the current operand size, or `0` at a call boundary where there +/// is nothing to measure yet. +fn gcd_should_stop(bits: u64) -> bool { + if !ceilings_active() { + return false; + } + if gcd_stopped() { + return true; + } + if spend_gcd_work() { + note_gcd_stop(GcdStop::Work(gcd_work())); + return true; + } + if let Err(t) = crate::budget::check_all() { + note_gcd_stop(GcdStop::Budget(t)); + return true; + } + if bits > MAX_GCD_BITS { + note_gcd_stop(GcdStop::Size(bits)); + return true; + } + false +} + +/// Total bit-length of every coefficient of a `Z[n]` polynomial. +fn ipoly_bits(p: &IPoly) -> u64 { + p.c.iter() + .map(|z| u64::from(z.significant_bits()) + 1) + .sum() +} + +/// Total bit-length of every integer coefficient of `p`. +/// +/// The measure the ceiling is phrased in, because "how many limbs is this" +/// is the quantity that predicts both the time and the memory, and neither +/// the degree in `k` nor the degree in `n` does. +fn bipoly_bits(p: &BiPoly) -> u64 { + p.c.iter() + .map(|q| { + q.c.iter() + .map(|z| u64::from(z.significant_bits()) + 1) + .sum::() + }) + .sum() +} + /// `p` rewritten as `(α / dn(n)) · B(n, k)` with `B ∈ Z[n][k]` and `α ∈ Q`. /// /// Leaving `Q(n)` for `Z[n][k]` is what makes the gcd below cheap; the scalar diff --git a/alkahest-core/src/holonomic/qzeil/field.rs b/alkahest-core/src/holonomic/qzeil/field.rs index 2086d22e..e2cc5295 100644 --- a/alkahest-core/src/holonomic/qzeil/field.rs +++ b/alkahest-core/src/holonomic/qzeil/field.rs @@ -39,6 +39,86 @@ pub type PolyX = PolyK; /// `Q(q)(x)` — where the recurrence coefficients `a_i` live. pub type RatX = RatK; +// --------------------------------------------------------------------------- +// Cooperative refusal for the field arithmetic +// --------------------------------------------------------------------------- + +/// Largest a single `Q(q)(x)` element may grow to inside this tower's +/// arithmetic, counted by [`ratx_terms`]. +/// +/// The Euclidean gcd used by [`RatY::normalize`] has no content removal, which +/// is the classical way for coefficients to explode while the *degrees* stay +/// small — and small degrees are exactly what the shape ceilings in +/// [`super::search`] can see. Sized well above anything the working cases in +/// this module's tests reach (the largest observed is under 100). +pub const MAX_FIELD_ELEMENT_TERMS: usize = 6_000; + +/// Why the field arithmetic gave up part-way through, if it did. +#[derive(Clone, Copy, Debug)] +pub enum FieldRefusal { + /// A coefficient grew past [`MAX_FIELD_ELEMENT_TERMS`]; the payload is the + /// size it reached. + SizeCeiling(usize), + /// An active [`crate::budget`] stopped it. + Budget(crate::budget::BudgetTrip), +} + +thread_local! { + static REFUSAL: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Record a refusal for the caller to pick up. +/// +/// The arithmetic here is infallible by signature (`PolyY::gcd` returns a +/// `PolyY`, not a `Result`) and threading `Result` through every operator +/// would be a far larger change than the problem warrants, so a refusal +/// travels out of band and the operation returns a *correct but unhelpful* +/// answer — `div_rem` returns `None`, which `gcd` already handles by returning +/// `1`, which `normalize` already handles by not cancelling. Nothing becomes +/// wrong; the caller is expected to check [`take_field_refusal`] and stop. +fn note_refusal(r: FieldRefusal) { + REFUSAL.with(|c| c.set(Some(r))); +} + +/// Clear any recorded refusal. Call before each probe. +pub fn clear_field_refusal() { + REFUSAL.with(|c| c.set(None)); +} + +/// Take (and clear) the refusal recorded on this thread, if any. +pub fn take_field_refusal() -> Option { + REFUSAL.with(|c| c.take()) +} + +/// The largest coefficient of `p`, by [`ratx_terms`]. +fn widest_coeff(p: &PolyY) -> usize { + p.coeffs.iter().map(ratx_terms).max().unwrap_or(0) +} + +/// How many rational numbers it takes to write `r` down — a cheap, exact +/// proxy for how big an element of `Q(q)(x)` has become. +/// +/// The `q`-tower is three levels deep (`Q(q) ⊂ Q(q)(x) ⊂ Q(q)(x)(y)`), so a +/// single "coefficient" in the linear system is a quotient of polynomials in +/// `x` whose own coefficients are quotients of polynomials in `q`. Elimination +/// grows that nesting, and the growth is fragile in the *input* rather than in +/// `max_order`/`max_degree`: `Σ_k [n;k]_q` decides in half a second where +/// `Σ_k [2n;k]_q` does not return at the cheapest bounds the engine accepts. +/// A ceiling on the shape of the system cannot see that; this can. +/// +/// Counts coefficient slots rather than bit-lengths: it is `O(size)` with no +/// allocation and no GMP calls, which is what makes it affordable in the +/// inner elimination loop. +pub fn ratx_terms(r: &RatX) -> usize { + fn polyx_terms(p: &PolyX) -> usize { + p.coeffs + .iter() + .map(|c| c.num.coeffs.len() + c.den.coeffs.len()) + .sum() + } + polyx_terms(&r.num) + polyx_terms(&r.den) +} + /// `q^i ∈ Q(q)`, for any sign of `i`. pub fn qq_pow(i: i64) -> Qq { if i == 0 { @@ -223,6 +303,26 @@ impl PolyY { let mut rem = a.clone().trim(); let mut quot: Vec = Vec::new(); while !rem.is_zero() && rem.degree() >= db { + // The hot loop of `gcd`, and the one place in this tower where a + // small system can run for minutes: the remainder's *degree* falls + // every step while its coefficients grow without bound. + if let Err(t) = crate::budget::check_all() { + note_refusal(FieldRefusal::Budget(t)); + return None; + } + // The `Z[q][x]` gcd one level down has given up, so every `RatX` + // operation from here on runs on unreduced (and therefore growing) + // representations. Stop now rather than finish this division more + // slowly than it would have run with cancellation. + if super::super::qfield::gcd_stop_pending() { + note_refusal(FieldRefusal::SizeCeiling(widest_coeff(&rem))); + return None; + } + let widest = widest_coeff(&rem); + if widest > MAX_FIELD_ELEMENT_TERMS { + note_refusal(FieldRefusal::SizeCeiling(widest)); + return None; + } let shift = (rem.degree() - db) as usize; let t = rem.leading_coeff().mul(&lb_inv); if shift >= quot.len() { diff --git a/alkahest-core/src/holonomic/qzeil/mod.rs b/alkahest-core/src/holonomic/qzeil/mod.rs index be30a3f9..a0446850 100644 --- a/alkahest-core/src/holonomic/qzeil/mod.rs +++ b/alkahest-core/src/holonomic/qzeil/mod.rs @@ -600,6 +600,66 @@ mod tests { pool.func("qbinomial", vec![top, bot]) } + /// `Σ_k [2n; k]_q` — the summand issue #10 (2026-08-19) was logged for. + /// + /// Class-legal and cheap to *state*; before this module had any ceiling it + /// ran for eight minutes at the documented defaults with no output and had + /// to be killed, while its near-twin `Σ_k [n; k]_q` decides in half a + /// second. The cost is fragile in the *input*, not in `max_order` / + /// `max_degree`, which is why a ceiling on the shape of the search is not + /// enough on its own. + fn q_central_2n(pool: &ExprPool, n: ExprId, k: ExprId) -> ExprId { + qbinom(pool, pool.mul(vec![pool.integer(2_i32), n]), k) + } + + #[test] + fn q_zeilberger_honours_a_wall_budget() { + let pool = ExprPool::new(); + let (q, n, k) = syms(&pool); + let term = q_central_2n(&pool, n, k); + let opts = QZeilbergerOpts::default(); + let _guard = crate::budget::enter( + crate::budget::Budget::new().with_wall(std::time::Duration::from_millis(300)), + ); + let start = std::time::Instant::now(); + let err = q_zeilberger(term, q, n, k, &pool, &opts) + .expect_err("a 300 ms budget cannot cover this search"); + // Loose by two orders of magnitude: the point is that the call + // *returns*, where before it consulted no budget at all. + assert!(start.elapsed().as_secs() < 60, "budget was not consulted"); + assert!( + matches!(err, QHolonomicError::SearchExhausted(_)), + "{err:?}" + ); + let trip = crate::budget::take_trip().expect("the budget trip must be recorded"); + assert_eq!(trip.code(), "E-BUDGET-001"); + } + + #[test] + fn q_zeilberger_refuses_at_a_resource_ceiling_rather_than_running_unbounded() { + let pool = ExprPool::new(); + let (q, n, k) = syms(&pool); + let term = q_central_2n(&pool, n, k); + // The cheapest bounds the engine accepts above the trivial ones. + let opts = QZeilbergerOpts { + max_order: 2, + max_degree: 2, + ..QZeilbergerOpts::default() + }; + let err = q_zeilberger(term, q, n, k, &pool, &opts) + .expect_err("no q-recurrence of this shape is found for this summand"); + let QHolonomicError::SearchExhausted(msg) = &err else { + panic!("expected SearchExhausted, got {err:?}"); + }; + assert!( + msg.contains("resource ceilings"), + "a ceiling refusal must say so — a caller that reads this as 'the grid was covered \ + and nothing exists' records a false negative. Got: {msg}" + ); + // No budget was active, so this is the module's own ceiling, not a trip. + assert_eq!(crate::budget::take_trip(), None); + } + /// `(q;q)_m` at an integer `m ≥ 0`, built straight from the definition — /// the independent yardstick the recurrence is checked against. fn q_poch_int(m: i64) -> Rn { diff --git a/alkahest-core/src/holonomic/qzeil/search.rs b/alkahest-core/src/holonomic/qzeil/search.rs index b5990825..4323b5ce 100644 --- a/alkahest-core/src/holonomic/qzeil/search.rs +++ b/alkahest-core/src/holonomic/qzeil/search.rs @@ -36,14 +36,72 @@ //! as the classical module's, and it is the only thing that makes a returned //! result a proof. -use super::field::{clear_denominators_x, qq_pow, PolyX, PolyY, RatX, RatY}; +use super::field::{ + clear_denominators_x, clear_field_refusal, qq_pow, ratx_terms, take_field_refusal, + FieldRefusal, PolyX, PolyY, RatX, RatY, MAX_FIELD_ELEMENT_TERMS, +}; use super::term::QProperTerm; use super::QHolonomicError; use crate::holonomic::hyperterm::rn_to_expr; -use crate::holonomic::qfield::{clear_denominators, rn_div, rn_is_zero, rn_poly, Rn}; +use crate::holonomic::qfield::{ + clear_denominators, clear_gcd_stop, enter_gcd_work_scope, rn_div, rn_is_zero, rn_poly, + take_gcd_stop, GcdStop, Rn, +}; use crate::holonomic::zeilberger::OrderSearch; use crate::kernel::{ExprId, ExprPool}; +// --------------------------------------------------------------------------- +// Resource ceilings +// --------------------------------------------------------------------------- +// +// `telescope_md` has had ceilings since it was written; this module was +// written *after* the 2026-08-13 report on `zeilberger`'s unbounded search and +// still had none, so at its own documented defaults a class-legal summand +// (`Σ_k [2n;k]_q`) ran for eight minutes with no output and had to be killed. +// The two below bound the two things that can grow, and they are deliberately +// different in kind, because the cost of this search is fragile in the *input* +// and only partly in the knobs: +// +// * the **shape** of the linear system — equations × unknowns — which +// `max_order`/`max_degree` already influence but do not bound, because the +// equation count comes from the degree of the key equation, not from the +// caller's `max_degree`; and +// * the **size of the numbers** in it, which nothing about the shape +// predicts. `Σ_k [n;k]_q` and `Σ_k [2n;k]_q` present systems of similar +// shape; only the second one's coefficients explode. +// +// Both refuse a probe *before* it is attempted (or, for the size ceiling, the +// moment an entry crosses the line) and are reported in the exhaustion message +// so that a refusal is never mistaken for "the grid was covered and nothing +// was found" — see `SearchExhausted`'s text at the end of +// `q_zeilberger_on_term`. + +/// Largest linear system, in cells (equations × unknowns), that one +/// `(order, degree)` probe may assemble. +/// +/// The system is `n_eq × n_var` over `Q(q)(x)`, and elimination is +/// `O(n_eq · n_var²)` *field* operations, each of which is a rational-function +/// arithmetic operation, not a machine one. 4 000 cells covers every probe the +/// module's own test suite makes by a wide margin (the largest is under 400). +const MAX_SYSTEM_CELLS: usize = 4_000; + +/// Largest total across every probe of one search call, so that a caller +/// cannot pay the per-probe ceiling once for each of `max_order × max_degree` +/// combinations. +const MAX_CUMULATIVE_SYSTEM_CELLS: usize = 20_000; + +/// The outcome of one `(order, degree)` probe. +enum Probe { + /// `(X coefficients, a_0..a_{order−1})`. + Solved(Vec, Vec), + /// The system has no solution of this shape — an ordinary miss. + NoSolution, + /// Refused by a resource ceiling before (or during) the solve. Distinct + /// from [`Probe::NoSolution`] on purpose: it must not be reported as + /// evidence that no certificate of this shape exists. + Refused, +} + /// Everything [`super::q_zeilberger`] takes beyond the term itself. /// /// `max_order` and `max_degree` are upper **bounds**: the `(order, degree)` @@ -168,10 +226,18 @@ fn q_gosper_normal_form(mut p: PolyY, mut r: PolyY) -> Option<(PolyY, PolyY, Pol } /// Gaussian elimination over the field `Q(q)(x)`. -fn field_solve(mut mat: Vec>, mut rhs: Vec) -> Option> { +/// +/// `Ok(None)` is an ordinary "no solution"; `Err` is a refusal — either an +/// active [`crate::budget`] (wall clock, steps, memory) or +/// [`MAX_FIELD_ELEMENT_TERMS`], the ceiling on how big one field element may +/// grow. Both must stay distinguishable from "no solution" all the way out. +fn field_solve( + mut mat: Vec>, + mut rhs: Vec, +) -> Result>, QHolonomicError> { let nrows = mat.len(); if nrows == 0 { - return Some(vec![]); + return Ok(Some(vec![])); } let ncols = mat[0].len(); let mut row = 0; @@ -179,12 +245,15 @@ fn field_solve(mut mat: Vec>, mut rhs: Vec) -> Option> if row >= nrows { break; } + checkpoint()?; let Some(pr) = (row..nrows).find(|&r| !mat[r][col].is_zero()) else { continue; }; mat.swap(row, pr); rhs.swap(row, pr); - let inv = mat[row][col].inv()?; + let Some(inv) = mat[row][col].inv() else { + return Ok(None); + }; for entry in mat[row].iter_mut().skip(col) { *entry = entry.mul(&inv); } @@ -199,8 +268,14 @@ fn field_solve(mut mat: Vec>, mut rhs: Vec) -> Option> if v.is_zero() { continue; } + // Per row, not per pivot: it is the entries, not the shape, that + // grow here. + checkpoint()?; for (entry, pivot) in mat[r].iter_mut().zip(pivot_row.iter()).skip(col) { *entry = entry.sub(&pivot.mul(&v)); + if ratx_terms(entry) > MAX_FIELD_ELEMENT_TERMS { + return Err(size_ceiling_error(ratx_terms(entry))); + } } rhs[r] = rhs[r].sub(&pivot_rhs.mul(&v)); } @@ -208,20 +283,63 @@ fn field_solve(mut mat: Vec>, mut rhs: Vec) -> Option> } for (r, mrow) in mat.iter().enumerate() { if mrow.iter().all(RatX::is_zero) && !rhs[r].is_zero() { - return None; + return Ok(None); } } let mut sol = vec![RatX::zero(); ncols]; for r in (0..nrows).rev() { if let Some(j) = mat[r].iter().position(|e| !e.is_zero()) { + checkpoint()?; let mut sum = rhs[r].clone(); for cidx in (j + 1)..ncols { sum = sum.sub(&mat[r][cidx].mul(&sol[cidx])); } - sol[j] = sum.div(&mat[r][j])?; + let Some(q) = sum.div(&mat[r][j]) else { + return Ok(None); + }; + sol[j] = q; } } - Some(sol) + Ok(Some(sol)) +} + +/// Turn a [`crate::budget`] trip into this module's exhaustive error type, +/// recording the real cause out of band for the bindings to raise as +/// `BudgetExceededError` — see [`crate::budget::record_trip`]. +fn trip_to_error(trip: crate::budget::BudgetTrip) -> QHolonomicError { + use crate::errors::AlkahestError; + crate::budget::record_trip(trip); + QHolonomicError::SearchExhausted(format!( + "the q-Zeilberger search was stopped before it finished, and no certificate was found \ + up to that point — this is NOT a statement that none exists: {} [{}]", + trip, + trip.code() + )) +} + +/// Cooperative checkpoint: wall clock, steps, cancellation, and the memory +/// ceilings of [`crate::budget::memory`]. Before this existed, `q_zeilberger` +/// honoured no budget at all — `Budget(wall_ms=...)` did not stop it. +fn checkpoint() -> Result<(), QHolonomicError> { + crate::budget::check_all().map_err(trip_to_error) +} + +/// Marker error for a [`MAX_FIELD_ELEMENT_TERMS`] trip. +/// +/// Carried as `SearchExhausted` because `QHolonomicError` is public and +/// exhaustive; the caller recognises it by [`is_size_ceiling`] and reports the +/// probe as [`Probe::Refused`], never as "no solution". +fn size_ceiling_error(terms: usize) -> QHolonomicError { + QHolonomicError::SearchExhausted(format!( + "{SIZE_CEILING_TAG}: a coefficient of the linear system reached {terms} rational \ + numbers, past this module's MAX_FIELD_ELEMENT_TERMS = {MAX_FIELD_ELEMENT_TERMS}" + )) +} + +const SIZE_CEILING_TAG: &str = "q-Zeilberger field-element ceiling"; + +fn is_size_ceiling(e: &QHolonomicError) -> bool { + matches!(e, QHolonomicError::SearchExhausted(m) if m.starts_with(SIZE_CEILING_TAG)) } /// Solve `A(y)·X(q·y) − B(y/q)·X(y) = C(y)·N(y)` for a degree-`d` `X` and the @@ -232,7 +350,8 @@ fn try_solve( c_ci: &[PolyY], order: usize, d: usize, -) -> Option<(Vec, Vec)> { + cumulative_cells: &mut usize, +) -> Result { let mut bx: Vec = Vec::with_capacity(d + 1); for j in 0..=d { let yj = y_mono(j); @@ -248,6 +367,18 @@ fn try_solve( let n_eq = (max_deg.max(0) as usize) + 1; let n_var = (d + 1) + order; + // Shape ceilings, both purely arithmetic and both *before* the system is + // assembled: `n_eq` comes from the degree of the key equation, so it is + // not bounded by the caller's `max_degree`. + let cells = n_eq.saturating_mul(n_var); + if cells > MAX_SYSTEM_CELLS { + return Ok(Probe::Refused); + } + if cumulative_cells.saturating_add(cells) > MAX_CUMULATIVE_SYSTEM_CELLS { + return Ok(Probe::Refused); + } + *cumulative_cells += cells; + let mut mat = vec![vec![RatX::zero(); n_var]; n_eq]; let mut rhs = vec![RatX::zero(); n_eq]; for (m, row) in mat.iter_mut().enumerate() { @@ -260,8 +391,13 @@ fn try_solve( rhs[m] = c_ci[order].coeff(m); } - let sol = field_solve(mat, rhs)?; - Some((sol[..=d].to_vec(), sol[(d + 1)..].to_vec())) + let sol = match field_solve(mat, rhs) { + Ok(Some(sol)) => sol, + Ok(None) => return Ok(Probe::NoSolution), + Err(e) if is_size_ceiling(&e) => return Ok(Probe::Refused), + Err(e) => return Err(e), + }; + Ok(Probe::Solved(sol[..=d].to_vec(), sol[(d + 1)..].to_vec())) } /// Rescale a `Q(q)[x]` family by one common element of `Q(q)` so that every @@ -389,21 +525,71 @@ pub fn q_zeilberger_on_term( "max_order and max_degree must both be at least 1".into(), )); } + // Only this call's trip may be attributed to this call. + crate::budget::clear_trip(); + // The `Z[q][x]` gcd under every `Q(q)(x)` operation is bounded for the + // whole search, not per probe: once the ceiling is reached, the remaining + // probes refuse immediately instead of each paying it again. + let _gcd_scope = enter_gcd_work_scope(); let p = f.ratio_k()?; let mut states: Vec> = Vec::with_capacity(opts.max_order); let mut degrees_failed = vec![0usize; opts.max_order]; + let mut cumulative_cells: usize = 0; + let mut refused_by_ceiling = false; + // `PolyY::gcd` (via `RatY::normalize`) is infallible by signature, so it + // reports a ceiling or budget stop out of band; this expands to the `?`, + // `continue` and flag update that every heavy step below needs, without + // threading a `Result` through the whole coefficient tower. + macro_rules! bail_on_field_refusal { + () => { + match take_field_refusal() { + None => {} + Some(FieldRefusal::Budget(t)) => return Err(trip_to_error(t)), + Some(FieldRefusal::SizeCeiling(_)) => { + refused_by_ceiling = true; + continue; + } + } + match take_gcd_stop() { + None => {} + Some(GcdStop::Budget(t)) => return Err(trip_to_error(t)), + Some(GcdStop::Size(_)) | Some(GcdStop::Work(_)) => { + refused_by_ceiling = true; + continue; + } + } + }; + } for (order, d) in search_plan(opts.max_order, opts.max_degree, opts.search) { degrees_failed[order - 1] += 1; + checkpoint()?; + clear_field_refusal(); + clear_gcd_stop(); while states.len() < order { states.push(order_state(f, &p, states.len() + 1)?); } let Some(state) = &states[order - 1] else { continue; }; - let Some((x_coeffs, lam_below)) = try_solve(&state.aa, &state.b_eq, &state.c_ci, order, d) - else { - continue; + let probe = try_solve( + &state.aa, + &state.b_eq, + &state.c_ci, + order, + d, + &mut cumulative_cells, + )?; + // A ceiling that fired inside the solve makes this probe a refusal, + // not a miss — checked before the `NoSolution` arm can swallow it. + bail_on_field_refusal!(); + let (x_coeffs, lam_below) = match probe { + Probe::Solved(x, lam) => (x, lam), + Probe::NoSolution => continue, + Probe::Refused => { + refused_by_ceiling = true; + continue; + } }; let mut lam_full = lam_below; @@ -416,6 +602,7 @@ pub fn q_zeilberger_on_term( } .normalize(); + bail_on_field_refusal!(); let (a_int, scale) = clear_denominators_x(&lam_full); if a_int.iter().all(PolyX::is_zero) || a_int[order].is_zero() { continue; @@ -433,6 +620,7 @@ pub fn q_zeilberger_on_term( // (`q^{n+1} − 1`) rather than as quotients (`qⁿ − 1/q`). The scale is // one common element of `Q(q)`, and it multiplies the certificate too, // so the identity is untouched — and it is re-verified below either way. + bail_on_field_refusal!(); let a_int = primitive_family(&a_int) .map(|(family, s)| { r_final = r_final.mul(&RatY::from_ratx(RatX::from_rn(s))); @@ -446,6 +634,7 @@ pub fn q_zeilberger_on_term( for (i, ci) in state.c.iter().enumerate() { lhs = lhs.add(&RatY::from_ratx(RatX::from_poly(a_int[i].clone())).mul(ci)); } + bail_on_field_refusal!(); let rhs_check = r_final.qshift_y(1).mul(&p).sub(&r_final); if !lhs.sub(&rhs_check).is_zero() { continue; @@ -454,6 +643,7 @@ pub fn q_zeilberger_on_term( // Rendered forms are simplified once, here: the builders emit // `1*q^n + -1` shapes that are correct but unreadable, and a caller // reading `a_1` off a returned certificate should not have to. + bail_on_field_refusal!(); let simp = |e: ExprId| crate::simplify::simplify(e, pool).value; let coeffs: Vec = a_int .iter() @@ -475,8 +665,22 @@ pub fn q_zeilberger_on_term( }); } + let ceiling_note = if refused_by_ceiling { + format!( + " (at least one (order, degree) combination within these bounds was refused by this \ + module's resource ceilings rather than attempted — MAX_SYSTEM_CELLS = \ + {MAX_SYSTEM_CELLS} equations x unknowns for any single probe, \ + MAX_CUMULATIVE_SYSTEM_CELLS = {MAX_CUMULATIVE_SYSTEM_CELLS} across the whole \ + search, MAX_FIELD_ELEMENT_TERMS = {MAX_FIELD_ELEMENT_TERMS} rational numbers in \ + any one coefficient of the linear system; so this is NOT a statement that no \ + q-recurrence exists within these bounds)" + ) + } else { + String::new() + }; Err(QHolonomicError::SearchExhausted(format!( - "no verified q-recurrence of order <= {} with certificate degree <= {} in q^k was found", + "no verified q-recurrence of order <= {} with certificate degree <= {} in q^k was \ + found{ceiling_note}", opts.max_order, opts.max_degree ))) } diff --git a/alkahest-core/src/holonomic/telescoping2d/mod.rs b/alkahest-core/src/holonomic/telescoping2d/mod.rs index 310b5261..e06fcde5 100644 --- a/alkahest-core/src/holonomic/telescoping2d/mod.rs +++ b/alkahest-core/src/holonomic/telescoping2d/mod.rs @@ -249,6 +249,7 @@ pub fn telescope_md( #[cfg(test)] mod tests { use super::*; + use crate::errors::AlkahestError as _; use crate::kernel::{Domain, ExprId, ExprPool}; use rug::ops::Pow as _; use rug::Integer; @@ -687,6 +688,45 @@ mod tests { assert_eq!(result.certs.len(), 1); } + /// The exact-rational elimination used to `abort()` the whole process when + /// it ran out of memory — `GNU MP: Cannot allocate memory (size=8)`, no + /// exception, no `BudgetExceededError`, nothing an `except` clause could + /// catch (2026-08-19 issue #5). `Budget`'s `max_bytes` ceiling turns that + /// into an ordinary coded error, checked *before* the allocation GMP has + /// no failure path for. + #[test] + fn a_memory_ceiling_refuses_instead_of_aborting_the_process() { + let pool = ExprPool::new(); + let n = pool.symbol("n", Domain::Real); + let x = pool.symbol("x", Domain::Real); + let y = pool.symbol("y", Domain::Real); + let f = pool.mul(vec![ + pool.func("binomial", vec![n, x]), + pool.func("binomial", vec![x, y]), + ]); + let _guard = crate::budget::enter_with_memory(crate::budget::Budget::new(), Some(1024)); + let err = search::telescope_md_search(f, n, &[x, y], &pool, &TelescopingMdOpts::default()) + .expect_err("1 KiB of exact-rational memory cannot cover this search"); + assert!( + matches!(err, Telescoping2dError::SearchExhausted(_)), + "{err:?}" + ); + let trip = crate::budget::take_trip().expect("the memory trip must be recorded"); + assert_eq!(trip.code(), "E-BUDGET-004"); + } + + /// A generous ceiling must not change the answer — the point is a ceiling, + /// not a refusal. + #[test] + fn a_generous_memory_ceiling_still_returns_the_certificate() { + let pool = ExprPool::new(); + let (n, k, _) = njk(&pool); + let f = pool.func("binomial", vec![n, k]); + let _guard = crate::budget::enter_with_memory(crate::budget::Budget::new(), Some(1 << 40)); + let res = telescope_md(f, n, &[k], &pool).expect("a 1 TiB ceiling cannot refuse this"); + assert!(res.order >= 1); + } + /// Regression test for the two resource ceilings in `search` /// (`MAX_ANSATZ_UNKNOWNS`, the per-probe ceiling, and /// `MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS`, the whole-search budget that @@ -748,21 +788,29 @@ mod tests { max_a_degree: 2, max_cert_degree: 3, }; - let start = std::time::Instant::now(); let err = search::telescope_md_search(f, n, &[x, y, z], &pool, &opts) .expect_err("this combination must be refused via the resource ceiling, not solved"); - let elapsed = start.elapsed(); - // 900s, not 180s: this bound exists to catch a genuine hang (the - // pre-fix behavior was still running after several minutes and - // growing), not to pin CI wall-clock precisely. Windows CI runners - // measured ~480s here against ~76s on Linux for the same exact- - // rational elimination — a real, expected platform gap for GMP- - // backed arithmetic under MSYS2/mingw, not evidence the ceiling - // isn't working. + // Deterministic, where the wall-clock bound this assertion replaces was + // not: the property under test is "bounded to roughly one expensive + // elimination, not one per (order, a_degree) combination", and that is + // a *count of probes the ceilings let through*, not a latency. The old + // 900 s bound measured the machine — four separate measurements put the + // same elimination at ~76 s idle, 312-508 s isolated-but-busy and + // 1100-1300 s under load, so it failed on a loaded runner while the + // ceiling was working perfectly. + let spent = search::large_probe_unknowns(); + assert!( + spent <= search::MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS, + "the whole-search budget is {} unknowns across every probe at or above {}; the \ + search spent {spent}", + search::MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS, + search::LARGE_PROBE_THRESHOLD + ); assert!( - elapsed.as_secs() < 900, - "expected a bounded refusal (roughly one expensive elimination, not several), took \ - {elapsed:?}" + spent > 0, + "the ceiling must bound an expensive search, not skip it entirely — no probe at or \ + above {} unknowns was attempted at all", + search::LARGE_PROBE_THRESHOLD ); assert!(matches!(err, Telescoping2dError::SearchExhausted(_))); let Telescoping2dError::SearchExhausted(msg) = &err else { diff --git a/alkahest-core/src/holonomic/telescoping2d/search.rs b/alkahest-core/src/holonomic/telescoping2d/search.rs index bae3e7e3..86b95dfd 100644 --- a/alkahest-core/src/holonomic/telescoping2d/search.rs +++ b/alkahest-core/src/holonomic/telescoping2d/search.rs @@ -92,7 +92,7 @@ use rug::{Integer, Rational}; /// message says so explicitly when at least one probe was skipped for this /// reason, so a caller sees a fast, clearly-explained refusal instead of a /// silent guess about whether raising the bounds would even help. -const MAX_ANSATZ_UNKNOWNS: usize = 400; +pub(crate) const MAX_ANSATZ_UNKNOWNS: usize = 400; /// A probe's own unknown count must reach this before it counts against /// [`MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS`] at all. Below this, a probe is @@ -100,7 +100,7 @@ const MAX_ANSATZ_UNKNOWNS: usize = 400; /// exceeds ~140 unknowns for any probe it tries — see the constant's own /// docs) and is exempted from the cumulative accounting entirely, so this /// budget cannot regress the existing two-index search in any way. -const LARGE_PROBE_THRESHOLD: usize = 150; +pub(crate) const LARGE_PROBE_THRESHOLD: usize = 150; /// A single probe under [`MAX_ANSATZ_UNKNOWNS`] can still be individually /// slow (the `m = 3` multinomial-coefficient worked example's `cols = 245` @@ -123,7 +123,7 @@ const LARGE_PROBE_THRESHOLD: usize = 150; /// `max_order` / `max_a_degree` / `max_cert_degree` are. `300` admits /// exactly one probe the size of the multinomial example (`245`) before /// refusing further ones of that size. -const MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS: usize = 300; +pub(crate) const MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS: usize = 300; use std::collections::BTreeMap; @@ -268,6 +268,66 @@ pub fn telescope2d_search( }) } +/// Turn a [`crate::budget`] trip into this module's exhaustive error type, +/// recording the real cause out of band for the bindings. +/// +/// [`Telescoping2dError`] is public and exhaustive, so it cannot grow a +/// `Budget` variant without a major semver break; the trip is left in +/// [`crate::budget::take_trip`] and the message says plainly that the search +/// was *stopped*, never that it was completed. A caller must be able to tell a +/// budget stop from a search that genuinely covered its grid. +fn trip_to_error(trip: crate::budget::BudgetTrip) -> Telescoping2dError { + use crate::errors::AlkahestError; + crate::budget::record_trip(trip); + Telescoping2dError::SearchExhausted(format!( + "the multi-index telescoping search was stopped before it finished, and no certificate \ + was found up to that point — this is NOT a statement that none exists: {} [{}]", + trip, + trip.code() + )) +} + +#[cfg(test)] +thread_local! { + /// Unknowns spent on probes at or above [`LARGE_PROBE_THRESHOLD`] that the + /// ceilings actually let through. + /// + /// This is exactly what [`MAX_CUMULATIVE_LARGE_PROBE_UNKNOWNS`] bounds, so + /// it is what the regression test asserts on — a deterministic quantity + /// rather than a wall-clock bound, which on a loaded machine measures the + /// machine and not the ceiling. Cheap probes are excluded deliberately: + /// they are not what the ceilings exist to stop. + static LARGE_PROBE_UNKNOWNS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn large_probe_unknowns() -> usize { + LARGE_PROBE_UNKNOWNS.with(|c| c.get()) +} + +#[allow(unused_variables)] +fn note_probe_attempted(unknowns: usize) { + #[cfg(test)] + if unknowns >= LARGE_PROBE_THRESHOLD { + LARGE_PROBE_UNKNOWNS.with(|c| c.set(c.get() + unknowns)); + } +} + +fn reset_probes_attempted() { + #[cfg(test)] + LARGE_PROBE_UNKNOWNS.with(|c| c.set(0)); +} + +/// Cooperative checkpoint: wall clock, steps, cancellation, and the memory +/// ceilings of [`crate::budget::memory`]. +/// +/// Called at the head of every probe and inside the exact-rational elimination +/// — the two places where this module can run for minutes and, at `m = 4`, +/// where GMP used to `abort()` the whole process rather than refuse. +fn checkpoint() -> Result<(), Telescoping2dError> { + crate::budget::check_all().map_err(trip_to_error) +} + /// Apagodu–Zeilberger search for general `m ≥ 1`: find and verify a /// creative-telescoping certificate for `term`, a proper hypergeometric /// `F(n, x_1, …, x_m)` with `indices = [x_1, …, x_m]`. @@ -300,6 +360,9 @@ pub fn telescope_md_search( )); } let num_axes = m + 1; + // Only this call's trip may be attributed to this call. + crate::budget::clear_trip(); + reset_probes_attempted(); let f = ProperTermM::parse(term, n, indices, pool)?; let mut rhos: Vec = Vec::with_capacity(m); @@ -361,6 +424,8 @@ pub fn telescope_md_search( cumulative_large_unknowns += total; } + checkpoint()?; + note_probe_attempted(total); if let Some(cand) = solve_ansatz_md(order, m, a_degree, cert_degree, &nn, &dn, &rhos, num_axes)? { @@ -569,6 +634,7 @@ fn solve_ansatz_md( let total_combos = box_len.pow(num_axes as u32); for combo in 0..total_combos { + checkpoint()?; let exps = unflatten(combo, num_axes, box_len); let p = exps[0]; let np = n_pow[p].clone(); @@ -597,7 +663,7 @@ fn solve_ansatz_md( } let matrix: Vec> = rows.into_values().collect(); - let basis_vecs = rational_nullspace(matrix, total); + let basis_vecs = rational_nullspace(matrix, total)?; if basis_vecs.is_empty() { return Ok(None); } @@ -683,7 +749,10 @@ fn primitive_scale_rationals(v: &mut [Rational]) { /// Nullspace basis of `mat` (rows = equations, `ncols` unknowns) over `Q`, by /// plain Gaussian elimination to row-echelon form. #[allow(clippy::needless_range_loop)] -fn rational_nullspace(mut mat: Vec>, ncols: usize) -> Vec> { +fn rational_nullspace( + mut mat: Vec>, + ncols: usize, +) -> Result>, Telescoping2dError> { let nrows = mat.len(); let mut pivot_cols: Vec = Vec::new(); let mut row = 0; @@ -691,6 +760,7 @@ fn rational_nullspace(mut mat: Vec>, ncols: usize) -> Vec= nrows { break; } + checkpoint()?; let Some(pr) = (row..nrows).find(|&r| mat[r][col] != 0) else { continue; }; @@ -707,6 +777,10 @@ fn rational_nullspace(mut mat: Vec>, ncols: usize) -> Vec>, ncols: usize) -> Vec bool { + match crate::budget::check_all() { + Ok(()) => true, + Err(trip) => { + crate::budget::record_trip(trip); + false + } + } +} + /// Try `p·(x_1²+…+x_n²)^N` for `N = 1, 2, …, max_power` (Reznick /// multipliers): a positive semidefinite form can fail to be SOS itself /// (Motzkin, Choi–Lam, Robinson's form) yet become SOS after multiplying by @@ -275,6 +295,9 @@ fn multiplier_search( log: &mut Vec, ) -> Option<(RatPoly, SosPoly, u32)> { for n in 1..=max_power { + if !budget_ok() { + return None; + } let sigma = RatPoly::sum_of_squares(nvars).pow(n); let q = target.mul(&sigma); let qdeg = q.total_degree(); @@ -318,6 +341,8 @@ pub fn sos_decompose( "at least one variable is required".into(), )); } + // Only this call's trip may be attributed to this call. + crate::budget::clear_trip(); let names = var_names(vars, pool); let target = RatPoly::from_expr(expr, vars, pool).map_err(SosError::NotPolynomial)?; let nvars = vars.len(); @@ -483,6 +508,7 @@ pub fn prove_nonneg( if constraints.is_empty() { return sos_decompose(expr, vars, pool, opts); } + crate::budget::clear_trip(); if vars.is_empty() { return Err(SosError::InvalidInput( "at least one variable is required".into(), @@ -591,6 +617,44 @@ mod tests { (pool, x, y) } + /// The Motzkin form `x⁴y² + x²y⁴ − 3x²y²z² + z⁶`: non-negative, not SOS, + /// and the case 2026-08-19 item 26d measured at **418.5 s inside a + /// `Budget(wall_ms=3000)`** — 140x over, ending in `E-SOS-002`. + /// + /// `E-SOS-002` says "record this as unknown, not as a closed branch", but + /// an unattended loop that reads a *timeout* as "not SOS" files a false + /// negative. So the budget stop must stay distinguishable: the search still + /// returns `NoCertificate`, and the trip is recorded out of band for the + /// bindings to raise as `BudgetExceededError`. + fn motzkin(pool: &ExprPool, x: ExprId, y: ExprId, z: ExprId) -> ExprId { + let sq = |e: ExprId| pool.mul(vec![e, e]); + pool.add(vec![ + pool.mul(vec![sq(sq(x)), sq(y)]), + pool.mul(vec![sq(x), sq(sq(y))]), + pool.mul(vec![pool.integer(-3_i32), sq(x), sq(y), sq(z)]), + sq(sq(z)), + ]) + } + + #[test] + fn sos_decompose_honours_a_wall_budget() { + let (pool, x, y) = setup(); + let z = pool.symbol("z", Domain::Real); + let p = motzkin(&pool, x, y, z); + let _guard = crate::budget::enter( + crate::budget::Budget::new().with_wall(std::time::Duration::from_millis(200)), + ); + let start = std::time::Instant::now(); + let err = sos_decompose(p, &[x, y, z], &pool, &SosOpts::default()) + .expect_err("no SOS certificate exists for the Motzkin form"); + // Loose by two orders of magnitude against the 418.5 s that a 3 s + // budget used to buy: this asserts the budget is consulted at all. + assert!(start.elapsed().as_secs() < 60, "budget was not consulted"); + assert!(matches!(err, SosError::NoCertificate(_)), "{err:?}"); + let trip = crate::budget::take_trip().expect("the budget trip must be recorded"); + assert_eq!(trip.code(), "E-BUDGET-001"); + } + #[test] fn perfect_square_is_certified() { let (pool, x, y) = setup(); diff --git a/alkahest-core/src/real/sos/psd.rs b/alkahest-core/src/real/sos/psd.rs index 17a975d7..9da9308b 100644 --- a/alkahest-core/src/real/sos/psd.rs +++ b/alkahest-core/src/real/sos/psd.rs @@ -366,6 +366,12 @@ const DR_POLISH_CANDIDATES: usize = 2; fn anneal_from(family: &Family, start: Vec) -> Vec { let mut t = start; for &floor in FLOOR_SCHEDULE { + // Returning the best point so far is always safe: these are only + // *candidates*, and every one of them is re-verified exactly before it + // can become a certificate. + if !super::budget_ok() { + return t; + } if let Some(next) = family.search_from(t.clone(), floor, 150) { t = next; } @@ -414,6 +420,9 @@ fn multistart_anneal(family: &Family, dim: usize) -> Vec> { results.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); for (eig, t) in results.iter_mut().take(DR_POLISH_CANDIDATES) { + if !super::budget_ok() { + break; + } for &lambda in DR_LAMBDAS { if let Some(cand) = family.douglas_rachford_from(t.clone(), 0.0, lambda, DR_ITERS) { let cand_eig = min_eigenvalue(&family.at(&cand)); @@ -542,6 +551,9 @@ fn search_rational_family( let candidate_matrices: Vec>> = candidates.iter().map(|s| family.at(s)).collect(); for s in candidates.iter().take(ROUNDING_CANDIDATES) { + if !super::budget_ok() { + return (None, candidate_matrices); + } let Some(t) = back_substitute_upper(&r, s) else { continue; }; diff --git a/alkahest-py/src/lib.rs b/alkahest-py/src/lib.rs index 97791b09..58b0f1c7 100644 --- a/alkahest-py/src/lib.rs +++ b/alkahest-py/src/lib.rs @@ -709,8 +709,13 @@ thread_local! { /// both around its `with` block. #[pyfunction] #[pyo3(name = "push_budget")] -#[pyo3(signature = (wall_ms=None, max_steps=None, seed=None))] -fn py_push_budget(wall_ms: Option, max_steps: Option, seed: Option) -> PyResult<()> { +#[pyo3(signature = (wall_ms=None, max_steps=None, seed=None, max_bytes=None))] +fn py_push_budget( + wall_ms: Option, + max_steps: Option, + seed: Option, + max_bytes: Option, +) -> PyResult<()> { let mut budget = alkahest_core::budget::Budget::new(); if let Some(ms) = wall_ms { // `Duration::from_secs_f64` panics above `u64::MAX` seconds, and @@ -727,7 +732,11 @@ fn py_push_budget(wall_ms: Option, max_steps: Option, seed: Option Option { alkahest_core::budget::seed() } +/// Bytes of GMP (exact integer/rational) memory held live by the innermost +/// active budget frame — what `Budget(max_bytes=...)` is measured against. +/// +/// Deliberately *not* in `alkahest.__all__`: it exists so a caller (and the +/// regression tests) can see what the ceiling is counting, not as a promise +/// about how exact arithmetic allocates. +#[pyfunction] +#[pyo3(name = "budget_bytes_used")] +fn py_budget_bytes_used() -> u64 { + alkahest_core::budget::bytes_used() +} + +/// Total bytes of GMP memory live in this process, or `0` when accounting is +/// not installed. See `py_budget_bytes_used`. +#[pyfunction] +#[pyo3(name = "gmp_live_bytes")] +fn py_gmp_live_bytes() -> u64 { + alkahest_core::budget::gmp_live_bytes() +} + /// Request cancellation of the current cooperative operation(s), process-wide. #[pyfunction] #[pyo3(name = "request_cancel")] @@ -4727,11 +4756,30 @@ fn py_verify_wz_pair( fn holonomic_error_to_py(e: CoreHolonomicError) -> PyErr { Python::with_gil(|py| { + if let Some(err) = budget_trip_to_py(py) { + return err; + } let exc_type = py.get_type_bound::(); make_structured_err(py, &exc_type, &e) }) } +/// `BudgetExceededError` for a call that an `alkahest_core::budget` +/// checkpoint stopped, or `None` if no trip was recorded on this thread. +/// +/// Engines whose error enums are public and exhaustive (`HolonomicError`, +/// `Telescoping2dError`, `SosError`, …) cannot grow a `Budget` variant without +/// a major semver break, so they return their own "gave up" variant and leave +/// the real cause in `budget::take_trip()`. Recovering it here is what keeps a +/// resource stop **distinguishable** from a search that genuinely covered its +/// grid: a loop that read `E-HOLO-041` or `E-SOS-002` as "no certificate +/// exists" would record a false negative. +fn budget_trip_to_py(py: Python<'_>) -> Option { + let trip = alkahest_core::budget::take_trip()?; + let exc_type = py.get_type_bound::(); + Some(make_structured_err(py, &exc_type, &trip)) +} + /// Modular-evaluation errors, raised as the *same* Python `HolonomicError`. /// /// `ModularError` is a separate Rust enum only because `HolonomicError` is @@ -5773,6 +5821,9 @@ fn py_cyclotomic_polynomial( fn q_holonomic_error_to_py(e: CoreQHolonomicError) -> PyErr { Python::with_gil(|py| { + if let Some(err) = budget_trip_to_py(py) { + return err; + } let exc_type = py.get_type_bound::(); make_structured_err(py, &exc_type, &e) }) @@ -5863,6 +5914,9 @@ fn py_q_zeilberger( fn telescoping2d_error_to_py(e: CoreTelescoping2dError) -> PyErr { Python::with_gil(|py| { + if let Some(err) = budget_trip_to_py(py) { + return err; + } let exc_type = py.get_type_bound::(); make_structured_err(py, &exc_type, &e) }) @@ -11125,6 +11179,12 @@ fn validated_error_to_py(e: CoreValidatedError) -> PyErr { fn sos_error_to_py(e: CoreSosError) -> PyErr { Python::with_gil(|py| { + // Before `SosError`: `E-SOS-002` already conflates "exhausted" with + // "not attempted", and a loop that reads a budget stop as "not SOS" + // records a false negative. See `budget_trip_to_py`. + if let Some(err) = budget_trip_to_py(py) { + return err; + } let exc_type = py.get_type_bound::(); make_structured_err(py, &exc_type, &e) }) @@ -15312,6 +15372,13 @@ fn py_binomial_mod(a: u64, b: i128, p: u64, k: u32) -> PyResult { #[pymodule] fn alkahest(m: &Bound<'_, PyModule>) -> PyResult<()> { + // Before anything in this module can reach GMP: install the allocation + // accounting that `Budget(max_bytes=...)` and the address-space guard are + // measured against. Idempotent, and cheap enough to be unconditional — + // two relaxed atomics per GMP allocation. Without it an exact-rational + // solve that outgrows the machine `abort()`s the interpreter with + // `GNU MP: Cannot allocate memory`, which no `except` clause can catch. + alkahest_core::budget::install_memory_accounting(); m.add_function(wrap_pyfunction!(version, m)?)?; m.add_function(wrap_pyfunction!(py_derived_result_context_simplify, m)?)?; m.add_function(wrap_pyfunction!(py_simplify, m)?)?; @@ -15690,6 +15757,8 @@ fn alkahest(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(py_pop_budget, m)?)?; m.add_function(wrap_pyfunction!(py_is_budget_active, m)?)?; m.add_function(wrap_pyfunction!(py_budget_seed, m)?)?; + m.add_function(wrap_pyfunction!(py_budget_bytes_used, m)?)?; + m.add_function(wrap_pyfunction!(py_gmp_live_bytes, m)?)?; m.add_function(wrap_pyfunction!(py_request_cancel, m)?)?; m.add_function(wrap_pyfunction!(py_clear_cancel, m)?)?; m.add_function(wrap_pyfunction!(py_is_cancelled, m)?)?; diff --git a/alkahest-skill/alkahest.md b/alkahest-skill/alkahest.md index a48b444d..37e14585 100644 --- a/alkahest-skill/alkahest.md +++ b/alkahest-skill/alkahest.md @@ -999,7 +999,15 @@ ak.get_context_value("any_key") ## Budgets, cancellation, and determinism Use these whenever you write a loop that calls Alkahest many times. A `Budget` is an -immutable `(wall_ms, max_steps, seed)` triple pushed by `context(budget=…)`. +immutable `(wall_ms, max_steps, seed, max_bytes)` tuple pushed by `context(budget=…)`. + +`max_bytes` is the memory analogue of `wall_ms`, and it is the one you cannot skip in an +unattended loop: without it, an exact-rational computation that outgrows the machine is +**not catchable at all** — GMP prints `GNU MP: Cannot allocate memory` and calls +`abort()`, so the whole interpreter dies and every result it was holding is lost, not +just the offending call. Alkahest additionally refuses (`E-BUDGET-005`) when the process +is about to exhaust a finite `RLIMIT_AS` (`ulimit -v`, a container limit), with or +without a budget. ```python import alkahest as ak @@ -1009,6 +1017,7 @@ with ak.context(pool=pool, budget=ak.Budget(wall_ms=300, max_steps=50_000, seed= r = ak.integrate(hard_expr, x) except ak.BudgetExceededError as e: e.code # E-BUDGET-001 wall clock | -002 max_steps | -003 cancelled + # -004 max_bytes | -005 process address-space limit # deprioritise this candidate; DO NOT record it as "no antiderivative" ak.request_cancel() # process-wide flag, e.g. from a watchdog thread diff --git a/docs/mdbook/src/budgets.md b/docs/mdbook/src/budgets.md index 940bcecb..472e13d6 100644 --- a/docs/mdbook/src/budgets.md +++ b/docs/mdbook/src/budgets.md @@ -23,9 +23,9 @@ with ak.context(pool=p, budget=ak.Budget(wall_ms=50, max_steps=10_000, seed=7)): ## Model -A `Budget` is an immutable `(wall_ms, max_steps, seed)` triple. Every field is optional; -`Budget()` never trips a check on its own — only `alkahest.request_cancel()` can stop a -call entered with a bare `Budget()`. +A `Budget` is an immutable `(wall_ms, max_steps, seed, max_bytes)` tuple. Every field is +optional; `Budget()` never trips a check on its own — only `alkahest.request_cancel()` +can stop a call entered with a bare `Budget()`. `context(budget=...)` pushes the budget into a **thread-local** stack on the Rust side (`alkahest_cas::budget`) for the scope of the `with` block, and pops it on exit — @@ -337,6 +337,63 @@ The `jit` (LLVM) feature leaks a whole LLVM `Context` per compile — a true lea pool to drop, on the error paths as well as the success path. Cranelift (the default wheel's JIT) is unaffected. Do not compile in a loop under a `+jit` / `+full` build. +## Memory: `max_bytes`, and why an OOM used to kill the process + +`wall_ms` bounds how long a call may run. Nothing bounded how much memory it could ask +for — and the answer to *that* question was not an exception: + +``` +GNU MP: Cannot allocate memory (size=8) +timeout: the monitored command dumped core # SIGABRT, exit 134 +``` + +GMP's reaction to a failed allocation is to print that line and call `abort()`. The Rust +allocator's is `memory allocation of N bytes failed` and the same `abort()`. Neither is +catchable, so an exact-rational solve that outgrew the machine took the whole +interpreter with it — and an unattended loop lost every result it was holding, not just +the offending call. It was reachable from *default arguments* (`telescope_md` on the +`m = 4` multinomial). + +This cannot be fixed inside the allocator. GMP's contract for a replacement allocation +function forbids returning `NULL` — the library has no failure path to take — and a Rust +`panic!` may not cross a C frame. So Alkahest **counts** GMP's allocations (a wrapper +that delegates to GMP's own functions and does nothing else) and **refuses at its own +cooperative checkpoints**, before the allocation that would have died: + +```python +with ak.context(budget=ak.Budget(max_bytes=512 * 1024 * 1024)): + try: + telescope_md(term, n, xs) + except ak.BudgetExceededError as e: + assert e.code == "E-BUDGET-004" +``` + +`max_bytes` counts bytes of GMP (exact integer/rational) memory held live *by the +guarded block* — the total now, less the total when the block was entered. It is the +missing size budget: the engines' existing ceilings bound the *shape* of a linear system +(how many unknowns), and it is the *size of its numbers* that exhausts memory. + +### The address-space guard (`E-BUDGET-005`), which needs no budget at all + +If the process runs under a finite `RLIMIT_AS` — `ulimit -v`, a container memory limit, +a batch scheduler — Alkahest refuses when it climbs to within a reserve of that limit, +**whether or not a budget is active**. The operator already said how much the process +may have; stopping inside that number is strictly better than dying at it, and it is +what makes the default-arguments case survivable. With no limit set the guard is inert +and behaviour is unchanged. + +The reserve is a flat floor (32 MiB, up to 256 MiB for large limits) widened by the +address-space growth observed between the last two probes, so it tightens exactly when a +workload accelerates. + +### What this does not do + +The guard is checkpoint-granular. A single allocation large enough to cross the whole +reserve between two consecutive checkpoints still aborts — nothing short of a fallible +allocator can prevent that, and GMP does not have one. Address-space *usage* is only +observable on Linux (`/proc/self/statm`); elsewhere the guard degrades to `max_bytes` +alone. And `max_bytes` counts GMP memory, not Rust-side allocations. + ## Error codes | Code | Cause | @@ -344,8 +401,10 @@ wheel's JIT) is unaffected. Do not compile in a loop under a `+jit` / `+full` bu | `E-BUDGET-001` | The active budget's wall-clock limit elapsed | | `E-BUDGET-002` | The active budget's step counter exceeded `max_steps` | | `E-BUDGET-003` | `request_cancel()` was called and not yet cleared | +| `E-BUDGET-004` | The active budget's `max_bytes` ceiling was reached | +| `E-BUDGET-005` | The process is about to exhaust its address-space limit | -All three are `Cause::Resource` in the Rust registry (`alkahest_cas::errors::codes`) — +All five are `Cause::Resource` in the Rust registry (`alkahest_cas::errors::codes`) — a budget/cancellation trip is an environment/policy limit, not a statement about the mathematics, so it is never conflated with e.g. `IntegrationError::NonElementary` (a proof that no elementary antiderivative exists). `alkahest.integrate` and diff --git a/docs/mdbook/src/errors.md b/docs/mdbook/src/errors.md index 071402ec..b52b7878 100644 --- a/docs/mdbook/src/errors.md +++ b/docs/mdbook/src/errors.md @@ -138,7 +138,7 @@ loop must record as **undecided**, never as a negative result. | `E-PSLQ-004` | `PslqError` | `guess_relation` found an integer relation the inputs' precision cannot justify — pinning down `n` coefficients bounded by `H` costs about `n·log10(2H+1)` digits of agreement, and the inputs do not carry that many. **Record it as `undecided`, not as "no relation exists":** the same constants at higher precision may well admit one. `relation_confidence` reports the same judgement as data, including a three-valued `credible` whose `None` means *the inputs' precision is not knowable*, never a pass | | `E-PSLQ-005` | `PslqError` | The constants are exact rationals and `Σ aᵢ·cᵢ` is not zero in exact arithmetic. **This one is a verdict, not a refusal** — the relation is refuted for the numbers supplied | | `E-INT-004` | `IntegrationError` | Proven non-elementary. **This one is a verdict, not a refusal** — keep it apart from the rest | -| `E-BUDGET-001..003` | `BudgetExceededError` | Ran out of the time/steps it was given, or was cancelled | +| `E-BUDGET-001..005` | `BudgetExceededError` | Ran out of the time, steps or memory it was given, was cancelled, or is about to exhaust the process address-space limit | `E-SERIES-003` travels out of band for the same reason (`SeriesError` is exhaustive) but *is* wired into the bindings: `series` returns `SeriesError::InvalidOrder` with diff --git a/python/alkahest/_budget.py b/python/alkahest/_budget.py index 027e64bf..68ad8005 100644 --- a/python/alkahest/_budget.py +++ b/python/alkahest/_budget.py @@ -8,7 +8,11 @@ OS-level kill. This module is the Python front door for that: :class:`Budget` - An immutable ``(wall_ms, max_steps, seed)`` triple. + An immutable ``(wall_ms, max_steps, seed, max_bytes)`` tuple. ``max_bytes`` + is the memory analogue of ``wall_ms``: without it, an exact-rational + computation that outgrows the machine is not catchable at all — GMP prints + ``GNU MP: Cannot allocate memory`` and calls ``abort()``, taking the + interpreter with it. See ``docs/mdbook/src/budgets.md``. ``alkahest.context(budget=...)`` Pushes the budget into the Rust-side cooperative checkpoint @@ -109,6 +113,20 @@ class Budget: Determinism seed available to RNG-consuming samplers via :func:`budget_seed` — two runs entering the same ``Budget(seed=7)`` observe the same seed at every call site that consults it. + max_bytes : int, optional + Ceiling, in bytes, on the exact-arithmetic (GMP) memory the guarded + block may hold live. The memory analogue of ``wall_ms``: the existing + engine ceilings bound the *shape* of a linear system (how many + unknowns) but not the *size* of its numbers, and it is coefficient + growth that exhausts memory. Exceeding it raises + :class:`~alkahest.BudgetExceededError` with code ``E-BUDGET-004``. + + Without it, an exact solve that outgrows the machine is **not** + catchable at all: GMP's reaction to a failed allocation is + ``GNU MP: Cannot allocate memory`` followed by ``abort()``, which takes + the interpreter down with it. Independently of this setting, Alkahest + also refuses (``E-BUDGET-005``) when the process is about to exhaust a + finite ``RLIMIT_AS`` (``ulimit -v``, or a container memory limit). Examples -------- @@ -123,6 +141,9 @@ class Budget: wall_ms: float | None = None max_steps: int | None = None seed: int | None = None + # Appended, not inserted: a defaulted trailing field keeps every existing + # positional `Budget(wall_ms, max_steps, seed)` call valid. + max_bytes: int | None = None def __post_init__(self) -> None: if self.wall_ms is not None and (not math.isfinite(self.wall_ms) or self.wall_ms < 0): @@ -131,6 +152,8 @@ def __post_init__(self) -> None: raise ValueError("Budget.max_steps must be a non-negative integer") if self.seed is not None and self.seed < 0: raise ValueError("Budget.seed must be a non-negative integer") + if self.max_bytes is not None and self.max_bytes < 0: + raise ValueError("Budget.max_bytes must be a non-negative integer") def _native(): @@ -199,11 +222,16 @@ class BudgetHandoff: seed : int or None Carried through so :func:`budget_seed` reads the same value on a worker as it does on the calling thread. + max_bytes : int or None + Carried through as-is, and — like ``max_steps`` — per worker rather + than batch-wide: the live-byte baseline is captured when each worker + pushes its frame, so each item gets its own ceiling. """ deadline: float | None max_steps: int | None seed: int | None + max_bytes: int | None = None def remaining_ms(self) -> float | None: """Milliseconds left until :attr:`deadline`, clamped at ``0.0``. @@ -223,7 +251,12 @@ def applied(self) -> Iterator[None]: from is thread-local). """ native = _native() - native.push_budget(wall_ms=self.remaining_ms(), max_steps=self.max_steps, seed=self.seed) + native.push_budget( + wall_ms=self.remaining_ms(), + max_steps=self.max_steps, + seed=self.seed, + max_bytes=self.max_bytes, + ) try: yield finally: @@ -263,7 +296,12 @@ def capture_budget(budget: Budget | None = None) -> BudgetHandoff | None: if budget is None: return None deadline = None if budget.wall_ms is None else time.perf_counter() + budget.wall_ms / 1000.0 - return BudgetHandoff(deadline=deadline, max_steps=budget.max_steps, seed=budget.seed) + return BudgetHandoff( + deadline=deadline, + max_steps=budget.max_steps, + seed=budget.seed, + max_bytes=budget.max_bytes, + ) def run_with_wall_fallback( diff --git a/python/alkahest/_context.py b/python/alkahest/_context.py index ae84dc83..6e1b4032 100644 --- a/python/alkahest/_context.py +++ b/python/alkahest/_context.py @@ -215,7 +215,12 @@ def context( if budget is not None: from . import alkahest as _native - _native.push_budget(wall_ms=budget.wall_ms, max_steps=budget.max_steps, seed=budget.seed) + _native.push_budget( + wall_ms=budget.wall_ms, + max_steps=budget.max_steps, + seed=budget.seed, + max_bytes=budget.max_bytes, + ) budget_pushed = True try: yield diff --git a/tests/test_resource_budgets.py b/tests/test_resource_budgets.py new file mode 100644 index 00000000..8b81727d --- /dev/null +++ b/tests/test_resource_budgets.py @@ -0,0 +1,267 @@ +"""Memory ceilings: `Budget.max_bytes`, and refusing before an out-of-memory abort. + +2026-08-19 issue #5. Three subsystems reached an out-of-memory condition on +three different code paths — `telescope_md` at *its own default arguments*, +a 7-parameter parametric Gröbner stage, and a jet-order-4 elimination — and +all three died the same way:: + + GNU MP: Cannot allocate memory (size=8) + timeout: the monitored command dumped core # SIGABRT, exit 134 + +No exception, no ``BudgetExceededError``, no ``try``/``except`` that could +help, and no memory analogue of ``Budget.wall_ms``. An unattended research +loop lost the whole interpreter, and with it every result it was holding — +not just the offending call. + +GMP cannot be made to *fail* an allocation: its contract for a replacement +allocator forbids returning ``NULL``, and a Rust ``panic!`` may not cross a C +frame. So the refusal happens *before* the allocation, at Alkahest's own +cooperative checkpoints, against two ceilings: + +``E-BUDGET-004`` + the caller's ``Budget(max_bytes=...)``. +``E-BUDGET-005`` + the process is about to exhaust a finite ``RLIMIT_AS`` (``ulimit -v``, a + container limit). Active with **no budget at all**, which is what makes + the default-arguments case survivable. + +Also covers the two sibling resource gaps from the same round: `q_zeilberger` +having no ceiling (#10) and `prove_nonneg` honouring no budget (26d). +""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import alkahest as ak +import pytest + +#: A call that has to *finish* to prove anything here takes well under a +#: minute; this only distinguishes "refused" from "hung". +HEAVY_TIMEOUT = 300 + + +@pytest.fixture +def pool() -> ak.ExprPool: + return ak.ExprPool() + + +def multinomial(pool: ak.ExprPool, m: int): + """``n! / (x₁!·…·x_m!·(n−Σxᵢ)!)`` — the m-index multinomial coefficient. + + At ``m = 4`` and ``telescope_md``'s own defaults this ran ~20 minutes and + then aborted the process; ``m = 3`` aborts too, under a memory cap that + ``m = 2`` fits inside comfortably. + """ + one = pool.integer(1) + n = pool.symbol("n") + xs = [pool.symbol(f"x{t + 1}") for t in range(m)] + rest = n + for x in xs: + rest = rest - x + den = ak.gamma(rest + one) + for x in xs: + den = den * ak.gamma(x + one) + return ak.gamma(n + one) / den, n, xs + + +# --------------------------------------------------------------------------- +# #5 — the abort itself +# --------------------------------------------------------------------------- + +#: Run in a **subprocess** with its own `RLIMIT_AS`, for two reasons: an +#: out-of-memory abort cannot be caught in-process (that is the whole bug), and +#: a test that deliberately exhausts memory must not be able to take the test +#: runner with it. The cap is set *relative to the address space the import +#: already mapped*, so this does not depend on how much a given build reserves. +_OOM_CHILD = textwrap.dedent( + """ + import resource, sys + import alkahest as ak + from alkahest.experimental import telescope_md + + def vsz(): + return int(open("/proc/self/statm").read().split()[0]) * resource.getpagesize() + + # 96 MB of headroom: more than the m = 2 case needs, far less than m = 3. + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + resource.setrlimit(resource.RLIMIT_AS, (vsz() + 96 * 1024 * 1024, hard)) + + pool = ak.ExprPool() + one = pool.integer(1) + n = pool.symbol("n") + xs = [pool.symbol("x%d" % (t + 1)) for t in range(3)] + rest = n + for x in xs: + rest = rest - x + den = ak.gamma(rest + one) + for x in xs: + den = den * ak.gamma(x + one) + term = ak.gamma(n + one) / den + + try: + telescope_md(term, n, xs) + except ak.BudgetExceededError as e: + print("REFUSED", e.code) + sys.exit(0) + except ak.HolonomicError as e: + # Also acceptable: a refusal is a refusal, as long as it is catchable. + print("REFUSED", e.code) + sys.exit(0) + print("COMPLETED") + """ +) + + +@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS + /proc/self/statm are Linux-only") +def test_out_of_memory_is_a_catchable_error_not_a_process_abort(): + """The headline of issue #5: no ``SIGABRT``, and an ``except`` clause works. + + Before the fix this child died with ``GNU MP: Cannot allocate memory + (size=8)`` and ``returncode == -6``; the ``print`` after the ``try`` was + never reached, and neither was any ``except``. + """ + proc = subprocess.run( + [sys.executable, "-c", _OOM_CHILD], + capture_output=True, + text=True, + timeout=HEAVY_TIMEOUT, + ) + assert proc.returncode >= 0, ( + f"the child died on signal {-proc.returncode} instead of raising — " + f"stderr: {proc.stderr[-800:]}" + ) + assert "Cannot allocate memory" not in proc.stderr, proc.stderr[-800:] + assert proc.returncode == 0, f"stdout: {proc.stdout!r} stderr: {proc.stderr[-800:]}" + assert proc.stdout.startswith("REFUSED E-BUDGET-"), proc.stdout + + +# --------------------------------------------------------------------------- +# #5 — Budget.max_bytes +# --------------------------------------------------------------------------- + + +def test_budget_max_bytes_defaults_to_none_and_validates(): + assert ak.Budget().max_bytes is None + assert ak.Budget(max_bytes=1024).max_bytes == 1024 + with pytest.raises(ValueError, match="max_bytes"): + ak.Budget(max_bytes=-1) + + +def test_max_bytes_is_a_trailing_field_so_positional_construction_still_works(): + """``Budget`` is a public dataclass; the field had to be *appended*.""" + b = ak.Budget(50.0, 10, 7) + assert (b.wall_ms, b.max_steps, b.seed, b.max_bytes) == (50.0, 10, 7, None) + + +def test_max_bytes_trips_telescope_md_with_a_coded_error(pool): + """``E-BUDGET-004``, and it is a ``BudgetExceededError`` — not a + ``HolonomicError`` that a loop would read as "no certificate exists".""" + from alkahest.experimental import telescope_md + + term, n, xs = multinomial(pool, 3) + with ak.context(budget=ak.Budget(max_bytes=1 << 20)): + with pytest.raises(ak.BudgetExceededError) as exc: + telescope_md(term, n, xs) + assert exc.value.code == "E-BUDGET-004" + assert "max_bytes" in (exc.value.remediation or "") + + +def test_a_generous_max_bytes_does_not_change_the_answer(pool): + """The ceiling is a ceiling, not an unconditional refusal.""" + from alkahest.experimental import telescope_md + + term, n, xs = multinomial(pool, 2) + with ak.context(budget=ak.Budget(max_bytes=1 << 40)): + cert = telescope_md(term, n, xs) + assert cert.order >= 1 + + +def test_gmp_accounting_is_installed_and_counts(pool): + """``max_bytes`` is measured against GMP's own live-byte total.""" + native = ak.alkahest + before = native.gmp_live_bytes() + big = pool.integer(2) ** pool.integer(200_000) + ak.simplify_expanded(big) + assert native.gmp_live_bytes() > 0 + assert before >= 0 + + +# --------------------------------------------------------------------------- +# #10 — q_zeilberger's resource ceiling +# --------------------------------------------------------------------------- + + +def test_q_zeilberger_refuses_at_a_ceiling_rather_than_running_unbounded(pool): + """``Σ_k [2n;k]_q`` — class-legal, and previously unbounded. + + Its near-twin ``Σ_k [n;k]_q`` decides in half a second; this one ran 8+ + minutes at the documented defaults with no output and had to be killed. + The refusal must *say* it is a ceiling, or a loop records a false negative. + """ + from alkahest.experimental import q_zeilberger, qbinomial + + q, n, k = pool.symbol("q"), pool.symbol("n"), pool.symbol("k") + term = qbinomial(pool, pool.integer(2) * n, k) + with pytest.raises(ak.HolonomicError) as exc: + q_zeilberger(term, q, n, k, max_order=2, max_degree=2) + assert exc.value.code == "E-HOLO-021" + assert "resource ceilings" in str(exc.value) + + +def test_q_zeilberger_still_solves_the_cheap_sibling(pool): + """``Σ_k [n;k]_q`` must be unaffected by the ceilings.""" + from alkahest.experimental import q_zeilberger, qbinomial + + q, n, k = pool.symbol("q"), pool.symbol("n"), pool.symbol("k") + cert = q_zeilberger(qbinomial(pool, n, k), q, n, k) + assert cert.order >= 1 + + +def test_q_zeilberger_honours_a_wall_budget(pool): + """It consulted no budget at all before — ``wall_ms`` did nothing.""" + from alkahest.experimental import q_zeilberger, qbinomial + + q, n, k = pool.symbol("q"), pool.symbol("n"), pool.symbol("k") + term = qbinomial(pool, pool.integer(2) * n, k) + with ak.context(budget=ak.Budget(wall_ms=300)): + with pytest.raises(ak.BudgetExceededError) as exc: + q_zeilberger(term, q, n, k, max_order=3, max_degree=6) + assert exc.value.code == "E-BUDGET-001" + + +# --------------------------------------------------------------------------- +# 26d — prove_nonneg honours a budget +# --------------------------------------------------------------------------- + + +def test_prove_nonneg_honours_a_wall_budget(pool): + """418.5 s inside ``Budget(wall_ms=3000)``, ending in ``E-SOS-002``. + + The error class matters as much as the timing: ``E-SOS-002`` already + conflates "exhausted", "budget-limited" and "never attempted", so a loop + that reads it as "not SOS" files a false negative. A budget stop has to be + a ``BudgetExceededError``. + """ + x, y, z = pool.symbol("x"), pool.symbol("y"), pool.symbol("z") + sq = lambda e: e * e # noqa: E731 + motzkin = ( + sq(x) * sq(x) * sq(y) + + sq(x) * sq(y) * sq(y) + - pool.integer(3) * sq(x) * sq(y) * sq(z) + + sq(z) * sq(z) * sq(z) + ) + with ak.context(budget=ak.Budget(wall_ms=1000)): + with pytest.raises(ak.BudgetExceededError) as exc: + ak.prove_nonneg(motzkin, [x, y, z]) + assert exc.value.code == "E-BUDGET-001" + + +def test_prove_nonneg_without_a_budget_still_reports_no_certificate(pool): + """No budget, no change: the Motzkin form is still an honest ``E-SOS-002``.""" + x, y = pool.symbol("x"), pool.symbol("y") + p = (x * x - pool.integer(2) * x * y + y * y) + pool.integer(1) + cert = ak.prove_nonneg(p, [x, y]) + assert cert.kind in {"sos", "handelman", "putinar"}