diff --git a/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/case.txt b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/case.txt new file mode 100644 index 0000000..84fa3ec --- /dev/null +++ b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/case.txt @@ -0,0 +1,4 @@ +name: A closing tag spelled with a dotted capital I does not close the block +why: The other character outside ASCII whose lowercase reaches into it, and it reaches differently: `U+0130` folds to two code points, an `i` and a combining dot, so `` becomes `` with the dot between and never matches ``. The block therefore stays open and nothing after it joins. This is the case that says the fold is a containment test rather than an offset: a length-changing fold is safe here only because no position in the folded line is used to slice the original, and a port that folded in place and then indexed by the result would read past its own line. It sits beside the Kelvin case because the two together are every character that can reach this path. +paragraphs_unwrapped: 0 +line_breaks_removed: 0 diff --git a/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/expected.md b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/expected.md new file mode 100644 index 0000000..3631245 --- /dev/null +++ b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/expected.md @@ -0,0 +1,5 @@ +
+raw html here + +wrapped prose +here. diff --git a/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/input.md b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/input.md new file mode 100644 index 0000000..3631245 --- /dev/null +++ b/corpus/cases/a-dotted-capital-i-does-not-close-an-html-block/input.md @@ -0,0 +1,5 @@ +
+raw html here + +wrapped prose +here. diff --git a/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/case.txt b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/case.txt new file mode 100644 index 0000000..816a0c2 --- /dev/null +++ b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/case.txt @@ -0,0 +1,4 @@ +name: A closing tag spelled with a Kelvin sign still closes the block +why: An HTML block ends where its closing tag appears, and the tool finds that tag by lowercasing the whole line and looking for an ASCII needle. Lowercasing is the one thing this transform still takes from the runtime rather than stating itself, and the reason it cannot be narrowed to ASCII is here: `U+212A` is not ASCII and its lowercase is, so `` folds to `` and closes the block, and the prose after it is prose. An implementation folding only ASCII leaves the block open to the end of the file and joins nothing, which is what the counts on this case separate. Only two characters outside ASCII fold into it -- this one and the dotted capital I in the sibling case -- so the pair covers the whole of what that dependency can reach. +paragraphs_unwrapped: 1 +line_breaks_removed: 1 diff --git a/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/expected.md b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/expected.md new file mode 100644 index 0000000..a01b84c --- /dev/null +++ b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/expected.md @@ -0,0 +1,4 @@ +
+raw html here + +wrapped prose here. diff --git a/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/input.md b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/input.md new file mode 100644 index 0000000..70a53d4 --- /dev/null +++ b/corpus/cases/a-kelvin-sign-folds-into-an-html-closing-tag/input.md @@ -0,0 +1,5 @@ +
+raw html here + +wrapped prose +here. diff --git a/src/scan.rs b/src/scan.rs index 02541e2..ddfb44c 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -13,8 +13,9 @@ /// Python's `str.isspace()`, `str.strip()`, and `\s` on a `str` pattern. /// -/// One set of 29 code points, verified identical across all three on 3.10 and -/// 3.13: the Unicode `White_Space` property plus the four C0 separators +/// One set of 29 code points, verified identical across all three on 3.10, 3.11, +/// 3.12, 3.13 and 3.14, re-measured 2026-09-04: the Unicode `White_Space` +/// property plus the four C0 separators /// `U+001C`-`U+001F`. Rust's `char::is_whitespace` is `White_Space` alone, 25 /// points, so `str::trim` is not `str.strip()` and is never used to port it. /// @@ -589,6 +590,41 @@ mod tests { assert_eq!(py_trim_end("\u{1c}a\u{1e}"), "\u{1c}a"); } + #[test] + fn the_characters_that_fold_into_ascii_have_not_moved() { + // `match_opening_html_block` lowercases a whole line and looks for an ASCII needle such + // as ``, so a character whose lowercase *contains* ASCII can complete one. + // Twenty-six of those are `A`-`Z`. The other two are why the line takes `to_lowercase` + // and not `to_ascii_lowercase`, and why the set is worth pinning here: it comes from + // the Unicode tables the compiler shipped with, and the Python side pins the same list + // against its own runtime, so the two agree by both answering to this list rather than + // by two runtimes happening to match. + // + // `U+0130` folds to two code points. That is safe here only because the + // fold feeds a containment test and never an offset. + let folded: Vec = (0..=0x10_FFFFu32) + .filter_map(char::from_u32) + .filter(|c| { + // The whole mapping against the whole original, not the first code + // point against the char: a fold that expands while keeping the + // original first -- `X` -> `Xy` -- differs from the char but leaves + // `next()` equal to it, so a `next()` test would drop it here and keep + // it on the Python side, where the comparison is `c.lower() != c`. + // Folded once and reused, since `to_lowercase` walks the tables. + let lowered: String = c.to_lowercase().collect(); + lowered != c.to_string() && lowered.chars().any(|x| x.is_ascii()) + }) + .map(u32::from) + .collect(); + let expected: Vec = (u32::from('A')..=u32::from('Z')) + .chain([0x130, 0x212A]) + .collect(); + assert_eq!( + folded, expected, + "lowercasing now maps a different set onto ASCII" + ); + } + #[test] fn an_ordered_list_marker_is_ascii_digits_only() { // The specification narrowed to `[0-9]`; `\d` was 650 code points on diff --git a/tests/test_unwrap.py b/tests/test_unwrap.py index 6554414..c678778 100644 --- a/tests/test_unwrap.py +++ b/tests/test_unwrap.py @@ -9,6 +9,7 @@ import io import json +import re import sys from pathlib import Path @@ -463,3 +464,117 @@ def test_the_negative_number_rule_is_the_tools_own(tmp_path: Path) -> None: main(['--write', token, str(doc)]) assert raised.value.code == 2, token assert doc.read_text(encoding='utf-8') == original, token + + +def test_the_whitespace_set_has_not_moved_under_the_interpreter() -> None: + r"""`str.strip()` still removes exactly the 29 code points the Rust writes out. + + `str.strip()`, `str.isspace()` and `\s` on a `str` pattern share one set: + Unicode `White_Space` plus the four C0 separators. All three are checked, + not just the one the tool calls most: `is_python_space` is pinned as the + equivalent of all three, and the matcher constants use `\s` directly, so a + runtime that moved one of them without the others would leave a + `strip()`-only guard green while the transform had already diverged. + `is_python_space` in `src/scan.rs` writes those 29 out by hand rather than + delegating to + `char::is_whitespace`, which is `White_Space` alone and so 25, and that file + carries the matching drift detector for its own side. This is the detector for + this one: the two implementations agree because both are pinned to this list, + not because two runtimes happen to define it the same way, so an interpreter + that moves it has to fail here rather than quietly change what the tool reads + as a blank line. + + Measured identical on 3.10, 3.11, 3.12, 3.13 and 3.14 on 2026-09-04. + """ + expected = ( + # tab, newline, vertical tab, form feed, carriage return + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + # file, group, record and unit separator: the four `char::is_whitespace` + # omits, which is why `is_python_space` writes the set out instead + 0x1C, + 0x1D, + 0x1E, + 0x1F, + 0x20, # space + 0x85, # next line + 0xA0, # no-break space + 0x1680, # ogham space mark + 0x2000, + 0x2001, + 0x2002, + 0x2003, + 0x2004, + 0x2005, + 0x2006, + 0x2007, + 0x2008, + 0x2009, + 0x200A, + 0x2028, # line separator + 0x2029, # paragraph separator + 0x202F, # narrow no-break space + 0x205F, # medium mathematical space + 0x3000, # ideographic space + ) + # One sweep, one `chr` per code point, the pattern compiled once. The guard + # has to cover the whole range to catch an addition anywhere in it, so the + # sweep is the cost of the test and paying it three times is three times too + # much for the same answer. + matches_pattern = re.compile(r'\s').fullmatch + strip_hits: list[int] = [] + isspace_hits: list[int] = [] + pattern_hits: list[int] = [] + for cp in range(0x110000): + char = chr(cp) + if char.strip() == '': + strip_hits.append(cp) + if char.isspace(): + isspace_hits.append(cp) + if matches_pattern(char) is not None: + pattern_hits.append(cp) + + for name, found in ( + ('str.strip()', tuple(strip_hits)), + ('str.isspace()', tuple(isspace_hits)), + (r'\s', tuple(pattern_hits)), + ): + assert found == expected, ( + f'{name} no longer selects exactly this set; ' + f'gained {[f"U+{c:04X}" for c in set(found) - set(expected)]}, ' + f'lost {[f"U+{c:04X}" for c in set(expected) - set(found)]}' + ) + + +def test_the_characters_that_fold_into_ascii_have_not_moved() -> None: + """Lowercasing still maps only these onto ASCII. + + The HTML block matcher lowercases a whole line and looks for an ASCII needle + such as ``, so a character whose lowercase *contains* ASCII can + complete one. Twenty-six of those are `A`-`Z`. The other two are the reason the + line cannot simply be folded with `str.lower`'s ASCII-only counterpart, and the + reason this set is worth pinning: it is defined by the runtime's case tables, + which do move -- 1393 code points gained a lowercase mapping by 3.11 and 1460 + by 3.14 -- and none of those additions landed in this set only because none of + them folded into ASCII. + + `U+0130` is the length-changing one, folding to two code points, which is safe + here only because the fold feeds a containment test and never an offset. + + Measured on 3.10 through 3.14 on 2026-09-04, and matched by the Rust detector + in `src/scan.rs`. + """ + expected = (*range(ord('A'), ord('Z') + 1), 0x0130, 0x212A) + found = tuple( + cp + for cp in range(0x110000) + if (lowered := chr(cp).lower()) != chr(cp) and any(c.isascii() for c in lowered) + ) + assert found == expected, ( + 'lowercasing now maps a different set onto ASCII; ' + f'gained {[f"U+{c:04X}" for c in set(found) - set(expected)]}, ' + f'lost {[f"U+{c:04X}" for c in set(expected) - set(found)]}' + )