Skip to content

Commit 054bedf

Browse files
arvclaude
arv
and
claude
committed
unicode case: implement the Greek final-sigma rule
Adds the one context-sensitive rule that JS toLowerCase applies in the default (locale-independent) algorithm: Σ lowercases to ς when preceded by a cased letter (skipping case-ignorable chars) and not followed by one, else σ. - Generator emits Cased / Case_Ignorable as code-point ranges (from the \p{Cased} / \p{Case_Ignorable} escapes, so they track V8's Unicode version). - lower() tracks prevCased and, for Σ, scans forward for a following cased letter (LowerSigma). upper() is unchanged. - Tests: add final-sigma cases and an exhaustive guard that lower()/upper() equal JS for every code point (verified: 0 mismatches across all 1,112,064). With this there is no remaining context divergence from JS toLowerCase/ toUpperCase within a Unicode version. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 2fbf38d commit 054bedf

4 files changed

Lines changed: 782 additions & 25 deletions

File tree

‎deps/gen-unicode-case.mjs‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,22 @@
22
// driver's Unicode-aware lower()/upper() SQL functions.
33
//
44
// The tables are generated from THIS Node's String.prototype.toLowerCase /
5-
// toUpperCase. That makes the C implementation match JavaScript's (context-free)
6-
// case conversion by construction — which is exactly the consistency the
5+
// toUpperCase and the Cased / Case_Ignorable Unicode property escapes. That
6+
// makes the C implementation match JavaScript's case conversion by construction
7+
// (including the Greek final-sigma rule) — exactly the consistency the
78
// client-side IVM matcher (toLowerCase) and the zqlite replica rely on. Case
8-
// mappings are stable across Unicode versions, so this only needs regenerating
9-
// on a deliberate Unicode bump.
9+
// data is stable across Unicode versions, so this only needs regenerating on a
10+
// deliberate Unicode bump.
1011
//
1112
// Usage: node deps/gen-unicode-case.mjs > src/util/unicode_case_data.h
1213

1314
const MAX = 0x10ffff;
15+
const isSurrogate = cp => cp >= 0xd800 && cp <= 0xdfff;
1416

1517
function mappings(method) {
1618
const rows = [];
1719
for (let cp = 0; cp <= MAX; cp++) {
18-
if (cp >= 0xd800 && cp <= 0xdfff) continue; // lone surrogates
20+
if (isSurrogate(cp)) continue; // lone surrogates
1921
const ch = String.fromCodePoint(cp);
2022
const mapped = ch[method]();
2123
if (mapped === ch) continue;
@@ -26,6 +28,33 @@ function mappings(method) {
2628
return rows;
2729
}
2830

31+
// Collapses the code points matching `re` into sorted [lo, hi] ranges. Used for
32+
// the Cased / Case_Ignorable properties needed by the final-sigma rule.
33+
function propertyRanges(re) {
34+
const ranges = [];
35+
let start = -1;
36+
for (let cp = 0; cp <= MAX; cp++) {
37+
const match = !isSurrogate(cp) && re.test(String.fromCodePoint(cp));
38+
if (match && start < 0) start = cp;
39+
else if (!match && start >= 0) {
40+
ranges.push([start, cp - 1]);
41+
start = -1;
42+
}
43+
}
44+
if (start >= 0) ranges.push([start, MAX]);
45+
return ranges;
46+
}
47+
48+
function emitRanges(name, ranges) {
49+
const lines = ranges.map(
50+
([lo, hi]) => ` {0x${lo.toString(16)}u, 0x${hi.toString(16)}u},`,
51+
);
52+
return (
53+
`static const ZeroRange ${name}[] = {\n${lines.join('\n')}\n};\n` +
54+
`static const int ${name}Len = ${ranges.length};\n`
55+
);
56+
}
57+
2958
function emit(name, rows) {
3059
const lines = rows.map(([from, to]) => {
3160
const t = [...to, 0, 0, 0].slice(0, 3).map(c => `0x${c.toString(16)}u`);
@@ -39,18 +68,26 @@ function emit(name, rows) {
3968

4069
const lower = mappings('toLowerCase');
4170
const upper = mappings('toUpperCase');
71+
const cased = propertyRanges(/\p{Cased}/u);
72+
const caseIgnorable = propertyRanges(/\p{Case_Ignorable}/u);
4273

4374
process.stdout.write(
4475
`// AUTO-GENERATED by deps/gen-unicode-case.mjs — DO NOT EDIT.\n` +
4576
`// Source: Node ${process.versions.node} (Unicode ${process.versions.unicode}).\n` +
46-
`// Each row maps one source code point to 1-3 target code points, matching\n` +
47-
`// JavaScript String.prototype.toLowerCase/toUpperCase (context-free).\n` +
77+
`// Case maps: one source code point to 1-3 target code points, matching\n` +
78+
`// JavaScript String.prototype.toLowerCase/toUpperCase. Cased / CaseIgnorable\n` +
79+
`// are property ranges used by the Greek final-sigma rule.\n` +
4880
`#ifndef ZERO_UNICODE_CASE_DATA_H\n#define ZERO_UNICODE_CASE_DATA_H\n\n` +
4981
`typedef struct ZeroCaseMap {\n` +
5082
` unsigned int from;\n unsigned int to[3];\n unsigned char n;\n` +
5183
`} ZeroCaseMap;\n\n` +
84+
`typedef struct ZeroRange {\n unsigned int lo;\n unsigned int hi;\n} ZeroRange;\n\n` +
5285
emit('kZeroLowerMap', lower) +
5386
`\n` +
5487
emit('kZeroUpperMap', upper) +
88+
`\n` +
89+
emitRanges('kZeroCased', cased) +
90+
`\n` +
91+
emitRanges('kZeroCaseIgnorable', caseIgnorable) +
5592
`\n#endif /* ZERO_UNICODE_CASE_DATA_H */\n`,
5693
);

‎src/util/unicode_case.cpp‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
// case conversion. Registered as an auto-extension, overriding SQLite's
66
// ASCII-only built-in lower()/upper() on every connection.
77
//
8-
// Scope: context-free case mapping (the common case). It does not implement
9-
// context-sensitive rules such as Greek final sigma; JavaScript's toLowerCase
10-
// applies those, so a word-final Σ can differ. The zqlite ILIKE parity test
11-
// guards the cases Zero relies on.
8+
// This matches JavaScript's default (locale-independent) case conversion for
9+
// all input: per-code-point full mappings plus the one context-sensitive rule
10+
// that algorithm applies — Greek final sigma (see LowerSigma). Locale-specific
11+
// rules (Turkish dotless i, Lithuanian) are not applied, and neither does
12+
// String.prototype.toLowerCase, so the two stay consistent.
1213

1314
#include "unicode_case_data.h"
1415

@@ -71,7 +72,47 @@ static const ZeroCaseMap* Lookup(const ZeroCaseMap* map, int len, unsigned int c
7172
return NULL;
7273
}
7374

74-
static void Apply(sqlite3_context* ctx, sqlite3_value* arg, const ZeroCaseMap* map, int len) {
75+
// Whether `cp` falls in one of the sorted, non-overlapping [lo, hi] ranges.
76+
static int InRanges(const ZeroRange* r, int len, unsigned int cp) {
77+
int lo = 0, hi = len - 1;
78+
while (lo <= hi) {
79+
int mid = (lo + hi) >> 1;
80+
if (cp < r[mid].lo) hi = mid - 1;
81+
else if (cp > r[mid].hi) lo = mid + 1;
82+
else return 1;
83+
}
84+
return 0;
85+
}
86+
87+
static int IsCased(unsigned int cp) {
88+
return InRanges(kZeroCased, kZeroCasedLen, cp);
89+
}
90+
static int IsCaseIgnorable(unsigned int cp) {
91+
return InRanges(kZeroCaseIgnorable, kZeroCaseIgnorableLen, cp);
92+
}
93+
94+
static const unsigned int kCapitalSigma = 0x3A3u; // Σ
95+
static const unsigned int kSmallSigma = 0x3C3u; // σ
96+
static const unsigned int kFinalSigma = 0x3C2u; // ς
97+
98+
// Lowercasing Σ is the one context-sensitive rule in the default (locale-
99+
// independent) algorithm that JS toLowerCase applies: Σ -> ς when it is preceded
100+
// by a cased letter (ignoring case-ignorable chars) and not followed by one;
101+
// otherwise Σ -> σ. `prevCased` is whether the last non-ignorable input char was
102+
// cased; `in`/`n`/`after` scan the input following the Σ.
103+
static unsigned int LowerSigma(const unsigned char* in, int n, int after, int prevCased) {
104+
int followedByCased = 0;
105+
int j = after;
106+
while (j < n) {
107+
unsigned int c = Utf8Decode(in, n, &j);
108+
if (IsCaseIgnorable(c)) continue;
109+
followedByCased = IsCased(c);
110+
break;
111+
}
112+
return (prevCased && !followedByCased) ? kFinalSigma : kSmallSigma;
113+
}
114+
115+
static void Apply(sqlite3_context* ctx, sqlite3_value* arg, const ZeroCaseMap* map, int len, int lower) {
75116
if (sqlite3_value_type(arg) == SQLITE_NULL) {
76117
sqlite3_result_null(ctx);
77118
return;
@@ -92,6 +133,7 @@ static void Apply(sqlite3_context* ctx, sqlite3_value* arg, const ZeroCaseMap* m
92133
}
93134
int outn = 0;
94135
int i = 0;
136+
int prevCased = 0; // was the last non-case-ignorable input char cased?
95137
while (i < n) {
96138
unsigned int cp = Utf8Decode(in, n, &i);
97139
// Reserve room for up to 3 mapped code points (4 bytes each).
@@ -105,24 +147,30 @@ static void Apply(sqlite3_context* ctx, sqlite3_value* arg, const ZeroCaseMap* m
105147
}
106148
out = grown;
107149
}
108-
const ZeroCaseMap* m = Lookup(map, len, cp);
109-
if (m) {
110-
for (int k = 0; k < m->n; k++) outn += Utf8Encode(m->to[k], out + outn);
150+
if (lower && cp == kCapitalSigma) {
151+
outn += Utf8Encode(LowerSigma(in, n, i, prevCased), out + outn);
111152
} else {
112-
outn += Utf8Encode(cp, out + outn);
153+
const ZeroCaseMap* m = Lookup(map, len, cp);
154+
if (m) {
155+
for (int k = 0; k < m->n; k++) outn += Utf8Encode(m->to[k], out + outn);
156+
} else {
157+
outn += Utf8Encode(cp, out + outn);
158+
}
113159
}
160+
// Context for final-sigma is evaluated on the original input.
161+
if (!IsCaseIgnorable(cp)) prevCased = IsCased(cp);
114162
}
115163
sqlite3_result_text(ctx, out, outn, sqlite3_free);
116164
}
117165

118166
static void LowerFunc(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
119167
(void)argc;
120-
Apply(ctx, argv[0], kZeroLowerMap, kZeroLowerMapLen);
168+
Apply(ctx, argv[0], kZeroLowerMap, kZeroLowerMapLen, 1);
121169
}
122170

123171
static void UpperFunc(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
124172
(void)argc;
125-
Apply(ctx, argv[0], kZeroUpperMap, kZeroUpperMapLen);
173+
Apply(ctx, argv[0], kZeroUpperMap, kZeroUpperMapLen, 0);
126174
}
127175

128176
} // namespace UnicodeCase

0 commit comments

Comments
 (0)