Skip to content

Commit 2fbf38d

Browse files
arvclaude
andcommitted
Replace ICU with a bundled Unicode case table for lower()/upper()
ICU's value for Zero was only Unicode lower()/upper() (zqlite compiles ILIKE to lower(col) LIKE lower(pattern)), but linking it has been a continual source of pain: no -fPIC distro static archives, soname coupling that breaks glibc consumers (libicui18n.so.67), QEMU arm builds, ~28MB, and a Windows exception. Drop the ICU dependency entirely. Register Unicode-aware lower()/upper() on every connection from an embedded case-mapping table generated from Node's toLowerCase/toUpperCase (so the SQL functions match the client-side IVM matcher). The result is self-contained on every platform (incl. Windows), with no runtime ICU, no -fPIC/soname/QEMU problems, and ~120KB of generated data. - deps/gen-unicode-case.mjs: generator -> src/util/unicode_case_data.h. - src/util/unicode_case.cpp: UTF-8 case transform + lower()/upper(), registered via sqlite3_auto_extension in the addon init. - Remove SQLITE_ENABLE_ICU, deps/icu.js, and the ICU gyp linking. - Rename test/52.icu.js -> test/52.unicode-case.js; assert lower()/upper() match JavaScript across scripts and 1:many mappings (ß->SS, İ->i̇). Scope: context-free case mapping only (no folding, no Greek final-sigma context). Follow-ups: the now-dead CI ICU installs can be removed (pairs with the native arm64 PR), and the zero_sqlite3 shell still uses ASCII lower()/upper(). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 000daaa commit 2fbf38d

10 files changed

Lines changed: 3368 additions & 244 deletions

File tree

binding.gyp

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,6 @@
2424
},
2525
},
2626
'conditions': [
27-
# ICU is statically linked into the SQLite static library on
28-
# non-Windows; the final .node must resolve its ICU symbols. (See
29-
# deps/sqlite3.gyp for why Windows is excluded.)
30-
['OS != "win"', {
31-
'libraries': ['<!@(node deps/icu.js libs)'],
32-
}],
3327
['OS=="linux"', {
3428
'ldflags': [
3529
'-Wl,-Bsymbolic',
@@ -75,13 +69,6 @@
7569
'defines': ['HAVE_EDITLINE=1'],
7670
'libraries': ['-ledit', '-lncurses'],
7771
}],
78-
# Unicode-aware LIKE/upper()/lower() via the bundled ICU extension,
79-
# statically linked. Excluded on Windows (see deps/sqlite3.gyp).
80-
['OS != "win"', {
81-
'defines': ['SQLITE_ENABLE_ICU'],
82-
'include_dirs': ['<!@(node deps/icu.js include)'],
83-
'libraries': ['<!@(node deps/icu.js libs)'],
84-
}],
8572
],
8673
'configurations': {
8774
'Debug': {

deps/download.sh

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,6 @@
2121
CHECKIN="0e862bc9ed7aa9ae"
2222

2323
# Defines below are sorted alphabetically.
24-
#
25-
# Note: SQLITE_ENABLE_ICU is intentionally NOT listed here. These defines are
26-
# applied unconditionally on every platform (they become defines.gypi and are
27-
# passed to every compile), but ICU is only available on non-Windows builds.
28-
# It is therefore defined conditionally (OS != "win") in deps/sqlite3.gyp
29-
# instead. The ICU extension code already ships in the amalgamation guarded by
30-
# #ifdef SQLITE_ENABLE_ICU, so it does not need to be set when generating
31-
# sqlite3.c here.
3224
DEFINES="
3325
HAVE_INT16_T=1
3426
HAVE_INT32_T=1

deps/gen-unicode-case.mjs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Generates src/util/unicode_case_data.h — the case-mapping tables used by the
2+
// driver's Unicode-aware lower()/upper() SQL functions.
3+
//
4+
// The tables are generated from THIS Node's String.prototype.toLowerCase /
5+
// toUpperCase. That makes the C implementation match JavaScript's (context-free)
6+
// case conversion by construction — which is exactly the consistency the
7+
// client-side IVM matcher (toLowerCase) and the zqlite replica rely on. Case
8+
// mappings are stable across Unicode versions, so this only needs regenerating
9+
// on a deliberate Unicode bump.
10+
//
11+
// Usage: node deps/gen-unicode-case.mjs > src/util/unicode_case_data.h
12+
13+
const MAX = 0x10ffff;
14+
15+
function mappings(method) {
16+
const rows = [];
17+
for (let cp = 0; cp <= MAX; cp++) {
18+
if (cp >= 0xd800 && cp <= 0xdfff) continue; // lone surrogates
19+
const ch = String.fromCodePoint(cp);
20+
const mapped = ch[method]();
21+
if (mapped === ch) continue;
22+
const to = Array.from(mapped, c => c.codePointAt(0));
23+
if (to.length > 3) throw new Error(`mapping for U+${cp.toString(16)} > 3 cps`);
24+
rows.push([cp, to]);
25+
}
26+
return rows;
27+
}
28+
29+
function emit(name, rows) {
30+
const lines = rows.map(([from, to]) => {
31+
const t = [...to, 0, 0, 0].slice(0, 3).map(c => `0x${c.toString(16)}u`);
32+
return ` {0x${from.toString(16)}u, {${t.join(', ')}}, ${to.length}},`;
33+
});
34+
return (
35+
`static const ZeroCaseMap ${name}[] = {\n${lines.join('\n')}\n};\n` +
36+
`static const int ${name}Len = ${rows.length};\n`
37+
);
38+
}
39+
40+
const lower = mappings('toLowerCase');
41+
const upper = mappings('toUpperCase');
42+
43+
process.stdout.write(
44+
`// AUTO-GENERATED by deps/gen-unicode-case.mjs — DO NOT EDIT.\n` +
45+
`// Source: Node ${process.versions.node} (Unicode ${process.versions.unicode}).\n` +
46+
`// Each row maps one source code point to 1-3 target code points, matching\n` +
47+
`// JavaScript String.prototype.toLowerCase/toUpperCase (context-free).\n` +
48+
`#ifndef ZERO_UNICODE_CASE_DATA_H\n#define ZERO_UNICODE_CASE_DATA_H\n\n` +
49+
`typedef struct ZeroCaseMap {\n` +
50+
` unsigned int from;\n unsigned int to[3];\n unsigned char n;\n` +
51+
`} ZeroCaseMap;\n\n` +
52+
emit('kZeroLowerMap', lower) +
53+
`\n` +
54+
emit('kZeroUpperMap', upper) +
55+
`\n#endif /* ZERO_UNICODE_CASE_DATA_H */\n`,
56+
);

deps/icu.js

Lines changed: 0 additions & 174 deletions
This file was deleted.

deps/sqlite3.gyp

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,6 @@
5858
'SQLITE_ENABLE_COLUMN_METADATA',
5959
],
6060
}],
61-
# Unicode-aware LIKE/upper()/lower() via SQLite's bundled ICU extension,
62-
# statically linked so the prebuilt binaries stay self-contained.
63-
# Not enabled on Windows yet: static ICU there means building it from
64-
# source (vcpkg), which is impractically slow in CI. Windows therefore
65-
# keeps SQLite's ASCII-only LIKE for now.
66-
['OS != "win"', {
67-
'defines': ['SQLITE_ENABLE_ICU'],
68-
'include_dirs': ['<!@(node icu.js include)'],
69-
}],
7061
],
7162
'configurations': {
7263
'Debug': {

src/better_sqlite3.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ class Backup;
4444
#include "objects/statement.cpp"
4545
#include "objects/database.cpp"
4646
#include "objects/statement-iterator.cpp"
47+
#include "util/unicode_case.cpp"
4748

4849
NODE_MODULE_INIT(/* exports, context */) {
4950
#if defined(NODE_MODULE_VERSION) && NODE_MODULE_VERSION >= 140
@@ -55,6 +56,9 @@ NODE_MODULE_INIT(/* exports, context */) {
5556
v8::HandleScope scope(isolate);
5657
Addon::ConfigureURI();
5758

59+
// Register Unicode-aware lower()/upper() on every connection (replaces ICU).
60+
sqlite3_auto_extension((void (*)(void))zeroRegisterUnicodeCase);
61+
5862
// Initialize addon instance.
5963
Addon* addon = new Addon(isolate);
6064
v8::Local<v8::External> data = v8::External::New(isolate, addon);

0 commit comments

Comments
 (0)