Integrate native structural storage with the legacy FastText model and CLI

.github/workflows/ci.yml

@@ -54,6 +54,21 @@ jobs:
           c++ -std=c++11 -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -Isrc -Inative -Imylib/include tests/structural_native_test.cc native/meft_structural.cc -Lmylib/lib -ltrie -lpub -o build/structural-sanitizers
           ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 build/structural-sanitizers
 
+  fasttext-native-fusion:
+    name: FastText native facade sanitizers
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Check original model IDs, row bytes and inference outputs
+        env:
+          MEFT_FUSION_SANITIZERS: "1"
+          ASAN_OPTIONS: detect_leaks=1:halt_on_error=1
+          UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
+        run: python -m unittest tests.test_fasttext_native_bundle -v
+
   legacy-roundtrip:
     name: Legacy artifact roundtrip
     runs-on: ubuntu-latest

Makefile

@@ -15,7 +15,13 @@ LIBS = -L./mylib/lib -ltrie -lpub
 LIBNAME = libfasttext
 PYTHON ?= python3
 
-.PHONY: opt debug test clean native structural-native
+.PHONY: opt debug test clean native structural-native fasttext-native
+
+# Optional Linux FastText facade; retain the original binary/library targets.
+fasttext-native:
+	mkdir -p build
+	$(CC) -std=c11 -O2 -fPIC -c native/meft_rows.c -o build/meft_rows_fasttext.o
+	$(CXX) $(CXXFLAGS) -O2 -DMEFT_NATIVE_FASTTEXT -Isrc -Inative $(INCLUDES) $(addprefix src/,$(OBJS:.o=.cc)) native/fasttext_bundle.cc src/main.cc build/meft_rows_fasttext.o $(LIBS) -lz -o build/fasttext-native
 
 # Optional original-algorithm bridge; bundled archives require Linux x86-64.
 structural-native:

README.md

@@ -25,6 +25,7 @@ records that separately; the legacy C++ exporter also supports approximate shari
 · [**Original DATrie + mark-compact native integration and trained ablations**](docs/structural_native_evidence_20261030/README.md)
 · [**1.85M trained-row CPU capacity controls and read-amplification limits**](docs/cpu_capacity_evidence_20261030/README.md)
 · [**Sparse native blocks: 16–27× lower miss latency on the same trained table**](docs/sparse_blocks_evidence_20261030/README.md)
+· [**Native storage in the original FastText model/CLI: trained fidelity and serving controls**](docs/fasttext_native_evidence_20261031/README.md)
 · [**Installable TinyEngram/Qwen checkpoint backend**](docs/tinyengram_qwen_evidence_20261030/README.md)
 · [**Host memory → device query and prefetch**](docs/device_memory_prefetch.md)
 
@@ -45,6 +46,14 @@ large-table whole-model memory gains remain unmeasured.
 
 This repository accompanies the paper [Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management](https://arxiv.org/abs/2506.01254).
 
+The optional [native FastText facade](docs/fasttext_native_bundle.md) converts
+this repository's dense `.bin` models through the original DATrie registry and
+mark-compact pipeline, then serves them with the existing CLI/query APIs.
+Layout/exact modes retain every input byte and float32 aggregation order.
+The first actual trained-model control saves 0.46% for the whole bundle; hot
+queries remain slower than dense, and structural ordering has no added storage
+benefit on that table. Historical valid word/sentence outputs match exactly.
+
 > Status: research / production-prototype code. The command-line interface remains close to the original fastText workflow, while the storage and serving path has been extended for large-vocabulary, memory-sensitive deployments.
 
 > Inference framing: `me_fasttext` can also be read as an early compact-inference prototype for lexical embeddings. It keeps subword identity exact during training, then rewrites the learned rows into a smaller mmap-friendly serving artifact for low-memory retrieval and feature-serving workloads.

docs/fasttext_native_bundle.md

@@ -0,0 +1,92 @@
+# Native storage through the original FastText model and CLI
+
+Dense **me_fasttext** `.bin` models can be converted into immutable native
+serving bundles. `loadModel(path)` and existing query commands then use the
+original dictionary and row IDs, with input vectors stored in compressed native
+blocks. The production DATrie prefix/suffix planner and mark-compact forwarding
+are part of this conversion. [Actual trained-model results and limits](fasttext_native_evidence_20261031/README.md)
+include a historical-main comparison and a matched identity-layout control.
+
+## Export and query
+
+The optional facade and bundled DATrie archives require Linux x86-64. Build from
+this repository; the standalone memory wheel does not bundle the legacy trainer.
+
+```bash
+make fasttext-native native structural-native
+
+# Use a NEW output directory. Keep the original .bin for training/quantization.
+python tools/export_fasttext_native_cli.py \
+  --model model.bin --output-dir model.native \
+  --fasttext-binary build/fasttext-native \
+  --structural-library build/libmeft_structural.so \
+  --native-library build/libmeft_rows.so --mode exact
+
+# The same command and arguments accept either a .bin or a bundle directory.
+printf 'example\nexamples\n' | build/fasttext-native print-word-vectors model.native
+printf 'an example sentence\n' | build/fasttext-native print-sentence-vectors model.native
+```
+
+For a supervised model, `predict`, `predict-prob` and `test` also use the native
+input backend. `nn`/`analogies` still precompute resident word vectors and are not
+a bounded-memory nearest-neighbor service. The original `make` targets remain
+available. The original `.z`/FTIndex path retains its own format and reader.
+
+After installing `meft-memory` from this checkout, the same converter is exposed
+as `meft-fasttext-export` and `meft_memory.export_fasttext_model`. The binary and
+structural-library paths remain explicit. No Torch dependency is required for
+FastText export/query.
+
+In an optional C++ source build, query calls keep their signatures:
+
+```cpp
+fasttext::FastText model;
+model.loadModel("model.native");
+fasttext::Vector word(model.getDimension());
+model.getWordVector(word, "example");
+```
+
+Link `native/fasttext_bundle.cc` and the C row reader with
+`-DMEFT_NATIVE_FASTTEXT`, as the Make target does. Recompile C++ callers: retained
+source signatures are not an old binary ABI guarantee.
+
+## Fidelity, compatibility and memory
+
+- `layout` retains every row; `exact` additionally shares byte-identical
+  structurally related rows when present. Both verify the full reconstructed
+  input SHA-256, including untrained rows. The source `.bin` is never rewritten.
+- `joint` is opt-in approximate sharing with cosine and norm-ratio guards.
+  Its manifest separates source bytes from reconstructed representative bytes.
+  It does not inherit lossless output guarantees.
+- The dictionary looks up the **original registered IDs** in `keys.das`;
+  `table.men` resolves their physical rows through forwarding. Aggregation
+  preserves the original request order/multiplicity and float32 adds, across
+  gather chunks. The FP64 native bag reducer is deliberately not used here.
+- Loaded dictionaries now skip unknown subwords instead of registering rows
+  beyond the trained input matrix. Direct subword helpers use real DATrie IDs;
+  missing keys return `-1`/zero. Empty or zero-vector string sentence queries
+  return zero instead of NaN. These fix prior undefined/invalid behavior;
+  equivalence is claimed for valid historical queries, not those bugs.
+- The default export caps decoded blocks at 8,192 bytes, uses a 4 MiB decoded
+  block cache and gathers at most 256 rows before ordered accumulation. These
+  are configurable via `--max-block-bytes`, `--cache-bytes` and
+  `--query-chunk-rows`. A block's row count also depends on vector dimension.
+- Input is frozen and each handle is single-worker. `saveModel`, legacy
+  `saveIndex`, `quantize` and `getInputMatrix` reject native input; reload a dense
+  model for those operations. Starting a new `train(args)` clears the backend.
+- The dictionary, its legacy 200 MB unused word hash allocation, exact key
+  index and dense output matrix remain resident. Neither loader builds an
+  inference negative-sampling table. Cache budget is not a total RSS limit;
+  touched mmap pages remain file-backed memory.
+
+The bundle contains `table.men`, `keys.das`, original args/dictionary/output in
+`model.meta`, reader settings and audit manifests. Open only trusted,
+self-produced bundles on little-endian IEEE float32 Linux x86-64: the legacy
+metadata/index loader is not an untrusted-input parser. The exporter records
+checksums, but the C++ facade does not authenticate the entire bundle at open.
+Conversion holds full source rows and planning maps offline.
+
+This converter supports this repository's dense, unquantized, unpruned models.
+Official fastText `.bin`, quantized `.ftz`, and old `.z` files are separate
+formats. Engram hash collisions do not provide this unique string-key registry;
+the numeric-row adapter retains its separate integration boundary.

docs/fasttext_native_evidence_20261031/README.md

@@ -0,0 +1,146 @@
+# Actual me_fasttext model → native FastText serving
+
+The native backend now integrates with the original FastText facade, rather
+than requiring a separate numeric-row application. A genuinely trained legacy
+`.bin` converts through the original DATrie registry and structural planner;
+the same CLI and C++ query APIs can load the native bundle. See the
+[build/export/query guide](../fasttext_native_bundle.md) and
+[PR #36](https://github.com/initial-d/me_fasttext/pull/36).
+
+**This establishes functional fusion and exact inference. Storage and latency
+gains are limited on this small model.** The complete native bundle saves only
+0.46%; warm queries are slower than resident dense queries. Structural layout
+is slightly worse than a matched identity layout here. These are CPU public-text
+storage/serving results, not Engram checkpoint or downstream quality evidence.
+
+## Fidelity and historical compatibility
+
+The actual legacy trainer produces **90,380 × 64 float32 input rows**, including
+14,813 word/source/position rows and 75,567 registered n-gram rows. Training
+changes 88,725 input rows and 75,545 n-gram rows relative to the exact seed-1
+initializer. Every input row is retained; unused rows are not filtered out.
+
+Layout and exact exports reconstruct the full source input SHA-256:
+
+`88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9`
+
+All nine fresh-process serving runs (dense/layout/exact, three each) produce
+the same 1,024 held-out word vectors and 128 held-out eight-word sentence
+vectors, with output SHA-256:
+
+`da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058`
+
+The **pre-change main** at `bea4302bb7fb1a6c0108fdde141ce667f2389920`, new
+dense facade and native facade additionally produce byte-identical outputs on
+another 1,024 registered-word and 128 sentence queries. See
+[historical-report.json](historical-report.json) and the recorded
+[build/run script](historical-run.sh). This control excludes prior OOV
+registration and hash-bucket helper bugs; their invalid/out-of-bounds behavior
+is intentionally corrected. C++ callers must recompile.
+
+The Linux correctness fixtures separately cover dimensions 7/32, UTF-8/OOV,
+all input IDs, ordered repeated IDs, chunks, softmax/HS class scores, failure
+preserving the loaded model, immutable guards and dense reload. Fixture rows
+are mathematical correctness data, never trained compression evidence.
+
+## Storage and matched structural control
+
+| Same trained source | Input row artifact, bytes | Saving vs raw input |
+| --- | ---: | ---: |
+| Raw float32 input | 23,137,280 | — |
+| Native, identity order | 22,298,237 | 3.63% |
+| Native, structural layout/exact | 22,401,608 | 3.18% |
+
+All native controls use the same codec, 32-row / 8,192-byte blocks, 723,040-byte
+original-ID forwarding map, source rows and 4 MiB block cache. The identity
+control retains the **identical original-ID DASTrie**, model metadata and reader
+configuration; only physical row order changes. Its outputs match the same
+dense oracle. See [identity-layout-report.json](identity-layout-report.json).
+
+Exact structural sharing finds **zero byte-identical rows to merge**. All
+90,380 rows survive mark-compact. Structural order adds 103,371 bytes (0.46%
+relative to the identity native table) rather than improving this codec on
+this training distribution. This experiment demonstrates that the algorithm
+is connected, not extra lossless sharing or a universal row-order advantage.
+[Earlier trained structural ablations](../structural_native_evidence_20261030/README.md)
+separately measure approximate joint sharing and key-index controls.
+
+The dense `.bin` is 27,204,018 bytes. Exact native **whole-bundle** size is
+27,078,261 bytes (0.46% saving), counting input table, 605,232-byte DASTrie,
+original dictionary/output metadata, config, manifests and extraction log.
+Conversion takes 11.94 seconds, including verification; layout takes 11.61.
+These are one export per mode, not a converter throughput distribution.
+
+## End-to-end query costs
+
+Three fresh processes per configuration, with one CPU affinity, prewarmed
+filesystem and identical query order. First-pass cache starts empty; each
+process then repeats the same trace three more times. Values below are medians
+of the three process-level p50s; p95 and all raw passes are in
+[trained-report.json](trained-report.json) and [summary.json](summary.json).
+
+| Query / state | Dense .bin | Native exact | Native identity control |
+| --- | ---: | ---: | ---: |
+| Word first-pass p50 | 2.79 µs | 7.96 µs | 8.10 µs |
+| Sentence first-pass p50 | 18.02 µs | 647.29 µs | 511.87 µs |
+| Word hot p50 | 2.44 µs | 4.33 µs | 4.33 µs |
+| Sentence hot p50 | 16.55 µs | 27.38 µs | 27.45 µs |
+
+Native exact first-pass p95 is 749.40 µs for words and 1,745.41 µs for sentences.
+Structural order improves neither cold sentence latency nor warm latency here.
+The repeated working set fits the cache, so hot numbers do not represent an
+unbounded changing query stream. Timed calls include original word/subword
+resolution, float32 row aggregation, sentence normalization and decoding;
+oracle file I/O and SHA verification are outside query timing. Batch totals also
+include loop/string/oracle-copy overhead and are not kernel throughput.
+
+After the trace, median anonymous RSS is **221.11 MiB dense → 202.26 MiB exact**;
+total RSS is **224.88 → 223.55 MiB** because native touches about 21.29 MiB of
+file-backed pages. The legacy 200 MB unused `word2int_` allocation and output
+matrix remain in both paths. Neither loader allocates training negatives.
+Anonymous memory shifts to mapped pages; this is not a large total-memory
+reduction. Warm-filesystem median load time is 0.187 → 0.124 seconds.
+
+## Reproduce and inspect failures
+
+Execution source: `bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8`; identity-control
+source: `355c299` (adds the ablation tool without changing serving). Python
+3.11.15, GCC/G++ 11.4.0, Linux 5.4.0-42 / glibc 2.35, AMD EPYC 7742; no Torch
+or GPU used by these drivers. The run is pinned to one available CPU core,
+nice level 10, OMP/BLAS/MKL threads 1, on a shared machine. Build commands,
+data/model/query hashes, affinity, raw timings and limitations are in the report.
+
+The existing enwik8 archive is split at raw byte 90,000,000 **before** processing.
+Each partition strips XML tags, lowercases and keeps `[a-z]+` words. The first
+65,536 train words become 1,024 lines of `p#0` plus 63 adjacent
+`query\x01source` pairs. This is necessary because this repository's custom
+positional/source reader does not train on ordinary whitespace text. The
+reference is finite SGNS: dim 64, n-gram lengths 2–12, minCount 2, window 3,
+five negatives, two epochs, one thread, learning rate 0.025 and subsampling
+threshold 1. It is not ordinary upstream fastText or the paper's large dataset.
+Held-out queries sample from the first 8,192 words in the independent 10% raw
+partition, seed 42. No quality target or threshold is selected on them.
+
+```bash
+export CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1
+meft_cpu=$(python -c 'import os; print(min(os.sched_getaffinity(0)))')
+# Existing archive; this script does not download data. Use a fresh directory.
+nice -n 10 taskset -c "$meft_cpu" python tools/benchmark_fasttext_native.py \
+  --enwik8-zip /path/to/enwik8.zip --output-dir artifacts/fasttext-native
+nice -n 10 taskset -c "$meft_cpu" python tools/ablate_fasttext_layout.py \
+  --reproduction-dir artifacts/fasttext-native --output-dir artifacts/fasttext-identity
+
+MEFT_FUSION_SANITIZERS=1 ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \
+  UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \
+  python -m unittest tests.test_fasttext_native_bundle -v
+```
+
+[tests-initial-failed.txt](tests-initial-failed.txt) preserves the initial
+registry-export failure: the DATrie callback returned stop after one key.
+Commit `900f548` fixes its continuation convention. The corrected
+[Linux roundtrip](tests-registry-fixed.txt) and
+[ASan/UBSan/LeakSanitizer run](sanitizers.txt) pass. Bundled trie archives are
+linked but uninstrumented. Final CI adds an explicit sanitized facade job,
+including HS and zero-vector string helpers. Metadata/index files remain
+trusted host-specific artifacts; a cache budget does not bound RSS, and offline
+planning copies the full input/maps. No GPU process was touched.

docs/fasttext_native_evidence_20261031/build-native.txt

@@ -0,0 +1,17 @@
+make: 进入目录“/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source”
+mkdir -p build
+cc -std=c11 -O2 -fPIC -c native/meft_rows.c -o build/meft_rows_fasttext.o
+c++ -pthread -std=c++0x -march=native -fPIC -O2 -DMEFT_NATIVE_FASTTEXT -Isrc -Inative -I. -I./mylib/include src/args.cc src/dictionary.cc src/productquantizer.cc src/matrix.cc src/qmatrix.cc src/vector.cc src/model.cc src/utils.cc src/fasttext.cc src/ftindex.cc native/fasttext_bundle.cc src/main.cc build/meft_rows_fasttext.o -L./mylib/lib -ltrie -lpub -lz -o build/fasttext-native
+src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’:
+./mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
+   16 | #define pub_calloc(num, size)                   pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__)
+      |                                                                                  ^~~~~~~~
+src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’
+  461 |         if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL;
+      |                               ^~~~~~~~~~
+mkdir -p build
+cc -std=c11 -O2 -Wall -Wextra -Werror -fPIC -shared native/meft_rows.c -lz -o build/libmeft_rows.so
+cc -std=c11 -O2 -Wall -Wextra -Werror -Inative examples/native_gather.c native/meft_rows.c -lz -o build/native_gather
+mkdir -p build
+c++ -std=c++11 -O2 -Wall -Wextra -Werror -fPIC -shared -Isrc -Imylib/include native/meft_structural.cc -L./mylib/lib -ltrie -lpub -o build/libmeft_structural.so
+make: 离开目录“/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source”

docs/fasttext_native_evidence_20261031/build-probe.txt

@@ -0,0 +1,7 @@
+/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’:
+/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
+   16 | #define pub_calloc(num, size)                   pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__)
+      |                                                                                  ^~~~~~~~
+/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’
+  461 |         if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL;
+      |                               ^~~~~~~~~~