Skip to content

Commit 7e180ef

Browse files
committed
fix(runtime): canonicalize copied bitstring keys
1 parent 66c2d9d commit 7e180ef

6 files changed

Lines changed: 156 additions & 12 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@ All notable milestones for the nera project are documented in this file.
1010
or collects garbage, as on BEAM.
1111
- Copying reads preserve the final partial byte's logical bit length and remain
1212
valid after the table is deleted or the persistent term is erased.
13-
- The `ets_stored_bitstring` regression exercises both writer lifetimes and
14-
both storage APIs unchanged natively and on BEAM.
13+
- Bitstring keys and bag duplicate checks compare significant bits and exact
14+
length, not descriptor addresses or unused trailing bits, so copies remain
15+
interchangeable as keys, including inside lists and tuples.
16+
- The `ets_stored_bitstring` and `ets_bitstring_keys` regressions exercise
17+
writer lifetimes, copying reads, boxed updates and copied-key operations
18+
unchanged natively and on BEAM.
1519

1620
## Receives nested in after bodies resume at their own wait
1721

‎crates/nera-runtime/src/ets.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,13 +221,20 @@ impl TermKey {
221221
out.extend_from_slice(&term.0.to_le_bytes());
222222
return;
223223
}
224-
// Binary — serialize content bytes
225-
if term.is_binary() {
224+
// Bitstring identity is its exact length and significant bits, not
225+
// the descriptor address or unused bits in the final backing byte.
226+
if term.is_bitstring() {
226227
out.push(b'B');
227228
if let Some((ptr, len)) = unsafe { crate::ffi::binary_data_pub(term) } {
228229
let slice = unsafe { std::slice::from_raw_parts(ptr, len) };
229-
out.extend_from_slice(&(len as u64).to_le_bytes());
230-
out.extend_from_slice(slice);
230+
let bits = unsafe { term.binary_bit_size().unwrap() };
231+
out.extend_from_slice(&(bits as u64).to_le_bytes());
232+
let whole_bytes = bits / 8;
233+
out.extend_from_slice(&slice[..whole_bytes]);
234+
let trailing_bits = bits % 8;
235+
if trailing_bits != 0 {
236+
out.push(slice[whole_bytes] & (u8::MAX << (8 - trailing_bits)));
237+
}
231238
}
232239
return;
233240
}

‎crates/nera-runtime/src/ets_bitstring_tests.rs‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
//! Ownership checks for the bitstring storage regression. The Erlang fixture
2-
//! owns observable lifetime/equality behavior; this test deterministically
3-
//! rejects descriptor aliasing before either source allocation is released.
1+
//! Focused invariants for the bitstring storage regressions. The Erlang
2+
//! fixtures own observable lifetime/key behavior; these tests deterministically
3+
//! check descriptor independence and insignificant backing-byte padding.
44
55
use super::*;
66
use crate::binary::BinarySlice;
@@ -77,3 +77,40 @@ fn bitstring_copies_own_descriptors_in_both_directions() {
7777
}
7878
}
7979
}
80+
81+
// Storage copies must not turn equal bitstring keys into
82+
// address-based keys. A real byte window can retain bits beyond its logical end.
83+
#[test]
84+
fn bitstring_keys_ignore_padding_but_preserve_significant_bits_and_length() {
85+
let mut heap = ProcessHeap::new();
86+
let bytes = b"abcdefghP"; // Last byte's high nibble is <<5:4>>.
87+
let bits = bytes.len() * 8 - 4;
88+
let make_heap_bits = |heap: &mut ProcessHeap, data: &[u8], bits: usize| {
89+
let term = heap.alloc_binary(data);
90+
// SAFETY: the test exclusively owns the descriptor and only shortens
91+
// its bit window within initialized bytes.
92+
unsafe {
93+
*term.as_boxed_ptr().unwrap().cast_mut() =
94+
BoxedHeader::new(BoxedSubtype::HeapBinary, bits as u64).0;
95+
}
96+
term
97+
};
98+
let original = make_heap_bits(&mut heap, bytes, bits);
99+
let prefix = b"prefix";
100+
let framed = [prefix.as_slice(), b"abcdefghZ", b"suffix"].concat();
101+
let window =
102+
BinarySlice::new_bits(Arc::new(BinaryData::new(framed)), prefix.len(), bits).unwrap();
103+
let same_bits = heap.alloc_binary_slice(window);
104+
let different_bits = make_heap_bits(&mut heap, b"abcdefgh@", bits);
105+
let different_length = make_heap_bits(&mut heap, bytes, bits + 1);
106+
let whole_bytes = heap.alloc_binary(bytes);
107+
108+
// SAFETY: heap keeps all immutable input descriptors and bytes live.
109+
unsafe {
110+
let key = TermKey::from_term(original);
111+
assert!(key == TermKey::from_term(same_bits));
112+
assert!(key != TermKey::from_term(different_bits));
113+
assert!(key != TermKey::from_term(different_length));
114+
assert!(key != TermKey::from_term(whole_bytes));
115+
}
116+
}

‎tests/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,8 @@ These are automatically skipped by the runner and show up as SKIP in output.
462462
| `ets_basic.erl` | ETS new/insert/lookup/delete/tab2list | ~15 |
463463
| `ets_keypos.erl` | ETS keypos and named tables | ~8 |
464464
| `ets_stored_fun.erl` | A fun stored in ETS (at the top of a row, nested in a list, tuple and map, after `update_element/3`) or `persistent_term` by a process that then exits or collects garbage keeps its captured list, binaries, tuple, map, float, bignum and inner fun when looked up and called; a fun read by `tab2list/1` or `persistent_term:get/1` still works after the table or term is deleted; completion marker; also run on BEAM | ~16 |
465-
| `ets_stored_bitstring.erl` | Non-byte-aligned heap/refc bitstrings, sub-byte values and extracted tails stored in ETS or `persistent_term` survive writer exit/GC; `update_element/3` and copying reads remain intact after table deletion or term erasure; completion marker; same fixture on BEAM | 4 lifetime scenarios |
465+
| `ets_stored_bitstring.erl` | Non-byte-aligned heap/refc bitstrings, sub-byte values and extracted tails stored in ETS or `persistent_term` survive writer exit/GC; boxed `update_element/3` replacements and copying reads remain intact after table deletion or term erasure; completion marker; same fixture on BEAM | 4 lifetime scenarios |
466+
| `ets_bitstring_keys.erl` | Partial-bitstring bag duplicate suppression; copied direct/list/tuple keys in hash-backed tables for lookup, membership, insert_new, boxed updates, counters, deletion and forward/backward traversal; copied values as persistent_term keys; significant bits and exact lengths remain distinct; completion marker; same fixture on BEAM | Table-driven |
466467
| `ets_named_table_return.erl` | ets:new/2 with `named_table` returns the name (Cowboy's `?MODULE = ets:new(?MODULE, [named_table, ...])`): every table type, apply, variable module and fun calls; a table without `named_table` does not take the name, and creating or deleting one leaves a named table with the same atom reachable; the name is released by ets:delete/1; a table created in another process is reached by its name; also run on BEAM | Table-driven |
467468

468469
### Stdlib

‎tests/ets_bitstring_keys.erl‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
%% EXPECT: ets_bitstring_keys_ok
2+
%% Regression for storage copies exposing pointer-based bitstring keys.
3+
%% Returned/copied keys and bag values must still compare by significant bits,
4+
%% including bitstrings nested in lists and tuples. Run unchanged on BEAM.
5+
-module(ets_bitstring_keys).
6+
-export([test/0]).
7+
8+
test() ->
9+
bag_duplicates(),
10+
[table_keys(Type) || Type <- [set, bag, duplicate_bag]],
11+
persistent_keys(),
12+
io:format("ets_bitstring_keys_ok~n"),
13+
ok.
14+
15+
keys() ->
16+
Small = <<"abcdefgh", (id(5)):4>>,
17+
Bytes = list_to_binary(lists:duplicate(id(200), $x)),
18+
Large = <<Bytes/binary, (id(3)):2>>,
19+
[Small, Large, [Small], {wrapped, [Large, Small]},
20+
<<(id(5)):3>>, <<(id(20)):5>>, <<(id(4)):3>>].
21+
22+
id(X) -> X.
23+
24+
bag_duplicates() ->
25+
[Small, Large | _] = keys(),
26+
Value = {Small, [Large, Small]},
27+
T = ets:new(bitstring_bag, [bag]),
28+
Row = {key, Value},
29+
true = ets:insert(T, Row),
30+
true = ets:insert(T, Row),
31+
[Row] = ets:lookup(T, key),
32+
true = ets:delete(T),
33+
ok.
34+
35+
table_keys(Type) ->
36+
T = ets:new(bitstring_keys, [Type]),
37+
Keys = keys(),
38+
[true = ets:insert(T, {K, 0, unchanged}) || K <- Keys],
39+
CopiedRows = ets:tab2list(T),
40+
Count = length(Keys),
41+
Count = length(CopiedRows),
42+
[check_copied_key(Type, T, Row) || Row <- CopiedRows],
43+
Sorted = lists:sort(Keys),
44+
Sorted = lists:sort(walk(T, ets:first(T), next)),
45+
Sorted = lists:sort(walk(T, ets:last(T), prev)),
46+
%% Keys from traversal also work for lookup and membership.
47+
true = ets:member(T, ets:first(T)),
48+
true = ets:member(T, ets:last(T)),
49+
[begin
50+
true = ets:delete(T, K),
51+
false = ets:member(T, K),
52+
[] = ets:lookup(T, K)
53+
end || {K, _, _} <- CopiedRows],
54+
[] = ets:tab2list(T),
55+
true = ets:delete(T),
56+
ok.
57+
58+
check_copied_key(Type, T, {K, 0, unchanged}) ->
59+
true = ets:member(T, K),
60+
[{K, 0, unchanged}] = ets:lookup(T, K),
61+
false = ets:insert_new(T, {K, 99, replaced}),
62+
case Type of
63+
set ->
64+
true = ets:update_element(T, K, {3, [updated]}),
65+
1 = ets:update_counter(T, K, {2, 1}),
66+
[{K, 1, [updated]}] = ets:lookup(T, K);
67+
_ -> ok
68+
end,
69+
ok.
70+
71+
walk(_T, '$end_of_table', _Direction) -> [];
72+
walk(T, K, Direction) ->
73+
[_] = ets:lookup(T, K),
74+
Next = case Direction of
75+
next -> ets:next(T, K);
76+
prev -> ets:prev(T, K)
77+
end,
78+
[K | walk(T, Next, Direction)].
79+
80+
persistent_keys() ->
81+
CopyKey = {?MODULE, copied_key},
82+
[begin
83+
%% Reading a stored value makes a new descriptor, which must be
84+
%% interchangeable with the original when either is used as a key.
85+
ok = persistent_term:put(CopyKey, K),
86+
Copied = persistent_term:get(CopyKey),
87+
ok = persistent_term:put(K, original),
88+
original = persistent_term:get(Copied),
89+
ok = persistent_term:put(Copied, replaced),
90+
replaced = persistent_term:get(K),
91+
true = persistent_term:erase(Copied),
92+
missing = persistent_term:get(K, missing)
93+
end || K <- keys()],
94+
true = persistent_term:erase(CopyKey),
95+
ok.

‎tests/ets_stored_bitstring.erl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ ets_lifetime(Lifetime) ->
2323
[{bits, Actual, unchanged}] = ets:lookup(T, bits),
2424
equal({ets, Lifetime}, values(), Actual),
2525
%% Updating another element copies the bitstrings again.
26-
true = ets:update_element(T, bits, {3, updated}),
27-
[{bits, Listed, updated}] = ets:tab2list(T),
26+
true = ets:update_element(T, bits, {3, [updated]}),
27+
[{bits, Listed, [updated]}] = ets:tab2list(T),
2828
true = ets:delete(T),
2929
churn(),
3030
equal({tab2list, Lifetime}, values(), Listed),

0 commit comments

Comments
 (0)