diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc7c92..050fb91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ All notable milestones for the nera project are documented in this file. +## Sending and storing deeply nested terms + +- A term nested hundreds of thousands of levels deep, such as an iolist + built by prepending (`[Acc, Chunk]`) or a chain of tuples or maps, can now + be sent to another process, captured in a spawned fun, stored in ETS and + read back with `ets:lookup/2` or `ets:tab2list/1`, as on BEAM. Previously + the program died with a bus error: in ETS from about 5,000 levels, when + sent or captured from about 14,000. +- Such a term also works as the key of a `set` table, and with + `persistent_term:put/2` and `persistent_term:get/1`. +- The message copier, the ETS copies and the encoding of ETS keys keep the + work still to do on a heap-allocated stack, as the garbage collector does, + so nesting depth costs memory instead of native stack. A message still + copies shared subterms once. +- The ETS copies no longer build a temporary vector for each list, tuple or + map. In a benchmark, inserting rows that hold lists, tuples or maps took + about 30% less time and reading the table with `ets:tab2list/1` about 50% + less; sending, spawning and rows of plain values were unchanged. +- The `copy_deep_nesting` fixture sends, captures and stores 200,000-level + iolists, tuples and maps, and sends and captures closure chains, natively + and on BEAM. Comparing such terms, using one as an `ordered_set` key, and + `term_to_binary/1`, `erlang:phash2/1` or `iolist_to_binary/1` on them can + still crash; [known limits](docs/KNOWN_LIMITS.md) gives the depths. + ## error/3 compiles and raises its reason - `error(Reason, Args, Options)` and `erlang:error(Reason, Args, Options)` diff --git a/crates/nera-runtime/src/copy_stack.rs b/crates/nera-runtime/src/copy_stack.rs new file mode 100644 index 0000000..dc296d1 --- /dev/null +++ b/crates/nera-runtime/src/copy_stack.rs @@ -0,0 +1,170 @@ +//! Depth-first copying of term graphs without native recursion. +//! +//! A copier allocates each object it meets in its target first, then queues +//! the object's term slots on a [`SlotStack`] instead of copying them in a +//! nested call. Nesting depth costs one queued entry per open level on the +//! heap rather than a native stack frame, so a term nested hundreds of +//! thousands of levels deep (an iolist built by prepending, a chain of +//! tuples) copies as on BEAM. Message passing, spawned closure environments, +//! ETS and persistent terms copy through [`copy_graph`]; the garbage +//! collector's copier in `heap.rs` runs the same loop inline. + +use std::cell::Cell; +use std::thread::LocalKey; + +use crate::heap::is_plain_immediate; + +/// Most entries a spare work-stack buffer keeps between uses; a buffer grown +/// past this by a deeply nested term is freed instead. +const SPARE_CAPACITY: usize = 256; + +/// A work stack that takes its thread's spare buffer on its first push and +/// gives it back when dropped, so working through an ordinary term +/// allocates nothing, and a term with nothing to queue does not touch the +/// spare. A nested stack of the same kind finds the spare taken and +/// allocates its own buffer. +pub(crate) struct WorkStack { + items: Vec, + spare: &'static LocalKey>>, +} + +impl WorkStack { + /// An empty stack that will use the spare buffer `spare` when it is free. + pub(crate) fn new(spare: &'static LocalKey>>) -> Self { + WorkStack { items: Vec::new(), spare } + } + + #[inline] + pub(crate) fn push(&mut self, item: T) { + if self.items.capacity() == 0 { + self.items = self.spare.try_with(Cell::take).unwrap_or_default(); + } + self.items.push(item); + } + + #[inline] + pub(crate) fn pop(&mut self) -> Option { + self.items.pop() + } + + #[inline] + pub(crate) fn len(&self) -> usize { + self.items.len() + } +} + +impl Drop for WorkStack { + fn drop(&mut self) { + let capacity = self.items.capacity(); + if capacity != 0 && capacity <= SPARE_CAPACITY { + self.items.clear(); + let items = std::mem::take(&mut self.items); + // During thread teardown the spare is gone; the buffer is freed. + let _ = self.spare.try_with(|spare| spare.set(items)); + } + } +} + +thread_local! { + static SPARE_SLOTS: Cell> = const { Cell::new(Vec::new()) }; +} + +/// Term slots of an object already allocated in the target whose values are +/// still to be copied: the `len` words at `dst` receive copies of the `len` +/// source words at `src`. +struct PendingSlots { + dst: *mut u64, + src: *const u64, + len: usize, +} + +/// The work stack of a copy in progress. +pub(crate) struct SlotStack { + pending: WorkStack, +} + +impl Default for SlotStack { + fn default() -> Self { + SlotStack { pending: WorkStack::new(&SPARE_SLOTS) } + } +} + +impl SlotStack { + /// Queue the `len` term slots at `src` for copying into `dst`, the + /// matching words of an object just allocated in the target. + #[inline] + pub(crate) fn push(&mut self, dst: *mut u64, src: *const u64, len: usize) { + if len != 0 { + self.pending.push(PendingSlots { dst, src, len }); + } + } +} + +/// A term copier that queues the slots of each object it allocates. +pub(crate) trait SlotCopier { + /// The copier's work stack. + fn slots(&mut self) -> &mut SlotStack; + + /// Copy one term into the target and return the word to store for it. + /// An object's copy is allocated here with its term slots queued on + /// [`SlotCopier::slots`], not copied; a list may copy a run of cells + /// along its tail, queuing the first cell whose head needs a copy. + /// [`copy_graph`] may store an immediate slot without calling this. + /// + /// # Safety + /// `word` and everything it reaches must stay live and unchanged until + /// the copy completes. + unsafe fn copy_word(&mut self, word: u64) -> u64; +} + +/// Copy `root` and everything it reaches with `copier`. +/// +/// Objects are copied depth first, in the order a recursive copy would use: +/// an object's slots in order, each slot's subtree before the next slot. +/// A list nested in head position, or a nested tuple, map or closure, costs +/// one queued entry per open level; a proper list needs at most one. +/// Re-entrant: a nested call only drains the entries it queued. +/// +/// # Safety +/// The complete source graph must stay live and unchanged during the copy, +/// and target allocation must not move or free words already allocated. +/// Every entry the copier queues must name `len` readable source words and +/// `len` writable words reserved in their copy. +pub(crate) unsafe fn copy_graph(copier: &mut C, root: u64) -> u64 { + let base = copier.slots().pending.len(); + // SAFETY: the caller keeps the source graph live and unchanged. + let copied = unsafe { copier.copy_word(root) }; + while copier.slots().pending.len() > base { + let mut slots = copier.slots().pending.pop().expect("pending slots above base"); + loop { + let queued = copier.slots().pending.len(); + // SAFETY: `src` is a term word of a live source object, which no + // copy modifies; `dst` is the matching word reserved in its copy, + // written exactly once here. + unsafe { + let word = *slots.src; + *slots.dst = if is_plain_immediate(word) { word } else { copier.copy_word(word) }; + } + slots.len -= 1; + if slots.len == 0 { + break; + } + // SAFETY: slots remain, so both pointers stay inside the source + // object and its reserved copy. + unsafe { + slots.dst = slots.dst.add(1); + slots.src = slots.src.add(1); + } + if copier.slots().pending.len() != queued { + // The slot's object queued its own slots: copy them first, as + // recursion would, then this object's remaining slots. + let stack = &mut copier.slots().pending; + let child = stack.pop().expect("child slots just queued"); + stack.push(slots); + stack.push(child); + break; + } + } + } + copied +} diff --git a/crates/nera-runtime/src/ets.rs b/crates/nera-runtime/src/ets.rs index 9961bd9..cba2209 100644 --- a/crates/nera-runtime/src/ets.rs +++ b/crates/nera-runtime/src/ets.rs @@ -16,6 +16,7 @@ use crate::process::{ProcessId, AtomicProcessId}; use crate::binary::BinaryData; +use std::cell::Cell; use std::collections::{btree_map::Entry, BTreeMap, HashMap, HashSet}; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, OnceLock, RwLock}; @@ -23,7 +24,8 @@ use std::sync::{Arc, Mutex, OnceLock, RwLock}; use crate::atom_table::ATOM_UNDEFINED; use crate::ets_pool::{EtsAllocation, EtsRegionPool}; use crate::ffi::compare::compare_terms; -use crate::heap::{MapPairs, ProcessHeap}; +use crate::copy_stack::{copy_graph, SlotCopier, SlotStack, WorkStack}; +use crate::heap::{is_plain_immediate, ProcessHeap}; use crate::term::{BoxedHeader, ErlTerm, BOXED_TUPLE}; /// A unique table identifier (small integer handle). @@ -174,10 +176,40 @@ impl TermKey { TermKey::Boxed(bytes) } + /// Append the canonical bytes of `term` to `out`. Parts still to + /// serialize wait on a heap-allocated stack, so nesting depth costs heap, + /// not native stack. /// # Safety /// All input terms and reachable objects must remain live and unmoved by GC /// for this call. Binary backing bytes must not be concurrently mutated. unsafe fn serialize_term(term: ErlTerm, out: &mut Vec) { + let mut pending = WorkStack::new(&SPARE_KEY_PARTS); + Self::serialize_part(KeyPart::Term(term), out, &mut pending); + while let Some(part) = pending.pop() { + Self::serialize_part(part, out, &mut pending); + } + } + + /// Serialize one term (or the next cell of a list), pushing the parts + /// it contains on `pending` in reverse, so they pop in order. + /// # Safety + /// As for `serialize_term`. + unsafe fn serialize_part(part: KeyPart, out: &mut Vec, pending: &mut WorkStack) { + let term = match part { + KeyPart::Term(term) => term, + KeyPart::ListRest(cursor) => { + if let Some(ptr) = cursor.as_cons_ptr() { + unsafe { + pending.push(KeyPart::ListRest((*ptr).tail)); + pending.push(KeyPart::Term((*ptr).head)); + } + } else { + out.push(b'E'); // end of list marker + pending.push(KeyPart::Term(cursor)); // tail (nil or improper) + } + return; + } + }; // Atoms, small ints, pids, nil — just use the raw u64 if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { out.extend_from_slice(&term.0.to_le_bytes()); @@ -193,22 +225,10 @@ impl TermKey { } return; } - // Cons (list) + // Cons (list): each head, then the end marker and the tail if term.is_cons() { out.push(b'L'); - let mut cursor = term; - while cursor.is_cons() { - if let Some(ptr) = cursor.as_cons_ptr() { - unsafe { - Self::serialize_term((*ptr).head, out); - cursor = (*ptr).tail; - } - } else { - break; - } - } - out.push(b'E'); // end of list marker - Self::serialize_term(cursor, out); // tail (nil or improper) + pending.push(KeyPart::ListRest(term)); return; } // Tuple @@ -225,9 +245,8 @@ impl TermKey { let arity = header.arity() as usize; out.push(b'T'); out.extend_from_slice(&(arity as u64).to_le_bytes()); - for i in 0..arity { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - Self::serialize_term(elem, out); + for i in (0..arity).rev() { + pending.push(KeyPart::Term(ErlTerm(unsafe { *ptr.add(1 + i) }))); } return; } @@ -238,6 +257,18 @@ impl TermKey { } } +/// A part of a key still to serialize: a whole term, or the rest of a list +/// from a cell (or its tail) onward. +enum KeyPart { + Term(ErlTerm), + ListRest(ErlTerm), +} + +thread_local! { + static SPARE_KEY_PARTS: Cell> = const { Cell::new(Vec::new()) }; + static SPARE_TERMS: Cell> = const { Cell::new(Vec::new()) }; +} + /// Compare a stored ErlTerm against a pre-computed TermKey. /// Short-circuits for immediate terms (atom/int/pid/nil) to avoid /// serialisation and allocation entirely. @@ -305,17 +336,47 @@ impl Ord for OrdTermKey { /// Count the number of heap words needed to deep-copy a term. /// This is a pre-pass before `deep_copy_into_compact` to determine the /// exact Vec capacity needed, preventing reallocation (and thus pointer -/// invalidation) during the copy phase. +/// invalidation) during the copy phase. Terms still to count wait on a +/// heap-allocated stack, so nesting depth costs heap, not native stack. /// # Safety /// All input terms and reachable objects must remain live and unmoved by GC /// for this call. Binary backing bytes must not be concurrently mutated. pub(crate) unsafe fn count_heap_words(term: ErlTerm) -> usize { + if is_unboxed(term) { + return 0; + } + let mut pending = WorkStack::new(&SPARE_TERMS); + // SAFETY: forwarded from the caller, for the root and every term it reaches. + let mut count = unsafe { object_heap_words(term, &mut pending) }; + while let Some(term) = pending.pop() { + count += unsafe { object_heap_words(term, &mut pending) }; + } + count +} + +/// True for the terms ETS copies stand alone: atoms, small integers, pids and nil. +#[inline] +fn is_unboxed(term: ErlTerm) -> bool { + term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() +} + +/// Words a term's own storage needs in a `CompactHeap` (for a list, every +/// cell along its tail), queueing the contained terms that need words of +/// their own on `pending`. +/// # Safety +/// As for `count_heap_words`. +unsafe fn object_heap_words(term: ErlTerm, pending: &mut WorkStack) -> usize { use crate::heap::REFC_BINARY_THRESHOLD; use crate::term::{BOXED_MAP, BOXED_FLOAT, BOXED_BIGINT, BOXED_REFERENCE, BoxedHeader, REFC_BINARY_TOTAL_WORDS, REFERENCE_TOTAL_WORDS}; - if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { + let mut queue = |term: ErlTerm| { + if !is_unboxed(term) { + pending.push(term); + } + }; + if is_unboxed(term) { return 0; } if term.is_cons() { @@ -324,14 +385,15 @@ pub(crate) unsafe fn count_heap_words(term: ErlTerm) -> usize { while cursor.is_cons() { if let Some(ptr) = cursor.as_cons_ptr() { unsafe { - count += 2 + count_heap_words((*ptr).head); + count += 2; + queue((*ptr).head); cursor = (*ptr).tail; } } else { break; } } - count += count_heap_words(cursor); // tail + queue(cursor); // tail return count; } if term.is_binary() { @@ -350,22 +412,18 @@ pub(crate) unsafe fn count_heap_words(term: ErlTerm) -> usize { match header.subtype() { BOXED_TUPLE => { let arity = header.arity() as usize; - let mut count = 1 + arity; // header + element slots for i in 0..arity { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - count += count_heap_words(elem); + queue(ErlTerm(unsafe { *ptr.add(1 + i) })); } - return count; + return 1 + arity; // header + element slots } BOXED_MAP => { let num_pairs = header.arity() as usize; let total_elems = num_pairs * 2; - let mut count = 1 + total_elems; for i in 0..total_elems { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - count += count_heap_words(elem); + queue(ErlTerm(unsafe { *ptr.add(1 + i) })); } - return count; + return 1 + total_elems; } BOXED_FLOAT => return 2, BOXED_BIGINT => { @@ -412,121 +470,97 @@ unsafe fn alloc_compact_for_elements(pool: &Mutex, elements: &[Er CompactHeap::from_pool(allocation) } -/// Deep-copy a term into a `CompactHeap` (flat word buffer). +/// Deep-copy a term into a `CompactHeap` (flat word buffer), which must hold +/// the `count_heap_words` words the term needs. /// # Safety /// All input terms and reachable objects must remain live and unmoved by GC /// for this call. Binary backing bytes must not be concurrently mutated. pub(crate) unsafe fn deep_copy_into_compact(term: ErlTerm, heap: &mut CompactHeap) -> ErlTerm { - use crate::heap::REFC_BINARY_THRESHOLD; - use crate::term::{BOXED_MAP, BOXED_FLOAT, BOXED_BIGINT, - BOXED_REFERENCE, BoxedHeader, - REFC_BINARY_TOTAL_WORDS, REFC_BINARY_PTR_WORD_INDEX, - REFC_BINARY_OFFSET_WORD_INDEX, REFERENCE_TOTAL_WORDS, - REFERENCE_ID_WORD_INDEX}; - - if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { + if is_unboxed(term) { return term; } - if term.is_cons() { - let mut elems = Vec::new(); - let mut cursor = term; - while cursor.is_cons() { - if let Some(ptr) = cursor.as_cons_ptr() { - unsafe { - elems.push(deep_copy_into_compact((*ptr).head, heap)); - cursor = (*ptr).tail; - } - } else { - break; - } - } - let tail = deep_copy_into_compact(cursor, heap); - // Build list in reverse using compact heap - let mut result = tail; - for elem in elems.into_iter().rev() { - let words = heap.alloc_words(2); - unsafe { - *words = elem.0; - *words.add(1) = result.0; - } - result = ErlTerm::cons_unchecked(words as *const crate::term::ConsCell); - } - return result; - } - if term.is_binary() { - if let Some((ptr, len)) = unsafe { crate::ffi::binary_data_pub(term) } { - let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }; - if len <= REFC_BINARY_THRESHOLD { - // Inline heap binary: [header | data words...] - let data_words = len.div_ceil(8); - let total_words = 1 + data_words; - let words = heap.alloc_words(total_words); - unsafe { - *words = BoxedHeader::new(crate::term::BoxedSubtype::HeapBinary, (len * 8) as u64).0; - std::ptr::copy_nonoverlapping( - bytes.as_ptr(), - (words.add(1)) as *mut u8, - len, - ); - } - return ErlTerm::boxed_unchecked(words as *const u64); - } else { - // Refc binary: [header | arc_ptr | offset] - let data: Arc = Arc::new(BinaryData::new(bytes.to_vec())); - let raw_arc_ptr = Arc::as_ptr(&data) as u64; - let words = heap.alloc_words(REFC_BINARY_TOTAL_WORDS); - unsafe { - *words = BoxedHeader::new(crate::term::BoxedSubtype::RefcBinary, (len * 8) as u64).0; - *words.add(REFC_BINARY_PTR_WORD_INDEX) = raw_arc_ptr; - *words.add(REFC_BINARY_OFFSET_WORD_INDEX) = 0; - } - heap.refc_binaries.push(data); - return ErlTerm::boxed_unchecked(words as *const u64); - } - } + // SAFETY: forwarded from the caller; compact heap words never move. + unsafe { EtsCopier::copy(term, heap) } +} + +/// Deep-copy a term into a `ProcessHeap` (used for lookup — copying stored +/// terms back into a process's bumpalo arena). +/// # Safety +/// All input terms and reachable objects must remain live and unmoved by GC +/// for this call. Binary backing bytes must not be concurrently mutated. +pub(crate) unsafe fn deep_copy_into_heap(term: ErlTerm, heap: &mut ProcessHeap) -> ErlTerm { + if is_unboxed(term) { return term; } - if term.is_boxed() { - if let Some(ptr) = term.as_boxed_ptr() { - let header = BoxedHeader(unsafe { *ptr }); - match header.subtype() { - BOXED_TUPLE => { - let arity = header.arity() as usize; - let mut elems = Vec::with_capacity(arity); - for i in 0..arity { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - elems.push(deep_copy_into_compact(elem, heap)); - } - // Allocate tuple in compact heap: [header | elem0 | elem1 | ...] - let words = heap.alloc_words(1 + arity); + // SAFETY: forwarded from the caller; process heap words never move + // during the copy. + unsafe { EtsCopier::copy(term, heap) } +} + +/// Where an ETS copy allocates: a stored entry's `CompactHeap`, or the +/// process heap a stored term is read back into. +trait CopyTarget { + /// Allocate `n` uninitialized words that stay in place. + fn alloc_words(&mut self, n: usize) -> *mut u64; + + /// Copy a boxed term that holds no other terms (a binary, float, bigint + /// or reference); any other term is kept as it is. + /// # Safety + /// As for `deep_copy_into_compact`. + unsafe fn copy_leaf(&mut self, term: ErlTerm) -> ErlTerm; +} + +impl CopyTarget for CompactHeap { + fn alloc_words(&mut self, n: usize) -> *mut u64 { + CompactHeap::alloc_words(self, n) + } + + unsafe fn copy_leaf(&mut self, term: ErlTerm) -> ErlTerm { + use crate::heap::REFC_BINARY_THRESHOLD; + use crate::term::{BOXED_FLOAT, BOXED_BIGINT, BOXED_REFERENCE, + REFC_BINARY_TOTAL_WORDS, REFC_BINARY_PTR_WORD_INDEX, + REFC_BINARY_OFFSET_WORD_INDEX, REFERENCE_TOTAL_WORDS, + REFERENCE_ID_WORD_INDEX}; + + if term.is_binary() { + if let Some((ptr, len)) = unsafe { crate::ffi::binary_data_pub(term) } { + let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }; + if len <= REFC_BINARY_THRESHOLD { + // Inline heap binary: [header | data words...] + let data_words = len.div_ceil(8); + let total_words = 1 + data_words; + let words = self.alloc_words(total_words); unsafe { - *words = BoxedHeader::new(crate::term::BoxedSubtype::Tuple, arity as u64).0; - for (i, &elem) in elems.iter().enumerate() { - *words.add(1 + i) = elem.0; - } + *words = BoxedHeader::new(crate::term::BoxedSubtype::HeapBinary, (len * 8) as u64).0; + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + (words.add(1)) as *mut u8, + len, + ); } return ErlTerm::boxed_unchecked(words as *const u64); - } - BOXED_MAP => { - let num_pairs = header.arity() as usize; - let total_elems = num_pairs * 2; - let mut flat = Vec::with_capacity(total_elems); - for i in 0..total_elems { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - flat.push(deep_copy_into_compact(elem, heap)); - } - let words = heap.alloc_words(1 + total_elems); + } else { + // Refc binary: [header | arc_ptr | offset] + let data: Arc = Arc::new(BinaryData::new(bytes.to_vec())); + let raw_arc_ptr = Arc::as_ptr(&data) as u64; + let words = self.alloc_words(REFC_BINARY_TOTAL_WORDS); unsafe { - *words = BoxedHeader::new(crate::term::BoxedSubtype::Map, num_pairs as u64).0; - for (i, &elem) in flat.iter().enumerate() { - *words.add(1 + i) = elem.0; - } + *words = BoxedHeader::new(crate::term::BoxedSubtype::RefcBinary, (len * 8) as u64).0; + *words.add(REFC_BINARY_PTR_WORD_INDEX) = raw_arc_ptr; + *words.add(REFC_BINARY_OFFSET_WORD_INDEX) = 0; } + self.refc_binaries.push(data); return ErlTerm::boxed_unchecked(words as *const u64); } + } + return term; + } + if let Some(ptr) = term.as_boxed_ptr() { + let header = BoxedHeader(unsafe { *ptr }); + match header.subtype() { BOXED_FLOAT => { let bits = unsafe { *ptr.add(1) }; - let words = heap.alloc_words(2); + let words = self.alloc_words(2); unsafe { *words = BoxedHeader::new(crate::term::BoxedSubtype::Float, 0).0; *words.add(1) = bits; @@ -539,13 +573,13 @@ pub(crate) unsafe fn deep_copy_into_compact(term: ErlTerm, heap: &mut CompactHea // transfer the words to compact heap. let mut tmp = ProcessHeap::new(); let tmp_term = crate::bigint::bigint_to_term(&mut tmp, big); - return copy_from_process_heap_to_compact(tmp_term, heap); + return copy_from_process_heap_to_compact(tmp_term, self); } return term; } BOXED_REFERENCE => { let ref_id = unsafe { *ptr.add(REFERENCE_ID_WORD_INDEX) }; - let words = heap.alloc_words(REFERENCE_TOTAL_WORDS); + let words = self.alloc_words(REFERENCE_TOTAL_WORDS); unsafe { *words = header.0; *words.add(REFERENCE_ID_WORD_INDEX) = ref_id; @@ -555,108 +589,39 @@ pub(crate) unsafe fn deep_copy_into_compact(term: ErlTerm, heap: &mut CompactHea _ => return term, } } + term } - term } -/// Deep-copy a term from a process heap into a `CompactHeap`. -/// Used for the bigint fallback path. -fn copy_from_process_heap_to_compact(term: ErlTerm, heap: &mut CompactHeap) -> ErlTerm { - if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { - return term; +impl CopyTarget for ProcessHeap { + fn alloc_words(&mut self, n: usize) -> *mut u64 { + ProcessHeap::alloc_words(self, n) } - if term.is_boxed() { - if let Some(ptr) = term.as_boxed_ptr() { - let header = BoxedHeader(unsafe { *ptr }); - let subtype = header.subtype(); - if subtype == crate::term::BOXED_BIGINT { - // header.arity() = limb_count for bigints - let limb_count = header.arity() as usize; - let total = crate::term::bigint_total_words(limb_count); - let words = heap.alloc_words(total); - unsafe { - std::ptr::copy_nonoverlapping(ptr, words, total); - } - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - return unsafe { ErlTerm::boxed_unchecked(words as *const u64) }; - } - } - } - term -} -/// Deep-copy a term into a `ProcessHeap` (used for lookup — copying stored -/// terms back into a process's bumpalo arena). -/// # Safety -/// All input terms and reachable objects must remain live and unmoved by GC -/// for this call. Binary backing bytes must not be concurrently mutated. -pub(crate) unsafe fn deep_copy_into_heap(term: ErlTerm, heap: &mut ProcessHeap) -> ErlTerm { - if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { - return term; - } - if term.is_cons() { - let mut elems = Vec::new(); - let mut cursor = term; - while cursor.is_cons() { - if let Some(ptr) = cursor.as_cons_ptr() { - unsafe { - elems.push(deep_copy_into_heap((*ptr).head, heap)); - cursor = (*ptr).tail; - } - } else { - break; + unsafe fn copy_leaf(&mut self, term: ErlTerm) -> ErlTerm { + if term.is_binary() { + if let Some((ptr, len)) = unsafe { crate::ffi::binary_data_pub(term) } { + let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }; + return self.alloc_binary_auto(bytes); } + return term; } - let tail = deep_copy_into_heap(cursor, heap); - let mut result = tail; - for elem in elems.into_iter().rev() { - result = heap.alloc_cons(elem, result); - } - return result; - } - if term.is_binary() { - if let Some((ptr, len)) = unsafe { crate::ffi::binary_data_pub(term) } { - let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }; - return heap.alloc_binary_auto(bytes); - } - return term; - } - if term.is_boxed() { if let Some(ptr) = term.as_boxed_ptr() { let header = BoxedHeader(unsafe { *ptr }); match header.subtype() { - BOXED_TUPLE => { - let arity = header.arity() as usize; - let mut elems = Vec::with_capacity(arity); - for i in 0..arity { - let elem = ErlTerm(unsafe { *ptr.add(1 + i) }); - elems.push(deep_copy_into_heap(elem, heap)); - } - return heap.alloc_tuple(&elems); - } - crate::term::BOXED_MAP => { - let num_pairs = header.arity() as usize; - let mut pairs = Vec::with_capacity(num_pairs); - for i in 0..num_pairs { - let key = ErlTerm(unsafe { *ptr.add(1 + i * 2) }); - let value = ErlTerm(unsafe { *ptr.add(2 + i * 2) }); - pairs.push([deep_copy_into_heap(key, heap), deep_copy_into_heap(value, heap)]); - } - return heap.alloc_map(MapPairs::new(&pairs)); - } crate::term::BOXED_FLOAT => { let bits = unsafe { *ptr.add(1) }; - return heap.alloc_float(f64::from_bits(bits)); + return self.alloc_float(f64::from_bits(bits)); } crate::term::BOXED_BIGINT => { if let Some(big) = crate::bigint::term_to_bigint(term) { - return crate::bigint::bigint_to_term(heap, big); + return crate::bigint::bigint_to_term(self, big); } return term; } crate::term::BOXED_REFERENCE => { let ref_id = unsafe { *ptr.add(crate::term::REFERENCE_ID_WORD_INDEX) }; - let words = heap.alloc_words(crate::term::REFERENCE_TOTAL_WORDS); + let words = self.alloc_words(crate::term::REFERENCE_TOTAL_WORDS); unsafe { *words = header.0; *words.add(crate::term::REFERENCE_ID_WORD_INDEX) = ref_id; @@ -666,6 +631,137 @@ pub(crate) unsafe fn deep_copy_into_heap(term: ErlTerm, heap: &mut ProcessHeap) _ => return term, } } + term + } +} + +/// Copies a term into a `CopyTarget` through `copy_graph`: work still to +/// copy waits on a heap-allocated stack, so nesting depth costs heap, not +/// native stack. Lists, tuples and maps are copied here; other terms through +/// `CopyTarget::copy_leaf`. A subterm shared within the term is copied once +/// for each place that holds it. +struct EtsCopier<'a, T: CopyTarget> { + target: &'a mut T, + slots: SlotStack, +} + +impl SlotCopier for EtsCopier<'_, T> { + fn slots(&mut self) -> &mut SlotStack { + &mut self.slots + } + + unsafe fn copy_word(&mut self, word: u64) -> u64 { + // SAFETY: forwarded from `copy_graph`'s caller. + unsafe { self.copy_object(ErlTerm(word)).0 } + } +} + +impl<'a, T: CopyTarget> EtsCopier<'a, T> { + /// # Safety + /// As for `deep_copy_into_compact`, and `target` never moves words it + /// has allocated. + unsafe fn copy(term: ErlTerm, target: &'a mut T) -> ErlTerm { + let mut copier = EtsCopier { target, slots: SlotStack::default() }; + // SAFETY: forwarded from the caller. + ErlTerm(unsafe { copy_graph(&mut copier, term.0) }) + } + + /// Copy one term, queueing the elements of tuples and maps instead of + /// copying them; a list copies a run of cells (see `copy_list`). + unsafe fn copy_object(&mut self, term: ErlTerm) -> ErlTerm { + if is_unboxed(term) { + return term; + } + if term.is_cons() { + return unsafe { self.copy_list(term) }; + } + if !unsafe { term.is_binary() } { + if let Some(ptr) = term.as_boxed_ptr() { + let header = BoxedHeader(unsafe { *ptr }); + let (subtype, slots) = match header.subtype() { + BOXED_TUPLE => (crate::term::BoxedSubtype::Tuple, header.arity() as usize), + crate::term::BOXED_MAP => (crate::term::BoxedSubtype::Map, 2 * header.arity() as usize), + _ => return unsafe { self.target.copy_leaf(term) }, + }; + // [header | slots...], with the elements (or interleaved + // keys and values) queued for copying. + let words = self.target.alloc_words(1 + slots); + unsafe { + *words = BoxedHeader::new(subtype, header.arity()).0; + self.slots.push(words.add(1), ptr.add(1), slots); + return ErlTerm::boxed_unchecked(words); + } + } + } + unsafe { self.target.copy_leaf(term) } + } + + /// Copy the cells of a list along its tail while their heads are + /// immediates. The first cell whose head needs copying queues its head + /// and tail and ends the run, so the head is copied before the rest of + /// the list. + unsafe fn copy_list(&mut self, list: ErlTerm) -> ErlTerm { + let mut first = None; + let mut prev_tail: *mut u64 = std::ptr::null_mut(); + let mut cursor = list; + loop { + let mut next = None; + let copied = match cursor.as_cons_ptr() { + // The terminator: nil or an improper tail. + None => unsafe { self.copy_object(cursor) }, + Some(cell) => { + let words = self.target.alloc_words(2); + // SAFETY: `cell` is a live source cons cell (head word + // first, then tail); `words` reserves its two-word copy. + unsafe { + let src = cell as *const u64; + let head = *src; + if is_plain_immediate(head) { + *words = head; + next = Some((words.add(1), (*cell).tail)); + } else { + self.slots.push(words, src, 2); + } + ErlTerm::cons_unchecked(words as *const crate::term::ConsCell) + } + } + }; + if first.is_none() { + first = Some(copied); + } else { + // SAFETY: the previous cell's reserved tail word, written once. + unsafe { *prev_tail = copied.0 }; + } + match next { + Some((tail_slot, tail)) => (prev_tail, cursor) = (tail_slot, tail), + None => return first.expect("first cell copied"), + } + } + } +} + +/// Deep-copy a term from a process heap into a `CompactHeap`. +/// Used for the bigint fallback path. +fn copy_from_process_heap_to_compact(term: ErlTerm, heap: &mut CompactHeap) -> ErlTerm { + if term.is_atom() || term.is_small_int() || term.is_pid() || term.is_nil() { + return term; + } + if term.is_boxed() { + if let Some(ptr) = term.as_boxed_ptr() { + let header = BoxedHeader(unsafe { *ptr }); + let subtype = header.subtype(); + if subtype == crate::term::BOXED_BIGINT { + // header.arity() = limb_count for bigints + let limb_count = header.arity() as usize; + let total = crate::term::bigint_total_words(limb_count); + let words = heap.alloc_words(total); + unsafe { + std::ptr::copy_nonoverlapping(ptr, words, total); + } + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + return unsafe { ErlTerm::boxed_unchecked(words as *const u64) }; + } + } } term } @@ -2211,6 +2307,7 @@ mod tests { } use super::*; use crate::atom_table::global_atom_table; + use crate::heap::MapPairs; /// Helper: create a unique ETS table for each test to avoid cross-test /// interference (the global registry is shared within a test binary). diff --git a/crates/nera-runtime/src/heap.rs b/crates/nera-runtime/src/heap.rs index cede02b..b40a3e9 100644 --- a/crates/nera-runtime/src/heap.rs +++ b/crates/nera-runtime/src/heap.rs @@ -995,7 +995,7 @@ enum GcTarget<'a> { /// An immediate without the external bit: the collector copies it as itself /// with no side effect. #[inline] -fn is_plain_immediate(word: u64) -> bool { +pub(crate) fn is_plain_immediate(word: u64) -> bool { word & EXTERNAL_BIT == 0 && matches!(word & TAG_MASK, TAG_SMALL_INT | TAG_ATOM | TAG_PID | TAG_NIL) } diff --git a/crates/nera-runtime/src/lib.rs b/crates/nera-runtime/src/lib.rs index 44e313b..8844ce4 100644 --- a/crates/nera-runtime/src/lib.rs +++ b/crates/nera-runtime/src/lib.rs @@ -54,6 +54,7 @@ pub mod binary; pub mod match_context; pub mod atom_table; pub mod heap; +mod copy_stack; pub mod process; pub mod message; pub mod scheduler; diff --git a/crates/nera-runtime/src/message.rs b/crates/nera-runtime/src/message.rs index 50fafb6..d4ddebc 100644 --- a/crates/nera-runtime/src/message.rs +++ b/crates/nera-runtime/src/message.rs @@ -9,11 +9,12 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::AtomicBool; -use crate::heap::{clone_arc_from_raw, FragmentHeap, ProcessHeap}; +use crate::copy_stack::{copy_graph, SlotCopier, SlotStack}; +use crate::heap::{clone_arc_from_raw, is_plain_immediate, FragmentHeap, ProcessHeap}; use crate::term::{ bigint_total_words, BoxedHeader, ConsCell, ErlTerm, BOXED_BIGINT, BOXED_BINARY, BOXED_CLOSURE, BOXED_FLOAT, BOXED_MAP, BOXED_MATCH_CTX, BOXED_REFC_BINARY, BOXED_REFERENCE, BOXED_TUPLE, - REFC_BINARY_OFFSET_WORD_INDEX, REFC_BINARY_PTR_WORD_INDEX, + EXTERNAL_BIT, REFC_BINARY_OFFSET_WORD_INDEX, REFC_BINARY_PTR_WORD_INDEX, REFC_BINARY_TOTAL_WORDS, REFERENCE_TOTAL_WORDS, TAG_ATOM, TAG_BOXED, TAG_CONS, TAG_MASK, TAG_NIL, TAG_PID, TAG_SMALL_INT, }; @@ -392,7 +393,10 @@ fn copy_simple_tuple_batch( /// /// Shared sub-structure within the source term is preserved (each source /// address is copied at most once), but the result is completely -/// independent of the source heap. +/// independent of the source heap. Work still to copy waits on a +/// heap-allocated stack (see `copy_stack`), so nesting depth costs heap, +/// not native stack. +/// /// Raw terms cannot establish source validity through a safe call: /// ```compile_fail,E0133 /// use nera_runtime::{message::deep_copy_term, heap::ProcessHeap, term::ErlTerm}; @@ -406,18 +410,36 @@ fn copy_simple_tuple_batch( /// `BorrowedTerm::copy_to_heap` when a source owner is available. pub unsafe fn deep_copy_term(term: ErlTerm, target_heap: &mut ProcessHeap) -> ErlTerm { let mut copier = TermCopier::new(target_heap); - copier.copy(term) + // SAFETY: the caller keeps the source graph live and immutable; the + // target heap never moves words it has already allocated. + ErlTerm(unsafe { copy_graph(&mut copier, term.0) }) } // --------------------------------------------------------------------------- // TermCopier // --------------------------------------------------------------------------- +/// Copies a term graph into a process heap through [`copy_graph`], so a term +/// nested hundreds of thousands of levels deep copies without native +/// recursion. struct TermCopier<'a> { target: &'a mut ProcessHeap, /// Maps source address -> copied ErlTerm (with tag) so that shared /// sub-structure is only copied once. forwarding: HashMap, + /// Copied objects whose term slots still need copying. + slots: SlotStack, +} + +impl SlotCopier for TermCopier<'_> { + fn slots(&mut self) -> &mut SlotStack { + &mut self.slots + } + + unsafe fn copy_word(&mut self, word: u64) -> u64 { + // SAFETY: forwarded from `copy_graph`'s caller. + unsafe { self.copy_object(ErlTerm(word)).0 } + } } impl<'a> TermCopier<'a> { @@ -425,6 +447,7 @@ impl<'a> TermCopier<'a> { TermCopier { target, forwarding: HashMap::new(), + slots: SlotStack::default(), } } @@ -440,266 +463,250 @@ impl<'a> TermCopier<'a> { self.forwarding.insert(addr, new_term); } - fn copy(&mut self, term: ErlTerm) -> ErlTerm { + /// Copy one object (or a run of list cells, see `copy_list`) into the + /// target heap, queueing its term slots instead of copying them. + /// Returns the copied term. + unsafe fn copy_object(&mut self, term: ErlTerm) -> ErlTerm { // External terms point into ETS storage — strip the external bit // and deep-copy from the ETS CompactHeap into the target heap. - if term.is_external() { - return self.copy(term.strip_external_bit()); - } + let term = if term.is_external() { term.strip_external_bit() } else { term }; match term.tag() { - TAG_SMALL_INT | TAG_ATOM | TAG_PID | TAG_NIL => { - // Immediates require no heap allocation. - term - } - TAG_CONS => { - // Iterative cons-tail traversal to avoid Rust stack - // overflow on long lists. - let mut cursor = term; - let mut first_new_term: Option = None; - let mut prev_cons_ptr: *mut ConsCell = std::ptr::null_mut(); - - while cursor.tag() == TAG_CONS { - let src_addr = (cursor.0 & !TAG_MASK) as usize; - if let Some(forwarded) = self.forwarded(src_addr) { - if first_new_term.is_none() { - return forwarded; - } - unsafe { (*prev_cons_ptr).tail = forwarded; } - return first_new_term.unwrap(); - } - - let src_cons = src_addr as *const ConsCell; - let (src_head, src_tail) = unsafe { ((*src_cons).head, (*src_cons).tail) }; + // Immediates require no heap allocation. + TAG_SMALL_INT | TAG_ATOM | TAG_PID | TAG_NIL => term, + TAG_CONS => unsafe { self.copy_list(term) }, + TAG_BOXED => unsafe { self.copy_boxed(term) }, + _ => term, + } + } - let new_term = self.target.alloc_cons(ErlTerm::nil(), ErlTerm::nil()); + /// Copy the cons cells of a list along its tail while their heads are + /// immediates, as most lists and strings need. The first cell whose head + /// needs copying queues its head and tail and ends the run, so the head's + /// subtree is copied before the rest of the list, as recursion would. A + /// cell copied before (shared structure) ends the run with its copy. + unsafe fn copy_list(&mut self, list: ErlTerm) -> ErlTerm { + let mut first = None; + let mut prev_tail: *mut u64 = std::ptr::null_mut(); + let mut cursor = list; + loop { + let mut next = None; + let copied = if cursor.tag() != TAG_CONS { + // The terminator: nil or an improper tail. + unsafe { self.copy_object(cursor) } + } else { + let src_addr = (cursor.0 & !(TAG_MASK | EXTERNAL_BIT)) as usize; + if let Some(forwarded) = self.forwarded(src_addr) { + forwarded + } else { + let new_ptr = self.target.alloc_words(2); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::cons_unchecked(new_ptr as *const ConsCell) }; self.record_forwarding(src_addr, new_term); - - // Copy head (recursive — heads are typically shallow). - let new_head = self.copy(src_head); + // SAFETY: `src_addr` is a live source cons cell (head word + // first, then tail); `new_ptr` reserves its two-word copy. unsafe { - let new_cons = new_term.as_cons_ptr().unwrap() as *mut ConsCell; - (*new_cons).head = new_head; - - if first_new_term.is_none() { - first_new_term = Some(new_term); + let src = src_addr as *const u64; + let head = *src; + if is_plain_immediate(head) { + *new_ptr = head; + next = Some((new_ptr.add(1), ErlTerm(*src.add(1)))); } else { - (*prev_cons_ptr).tail = new_term; + self.slots.push(new_ptr, src, 2); } - prev_cons_ptr = new_cons; } - - // Iterate on tail instead of recursing. - cursor = src_tail; + new_term } + }; + if first.is_none() { + first = Some(copied); + } else { + // SAFETY: the previous cell's reserved tail word, written once. + unsafe { *prev_tail = copied.0 }; + } + match next { + Some((tail_slot, tail)) => (prev_tail, cursor) = (tail_slot, tail), + None => return first.expect("first cell copied"), + } + } + } + + /// Copy one boxed object, queueing the term slots of tuples, maps and + /// closures. + unsafe fn copy_boxed(&mut self, term: ErlTerm) -> ErlTerm { + if term.0 == 0 { + return term; + } + let src_addr = (term.0 & !TAG_MASK) as usize; + if let Some(forwarded) = self.forwarded(src_addr) { + return forwarded; + } - // Copy the list terminator (nil or improper tail). - let new_tail = self.copy(cursor); - if first_new_term.is_none() { - return new_tail; + let src_header_ptr = src_addr as *const u64; + let header = unsafe { BoxedHeader(*src_header_ptr) }; + let arity = header.arity() as usize; + let subtype = header.subtype(); + + match subtype { + BOXED_TUPLE => { + // Allocate header + arity words. + let new_ptr = self.target.alloc_words(1 + arity); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + // SAFETY: elements follow the header in both objects. + unsafe { + *new_ptr = header.0; + self.slots.push(new_ptr.add(1), src_header_ptr.add(1), arity); } - unsafe { (*prev_cons_ptr).tail = new_tail; } - first_new_term.unwrap() + new_term } - TAG_BOXED => { - if term.0 == 0 { - return term; + BOXED_FLOAT => { + let new_ptr = self.target.alloc_words(2); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + *new_ptr = header.0; + *new_ptr.add(1) = *src_header_ptr.add(1); } - let src_addr = (term.0 & !TAG_MASK) as usize; - if let Some(forwarded) = self.forwarded(src_addr) { - return forwarded; + new_term + } + BOXED_BIGINT => { + // Reserved layout: + // [header(BIGINT, limb_count), sign, limbs...] + let total = bigint_total_words(arity); + let new_ptr = self.target.alloc_words(total); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); + } + new_term + } + BOXED_REFERENCE => { + // [header(REFERENCE, 0), unique_id] + let new_ptr = self.target.alloc_words(REFERENCE_TOTAL_WORDS); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, REFERENCE_TOTAL_WORDS); + } + new_term + } + BOXED_MAP => { + // arity = number of key-value pairs + // total elements = arity * 2 (interleaved k/v) + let total_elems = arity * 2; + let total = 1 + total_elems; + let new_ptr = self.target.alloc_words(total); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + // SAFETY: keys and values follow the header in both objects. + unsafe { + *new_ptr = header.0; + self.slots.push(new_ptr.add(1), src_header_ptr.add(1), total_elems); + } + new_term + } + BOXED_BINARY => { + // arity = bit count; derive byte count + let bit_count = arity; + let byte_count = bit_count.div_ceil(8); + let data_words = byte_count.div_ceil(8); + let total = 1 + data_words; + let new_ptr = self.target.alloc_words(total); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); + } + new_term + } + BOXED_REFC_BINARY => { + // Descriptor layout: + // [header(REFC_BINARY, byte_len), raw_arc_ptr, byte_offset] + let total = REFC_BINARY_TOTAL_WORDS; + let new_ptr = self.target.alloc_words(total); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); } - let src_header_ptr = src_addr as *const u64; - let header = unsafe { BoxedHeader(*src_header_ptr) }; - let arity = header.arity() as usize; - let subtype = header.subtype(); - - match subtype { - BOXED_TUPLE => { - // Allocate header + arity words. - let new_ptr = self.target.alloc_words(1 + arity); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - - // Write header. - unsafe { - *new_ptr = header.0; - } - // Recursively copy each element. - for i in 0..arity { - let elem = unsafe { ErlTerm(*src_header_ptr.add(1 + i)) }; - let copied = self.copy(elem); - unsafe { - *new_ptr.add(1 + i) = copied.0; - } - } - new_term - } - BOXED_FLOAT => { - let new_ptr = self.target.alloc_words(2); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - *new_ptr = header.0; - *new_ptr.add(1) = *src_header_ptr.add(1); - } - new_term - } - BOXED_BIGINT => { - // Reserved layout: - // [header(BIGINT, limb_count), sign, limbs...] - let total = bigint_total_words(arity); - let new_ptr = self.target.alloc_words(total); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); - } - new_term - } - BOXED_REFERENCE => { - // [header(REFERENCE, 0), unique_id] - let new_ptr = self.target.alloc_words(REFERENCE_TOTAL_WORDS); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, REFERENCE_TOTAL_WORDS); - } - new_term - } - BOXED_MAP => { - // arity = number of key-value pairs - // total elements = arity * 2 (interleaved k/v) - let total_elems = arity * 2; - let total = 1 + total_elems; - let new_ptr = self.target.alloc_words(total); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - - // Write header. - unsafe { - *new_ptr = header.0; - } - // Recursively copy each key and value. - for i in 0..total_elems { - let elem = unsafe { ErlTerm(*src_header_ptr.add(1 + i)) }; - let copied = self.copy(elem); - unsafe { - *new_ptr.add(1 + i) = copied.0; - } - } - new_term - } - BOXED_BINARY => { - // arity = bit count; derive byte count - let bit_count = arity; - let byte_count = bit_count.div_ceil(8); - let data_words = byte_count.div_ceil(8); - let total = 1 + data_words; - let new_ptr = self.target.alloc_words(total); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); - } - new_term - } - BOXED_REFC_BINARY => { - // Descriptor layout: - // [header(REFC_BINARY, byte_len), raw_arc_ptr, byte_offset] - let total = REFC_BINARY_TOTAL_WORDS; - let new_ptr = self.target.alloc_words(total); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); - } - - // Clone the backing Arc (atomic refcount increment) - // and register in the target heap's ownership list. - let raw_arc_ptr = unsafe { - *src_header_ptr.add(REFC_BINARY_PTR_WORD_INDEX) - }; - let cloned_arc = unsafe { clone_arc_from_raw(raw_arc_ptr) }; - self.target.retain_refc_binary(cloned_arc); - new_term - } - BOXED_CLOSURE => { - // Closure layout: [header(CLOSURE, env_count) | fn_ptr | arity | env0 | env1 | ...] - // header.arity() = env_count; fn_ptr and arity are raw words (not tagged terms). - // Env vars ARE tagged ErlTerms and must be recursively copied. - let env_count = arity; - let total_words = 3 + env_count; - let new_ptr = self.target.alloc_words(total_words); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - - // Copy header, fn_ptr, and closure arity as raw words. - unsafe { - *new_ptr = header.0; - *new_ptr.add(1) = *src_header_ptr.add(1); // fn_ptr - *new_ptr.add(2) = *src_header_ptr.add(2); // callable arity - } - // Recursively deep-copy each env var. - for i in 0..env_count { - let env_term = unsafe { ErlTerm(*src_header_ptr.add(3 + i)) }; - let copied = self.copy(env_term); - unsafe { - *new_ptr.add(3 + i) = copied.0; - } - } - new_term - } - BOXED_MATCH_CTX => { - // Materialize exactly the logical window before dropping - // source ownership. Large output bytes become the owned - // refc backing directly; small output lives in its heap. - unsafe { - let window = crate::match_context::MatchContext::view(term).unwrap(); - let count = window.remaining() as usize; - let length = count.div_ceil(8); - let (data, _) = binary_data(window.binary()).unwrap(); - let (new_term, subtype) = if length > crate::heap::REFC_BINARY_THRESHOLD { - let mut bytes = vec![0; length]; - crate::binary::bit_copy::copy_bits(bytes.as_mut_ptr(),0,data,window.offset() as usize,count); - let backing = Arc::new(crate::binary::BinaryData::new(bytes)); - (self.target.alloc_refc_binary_from_arc(backing,0,length), crate::term::BoxedSubtype::RefcBinary) - } else { - let result = self.target.alloc_binary_zeroed(length); - let target = result.as_boxed_ptr().unwrap().add(1).cast_mut().cast::(); - crate::binary::bit_copy::copy_bits(target,0,data,window.offset() as usize,count); - (result, crate::term::BoxedSubtype::HeapBinary) - }; - if !count.is_multiple_of(8) { - *new_term.as_boxed_ptr().unwrap().cast_mut() = BoxedHeader::new(subtype,count as u64).0; - } - self.record_forwarding(src_addr, new_term); - new_term - } - } - _ => { - // Generic: copy header + arity data words verbatim. - let total = 1 + arity; - let new_ptr = self.target.alloc_words(total); - // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). - let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; - self.record_forwarding(src_addr, new_term); - unsafe { - std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); - } - new_term + // Clone the backing Arc (atomic refcount increment) + // and register in the target heap's ownership list. + let raw_arc_ptr = unsafe { + *src_header_ptr.add(REFC_BINARY_PTR_WORD_INDEX) + }; + let cloned_arc = unsafe { clone_arc_from_raw(raw_arc_ptr) }; + self.target.retain_refc_binary(cloned_arc); + new_term + } + BOXED_CLOSURE => { + // Closure layout: [header(CLOSURE, env_count) | fn_ptr | arity | env0 | env1 | ...] + // header.arity() = env_count; fn_ptr and arity are raw words (not tagged terms). + // Env vars ARE tagged ErlTerms and are queued for copying. + let env_count = arity; + let total_words = 3 + env_count; + let new_ptr = self.target.alloc_words(total_words); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + + // Copy header, fn_ptr, and closure arity as raw words; the + // environment follows them in both objects. + unsafe { + *new_ptr = header.0; + *new_ptr.add(1) = *src_header_ptr.add(1); // fn_ptr + *new_ptr.add(2) = *src_header_ptr.add(2); // callable arity + self.slots.push(new_ptr.add(3), src_header_ptr.add(3), env_count); + } + new_term + } + BOXED_MATCH_CTX => { + // Materialize exactly the logical window before dropping + // source ownership. Large output bytes become the owned + // refc backing directly; small output lives in its heap. + unsafe { + let window = crate::match_context::MatchContext::view(term).unwrap(); + let count = window.remaining() as usize; + let length = count.div_ceil(8); + let (data, _) = binary_data(window.binary()).unwrap(); + let (new_term, subtype) = if length > crate::heap::REFC_BINARY_THRESHOLD { + let mut bytes = vec![0; length]; + crate::binary::bit_copy::copy_bits(bytes.as_mut_ptr(),0,data,window.offset() as usize,count); + let backing = Arc::new(crate::binary::BinaryData::new(bytes)); + (self.target.alloc_refc_binary_from_arc(backing,0,length), crate::term::BoxedSubtype::RefcBinary) + } else { + let result = self.target.alloc_binary_zeroed(length); + let target = result.as_boxed_ptr().unwrap().add(1).cast_mut().cast::(); + crate::binary::bit_copy::copy_bits(target,0,data,window.offset() as usize,count); + (result, crate::term::BoxedSubtype::HeapBinary) + }; + if !count.is_multiple_of(8) { + *new_term.as_boxed_ptr().unwrap().cast_mut() = BoxedHeader::new(subtype,count as u64).0; } + self.record_forwarding(src_addr, new_term); + new_term } } - _ => term, + _ => { + // Generic: copy header + arity data words verbatim. + let total = 1 + arity; + let new_ptr = self.target.alloc_words(total); + // SAFETY: the allocation reserves u64-aligned words (and ConsCell alignment is 8). + let new_term = unsafe { ErlTerm::boxed_unchecked(new_ptr as *const u64) }; + self.record_forwarding(src_addr, new_term); + unsafe { + std::ptr::copy_nonoverlapping(src_header_ptr, new_ptr, total); + } + new_term + } } } } diff --git a/docs/KNOWN_LIMITS.md b/docs/KNOWN_LIMITS.md index 26805e6..59154a1 100644 --- a/docs/KNOWN_LIMITS.md +++ b/docs/KNOWN_LIMITS.md @@ -89,14 +89,15 @@ platform and scope. These are real differences from the BEAM that we have reproduced. - Terms nested thousands of levels deep (an iolist built by prepending, for - example) survive garbage collection, but other operations on them still - recurse on the native stack and crash. Measured on macOS arm64 with a list - nested in head position: storing it in ETS from about 5,000 levels, - comparing it with `<` or `==` from about 8,000, `term_to_binary/1` from - about 11,000, sending it to another process or capturing it in a spawned - fun from about 14,000, `erlang:phash2/1` from about 27,000 and - `iolist_to_binary/1` from about 45,000. `=:=`, `iolist_size/1` and sending - to self work at 200,000 levels. The BEAM handles all of these. + example) survive garbage collection, can be sent to other processes, + captured in spawned funs, and stored in and read back from ETS and + `persistent_term`, but other operations on them still recurse on the + native stack and crash. Measured on macOS arm64 with a list nested in head + position: comparing it with `<` or `==`, or using it as an `ordered_set` + key, from about 8,000 levels, `term_to_binary/1` from about 11,000, + `erlang:phash2/1` from about 27,000 and `iolist_to_binary/1` from about + 45,000. `=:=` and `iolist_size/1` work at 200,000 levels. The BEAM handles + all of these. - Comprehension filters that are guard expressions, such as `[X || X <- L, map_size(X) > 0]`, are treated as plain expressions, so a filter that should be skipped can raise. diff --git a/docs/RUNTIME_DESIGN.md b/docs/RUNTIME_DESIGN.md index fe7a0ab..29b906e 100644 --- a/docs/RUNTIME_DESIGN.md +++ b/docs/RUNTIME_DESIGN.md @@ -558,15 +558,20 @@ the term is **deep-copied** from A's heap into B's heap. This guarantees: ### 10.2 Deep Copy Algorithm -The `deep_copy_term()` function walks the term recursively: +The `deep_copy_term()` function walks the term depth first. The slots it +still has to copy wait on a heap-allocated work stack instead of native +stack frames, so a term nested hundreds of thousands of levels deep copies +like a flat one. ETS and `persistent_term` copy terms in and out the same way. - **Immediates** (small integers, atoms, PIDs, nil): returned as-is. No allocation needed — the value is entirely in the 64-bit word. -- **Cons cells**: allocate a new cell on the target heap, recursively copy - head and tail. A forwarding table (`HashMap`) ensures - shared sub-structure is copied only once. +- **Cons cells**: allocate a new cell on the target heap and copy its head + before its tail; a run of cells with immediate heads is copied in one loop. + A forwarding table (`HashMap`) ensures shared + sub-structure is copied only once. - **Boxed objects** (tuples, floats, etc.): allocate header + payload words - on the target heap, recursively copy elements for tuples. + on the target heap, then copy the elements of tuples and maps and the + environment of closures. For the common case of simple messages (tuples of immediates + refc binaries), `direct_copy_message()` provides a fast path that skips the HashMap forwarding diff --git a/tests/README.md b/tests/README.md index ec6a047..3cd840e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -361,6 +361,7 @@ These are automatically skipped by the runner and show up as SKIP in output. | `process_flag_fullsweep.erl` | process_flag(fullsweep_after, N) | ~4 | | `garbage_collect_bif.erl` | Real garbage_collect/0 suspension, local/dictionary/mailbox roots and dynamic calls | ~9 | | `gc_deep_nesting.erl` | An iolist, tuple, map and closure chain nested 200,000 levels deep survive requested and allocation-driven collections with every level intact; completion marker; also run on BEAM | ~10 | +| `copy_deep_nesting.erl` | An iolist, tuple, map and closure chain nested 200,000 levels deep sent to another process and back and captured in spawned funs; the iolist, tuple and map stored in a `set` ETS table and read back by lookup, `tab2list/1` and from another process; a 200,000-level iolist as a key found with an equal term; every level checked; completion marker; also run on BEAM | ~20 | | `max_heap_size.erl` | Integer/map max_heap_size kills allocating workers; completion marker | 2 kills | | `max_heap_size_map.erl` | Default-policy size maps, nominal error_logger=true, previous flags and errors | ~13 | | `hibernate_basic.erl` | erlang:hibernate/3 wake on message | ~4 | diff --git a/tests/copy_deep_nesting.erl b/tests/copy_deep_nesting.erl new file mode 100644 index 0000000..9ce5989 --- /dev/null +++ b/tests/copy_deep_nesting.erl @@ -0,0 +1,108 @@ +%% EXPECT: copy_deep_nesting_ok +%% Regression: a term nested 200,000 levels deep can be sent to another +%% process, captured in a spawned fun, and stored in and read back from ETS, +%% as a value or as a key, as on BEAM. The message copier and the ETS copies +%% used to recurse on the native stack for list heads and tuple and map +%% elements, so an iolist built by prepending ([Acc, Chunk]) crashed the +%% program with a bus error: in ETS from about 5,000 levels, when sent or +%% captured from about 14,000. +%% Run this same fixture on native Nera and BEAM. +-module(copy_deep_nesting). +-export([test/0]). + +-define(DEPTH, 200000). + +test() -> + Terms = [{iolist, iolist(?DEPTH, [])}, + {tuple, tuple(?DEPTH, done)}, + {map, map(?DEPTH, done)}, + {closure, closure(?DEPTH, done)}], + + %% Send each term to another process, which checks its copy and sends + %% that copy back. + Echo = spawn(fun echo/0), + [ok = round_trip(Echo, Term) || Term <- Terms], + + %% Capture each term in a spawned fun. + [ok = spawn_capture(Term) || Term <- Terms], + + %% Store the iolist, tuple and map in ETS and read them back by key, + %% as the whole table, and from another process. + Stored = [Term || {Kind, _} = Term <- Terms, Kind =/= closure], + Table = ets:new(copy_deep_nesting, [set, public]), + true = ets:insert(Table, Stored), + Found = [Entry || {Kind, _} <- Stored, Entry <- ets:lookup(Table, Kind)], + 3 = length(Found), + [ok = check(Entry) || Entry <- Found], + [ok = round_trip(Echo, Entry) || Entry <- Found], + 3 = length([ok = check(Entry) || Entry <- ets:tab2list(Table)]), + Self = self(), + spawn(fun() -> Self ! {reader, [checked(Entry) || Entry <- ets:tab2list(Table)]} end), + receive {reader, Results} -> [ok, ok, ok] = Results end, + + %% A term this deep can be a key too, found again with an equal term + %% built separately. + {iolist, IoList} = lists:keyfind(iolist, 1, Terms), + true = ets:insert(Table, {IoList, deep_key}), + [{Key, deep_key}] = ets:lookup(Table, iolist(?DEPTH, [])), + ok = check({iolist, Key}), + + Echo ! stop, + io:format("copy_deep_nesting_ok~n"), + ok. + +echo() -> + receive + {From, Term} -> + From ! {echoed, checked(Term), Term}, + echo(); + stop -> + ok + end. + +round_trip(Echo, Term) -> + Echo ! {self(), Term}, + receive {echoed, Result, Back} -> ok = Result, check(Back) end. + +spawn_capture(Term) -> + Self = self(), + Pid = spawn(fun() -> Self ! {captured, self(), checked(Term)} end), + receive {captured, Pid, Result} -> Result end. + +%% The check's result in another process, which reports a failure instead +%% of dying silently. +checked(Term) -> + try check(Term) catch Class:Reason -> {Class, Reason} end. + +check({iolist, IoList}) -> ?DEPTH = iolist_depth(IoList, 0), ok; +check({tuple, Tuple}) -> ?DEPTH = tuple_depth(Tuple, 0), ok; +check({map, Map}) -> ?DEPTH = map_depth(Map, 0), ok; +check({closure, Fun}) -> ?DEPTH = closure_depth(Fun, 0), ok. + +iolist(0, Acc) -> Acc; +iolist(N, Acc) -> iolist(N - 1, [Acc, <<"chunk">>]). + +iolist_depth([], Depth) -> Depth; +iolist_depth([Inner, <<"chunk">>], Depth) -> iolist_depth(Inner, Depth + 1). + +tuple(0, Acc) -> Acc; +tuple(N, Acc) -> tuple(N - 1, {Acc, N}). + +tuple_depth(done, Depth) -> Depth; +tuple_depth({Inner, N}, Depth) when N =:= Depth + 1 -> tuple_depth(Inner, N). + +map(0, Acc) -> Acc; +map(N, Acc) -> map(N - 1, #{inner => Acc, level => N}). + +map_depth(done, Depth) -> Depth; +map_depth(#{inner := Inner, level := N}, Depth) when N =:= Depth + 1 -> + map_depth(Inner, N). + +closure(0, Acc) -> Acc; +closure(N, Acc) -> closure(N - 1, fun() -> {Acc, N} end). + +closure_depth(done, Depth) -> Depth; +closure_depth(Fun, Depth) -> + {Inner, N} = Fun(), + N = Depth + 1, + closure_depth(Inner, N).