|
| 1 | +//! Depth-first copying of term graphs without native recursion. |
| 2 | +//! |
| 3 | +//! A copier allocates each object it meets in its target first, then queues |
| 4 | +//! the object's term slots on a [`SlotStack`] instead of copying them in a |
| 5 | +//! nested call. Nesting depth costs one queued entry per open level on the |
| 6 | +//! heap rather than a native stack frame, so a term nested hundreds of |
| 7 | +//! thousands of levels deep (an iolist built by prepending, a chain of |
| 8 | +//! tuples) copies as on BEAM. Message passing, spawned closure environments, |
| 9 | +//! ETS and persistent terms copy through [`copy_graph`]; the garbage |
| 10 | +//! collector's copier in `heap.rs` runs the same loop inline. |
| 11 | +
|
| 12 | +use std::cell::Cell; |
| 13 | +use std::thread::LocalKey; |
| 14 | + |
| 15 | +use crate::heap::is_plain_immediate; |
| 16 | + |
| 17 | +/// Most entries a spare work-stack buffer keeps between uses; a buffer grown |
| 18 | +/// past this by a deeply nested term is freed instead. |
| 19 | +const SPARE_CAPACITY: usize = 256; |
| 20 | + |
| 21 | +/// A work stack that takes its thread's spare buffer on its first push and |
| 22 | +/// gives it back when dropped, so working through an ordinary term |
| 23 | +/// allocates nothing, and a term with nothing to queue does not touch the |
| 24 | +/// spare. A nested stack of the same kind finds the spare taken and |
| 25 | +/// allocates its own buffer. |
| 26 | +pub(crate) struct WorkStack<T: 'static> { |
| 27 | + items: Vec<T>, |
| 28 | + spare: &'static LocalKey<Cell<Vec<T>>>, |
| 29 | +} |
| 30 | + |
| 31 | +impl<T: 'static> WorkStack<T> { |
| 32 | + /// An empty stack that will use the spare buffer `spare` when it is free. |
| 33 | + pub(crate) fn new(spare: &'static LocalKey<Cell<Vec<T>>>) -> Self { |
| 34 | + WorkStack { items: Vec::new(), spare } |
| 35 | + } |
| 36 | + |
| 37 | + #[inline] |
| 38 | + pub(crate) fn push(&mut self, item: T) { |
| 39 | + if self.items.capacity() == 0 { |
| 40 | + self.items = self.spare.try_with(Cell::take).unwrap_or_default(); |
| 41 | + } |
| 42 | + self.items.push(item); |
| 43 | + } |
| 44 | + |
| 45 | + #[inline] |
| 46 | + pub(crate) fn pop(&mut self) -> Option<T> { |
| 47 | + self.items.pop() |
| 48 | + } |
| 49 | + |
| 50 | + #[inline] |
| 51 | + pub(crate) fn len(&self) -> usize { |
| 52 | + self.items.len() |
| 53 | + } |
| 54 | +} |
| 55 | + |
| 56 | +impl<T: 'static> Drop for WorkStack<T> { |
| 57 | + fn drop(&mut self) { |
| 58 | + let capacity = self.items.capacity(); |
| 59 | + if capacity != 0 && capacity <= SPARE_CAPACITY { |
| 60 | + self.items.clear(); |
| 61 | + let items = std::mem::take(&mut self.items); |
| 62 | + // During thread teardown the spare is gone; the buffer is freed. |
| 63 | + let _ = self.spare.try_with(|spare| spare.set(items)); |
| 64 | + } |
| 65 | + } |
| 66 | +} |
| 67 | + |
| 68 | +thread_local! { |
| 69 | + static SPARE_SLOTS: Cell<Vec<PendingSlots>> = const { Cell::new(Vec::new()) }; |
| 70 | +} |
| 71 | + |
| 72 | +/// Term slots of an object already allocated in the target whose values are |
| 73 | +/// still to be copied: the `len` words at `dst` receive copies of the `len` |
| 74 | +/// source words at `src`. |
| 75 | +struct PendingSlots { |
| 76 | + dst: *mut u64, |
| 77 | + src: *const u64, |
| 78 | + len: usize, |
| 79 | +} |
| 80 | + |
| 81 | +/// The work stack of a copy in progress. |
| 82 | +pub(crate) struct SlotStack { |
| 83 | + pending: WorkStack<PendingSlots>, |
| 84 | +} |
| 85 | + |
| 86 | +impl Default for SlotStack { |
| 87 | + fn default() -> Self { |
| 88 | + SlotStack { pending: WorkStack::new(&SPARE_SLOTS) } |
| 89 | + } |
| 90 | +} |
| 91 | + |
| 92 | +impl SlotStack { |
| 93 | + /// Queue the `len` term slots at `src` for copying into `dst`, the |
| 94 | + /// matching words of an object just allocated in the target. |
| 95 | + #[inline] |
| 96 | + pub(crate) fn push(&mut self, dst: *mut u64, src: *const u64, len: usize) { |
| 97 | + if len != 0 { |
| 98 | + self.pending.push(PendingSlots { dst, src, len }); |
| 99 | + } |
| 100 | + } |
| 101 | +} |
| 102 | + |
| 103 | +/// A term copier that queues the slots of each object it allocates. |
| 104 | +pub(crate) trait SlotCopier { |
| 105 | + /// The copier's work stack. |
| 106 | + fn slots(&mut self) -> &mut SlotStack; |
| 107 | + |
| 108 | + /// Copy one term into the target and return the word to store for it. |
| 109 | + /// An object's copy is allocated here with its term slots queued on |
| 110 | + /// [`SlotCopier::slots`], not copied; a list may copy a run of cells |
| 111 | + /// along its tail, queuing the first cell whose head needs a copy. |
| 112 | + /// [`copy_graph`] may store an immediate slot without calling this. |
| 113 | + /// |
| 114 | + /// # Safety |
| 115 | + /// `word` and everything it reaches must stay live and unchanged until |
| 116 | + /// the copy completes. |
| 117 | + unsafe fn copy_word(&mut self, word: u64) -> u64; |
| 118 | +} |
| 119 | + |
| 120 | +/// Copy `root` and everything it reaches with `copier`. |
| 121 | +/// |
| 122 | +/// Objects are copied depth first, in the order a recursive copy would use: |
| 123 | +/// an object's slots in order, each slot's subtree before the next slot. |
| 124 | +/// A list nested in head position, or a nested tuple, map or closure, costs |
| 125 | +/// one queued entry per open level; a proper list needs at most one. |
| 126 | +/// Re-entrant: a nested call only drains the entries it queued. |
| 127 | +/// |
| 128 | +/// # Safety |
| 129 | +/// The complete source graph must stay live and unchanged during the copy, |
| 130 | +/// and target allocation must not move or free words already allocated. |
| 131 | +/// Every entry the copier queues must name `len` readable source words and |
| 132 | +/// `len` writable words reserved in their copy. |
| 133 | +pub(crate) unsafe fn copy_graph<C: SlotCopier>(copier: &mut C, root: u64) -> u64 { |
| 134 | + let base = copier.slots().pending.len(); |
| 135 | + // SAFETY: the caller keeps the source graph live and unchanged. |
| 136 | + let copied = unsafe { copier.copy_word(root) }; |
| 137 | + while copier.slots().pending.len() > base { |
| 138 | + let mut slots = copier.slots().pending.pop().expect("pending slots above base"); |
| 139 | + loop { |
| 140 | + let queued = copier.slots().pending.len(); |
| 141 | + // SAFETY: `src` is a term word of a live source object, which no |
| 142 | + // copy modifies; `dst` is the matching word reserved in its copy, |
| 143 | + // written exactly once here. |
| 144 | + unsafe { |
| 145 | + let word = *slots.src; |
| 146 | + *slots.dst = if is_plain_immediate(word) { word } else { copier.copy_word(word) }; |
| 147 | + } |
| 148 | + slots.len -= 1; |
| 149 | + if slots.len == 0 { |
| 150 | + break; |
| 151 | + } |
| 152 | + // SAFETY: slots remain, so both pointers stay inside the source |
| 153 | + // object and its reserved copy. |
| 154 | + unsafe { |
| 155 | + slots.dst = slots.dst.add(1); |
| 156 | + slots.src = slots.src.add(1); |
| 157 | + } |
| 158 | + if copier.slots().pending.len() != queued { |
| 159 | + // The slot's object queued its own slots: copy them first, as |
| 160 | + // recursion would, then this object's remaining slots. |
| 161 | + let stack = &mut copier.slots().pending; |
| 162 | + let child = stack.pop().expect("child slots just queued"); |
| 163 | + stack.push(slots); |
| 164 | + stack.push(child); |
| 165 | + break; |
| 166 | + } |
| 167 | + } |
| 168 | + } |
| 169 | + copied |
| 170 | +} |
0 commit comments