diff --git a/majit/majit-gc/src/trace.rs b/majit/majit-gc/src/trace.rs index 36c67617594..b7b2a477c4d 100644 --- a/majit/majit-gc/src/trace.rs +++ b/majit/majit-gc/src/trace.rs @@ -307,16 +307,24 @@ pub struct TypeRegistry { impl TypeRegistry { /// Maximum number of types the backing `layout_table` can hold - /// without reallocating. Sized generously above the dozen-or-so - /// types pyre currently registers; bumps require recompiling - /// majit-gc. + /// without reallocating; bumps require recompiling majit-gc. /// /// RPython's `type_info_group` is bounded by the half-word width /// of the inline `GROUP_MEMBER_OFFSET` (translator/c/src/llgroup.h) /// — 64KB on 32-bit and 4GB on 64-bit. majit's bound is set by /// the maximum number of distinct GC types we expect to register, - /// not by an addressing limit. - pub const MAX_TYPES: usize = 1024; + /// not by an addressing limit: `Header` gives the type id the whole + /// low 32 bits (`header::TYPE_ID_BITS`), so this constant costs + /// pre-allocated capacity and nothing else. + /// + /// It was 1024, sized for "the dozen-or-so types pyre registers". + /// Lowering the named-struct transparent constructors to `New` + /// (`jtransform`) turns each one into a registered GC type and put + /// the count past that: `PYRE_GC_TYPE_COUNT=1` reports 1046 frozen. + /// Read that number before choosing the next value rather than + /// inferring one from the `register` assert, which names the cap it + /// blew and not the total it needed. + pub const MAX_TYPES: usize = 4096; } /// Custom trace function type. @@ -975,6 +983,15 @@ impl TypeRegistry { return; } self.can_add_new_types = false; + // The `register` assert says to bump `MAX_TYPES` but not to what; + // this is the number to size it against. + if std::env::var_os("PYRE_GC_TYPE_COUNT").is_some() { + eprintln!( + "[gc-type-count] frozen={} max={}", + self.entries.len(), + Self::MAX_TYPES + ); + } self.assign_inheritance_ids(); // Refresh layout_table rows for object types whose // subclassrange_{min,max} just changed. diff --git a/majit/majit-translate/src/codewriter/assembler.rs b/majit/majit-translate/src/codewriter/assembler.rs index 2d4323705d5..8673360cbca 100644 --- a/majit/majit-translate/src/codewriter/assembler.rs +++ b/majit/majit-translate/src/codewriter/assembler.rs @@ -3596,7 +3596,7 @@ impl crate::translator::rtyper::lltypesystem::llmemory::OffsetLayout for NoStruc } } -fn bh_size_spec_from_callcontrol( +pub(crate) fn bh_size_spec_from_callcontrol( cc: &CallControl, owner: &str, ) -> Option { @@ -3630,10 +3630,17 @@ fn bh_size_spec_from_callcontrol( // `index_in_parent`; without the base tag in this list both the tag and // payload are slot 0 and the payload replaces the tag despite living at // byte offset 8. + // `Option::Some` is on this list for the same reason its two `Result` + // siblings are: `front/mir.rs` gives it the same explicit shell, so the + // blackhole must reconstruct the same inherited tag slot or the payload + // takes slot 0 here while living at byte 8 there. `None` carries no + // payload row, so it needs no reconstruction. let result_variant = layout_owner.ends_with("result::Result::Ok") || layout_owner.ends_with("result::Result::Err") + || layout_owner.ends_with("option::Option::Some") || layout_owner == "Result::Ok" - || layout_owner == "Result::Err"; + || layout_owner == "Result::Err" + || layout_owner == "Option::Some"; // The bare Charon variant can contain only the inherited enum tag while // the annotator keeps the payload row on the concrete instantiation // (`Result::Ok`). RPython has one concrete low-level @@ -3759,10 +3766,10 @@ fn bh_all_field_specs_for_struct( specs } -/// Build the variant-owned portion of the explicit `Result` -/// shell. The front end deliberately represents this as one inherited tag -/// word followed by one-word payload fields (`front/mir.rs`'s -/// `synthetic_result_shell`), rather than as Rust's native enum layout. The +/// Build the variant-owned portion of the explicit `Result` / +/// `Option` shell. The front end deliberately represents these as one +/// inherited tag word followed by one-word payload fields (`front/mir.rs`'s +/// `explicit_sum_shell`), rather than as Rust's native enum layout. The /// shared nominal layout registry cannot describe each generic payload at /// once, so read the concrete instantiation's rows directly and place them /// after the inherited tag, matching RPython's concrete low-level STRUCT. diff --git a/majit/majit-translate/src/codewriter/format.rs b/majit/majit-translate/src/codewriter/format.rs index b70484a1173..8b7fa96021e 100644 --- a/majit/majit-translate/src/codewriter/format.rs +++ b/majit/majit-translate/src/codewriter/format.rs @@ -446,7 +446,9 @@ fn call_target_repr(target: &crate::model::CallTarget) -> String { CallTarget::FunctionPath { segments } => { format!("$<* function '{}'>", segments.join(".")) } - CallTarget::SyntheticTransparentCtor { name, owner_path } => { + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => { if owner_path.is_empty() { format!("$<* synthetic-transparent-ctor '{name}'>") } else { diff --git a/majit/majit-translate/src/codewriter/jtransform.rs b/majit/majit-translate/src/codewriter/jtransform.rs index 62613fcca1e..9b510a852b9 100644 --- a/majit/majit-translate/src/codewriter/jtransform.rs +++ b/majit/majit-translate/src/codewriter/jtransform.rs @@ -3702,7 +3702,9 @@ impl<'a> Transformer<'a> { // Keep the owner-path gate: the universal MIR tuple aggregate has no // Rust nominal owner, whereas a user ADT whose leaf happens to start // with `Tuple` is a different allocation policy. - if let CallTarget::SyntheticTransparentCtor { name, owner_path } = target + if let CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } = target && owner_path.is_empty() && majit_ir::descr::is_shaped_tuple_name(name) && args.is_empty() @@ -3725,7 +3727,9 @@ impl<'a> Transformer<'a> { // Bare `Array` is deliberately excluded: only `Array` carries a // complete low-level owner identity. The owner-path gate keeps nominal // user types out of this builtin aggregate policy. - if let CallTarget::SyntheticTransparentCtor { name, owner_path } = target + if let CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } = target && owner_path.is_empty() && majit_ir::descr::is_shaped_array_name(name) && args.is_empty() @@ -3742,6 +3746,60 @@ impl<'a> Transformer<'a> { }, }]); } + // A niladic named-struct ctor is an allocation, exactly like the tuple + // and array forms above: `front::mir` emits the constructor with empty + // args and the operands follow it as `FieldWrite`s in program order, + // which is `rtyper`'s `malloc(GcStruct)` + one `setfield` per field. + // Left as a call it has no registered function address and puts its + // stable hash in JitCode as though it were executable code, which is + // what stops a descent that reaches one. + // + // `is_struct` is the whole gate, and it is why this cannot be spelled + // by inspecting the name. A sum-type variant carries a tag Charon does + // not spell as a MIR operand, so allocating one here without stamping + // it would leave the discriminant reading whatever the payload wrote; + // only `Result` and `Option`, whose variant order the language fixes, + // can have one stamped back (below). `front::mir` sets the flag only + // where it resolved a `TypeDeclKind::Struct`, and every other + // construction path leaves it false. + // + // `is_struct` alone is not enough to allocate. Charon models a + // closure environment as an Adt whose decl is a `Struct`, so the flag + // is true for one, and a closure has no registered layout at all. Ask + // the assembler's own question instead -- `bh_size_spec_from_callcontrol` + // is exactly what `OpKind::New` calls at emit time, and it panics there + // rather than declining, so anything it cannot answer must not become a + // `New` here. + // + // A layout answer is still not enough: the spec must also list fields. + // A field-less spec is materialized by the bare + // `make_size_descr_with_type_and_vtable` mint rather than + // `simple_descr_group_from_bh_size`, so it never enters + // `gc_cache._cache_size`, `register_unresolved_struct_tids` never walks + // it, and its allocation tid stays the structural hash truncated to + // u32 -- which the walker refuses as `UnregisteredNewGcType` after the + // descent has already run. Registering it here is not the alternative: + // an empty field list carries no ref offsets, so the shape a + // registration would publish says "no pointers" for a struct whose + // size says otherwise, and the collector would stop tracing through it. + // The payload-less sum variant below declines for this same reason. + if let CallTarget::SyntheticTransparentCtor { + is_struct: true, .. + } = target + && args.is_empty() + && let ValueType::Ref(Some(owner)) = result_ty + && self.callcontrol.as_deref().is_some_and(|cc| { + crate::codewriter::assembler::bh_size_spec_from_callcontrol(cc, owner) + .is_some_and(|spec| !spec.all_fielddescrs.is_empty()) + }) + { + return RewriteResult::Replace(vec![SpaceOperation { + result: op.result.clone(), + kind: OpKind::New { + owner: owner.clone(), + }, + }]); + } // RPython `rtyper` lowers a heap-carried sum-type variant to // `malloc(GcStruct)` plus its discriminant/payload `setfield`s before // `jtransform`; `rewrite_op_malloc` then emits `new(descr)`. @@ -6319,6 +6377,7 @@ impl<'a> Transformer<'a> { let CallTarget::SyntheticTransparentCtor { name, owner_path: _, + is_struct: _, } = target else { return false; @@ -6871,7 +6930,9 @@ fn target_to_call_path(target: &CallTarget) -> crate::parse::CallPath { crate::parse::CallPath::from_segments(segments.iter().map(String::as_str)) } CallTarget::Method { name, .. } => crate::parse::CallPath::from_segments([name.as_str()]), - CallTarget::SyntheticTransparentCtor { name, owner_path } => { + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => { let mut segs: Vec<&str> = owner_path.iter().map(String::as_str).collect(); segs.push(name.as_str()); crate::parse::CallPath::from_segments(segs) diff --git a/majit/majit-translate/src/front/mir.rs b/majit/majit-translate/src/front/mir.rs index bfdc75862f8..ecc50175477 100644 --- a/majit/majit-translate/src/front/mir.rs +++ b/majit/majit-translate/src/front/mir.rs @@ -1224,7 +1224,10 @@ fn register_synthetic_positional_metadata( .flat_map(|block| &block.operations) { let OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { name, owner_path }, + target: + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + }, args, .. } = &op.kind @@ -1273,7 +1276,8 @@ fn register_synthetic_positional_metadata( } } -/// Byte size of the explicit `Result` shell that carries `field_offsets`. +/// Byte size of the explicit `Result` / `Option` shell that carries +/// `field_offsets`. /// /// The shell is a non-overlapping `[tag@0 | payload@8 | ...]` laid out by this /// module rather than borrowed from Rust's enum layout, so its size follows @@ -1281,7 +1285,7 @@ fn register_synthetic_positional_metadata( /// A fixed 16 would truncate any shell that ever records more than one payload /// word. The floor keeps a tag-only shell at the full `[tag | payload]` width, /// matching the `size.max(16)` the codewriter applies to the same shell. -fn result_shell_size(field_offsets: &std::collections::HashMap) -> u64 { +fn sum_shell_size(field_offsets: &std::collections::HashMap) -> u64 { field_offsets .values() .copied() @@ -1646,10 +1650,14 @@ fn derive_program_metadata( // `{enum_leaf}::{variant}`. No cross-variant dedup: each // variant owns its field namespace. let enum_layout = td.layout_for_target(&target); - // `Result` is not materialised as Rust's native enum in - // translated code. The inverse exception transform removes - // ordinary `?` paths; a hand-written `match` that remains is - // represented by the explicit rtyper shell emitted below: + // `Result` and `Option` are not materialised as + // Rust's native enums in translated code. The inverse + // exception transform removes ordinary `?` paths; a + // hand-written `match` that remains, and every + // runtime-discriminant pair `emit_tagged_pair_aggregate` + // synthesises (`checked_neg`, `usize::try_from`, + // `i64::try_from(&RBigInt)`), are represented by the explicit + // rtyper shell emitted below: // `{ __discriminant: Signed, __pos_0: payload }`. Rust's // niche layout commonly places both the implicit tag and the // payload at offset 0, but applying that host layout to this @@ -1659,7 +1667,20 @@ fn derive_program_metadata( // the graph declares, so give that synthetic struct its own // non-overlapping layout instead of borrowing Rust's enum // layout. - let synthetic_result_shell = name == "core::result::Result"; + // + // `Option` needs it for the same reason and did not have it: + // `Option` registered `__discriminant` and + // `Some.__pos_0` both at offset 0 of an 8-byte parent, so a + // descent into any body holding a `checked_neg` read `-v` as + // the tag and aborted on the match's unreachable arm. A + // pointer-niche `Option` is diverted to a bare nullable + // pointer before any constructor is emitted + // (`tyref_is_niche_option_ptr`) and never allocates, and the + // base tag stays at offset 0 either way, so the shell only + // moves the payload of an Option this front end actually + // builds. + let explicit_sum_shell = + name == "core::result::Result" || name == "core::option::Option"; // Register the enum BASE in `exact_layouts`: a single // `__discriminant` field at the tag's real byte position // (`discriminator.Branch.offset` via `discriminant_offset`). @@ -1671,13 +1692,13 @@ fn derive_program_metadata( // tag (`discriminant_offset` → `None`) and also registers 0. // Fieldless enums skip this (int-valued, no base ClassDef). if !fieldless { - if synthetic_result_shell { + if explicit_sum_shell { let mut base_offsets = std::collections::HashMap::new(); base_offsets.insert("__discriminant".to_string(), 0); exact_layouts.insert( base_sid, crate::front::semantic::ExactLayout { - size: Some(result_shell_size(&base_offsets)), + size: Some(sum_shell_size(&base_offsets)), align: Some(8), field_offsets: base_offsets, }, @@ -1724,7 +1745,7 @@ fn derive_program_metadata( tyref_to_attr_value_type(&f.ty, llbc), ) }; - let field_offset = if synthetic_result_shell { + let field_offset = if explicit_sum_shell { Some(8 + (i as u64) * 8) } else { enum_layout.as_ref().and_then(|l| l.field_offset(vidx, i)) @@ -1748,9 +1769,9 @@ fn derive_program_metadata( record_struct_id(&mut struct_ids, variant_qual.clone(), vsid); record_struct_id(&mut struct_ids, variant_leaf.clone(), vsid); record_struct_id(&mut struct_ids, variant_canon.clone(), vsid); - if synthetic_result_shell { + if explicit_sum_shell { let exact = crate::front::semantic::ExactLayout { - size: Some(result_shell_size(&voffsets)), + size: Some(sum_shell_size(&voffsets)), align: Some(8), field_offsets: voffsets, }; @@ -5134,38 +5155,41 @@ impl<'a> Lowering<'a> { // use `variant_idx = null`, enum variants index into the // `TypeDeclKind::Enum` variant list. let resolved = self.resolve_aggregate_adt(&kind); - let (owner_path, ctor_name, field_rows, aggregate_owner_id) = match resolved { - Some((owner_path, ctor_name, field_rows, owner_id)) => { - (owner_path, ctor_name, field_rows, Some(owner_id)) - } - None => { - // Synthetic placeholders for non-Adt aggregates - // (`Tuple`, `Array`, `Closure`) — they have no - // user-defined class to resolve into. A non-empty - // tuple or array carries its per-shape `<…>` suffix - // so its `__pos_N` attrs do not collide with other - // shapes on one global class; the suffix matches - // `positional_aggregate_owner` (Site-A reads) and - // `tyref_positional_aggregate_suffix` (Site-B reads). - // The per-shape `<…>` suffix is rendered from the - // destination place type, not the `AggregateKind` head: - // Charon's `AggregateKind::Adt(Tuple, …)` carries no - // element `types`, so keying off it would spell a bare - // `Tuple` on the write while the `.N` projection reads - // (which do see `place.ty`'s element types) spell the - // suffixed owner — a write/read owner split. `dest_ty` - // is that same `place.ty`, so both sides agree. - let leaf = format!( - "{}{}", - aggregate_ctor_name(&kind), - tyref_positional_aggregate_suffix(dest_ty, self.llbc) - ); - let positional = (0..arg_vars.len()) - .map(|i| (format!("__pos_{i}"), String::new())) - .collect(); - (Vec::new(), leaf, positional, None) - } - }; + let (owner_path, ctor_name, field_rows, aggregate_owner_id, adt_is_struct) = + match resolved { + Some((owner_path, ctor_name, field_rows, owner_id, is_struct)) => { + (owner_path, ctor_name, field_rows, Some(owner_id), is_struct) + } + None => { + // Synthetic placeholders for non-Adt aggregates + // (`Tuple`, `Array`, `Closure`) — they have no + // user-defined class to resolve into. A non-empty + // tuple or array carries its per-shape `<…>` suffix + // so its `__pos_N` attrs do not collide with other + // shapes on one global class; the suffix matches + // `positional_aggregate_owner` (Site-A reads) and + // `tyref_positional_aggregate_suffix` (Site-B reads). + // The per-shape `<…>` suffix is rendered from the + // destination place type, not the `AggregateKind` head: + // Charon's `AggregateKind::Adt(Tuple, …)` carries no + // element `types`, so keying off it would spell a bare + // `Tuple` on the write while the `.N` projection reads + // (which do see `place.ty`'s element types) spell the + // suffixed owner — a write/read owner split. `dest_ty` + // is that same `place.ty`, so both sides agree. + let leaf = format!( + "{}{}", + aggregate_ctor_name(&kind), + tyref_positional_aggregate_suffix(dest_ty, self.llbc) + ); + let positional = (0..arg_vars.len()) + .map(|i| (format!("__pos_{i}"), String::new())) + .collect(); + // Not an Adt at all, so there is no struct decl to + // stand behind an allocation rewrite. + (Vec::new(), leaf, positional, None, false) + } + }; let result_ty_owner = if owner_path.is_empty() { ctor_name.clone() } else { @@ -5185,6 +5209,14 @@ impl<'a> Lowering<'a> { // the underlying `SomeInstance(classdef)`. let ctor_target = if owner_path.is_empty() { CallTarget::synthetic_transparent_ctor(ctor_name.clone()) + } else if adt_is_struct { + // Resolved to a `TypeDeclKind::Struct`, so the value is + // its field writes and nothing else; `jtransform` may + // lower the constructor to a bare allocation. + CallTarget::synthetic_transparent_struct_ctor( + owner_path.clone(), + ctor_name.clone(), + ) } else { CallTarget::synthetic_transparent_ctor_with_owner( owner_path.clone(), @@ -6377,6 +6409,11 @@ impl<'a> Lowering<'a> { variants.get(variant_idx)?.discriminant_i64() } + /// The trailing `bool` is `true` when the decl is a `TypeDeclKind::Struct`. + /// A struct's value is its fields; a variant additionally carries a tag + /// Charon does not spell as an operand, and the two must stay + /// distinguishable downstream — see `CallTarget::SyntheticTransparentCtor`'s + /// `is_struct`. fn resolve_aggregate_adt( &self, kind: &serde_json::Value, @@ -6385,6 +6422,7 @@ impl<'a> Lowering<'a> { String, Vec<(String, String)>, majit_ir::descr::StructId, + bool, )> { let adt = kind.as_object()?.get("Adt")?.as_array()?; // `AggregateKind::Adt` head: either a bare `type_id` u64 or a @@ -6428,6 +6466,7 @@ impl<'a> Lowering<'a> { type_leaf, field_rows, concrete_adt_struct_id(template, head_adt, self.llbc), + true, )) } (TypeDeclKind::Enum(variants), Some(idx)) => { @@ -6469,6 +6508,7 @@ impl<'a> Lowering<'a> { v.name.clone(), field_rows, concrete_adt_struct_id(template, head_adt, self.llbc), + false, )) } _ => None, @@ -23013,10 +23053,7 @@ mod tests { #[test] fn block_emptied_after_the_head_collapse_is_still_rewired_past() { let unit_ctor = || OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Tuple".to_string(), - owner_path: Vec::new(), - }, + target: CallTarget::synthetic_transparent_ctor("Tuple"), args: Vec::new(), result_ty: ValueType::Ref(Some("Tuple".to_string())), }; @@ -23790,10 +23827,7 @@ mod tests { graph.block_mut(a).operations.push(SpaceOperation { result: Some(arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -23855,10 +23889,7 @@ mod tests { graph.block_mut(a).operations.push(SpaceOperation { result: Some(tuple.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Tuple".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Tuple"), args: vec![], result_ty: ValueType::Ref(Some("Tuple".to_string())), }, @@ -23910,10 +23941,7 @@ mod tests { graph.block_mut(b).operations.push(SpaceOperation { result: Some(args_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -23931,10 +23959,7 @@ mod tests { graph.block_mut(b).operations.push(SpaceOperation { result: Some(pieces_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24009,10 +24034,7 @@ mod tests { graph.block_mut(b0).operations.push(SpaceOperation { result: Some(tuple.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Tuple".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Tuple"), args: vec![], result_ty: ValueType::Ref(Some("Tuple".to_string())), }, @@ -24060,10 +24082,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(args_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24081,10 +24100,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(pieces_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24229,10 +24245,7 @@ mod tests { .push_op_var( bf, OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Arguments".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Arguments"), args: vec![], result_ty: ValueType::Ref(None), }, @@ -24425,10 +24438,7 @@ mod tests { graph.block_mut(b0).operations.push(SpaceOperation { result: Some(tuple.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Tuple".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Tuple"), args: vec![], result_ty: ValueType::Ref(Some("Tuple".to_string())), }, @@ -24502,10 +24512,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(args_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24525,10 +24532,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(pieces_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24715,10 +24719,7 @@ mod tests { graph.block_mut(b0).operations.push(SpaceOperation { result: Some(tuple.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Tuple".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Tuple"), args: vec![], result_ty: ValueType::Ref(Some("Tuple".to_string())), }, @@ -24791,10 +24792,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(args_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, @@ -24814,10 +24812,7 @@ mod tests { graph.block_mut(bp).operations.push(SpaceOperation { result: Some(pieces_arr.clone()), kind: OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { - name: "Array".to_string(), - owner_path: vec![], - }, + target: CallTarget::synthetic_transparent_ctor("Array"), args: vec![], result_ty: ValueType::Ref(Some("Array".to_string())), }, diff --git a/majit/majit-translate/src/front/option_ctor.rs b/majit/majit-translate/src/front/option_ctor.rs index 648b9e0819b..a7eb05daaf1 100644 --- a/majit/majit-translate/src/front/option_ctor.rs +++ b/majit/majit-translate/src/front/option_ctor.rs @@ -30,9 +30,9 @@ const SOME_TAG: i64 = 1; /// Splits a `CallTarget::SyntheticTransparentCtor` into `(owner_path, leaf)`. fn ctor_parts(target: &CallTarget) -> Option<(&[String], &str)> { match target { - CallTarget::SyntheticTransparentCtor { name, owner_path } => { - Some((owner_path.as_slice(), name.as_str())) - } + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => Some((owner_path.as_slice(), name.as_str())), _ => None, } } diff --git a/majit/majit-translate/src/front/result_exc.rs b/majit/majit-translate/src/front/result_exc.rs index 46ade06cf38..cde1655333c 100644 --- a/majit/majit-translate/src/front/result_exc.rs +++ b/majit/majit-translate/src/front/result_exc.rs @@ -330,7 +330,10 @@ pub(crate) fn widen_unit_return_to_void(graph: &mut FunctionGraph) { /// it before the bare-path compare so suffixed and bare Result ctors are /// recognised alike. pub(crate) fn result_ctor_kind(target: &CallTarget) -> Option { - let CallTarget::SyntheticTransparentCtor { name, owner_path } = target else { + let CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } = target + else { return None; }; let [head @ .., tail] = owner_path.as_slice() else { diff --git a/majit/majit-translate/src/model.rs b/majit/majit-translate/src/model.rs index 17fd161ec6f..fd88d205c1e 100644 --- a/majit/majit-translate/src/model.rs +++ b/majit/majit-translate/src/model.rs @@ -285,6 +285,21 @@ pub enum CallTarget { name: String, #[serde(default)] owner_path: Vec, + /// `true` when the constructed ADT is a `struct`, whose value is + /// described entirely by the field writes that follow it. + /// + /// A sum-type variant constructor sets `false`. Charon does not spell + /// the enum tag as a MIR operand, so a rewrite that turns the + /// constructor into a bare allocation would drop it, and only `Result` + /// and `Option` — whose variant order the language fixes — can have + /// one stamped back on ([`sum_variant_ctor`]). `false` is therefore + /// the safe answer and the one every constructor that cannot prove + /// otherwise records: a consumer that reads this decides whether to + /// allocate, and dropping a tag is silent. + /// + /// [`sum_variant_ctor`]: crate::codewriter::jtransform + #[serde(default)] + is_struct: bool, }, /// RPython: `indirect_call` opname. Receiver's static type is a /// `dyn Trait` (Rust fat pointer); at JIT time the actual callee @@ -348,6 +363,7 @@ impl CallTarget { Self::SyntheticTransparentCtor { name: name.into(), owner_path: Vec::new(), + is_struct: false, } } @@ -362,6 +378,26 @@ impl CallTarget { Self::SyntheticTransparentCtor { name: name.into(), owner_path, + is_struct: false, + } + } + + /// [`synthetic_transparent_ctor_with_owner`] for a constructor the caller + /// has resolved to a `struct` declaration rather than an enum variant. + /// + /// Separate from the general constructor so that "this is a struct" is + /// something a caller states from a resolved `TypeDeclKind`, never a + /// default a new call site inherits by omission. + /// + /// [`synthetic_transparent_ctor_with_owner`]: Self::synthetic_transparent_ctor_with_owner + pub fn synthetic_transparent_struct_ctor( + owner_path: Vec, + name: impl Into, + ) -> Self { + Self::SyntheticTransparentCtor { + name: name.into(), + owner_path, + is_struct: true, } } @@ -383,7 +419,9 @@ impl CallTarget { // make `Instruction::LoadFast` and `OtherEnum::LoadFast` // share segments and break any downstream caller that // uses `path_segments()` for qualified identity. - CallTarget::SyntheticTransparentCtor { name, owner_path } => { + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => { let mut segs: Vec<&str> = owner_path.iter().map(String::as_str).collect(); segs.push(name.as_str()); Some(segs) @@ -408,7 +446,9 @@ impl fmt::Display for CallTarget { .. } => f.write_str(name), CallTarget::FunctionPath { segments } => f.write_str(&segments.join("::")), - CallTarget::SyntheticTransparentCtor { name, owner_path } => { + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => { if owner_path.is_empty() { write!(f, "") } else { diff --git a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs index 551b4a4061b..4866545ff20 100644 --- a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs +++ b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs @@ -616,7 +616,10 @@ fn fmt_op_result(op: &SpaceOperation) -> String { /// `op_canraise` is false exactly when `translate_op` emits no op. fn is_elided_unit_variant_ctor(kind: &OpKind) -> bool { if let OpKind::Call { - target: crate::model::CallTarget::SyntheticTransparentCtor { name, owner_path }, + target: + crate::model::CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + }, args, .. } = kind @@ -646,6 +649,7 @@ fn is_array_aggregate_ctor(kind: &OpKind) -> bool { target: crate::model::CallTarget::SyntheticTransparentCtor { name, owner_path, + .. }, .. } if owner_path.is_empty() @@ -2432,7 +2436,9 @@ pub fn translate_op( call_args.extend(arg_hls); Ok(vec![FlowspaceOp::new("simple_call", call_args, result)]) } - CallTarget::SyntheticTransparentCtor { name, owner_path } => { + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + } => { // RPython parity: tagged-union ctor `Foo(x)` annotates as // `SomePBC([ClassDesc(Foo)])` then `pair_simple_call` // constructs `SomeInstance(classdef)` (`bookkeeper.py: @@ -3190,7 +3196,10 @@ fn legacy_const_define_hlvalue( // the same set consulted here — both layers agree on which // paths are unit-variant singletons. OpKind::Call { - target: crate::model::CallTarget::SyntheticTransparentCtor { name, owner_path }, + target: + crate::model::CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + }, args, .. } if args.is_empty() => { @@ -5352,10 +5361,7 @@ mod tests { let op = SpaceOperation { result: Some(vars[2].clone()), kind: OpKind::Call { - target: crate::model::CallTarget::SyntheticTransparentCtor { - name: "Point".into(), - owner_path: Vec::new(), - }, + target: crate::model::CallTarget::synthetic_transparent_ctor("Point"), args: vec![vars[1].clone()], result_ty: ValueType::Ref(None), }, diff --git a/majit/majit-translate/src/translator/rtyper/unit_variant_fold.rs b/majit/majit-translate/src/translator/rtyper/unit_variant_fold.rs index 10c5b3f60e2..bf201dc056c 100644 --- a/majit/majit-translate/src/translator/rtyper/unit_variant_fold.rs +++ b/majit/majit-translate/src/translator/rtyper/unit_variant_fold.rs @@ -132,7 +132,10 @@ pub fn fold_unit_variant_ctors(graph: &mut FunctionGraph) { for block in graph.blocks.iter_mut() { for op in block.operations.iter_mut() { let OpKind::Call { - target: CallTarget::SyntheticTransparentCtor { name, owner_path }, + target: + CallTarget::SyntheticTransparentCtor { + name, owner_path, .. + }, args, .. } = &op.kind diff --git a/pyre/design.md b/pyre/design.md index 4ae10bc44be..0eee4bcb5c0 100644 --- a/pyre/design.md +++ b/pyre/design.md @@ -531,11 +531,16 @@ recorder, or the green accounts for. ### 3.8 The fold layer: hand-written compensation for an opaque objspace -pyre records traces through 69 `try_walker_specialize_*` functions — 67 in +pyre records traces through 70 `try_walker_specialize_*` functions — 68 in `jitcode_dispatch/specialize.rs`, one each in `residual_call.rs` -(`load_deref`) and `inline_call.rs` (`instance_next`) — 9,865 lines of body -inside `specialize.rs`'s 16,933, described by the 74 rows of +(`load_deref`) and `inline_call.rs` (`instance_next`) — 9,992 lines of body +inside `specialize.rs`'s 17,298, described by the 77 rows of `SPEC_FOLD_ROWS` (one fold can back several rows, and row-less folds exist). +Three of those rows are not folds at all: `subscr_tuple_descent`, +`unary_invert_descent` and `unary_negative_descent` are orthodox sub-walks +of `w_tuple_getitem`, `invert_inner` and `neg_inner`, carrying a row only so +they can be suppressed and A/B'd like the folds they replaced. Counting them +as debt overstates it by three. Nothing in this charter named that layer before 2026-08-26, which is itself the finding: it is the largest single adaptation in the tree. @@ -543,18 +548,18 @@ Re-derive every number here before citing it; this section has published two miscounts, and both survived because the recipe beside them did not run. Every command below is quoted as it must be typed. -* Rows — `spec_folds!` opens at `diag.rs:342` and closes at `:418`: - `sed -n '342,418p' pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs | rg -cF '=> ("'`. +* Rows — `spec_folds!` opens at `diag.rs:342` and closes at `:421`: + `sed -n '342,421p' pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs | rg -cF '=> ("'`. `-F` is load-bearing: without it the `(` is an unclosed regex group and `rg` exits 2 rather than counting. -* Definitions — `rg -c` reports one count *per file*, so it answers 67/1/1 - rather than 69. Sum the matches instead: +* Definitions — `rg -c` reports one count *per file*, so it answers 68/1/1 + rather than 70. Sum the matches instead: `rg -o 'fn try_walker_specialize_' pyre/ majit/ -g '*.rs' | wc -l`. * Body lines — sum the brace-matched span of each `fn try_walker_specialize_*` in `specialize.rs`; no one-liner does it. * Corpus — `ls pyre/bench/synth/*.py | wc -l`. Non-recursive **on purpose**: a recursive walk sweeps `_pending`, `foriter57` and `iter57` and answers - 530, over-counting by 46. Do not "fix" this to a `find`. + 533, over-counting by 46. Do not "fix" this to a `find`. `specialize.rs`'s own line count moved seven times in seven commits and is not a usable identifier for a tree. @@ -576,6 +581,12 @@ favour of the ported optimizer" — is not available as stated. Group by group: | frame / execution-context introspection | 6 | none at any layer; PyPy forces the virtualizable instead | | function-object construction | 2 | none | +The `n` column sums to 73, not to the 77 rows above it: the split was taken +when the layer had 73 rows and no one has re-derived it since. Re-derive it +by classifying every row, not by apportioning the difference — the two +miscounts this section already published both came from adjusting a +published number instead of recounting. + Four groups have only downstream cleanup upstream, two have nothing at all, and exactly one has a counterpart that is a pass rather than a consumer. So the convergence target for this layer is **descent reach into the diff --git a/pyre/gate-triage.md b/pyre/gate-triage.md index ac5c224ab27..4872187433d 100644 --- a/pyre/gate-triage.md +++ b/pyre/gate-triage.md @@ -222,10 +222,12 @@ build. `PYRE_FBW_NO_SPECIALIZE` is the one entry here that changes behaviour rather than reporting it: its comma-separated selectors (or the reserved `all`) turn -off that many of the 74 hand-written trace-time specialization rows (the -`spec_folds!` invocation at `jitcode_dispatch/diag.rs:342-417`; count them -there rather than trusting this sentence), and an unset variable suppresses -none. It is a measurement instrument — suppressing a fold is how the descent +off that many of the 77 trace-time specialization rows (the `spec_folds!` +invocation at `jitcode_dispatch/diag.rs:342-421`; count them there rather +than trusting this sentence), and an unset variable suppresses none. Not all +75 are hand-written: `subscr_tuple_descent`, `unary_invert_descent` and +`unary_negative_descent` name orthodox sub-walks of the interpreter's own +body, and a row is what lets one be suppressed and A/B'd like any other. It is a measurement instrument — suppressing a fold is how the descent wall behind it is made to print — so it retires with the folds it selects, not before them. `PYRE_FBW_SPEC_CENSUS` in §6c is its read-only half: the per-fold @@ -348,6 +350,7 @@ under measurement. | gate | default polarity | what it gates / retirement condition | |---|---|---| | `PYRE_PROBE14` | OFF | reports discarded reference-constant relocations; retire when relocation preservation is covered by ordinary tests | +| `PYRE_GC_TYPE_COUNT` | OFF | prints `[gc-type-count] frozen=N max=M` when `freeze_types` runs, so the number to size `TypeRegistry::MAX_TYPES` against can be read instead of guessed -- the `register` assert names the cap it blew but not the total it needed; retire when the registry grows on demand rather than pre-allocating a fixed capacity for a stable base address | | `PYRE_GC_SIZE_AUDIT` | OFF | panics when a block is stamped with a type id whose declared payload is larger than the block's own extent, at the allocation that stamps it rather than in whichever later collection reads the neighbouring block as a field (varsize types are exempt); retire when every allocator derives the size from the type id it stamps, so the two cannot disagree | ## Summary diff --git a/pyre/pyre-interpreter/src/objspace/descroperation.rs b/pyre/pyre-interpreter/src/objspace/descroperation.rs index 1b7e3ea1620..c47a740cf3b 100644 --- a/pyre/pyre-interpreter/src/objspace/descroperation.rs +++ b/pyre/pyre-interpreter/src/objspace/descroperation.rs @@ -5525,6 +5525,25 @@ pub fn neg(a: PyObjectRef) -> PyResult { if let Some(result) = try_numeric_unaryop_override(a, "__neg__")? { return Ok(result); } + neg_inner(a) + } +} + +/// [`neg`] past its `__neg__` override probe. +/// +/// Split out so that a trace can descend this body rather than re-emit it by +/// hand. The probe is what stops such a descent: its +/// `needs_numeric_unaryop_dispatch` is `dont_look_inside` and is the second +/// operation the body executes. A caller that has already proven an exact +/// builtin receiver cannot take it, so entering here records the arm that +/// receiver selects and nothing else. Every other caller reaches this through +/// [`neg`] and is unaffected. +/// +/// Unlike [`invert_inner`] this keeps the `bool` operand: `neg` has no separate +/// bool slot to leave behind, and the integer arm below already answers `True` +/// and `False` through [`int_value`]. +pub fn neg_inner(a: PyObjectRef) -> PyResult { + unsafe { if is_int(a) || is_bool(a) { let v = int_value(a); return match v.checked_neg() { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index 288b6746fdd..9cf6489e1dc 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -346,6 +346,7 @@ spec_folds! { UnaryPositiveInt => ("unary_positive_int", "residual_call", "-"), UnaryNegativeInt => ("unary_negative_int", "residual_call", "-"), UnaryInvertDescent => ("unary_invert_descent", "residual_call", "-"), + UnaryNegativeDescent => ("unary_negative_descent", "residual_call", "-"), StoreSubscr => ("store_subscr", "residual_call", "-"), Setslice => ("setslice", "residual_call", "-"), GetIter => ("get_iter", "residual_call", "-"), diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index a51bfa548e8..8b5ea451cfe 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -6289,6 +6289,25 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } + // UNARY_NEGATIVE. Descend `neg_inner` -- `neg` past the override probe -- + // rather than re-emit its integer arm by hand, the same shape the invert + // descent below takes. Sits ahead of the `unary_negative_int` fold so that + // fold's `consulted` count reads whether the descent took the site. The + // fold is not dead and stays: besides the bool / subclass / non-int operand + // it never admitted, it is what serves the `INT_MIN` promotion, which the + // descent declines on an unlowered `PyObject` transparent ctor. + if ctx.is_authoritative_executor + && dst_bank == 'r' + && r_args.len() == 1 + && ei.runtime_helper == majit_ir::RuntimeHelperKind::UnaryNegative + && spec_gate(SpecFold::UnaryNegativeDescent, || { + try_walker_orthodox_unary_negative(ctx, op.pc, r_args[0], dst, dst_bank) + })? + .is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + // #61: UNARY_NEGATIVE `-int` fold. `-x` is `0 - x`; the fold emits // `IntSubOvf(0, x)` behind a `GUARD_CLASS INT` (reusing the binary-sub // overflow discipline), eliding the `CALL_MAY_FORCE`. A bool / subclass / diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 88c7e216d90..dccb23100c4 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -7590,6 +7590,133 @@ pub(crate) fn try_walker_orthodox_unary_invert( Ok(Some(())) } +/// Descend `neg_inner`'s compiled body for `-x` on an exact builtin `int` or +/// `long`, instead of re-emitting its integer arm by hand. +/// +/// The same orthodox shape as [`try_walker_orthodox_unary_invert`], and for the +/// same reason: upstream traces *through* `descr_neg`, which is ordinary +/// RPython. +/// +/// This does NOT retire `unary_negative_int`, and the census says why. Over +/// the 484-fixture corpus the descent fires on six fixtures and the fold on +/// one -- `unary_negative.py`'s `main_int_min`, the `INT_MIN` promotion. The +/// body reaches that through `w_long_new_fresh_rbigint_handle`, and the +/// sub-walk stops there on a `PyObject` synthetic transparent ctor this build +/// does not lower, so the descent declines and the fold's `guard_value` + +/// ovf2long tail still serves it. What the descent does take is every +/// non-promoting exact `int` and the exact `long` operand, which reached no +/// fold at all before. +/// +/// `neg` itself cannot be entered: its `__neg__` override probe is +/// `dont_look_inside` and is the second operation the body executes. +/// `neg_inner` is that body past it. Unlike `invert`, `neg` has no bool slot to +/// step over, so the split leaves the probe alone. +/// +/// The guards emitted here are what let the caller skip the probe: an `int` or +/// `long` subclass keeps the builtin `ob_type` but retags `w_class` and may +/// define `__neg__`, so it must side-exit to the residual, which still runs the +/// whole of `neg`. `bool` is excluded for a second reason as well -- see the +/// `walker_exact_builtin_class` read below. +pub(crate) fn try_walker_orthodox_unary_negative( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + operand: OpRef, + dst: usize, + dst_bank: char, +) -> Result, DispatchError> { + let Some(operand_obj) = walker_concrete_ref_object(ctx, operand) else { + return Ok(None); + }; + // SAFETY: `operand_obj` is a live concrete `PyObjectRef` from the walker + // shadow. + let is_long = unsafe { pyre_object::is_long(operand_obj) }; + let admitted = unsafe { + !pyre_object::is_bool(operand_obj) + && (is_long || pyre_object::is_int(operand_obj)) + && pyre_object::is_exact_builtin_instance(operand_obj) + }; + if !admitted { + return Ok(None); + } + // SAFETY: as above. + // + // This cannot decline for an admitted operand, by the argument + // [`try_walker_orthodox_unary_invert`] gives: the only exact builtins born + // with a null `w_class` are the five read-only singletons, and the + // admission above already rejects every one of them. + let Some(operand_class) = (unsafe { walker_exact_builtin_class(operand_obj) }) else { + return Ok(None); + }; + + // Resolve every possible decline before recording a guard. + let Some(jc_arc) = crate::jitcode_runtime::neg_inner_jitcode() else { + return Ok(None); + }; + let Some(sub_body) = sub_jitcode_body_by_index(jc_arc.index()) else { + return Ok(None); + }; + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() { + return Ok(None); + } + // SAFETY: set for the lifetime of the enclosing full-body walk. + if unsafe { (&*sym_ptr).jitcode().is_null() } { + return Ok(None); + } + let sym = unsafe { &*sym_ptr }; + + let pre_fold_pos = ctx.trace_ctx.get_trace_position(); + let type_addr = if is_long { + &pyre_object::pyobject::LONG_TYPE as *const _ as i64 + } else { + &pyre_object::pyobject::INT_TYPE as *const _ as i64 + }; + walker_guard_class(ctx, op_pc, operand, type_addr)?; + walker_guard_exact_w_class(ctx, op_pc, operand, operand_class)?; + ctx.trace_ctx.set_opref_concrete( + operand, + majit_ir::Value::Ref(majit_ir::GcRef(operand_obj as usize)), + ); + + let walk = run_orthodox_helper_subwalk( + ctx, + op_pc, + sym, + &sub_body, + "unary_negative_commit", + "neg_inner_call_site", + &[], + &[], + &[operand], + &[ConcreteValue::Ref(operand_obj)], + ); + let (walk_outcome, _walk_start) = match walk { + // The body reached a helper this build did not lower. Both admitted + // arms of `neg_inner` are pure reads that allocate their result, so + // nothing is committed -- cut the tentative IR, with its snapshots, and + // let the residual serve the operator. The two guards above are emitted + // with snapshots attached and name the discarded operation namespace, + // so leaving them behind would expose stale boxes to a later remap. + Err(DispatchError::OrthodoxSubWalkTraceUnsupported { pc, .. }) => { + if fbw_debug_abort_enabled() { + eprintln!("[decline-why] UNARY-NEGATIVE-SUBWALK pc={pc}"); + } + ctx.trace_ctx.cut_trace_with_snapshots(pre_fold_pos); + ctx.trace_ctx.heap_cache_mut().reset(); + return Ok(None); + } + Ok(pair) => pair, + Err(error) => return Err(error), + }; + let result = match walk_outcome { + DispatchOutcome::SubReturn { result } => finish_inline_callee_return(ctx, result) + .ok_or(DispatchError::UnexpectedVoidSubReturn { pc: op_pc })?, + _ => return Err(DispatchError::UnexpectedVoidSubReturn { pc: op_pc }), + }; + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; + Ok(Some(())) +} + /// `s[i]` on an exact `str` with an exact machine-`int` index: emit the /// guarded unbox plus one elidable [`pyre_object::jit_str_getitem`] call /// instead of the opaque `bh_binary_op_fn` residual. diff --git a/pyre/pyre-jit-trace/src/jitcode_runtime.rs b/pyre/pyre-jit-trace/src/jitcode_runtime.rs index e06e03cf0e1..e95d679997c 100644 --- a/pyre/pyre-jit-trace/src/jitcode_runtime.rs +++ b/pyre/pyre-jit-trace/src/jitcode_runtime.rs @@ -403,6 +403,9 @@ thread_local! { /// two of eight observed cache generations, so an index is not stable /// across builds and a path is. static INVERT_INNER_JITCODE_INDEX: OnceCell> = const { OnceCell::new() }; + /// Cached `ALL_JITCODES` index of `neg_inner` for the current thread. + /// Resolved by graph key, for the reason given above. + static NEG_INNER_JITCODE_INDEX: OnceCell> = const { OnceCell::new() }; } /// Scan the build-time names index for the unique entry equal to `name`. @@ -525,6 +528,25 @@ pub fn invert_inner_jitcode() -> Option> { get_jitcode_by_index(idx) } +/// The charon `neg_inner` body in `ALL_JITCODES`, resolved by the graph key the +/// codewriter allocated it under and cached per thread. `None` if the helper is +/// absent from the build-time pipeline. +/// +/// This is `descroperation.rs` `neg` past its `__neg__` override probe -- the +/// one arm a descent cannot take. What is left is the integer arm (including +/// the `checked_neg` promotion of `INT_MIN`), the `long`, `float` and `complex` +/// arms, the instance fallback and the terminal `TypeError`, so a caller that +/// has pinned an exact `int` or `long` receiver records one of the first two +/// and nothing else. +pub fn neg_inner_jitcode() -> Option> { + let idx = NEG_INNER_JITCODE_INDEX.with(|cell| { + *cell.get_or_init(|| { + compute_pathed_jitcode_index("pyre_interpreter::objspace::descroperation::neg_inner") + }) + })?; + get_jitcode_by_index(idx) +} + pub fn tuple_getitem_jitcode() -> Option> { let idx = TUPLE_GETITEM_JITCODE_INDEX.with(|cell| { *cell.get_or_init(|| {