diff --git a/runtime/interpreter-gen-x64.cpp b/runtime/interpreter-gen-x64.cpp index 2e560de1f..9358dbbba 100644 --- a/runtime/interpreter-gen-x64.cpp +++ b/runtime/interpreter-gen-x64.cpp @@ -1512,24 +1512,33 @@ void emitHandler(EmitEnv* env) { template <> void emitHandler(EmitEnv* env) { ScratchReg r_base(env); - ScratchReg r_layout_id(env); - ScratchReg r_cache_value(env); - ScratchReg r_caches(env); + ScratchReg r_packed_location(env); Label slow_path; - __ popq(r_base); - emitGetLayoutId(env, r_layout_id, r_base); - __ movq(r_caches, Address(env->frame, Frame::kCachesOffset)); - emitIcLookupMonomorphic(env, &slow_path, r_cache_value, r_layout_id, - r_caches); - emitConvertFromSmallInt(env, r_cache_value); - { - ScratchReg r_scratch(env); - emitLoadOverflowTuple(env, r_scratch, r_layout_id, r_base); - // The real tuple index is -offset - 1, which is the same as ~offset. - __ notq(r_cache_value); - __ popq(Address(r_scratch, r_cache_value, TIMES_8, heapObjectDisp(0))); + ScratchReg r_layout_id(env); + ScratchReg r_caches(env); + __ popq(r_base); + emitGetLayoutId(env, r_layout_id, r_base); + __ movq(r_caches, Address(env->frame, Frame::kCachesOffset)); + emitIcLookupMonomorphic(env, &slow_path, r_packed_location, r_layout_id, + r_caches); + } + + ScratchReg r_overflow_offset(env); + // uword packed_location = SmallInt::cast(cached).value(); + emitConvertFromSmallInt(env, r_packed_location); + __ movq(r_overflow_offset, r_packed_location); + // uword overflow_offset = packed_location & 0xffffffff; + __ andq(r_overflow_offset, Immediate(0xffffffff)); + // uword offset = packed_location >> 32; + __ shrq(r_packed_location, Immediate(32)); + { + ScratchReg r_overflow_tuple(env); + __ movq(r_overflow_tuple, + Address(r_base, r_overflow_offset, TIMES_1, heapObjectDisp(0))); + __ popq(Address(r_overflow_tuple, r_packed_location, TIMES_8, + heapObjectDisp(0))); emitNextOpcode(env); } diff --git a/runtime/interpreter.cpp b/runtime/interpreter.cpp index f4cbf396f..aa3c358be 100644 --- a/runtime/interpreter.cpp +++ b/runtime/interpreter.cpp @@ -3288,9 +3288,20 @@ Continue Interpreter::storeAttrUpdateCache(Thread* thread, word arg, icUpdateAttr(thread, caches, cache, saved_layout_id, location, name, dependent); } else { - rewriteCurrentBytecode(frame, STORE_ATTR_INSTANCE_OVERFLOW); - icUpdateAttr(thread, caches, cache, saved_layout_id, location, name, - dependent); + uword offset = -SmallInt::cast(*location).value() - 1; + Layout layout(&scope, thread->runtime()->layoutOf(*receiver)); + uword overflow_offset = layout.overflowOffset(); + // Save one bit for the SmallInt tag + uword packed_location = (offset << 32) | overflow_offset; + // We need to check that they fit in 32 bits each and 63 bits together. + if (Utils::fits(offset) && + Utils::fits(overflow_offset) && + SmallInt::isValid(packed_location)) { + location = SmallInt::fromWord(packed_location); + rewriteCurrentBytecode(frame, STORE_ATTR_INSTANCE_OVERFLOW); + icUpdateAttr(thread, caches, cache, saved_layout_id, location, name, + dependent); + } } } else { // Layout transition. @@ -3316,6 +3327,18 @@ Continue Interpreter::storeAttrUpdateCache(Thread* thread, word arg, currentBytecode(thread) == STORE_ATTR_INSTANCE_OVERFLOW || currentBytecode(thread) == STORE_ATTR_POLYMORPHIC, "unexpected opcode"); + if (ic_state == ICState::kMonomorphic && + currentBytecode(thread) == STORE_ATTR_INSTANCE_OVERFLOW) { + // Fix up the packed cache; save only the index into the overflow tuple. + // That's what STORE_ATTR_POLYMORPHIC expects. Also, make it negative + // again to indicate that it's an overflow offset. + word index = cache * kIcPointersPerEntry; + RawObject cached = caches.at(index + kIcEntryValueOffset); + uword packed_location = SmallInt::cast(cached).value(); + caches.atPut(index + kIcEntryValueOffset, + SmallInt::fromWord(~(packed_location >> 32))); + } + if (saved_layout_id == receiver_layout_id) { rewriteCurrentBytecode(frame, STORE_ATTR_POLYMORPHIC); icUpdateAttr(thread, caches, cache, saved_layout_id, location, name, @@ -3384,13 +3407,12 @@ HANDLER_INLINE Continue Interpreter::doStoreAttrInstanceOverflow(Thread* thread, EVENT_CACHE(STORE_ATTR_INSTANCE_OVERFLOW); return storeAttrUpdateCache(thread, arg, cache); } - word offset = SmallInt::cast(cached).value(); - DCHECK(offset < 0, "unexpected offset"); + uword packed_location = SmallInt::cast(cached).value(); + uword overflow_offset = packed_location & 0xffffffff; + uword offset = packed_location >> 32; RawInstance instance = Instance::cast(receiver); - RawLayout layout = Layout::cast(thread->runtime()->layoutOf(receiver)); - RawTuple overflow = - Tuple::cast(instance.instanceVariableAt(layout.overflowOffset())); - overflow.atPut(-offset - 1, thread->stackPeek(1)); + RawTuple overflow = Tuple::cast(instance.instanceVariableAt(overflow_offset)); + overflow.atPut(offset, thread->stackPeek(1)); thread->stackDrop(2); return Continue::NEXT; }