From 8dc25d2b294d8817b3ddd9daf9a044a83654cb20 Mon Sep 17 00:00:00 2001 From: "danno@chromium.org" Date: Mon, 11 Mar 2013 14:11:03 +0000 Subject: [PATCH] Use direct jump and call instruction for X64 when the deoptimization entries are in the code range. Review URL: https://codereview.chromium.org/11574027 Patch from Haitao Feng . git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@13903 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/arm/assembler-arm-inl.h | 13 ++++++++++ src/assembler.h | 7 +++++ src/ia32/assembler-ia32-inl.h | 25 +++++++++++++----- src/ia32/assembler-ia32.cc | 23 ++++++++++++----- src/mips/assembler-mips-inl.h | 13 ++++++++++ src/objects.cc | 7 ++++- src/spaces.h | 4 +++ src/x64/assembler-x64-inl.h | 58 +++++++++++++++++++++++++++--------------- src/x64/assembler-x64.cc | 30 ++++++++++++++++++++++ src/x64/assembler-x64.h | 5 ++++ src/x64/lithium-codegen-x64.cc | 8 +++--- 11 files changed, 155 insertions(+), 38 deletions(-) diff --git a/src/arm/assembler-arm-inl.h b/src/arm/assembler-arm-inl.h index af29bb8..85a523d 100644 --- a/src/arm/assembler-arm-inl.h +++ b/src/arm/assembler-arm-inl.h @@ -178,6 +178,19 @@ Address* RelocInfo::target_reference_address() { } +Address RelocInfo::target_runtime_entry(Assembler* origin) { + ASSERT(IsRuntimeEntry(rmode_)); + return target_address(); +} + + +void RelocInfo::set_target_runtime_entry(Address target, + WriteBarrierMode mode) { + ASSERT(IsRuntimeEntry(rmode_)); + if (target_address() != target) set_target_address(target, mode); +} + + Handle RelocInfo::target_cell_handle() { ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL); Address address = Memory::Address_at(pc_); diff --git a/src/assembler.h b/src/assembler.h index 0fee67a..f5ae611 100644 --- a/src/assembler.h +++ b/src/assembler.h @@ -316,6 +316,9 @@ class RelocInfo BASE_EMBEDDED { static inline bool IsEmbeddedObject(Mode mode) { return mode == EMBEDDED_OBJECT; } + static inline bool IsRuntimeEntry(Mode mode) { + return mode == RUNTIME_ENTRY; + } // Is the relocation mode affected by GC? static inline bool IsGCRelocMode(Mode mode) { return mode <= LAST_GCED_ENUM; @@ -379,6 +382,10 @@ class RelocInfo BASE_EMBEDDED { INLINE(Object** target_object_address()); INLINE(void set_target_object(Object* target, WriteBarrierMode mode = UPDATE_WRITE_BARRIER)); + INLINE(Address target_runtime_entry(Assembler* origin)); + INLINE(void set_target_runtime_entry(Address target, + WriteBarrierMode mode = + UPDATE_WRITE_BARRIER)); INLINE(JSGlobalPropertyCell* target_cell()); INLINE(Handle target_cell_handle()); INLINE(void set_target_cell(JSGlobalPropertyCell* cell, diff --git a/src/ia32/assembler-ia32-inl.h b/src/ia32/assembler-ia32-inl.h index 56d88b0..fbe2f24 100644 --- a/src/ia32/assembler-ia32-inl.h +++ b/src/ia32/assembler-ia32-inl.h @@ -51,7 +51,7 @@ static const byte kCallOpcode = 0xE8; // The modes possibly affected by apply must be in kApplyMask. void RelocInfo::apply(intptr_t delta) { - if (rmode_ == RUNTIME_ENTRY || IsCodeTarget(rmode_)) { + if (IsRuntimeEntry(rmode_) || IsCodeTarget(rmode_)) { int32_t* p = reinterpret_cast(pc_); *p -= delta; // Relocate entry. CPU::FlushICache(p, sizeof(uint32_t)); @@ -83,13 +83,13 @@ void RelocInfo::apply(intptr_t delta) { Address RelocInfo::target_address() { - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY); + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)); return Assembler::target_address_at(pc_); } Address RelocInfo::target_address_address() { - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || rmode_ == EMBEDDED_OBJECT || rmode_ == EXTERNAL_REFERENCE); return reinterpret_cast
(pc_); @@ -103,7 +103,7 @@ int RelocInfo::target_address_size() { void RelocInfo::set_target_address(Address target, WriteBarrierMode mode) { Assembler::set_target_address_at(pc_, target); - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY); + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)); if (mode == UPDATE_WRITE_BARRIER && host() != NULL && IsCodeTarget(rmode_)) { Object* target_code = Code::GetCodeFromTargetAddress(target); host()->GetHeap()->incremental_marking()->RecordWriteIntoCode( @@ -149,6 +149,19 @@ Address* RelocInfo::target_reference_address() { } +Address RelocInfo::target_runtime_entry(Assembler* origin) { + ASSERT(IsRuntimeEntry(rmode_)); + return reinterpret_cast
(*reinterpret_cast(pc_)); +} + + +void RelocInfo::set_target_runtime_entry(Address target, + WriteBarrierMode mode) { + ASSERT(IsRuntimeEntry(rmode_)); + if (target_address() != target) set_target_address(target, mode); +} + + Handle RelocInfo::target_cell_handle() { ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL); Address address = Memory::Address_at(pc_); @@ -262,7 +275,7 @@ void RelocInfo::Visit(ObjectVisitor* visitor) { Isolate::Current()->debug()->has_break_points()) { visitor->VisitDebugTarget(this); #endif - } else if (mode == RelocInfo::RUNTIME_ENTRY) { + } else if (IsRuntimeEntry(mode)) { visitor->VisitRuntimeEntry(this); } } @@ -291,7 +304,7 @@ void RelocInfo::Visit(Heap* heap) { IsPatchedDebugBreakSlotSequence()))) { StaticVisitor::VisitDebugTarget(heap, this); #endif - } else if (mode == RelocInfo::RUNTIME_ENTRY) { + } else if (IsRuntimeEntry(mode)) { StaticVisitor::VisitRuntimeEntry(this); } } diff --git a/src/ia32/assembler-ia32.cc b/src/ia32/assembler-ia32.cc index d926316..b48906e 100644 --- a/src/ia32/assembler-ia32.cc +++ b/src/ia32/assembler-ia32.cc @@ -1425,7 +1425,11 @@ void Assembler::call(byte* entry, RelocInfo::Mode rmode) { EnsureSpace ensure_space(this); ASSERT(!RelocInfo::IsCodeTarget(rmode)); EMIT(0xE8); - emit(entry - (pc_ + sizeof(int32_t)), rmode); + if (RelocInfo::IsRuntimeEntry(rmode)) { + emit(reinterpret_cast(entry), rmode); + } else { + emit(entry - (pc_ + sizeof(int32_t)), rmode); + } } @@ -1490,7 +1494,11 @@ void Assembler::jmp(byte* entry, RelocInfo::Mode rmode) { EnsureSpace ensure_space(this); ASSERT(!RelocInfo::IsCodeTarget(rmode)); EMIT(0xE9); - emit(entry - (pc_ + sizeof(int32_t)), rmode); + if (RelocInfo::IsRuntimeEntry(rmode)) { + emit(reinterpret_cast(entry), rmode); + } else { + emit(entry - (pc_ + sizeof(int32_t)), rmode); + } } @@ -1547,7 +1555,11 @@ void Assembler::j(Condition cc, byte* entry, RelocInfo::Mode rmode) { // 0000 1111 1000 tttn #32-bit disp. EMIT(0x0F); EMIT(0x80 | cc); - emit(entry - (pc_ + sizeof(int32_t)), rmode); + if (RelocInfo::IsRuntimeEntry(rmode)) { + emit(reinterpret_cast(entry), rmode); + } else { + emit(entry - (pc_ + sizeof(int32_t)), rmode); + } } @@ -2564,10 +2576,7 @@ void Assembler::GrowBuffer() { // Relocate runtime entries. for (RelocIterator it(desc); !it.done(); it.next()) { RelocInfo::Mode rmode = it.rinfo()->rmode(); - if (rmode == RelocInfo::RUNTIME_ENTRY) { - int32_t* p = reinterpret_cast(it.rinfo()->pc()); - *p -= pc_delta; // relocate entry - } else if (rmode == RelocInfo::INTERNAL_REFERENCE) { + if (rmode == RelocInfo::INTERNAL_REFERENCE) { int32_t* p = reinterpret_cast(it.rinfo()->pc()); if (*p != 0) { // 0 means uninitialized. *p += pc_delta; diff --git a/src/mips/assembler-mips-inl.h b/src/mips/assembler-mips-inl.h index 0499d36..1dad815 100644 --- a/src/mips/assembler-mips-inl.h +++ b/src/mips/assembler-mips-inl.h @@ -231,6 +231,19 @@ Address* RelocInfo::target_reference_address() { } +Address RelocInfo::target_runtime_entry(Assembler* origin) { + ASSERT(IsRuntimeEntry(rmode_)); + return target_address(); +} + + +void RelocInfo::set_target_runtime_entry(Address target, + WriteBarrierMode mode) { + ASSERT(IsRuntimeEntry(rmode_)); + if (target_address() != target) set_target_address(target, mode); +} + + Handle RelocInfo::target_cell_handle() { ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL); Address address = Memory::Address_at(pc_); diff --git a/src/objects.cc b/src/objects.cc index fe36002..72d0dad 100644 --- a/src/objects.cc +++ b/src/objects.cc @@ -8793,8 +8793,10 @@ void Code::CopyFrom(const CodeDesc& desc) { int mode_mask = RelocInfo::kCodeTargetMask | RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT) | RelocInfo::ModeMask(RelocInfo::GLOBAL_PROPERTY_CELL) | + RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) | RelocInfo::kApplyMask; - Assembler* origin = desc.origin; // Needed to find target_object on X64. + // Needed to find target_object and runtime_entry on X64 + Assembler* origin = desc.origin; for (RelocIterator it(this, mode_mask); !it.done(); it.next()) { RelocInfo::Mode mode = it.rinfo()->rmode(); if (mode == RelocInfo::EMBEDDED_OBJECT) { @@ -8810,6 +8812,9 @@ void Code::CopyFrom(const CodeDesc& desc) { Code* code = Code::cast(*p); it.rinfo()->set_target_address(code->instruction_start(), SKIP_WRITE_BARRIER); + } else if (RelocInfo::IsRuntimeEntry(mode)) { + Address p = it.rinfo()->target_runtime_entry(origin); + it.rinfo()->set_target_runtime_entry(p, SKIP_WRITE_BARRIER); } else { it.rinfo()->apply(delta); } diff --git a/src/spaces.h b/src/spaces.h index 8365312..6847b31 100644 --- a/src/spaces.h +++ b/src/spaces.h @@ -894,6 +894,10 @@ class CodeRange { void TearDown(); bool exists() { return this != NULL && code_range_ != NULL; } + Address start() { + if (this == NULL || code_range_ == NULL) return NULL; + return static_cast
(code_range_->address()); + } bool contains(Address address) { if (this == NULL || code_range_ == NULL) return false; Address start = static_cast
(code_range_->address()); diff --git a/src/x64/assembler-x64-inl.h b/src/x64/assembler-x64-inl.h index 67acbf0..91bc528 100644 --- a/src/x64/assembler-x64-inl.h +++ b/src/x64/assembler-x64-inl.h @@ -86,6 +86,14 @@ void Assembler::emit_code_target(Handle target, } +void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) { + ASSERT(RelocInfo::IsRuntimeEntry(rmode)); + ASSERT(isolate()->code_range()->exists()); + RecordRelocInfo(rmode); + emitl(static_cast(entry - isolate()->code_range()->start())); +} + + void Assembler::emit_rex_64(Register reg, Register rm_reg) { emit(0x48 | reg.high_bit() << 2 | rm_reg.high_bit()); } @@ -208,6 +216,12 @@ Handle Assembler::code_target_object_handle_at(Address pc) { return code_targets_[Memory::int32_at(pc)]; } + +Address Assembler::runtime_entry_at(Address pc) { + ASSERT(isolate()->code_range()->exists()); + return Memory::int32_at(pc) + isolate()->code_range()->start(); +} + // ----------------------------------------------------------------------------- // Implementation of RelocInfo @@ -217,7 +231,7 @@ void RelocInfo::apply(intptr_t delta) { // absolute code pointer inside code object moves with the code object. Memory::Address_at(pc_) += static_cast(delta); CPU::FlushICache(pc_, sizeof(Address)); - } else if (IsCodeTarget(rmode_)) { + } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) { Memory::int32_at(pc_) -= static_cast(delta); CPU::FlushICache(pc_, sizeof(int32_t)); } else if (rmode_ == CODE_AGE_SEQUENCE) { @@ -231,17 +245,13 @@ void RelocInfo::apply(intptr_t delta) { Address RelocInfo::target_address() { - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY); - if (IsCodeTarget(rmode_)) { - return Assembler::target_address_at(pc_); - } else { - return Memory::Address_at(pc_); - } + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)); + return Assembler::target_address_at(pc_); } Address RelocInfo::target_address_address() { - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || rmode_ == EMBEDDED_OBJECT || rmode_ == EXTERNAL_REFERENCE); return reinterpret_cast
(pc_); @@ -258,17 +268,12 @@ int RelocInfo::target_address_size() { void RelocInfo::set_target_address(Address target, WriteBarrierMode mode) { - ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY); - if (IsCodeTarget(rmode_)) { - Assembler::set_target_address_at(pc_, target); + ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)); + Assembler::set_target_address_at(pc_, target); + if (mode == UPDATE_WRITE_BARRIER && host() != NULL && IsCodeTarget(rmode_)) { Object* target_code = Code::GetCodeFromTargetAddress(target); - if (mode == UPDATE_WRITE_BARRIER && host() != NULL) { - host()->GetHeap()->incremental_marking()->RecordWriteIntoCode( - host(), this, HeapObject::cast(target_code)); - } - } else { - Memory::Address_at(pc_) = target; - CPU::FlushICache(pc_, sizeof(Address)); + host()->GetHeap()->incremental_marking()->RecordWriteIntoCode( + host(), this, HeapObject::cast(target_code)); } } @@ -314,6 +319,19 @@ void RelocInfo::set_target_object(Object* target, WriteBarrierMode mode) { } +Address RelocInfo::target_runtime_entry(Assembler* origin) { + ASSERT(IsRuntimeEntry(rmode_)); + return origin->runtime_entry_at(pc_); +} + + +void RelocInfo::set_target_runtime_entry(Address target, + WriteBarrierMode mode) { + ASSERT(IsRuntimeEntry(rmode_)); + if (target_address() != target) set_target_address(target, mode); +} + + Handle RelocInfo::target_cell_handle() { ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL); Address address = Memory::Address_at(pc_); @@ -443,7 +461,7 @@ void RelocInfo::Visit(ObjectVisitor* visitor) { Isolate::Current()->debug()->has_break_points()) { visitor->VisitDebugTarget(this); #endif - } else if (mode == RelocInfo::RUNTIME_ENTRY) { + } else if (RelocInfo::IsRuntimeEntry(mode)) { visitor->VisitRuntimeEntry(this); } } @@ -472,7 +490,7 @@ void RelocInfo::Visit(Heap* heap) { IsPatchedDebugBreakSlotSequence()))) { StaticVisitor::VisitDebugTarget(heap, this); #endif - } else if (mode == RelocInfo::RUNTIME_ENTRY) { + } else if (RelocInfo::IsRuntimeEntry(mode)) { StaticVisitor::VisitRuntimeEntry(this); } } diff --git a/src/x64/assembler-x64.cc b/src/x64/assembler-x64.cc index 97b9075..57d40f7 100644 --- a/src/x64/assembler-x64.cc +++ b/src/x64/assembler-x64.cc @@ -841,6 +841,16 @@ void Assembler::call(Label* L) { } +void Assembler::call(Address entry, RelocInfo::Mode rmode) { + ASSERT(RelocInfo::IsRuntimeEntry(rmode)); + positions_recorder()->WriteRecordedPositions(); + EnsureSpace ensure_space(this); + // 1110 1000 #32-bit disp. + emit(0xE8); + emit_runtime_entry(entry, rmode); +} + + void Assembler::call(Handle target, RelocInfo::Mode rmode, TypeFeedbackId ast_id) { @@ -1247,6 +1257,16 @@ void Assembler::j(Condition cc, Label* L, Label::Distance distance) { } +void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) { + ASSERT(RelocInfo::IsRuntimeEntry(rmode)); + EnsureSpace ensure_space(this); + ASSERT(is_uint4(cc)); + emit(0x0F); + emit(0x80 | cc); + emit_runtime_entry(entry, rmode); +} + + void Assembler::j(Condition cc, Handle target, RelocInfo::Mode rmode) { @@ -1309,6 +1329,15 @@ void Assembler::jmp(Handle target, RelocInfo::Mode rmode) { } +void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { + ASSERT(RelocInfo::IsRuntimeEntry(rmode)); + EnsureSpace ensure_space(this); + ASSERT(RelocInfo::IsRuntimeEntry(rmode)); + emit(0xE9); + emit_runtime_entry(entry, rmode); +} + + void Assembler::jmp(Register target) { EnsureSpace ensure_space(this); // Opcode FF/4 r64. @@ -3049,6 +3078,7 @@ void Assembler::RecordComment(const char* msg, bool force) { const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask | + 1 << RelocInfo::RUNTIME_ENTRY | 1 << RelocInfo::INTERNAL_REFERENCE | 1 << RelocInfo::CODE_AGE_SEQUENCE; diff --git a/src/x64/assembler-x64.h b/src/x64/assembler-x64.h index 03e742d..49a57e6 100644 --- a/src/x64/assembler-x64.h +++ b/src/x64/assembler-x64.h @@ -561,6 +561,7 @@ class Assembler : public AssemblerBase { } inline Handle code_target_object_handle_at(Address pc); + inline Address runtime_entry_at(Address pc); // Number of bytes taken up by the branch target in the code. static const int kSpecialTargetSize = 4; // Use 32-bit displacement. // Distance between the address of the code target in the call instruction @@ -1180,6 +1181,7 @@ class Assembler : public AssemblerBase { // Calls // Call near relative 32-bit displacement, relative to next instruction. void call(Label* L); + void call(Address entry, RelocInfo::Mode rmode); void call(Handle target, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET, TypeFeedbackId ast_id = TypeFeedbackId::None()); @@ -1201,6 +1203,7 @@ class Assembler : public AssemblerBase { // Use a 32-bit signed displacement. // Unconditional jump to L void jmp(Label* L, Label::Distance distance = Label::kFar); + void jmp(Address entry, RelocInfo::Mode rmode); void jmp(Handle target, RelocInfo::Mode rmode); // Jump near absolute indirect (r64) @@ -1213,6 +1216,7 @@ class Assembler : public AssemblerBase { void j(Condition cc, Label* L, Label::Distance distance = Label::kFar); + void j(Condition cc, Address entry, RelocInfo::Mode rmode); void j(Condition cc, Handle target, RelocInfo::Mode rmode); // Floating-point operations @@ -1430,6 +1434,7 @@ class Assembler : public AssemblerBase { inline void emit_code_target(Handle target, RelocInfo::Mode rmode, TypeFeedbackId ast_id = TypeFeedbackId::None()); + inline void emit_runtime_entry(Address entry, RelocInfo::Mode rmode); void emit(Immediate x) { emitl(x.value_); } // Emits a REX prefix that encodes a 64-bit operand size and diff --git a/src/x64/lithium-codegen-x64.cc b/src/x64/lithium-codegen-x64.cc index 5d06c60..81976a6 100644 --- a/src/x64/lithium-codegen-x64.cc +++ b/src/x64/lithium-codegen-x64.cc @@ -353,9 +353,9 @@ bool LCodeGen::GenerateJumpTable() { } } else { if (is_lazy_deopt) { - __ Call(entry, RelocInfo::RUNTIME_ENTRY); + __ call(entry, RelocInfo::RUNTIME_ENTRY); } else { - __ Jump(entry, RelocInfo::RUNTIME_ENTRY); + __ jmp(entry, RelocInfo::RUNTIME_ENTRY); } } } @@ -754,9 +754,9 @@ void LCodeGen::DeoptimizeIf(Condition cc, LEnvironment* environment) { bool needs_lazy_deopt = info()->IsStub(); if (cc == no_condition && frame_is_built_) { if (needs_lazy_deopt) { - __ Call(entry, RelocInfo::RUNTIME_ENTRY); + __ call(entry, RelocInfo::RUNTIME_ENTRY); } else { - __ Jump(entry, RelocInfo::RUNTIME_ENTRY); + __ jmp(entry, RelocInfo::RUNTIME_ENTRY); } } else { // We often have several deopts to the same entry, reuse the last -- 2.7.4