1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef V8_X64_ASSEMBLER_X64_INL_H_
6 #define V8_X64_ASSEMBLER_X64_INL_H_
8 #include "src/x64/assembler-x64.h"
10 #include "src/base/cpu.h"
11 #include "src/debug.h"
12 #include "src/v8memory.h"
17 bool CpuFeatures::SupportsCrankshaft() { return true; }
20 // -----------------------------------------------------------------------------
21 // Implementation of Assembler
24 static const byte kCallOpcode = 0xE8;
25 // The length of pushq(rbp), movp(rbp, rsp), Push(rsi) and Push(rdi).
26 static const int kNoCodeAgeSequenceLength = kPointerSize == kInt64Size ? 6 : 17;
29 void Assembler::emitl(uint32_t x) {
30 Memory::uint32_at(pc_) = x;
31 pc_ += sizeof(uint32_t);
35 void Assembler::emitp(void* x, RelocInfo::Mode rmode) {
36 uintptr_t value = reinterpret_cast<uintptr_t>(x);
37 Memory::uintptr_at(pc_) = value;
38 if (!RelocInfo::IsNone(rmode)) {
39 RecordRelocInfo(rmode, value);
41 pc_ += sizeof(uintptr_t);
45 void Assembler::emitq(uint64_t x) {
46 Memory::uint64_at(pc_) = x;
47 pc_ += sizeof(uint64_t);
51 void Assembler::emitw(uint16_t x) {
52 Memory::uint16_at(pc_) = x;
53 pc_ += sizeof(uint16_t);
57 void Assembler::emit_code_target(Handle<Code> target,
58 RelocInfo::Mode rmode,
59 TypeFeedbackId ast_id) {
60 DCHECK(RelocInfo::IsCodeTarget(rmode) ||
61 rmode == RelocInfo::CODE_AGE_SEQUENCE);
62 if (rmode == RelocInfo::CODE_TARGET && !ast_id.IsNone()) {
63 RecordRelocInfo(RelocInfo::CODE_TARGET_WITH_ID, ast_id.ToInt());
65 RecordRelocInfo(rmode);
67 int current = code_targets_.length();
68 if (current > 0 && code_targets_.last().is_identical_to(target)) {
69 // Optimization if we keep jumping to the same code target.
72 code_targets_.Add(target);
78 void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) {
79 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
80 RecordRelocInfo(rmode);
81 emitl(static_cast<uint32_t>(entry - isolate()->code_range()->start()));
85 void Assembler::emit_rex_64(Register reg, Register rm_reg) {
86 emit(0x48 | reg.high_bit() << 2 | rm_reg.high_bit());
90 void Assembler::emit_rex_64(XMMRegister reg, Register rm_reg) {
91 emit(0x48 | (reg.code() & 0x8) >> 1 | rm_reg.code() >> 3);
95 void Assembler::emit_rex_64(Register reg, XMMRegister rm_reg) {
96 emit(0x48 | (reg.code() & 0x8) >> 1 | rm_reg.code() >> 3);
100 void Assembler::emit_rex_64(Register reg, const Operand& op) {
101 emit(0x48 | reg.high_bit() << 2 | op.rex_);
105 void Assembler::emit_rex_64(XMMRegister reg, const Operand& op) {
106 emit(0x48 | (reg.code() & 0x8) >> 1 | op.rex_);
110 void Assembler::emit_rex_64(Register rm_reg) {
111 DCHECK_EQ(rm_reg.code() & 0xf, rm_reg.code());
112 emit(0x48 | rm_reg.high_bit());
116 void Assembler::emit_rex_64(const Operand& op) {
117 emit(0x48 | op.rex_);
121 void Assembler::emit_rex_32(Register reg, Register rm_reg) {
122 emit(0x40 | reg.high_bit() << 2 | rm_reg.high_bit());
126 void Assembler::emit_rex_32(Register reg, const Operand& op) {
127 emit(0x40 | reg.high_bit() << 2 | op.rex_);
131 void Assembler::emit_rex_32(Register rm_reg) {
132 emit(0x40 | rm_reg.high_bit());
136 void Assembler::emit_rex_32(const Operand& op) {
137 emit(0x40 | op.rex_);
141 void Assembler::emit_optional_rex_32(Register reg, Register rm_reg) {
142 byte rex_bits = reg.high_bit() << 2 | rm_reg.high_bit();
143 if (rex_bits != 0) emit(0x40 | rex_bits);
147 void Assembler::emit_optional_rex_32(Register reg, const Operand& op) {
148 byte rex_bits = reg.high_bit() << 2 | op.rex_;
149 if (rex_bits != 0) emit(0x40 | rex_bits);
153 void Assembler::emit_optional_rex_32(XMMRegister reg, const Operand& op) {
154 byte rex_bits = (reg.code() & 0x8) >> 1 | op.rex_;
155 if (rex_bits != 0) emit(0x40 | rex_bits);
159 void Assembler::emit_optional_rex_32(XMMRegister reg, XMMRegister base) {
160 byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
161 if (rex_bits != 0) emit(0x40 | rex_bits);
165 void Assembler::emit_optional_rex_32(XMMRegister reg, Register base) {
166 byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
167 if (rex_bits != 0) emit(0x40 | rex_bits);
171 void Assembler::emit_optional_rex_32(Register reg, XMMRegister base) {
172 byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
173 if (rex_bits != 0) emit(0x40 | rex_bits);
177 void Assembler::emit_optional_rex_32(Register rm_reg) {
178 if (rm_reg.high_bit()) emit(0x41);
182 void Assembler::emit_optional_rex_32(XMMRegister rm_reg) {
183 if (rm_reg.high_bit()) emit(0x41);
187 void Assembler::emit_optional_rex_32(const Operand& op) {
188 if (op.rex_ != 0) emit(0x40 | op.rex_);
192 // byte 1 of 3-byte VEX
193 void Assembler::emit_vex3_byte1(XMMRegister reg, XMMRegister rm,
195 byte rxb = ~((reg.high_bit() << 2) | rm.high_bit()) << 5;
200 // byte 1 of 3-byte VEX
201 void Assembler::emit_vex3_byte1(XMMRegister reg, const Operand& rm,
203 byte rxb = ~((reg.high_bit() << 2) | rm.rex_) << 5;
208 // byte 1 of 2-byte VEX
209 void Assembler::emit_vex2_byte1(XMMRegister reg, XMMRegister v, VectorLength l,
211 byte rv = ~((reg.high_bit() << 4) | v.code()) << 3;
216 // byte 2 of 3-byte VEX
217 void Assembler::emit_vex3_byte2(VexW w, XMMRegister v, VectorLength l,
219 emit(w | ((~v.code() & 0xf) << 3) | l | pp);
223 void Assembler::emit_vex_prefix(XMMRegister reg, XMMRegister vreg,
224 XMMRegister rm, VectorLength l, SIMDPrefix pp,
225 LeadingOpcode mm, VexW w) {
226 if (rm.high_bit() || mm != k0F || w != kW0) {
228 emit_vex3_byte1(reg, rm, mm);
229 emit_vex3_byte2(w, vreg, l, pp);
232 emit_vex2_byte1(reg, vreg, l, pp);
237 void Assembler::emit_vex_prefix(XMMRegister reg, XMMRegister vreg,
238 const Operand& rm, VectorLength l,
239 SIMDPrefix pp, LeadingOpcode mm, VexW w) {
240 if (rm.rex_ || mm != k0F || w != kW0) {
242 emit_vex3_byte1(reg, rm, mm);
243 emit_vex3_byte2(w, vreg, l, pp);
246 emit_vex2_byte1(reg, vreg, l, pp);
251 Address Assembler::target_address_at(Address pc,
252 ConstantPoolArray* constant_pool) {
253 return Memory::int32_at(pc) + pc + 4;
257 void Assembler::set_target_address_at(Address pc,
258 ConstantPoolArray* constant_pool,
260 ICacheFlushMode icache_flush_mode) {
261 Memory::int32_at(pc) = static_cast<int32_t>(target - pc - 4);
262 if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
263 CpuFeatures::FlushICache(pc, sizeof(int32_t));
268 void Assembler::deserialization_set_target_internal_reference_at(
269 Address pc, Address target, RelocInfo::Mode mode) {
270 Memory::Address_at(pc) = target;
274 Address Assembler::target_address_from_return_address(Address pc) {
275 return pc - kCallTargetAddressOffset;
279 Address Assembler::break_address_from_return_address(Address pc) {
280 return pc - Assembler::kPatchDebugBreakSlotReturnOffset;
284 Handle<Object> Assembler::code_target_object_handle_at(Address pc) {
285 return code_targets_[Memory::int32_at(pc)];
289 Address Assembler::runtime_entry_at(Address pc) {
290 return Memory::int32_at(pc) + isolate()->code_range()->start();
293 // -----------------------------------------------------------------------------
294 // Implementation of RelocInfo
296 // The modes possibly affected by apply must be in kApplyMask.
297 void RelocInfo::apply(intptr_t delta, ICacheFlushMode icache_flush_mode) {
298 bool flush_icache = icache_flush_mode != SKIP_ICACHE_FLUSH;
299 if (IsInternalReference(rmode_)) {
300 // absolute code pointer inside code object moves with the code object.
301 Memory::Address_at(pc_) += delta;
302 if (flush_icache) CpuFeatures::FlushICache(pc_, sizeof(Address));
303 } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) {
304 Memory::int32_at(pc_) -= static_cast<int32_t>(delta);
305 if (flush_icache) CpuFeatures::FlushICache(pc_, sizeof(int32_t));
306 } else if (rmode_ == CODE_AGE_SEQUENCE) {
307 if (*pc_ == kCallOpcode) {
308 int32_t* p = reinterpret_cast<int32_t*>(pc_ + 1);
309 *p -= static_cast<int32_t>(delta); // Relocate entry.
310 if (flush_icache) CpuFeatures::FlushICache(p, sizeof(uint32_t));
316 Address RelocInfo::target_address() {
317 DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
318 return Assembler::target_address_at(pc_, host_);
322 Address RelocInfo::target_address_address() {
323 DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)
324 || rmode_ == EMBEDDED_OBJECT
325 || rmode_ == EXTERNAL_REFERENCE);
326 return reinterpret_cast<Address>(pc_);
330 Address RelocInfo::constant_pool_entry_address() {
336 int RelocInfo::target_address_size() {
337 if (IsCodedSpecially()) {
338 return Assembler::kSpecialTargetSize;
345 void RelocInfo::set_target_address(Address target,
346 WriteBarrierMode write_barrier_mode,
347 ICacheFlushMode icache_flush_mode) {
348 DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
349 Assembler::set_target_address_at(pc_, host_, target, icache_flush_mode);
350 if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != NULL &&
351 IsCodeTarget(rmode_)) {
352 Object* target_code = Code::GetCodeFromTargetAddress(target);
353 host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(
354 host(), this, HeapObject::cast(target_code));
359 Object* RelocInfo::target_object() {
360 DCHECK(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
361 return Memory::Object_at(pc_);
365 Handle<Object> RelocInfo::target_object_handle(Assembler* origin) {
366 DCHECK(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
367 if (rmode_ == EMBEDDED_OBJECT) {
368 return Memory::Object_Handle_at(pc_);
370 return origin->code_target_object_handle_at(pc_);
375 Address RelocInfo::target_external_reference() {
376 DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
377 return Memory::Address_at(pc_);
381 Address RelocInfo::target_internal_reference() {
382 DCHECK(rmode_ == INTERNAL_REFERENCE);
383 return Memory::Address_at(pc_);
387 Address RelocInfo::target_internal_reference_address() {
388 DCHECK(rmode_ == INTERNAL_REFERENCE);
389 return reinterpret_cast<Address>(pc_);
393 void RelocInfo::set_target_object(Object* target,
394 WriteBarrierMode write_barrier_mode,
395 ICacheFlushMode icache_flush_mode) {
396 DCHECK(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
397 Memory::Object_at(pc_) = target;
398 if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
399 CpuFeatures::FlushICache(pc_, sizeof(Address));
401 if (write_barrier_mode == UPDATE_WRITE_BARRIER &&
403 target->IsHeapObject()) {
404 host()->GetHeap()->incremental_marking()->RecordWrite(
405 host(), &Memory::Object_at(pc_), HeapObject::cast(target));
410 Address RelocInfo::target_runtime_entry(Assembler* origin) {
411 DCHECK(IsRuntimeEntry(rmode_));
412 return origin->runtime_entry_at(pc_);
416 void RelocInfo::set_target_runtime_entry(Address target,
417 WriteBarrierMode write_barrier_mode,
418 ICacheFlushMode icache_flush_mode) {
419 DCHECK(IsRuntimeEntry(rmode_));
420 if (target_address() != target) {
421 set_target_address(target, write_barrier_mode, icache_flush_mode);
426 Handle<Cell> RelocInfo::target_cell_handle() {
427 DCHECK(rmode_ == RelocInfo::CELL);
428 Address address = Memory::Address_at(pc_);
429 return Handle<Cell>(reinterpret_cast<Cell**>(address));
433 Cell* RelocInfo::target_cell() {
434 DCHECK(rmode_ == RelocInfo::CELL);
435 return Cell::FromValueAddress(Memory::Address_at(pc_));
439 void RelocInfo::set_target_cell(Cell* cell,
440 WriteBarrierMode write_barrier_mode,
441 ICacheFlushMode icache_flush_mode) {
442 DCHECK(rmode_ == RelocInfo::CELL);
443 Address address = cell->address() + Cell::kValueOffset;
444 Memory::Address_at(pc_) = address;
445 if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
446 CpuFeatures::FlushICache(pc_, sizeof(Address));
448 if (write_barrier_mode == UPDATE_WRITE_BARRIER &&
450 // TODO(1550) We are passing NULL as a slot because cell can never be on
451 // evacuation candidate.
452 host()->GetHeap()->incremental_marking()->RecordWrite(
458 void RelocInfo::WipeOut() {
459 if (IsEmbeddedObject(rmode_) || IsExternalReference(rmode_) ||
460 IsInternalReference(rmode_)) {
461 Memory::Address_at(pc_) = NULL;
462 } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) {
463 // Effectively write zero into the relocation.
464 Assembler::set_target_address_at(pc_, host_, pc_ + sizeof(int32_t));
471 bool RelocInfo::IsPatchedReturnSequence() {
472 // The recognized call sequence is:
473 // movq(kScratchRegister, address); call(kScratchRegister);
474 // It only needs to be distinguished from a return sequence
475 // movq(rsp, rbp); pop(rbp); ret(n); int3 *6
476 // The 11th byte is int3 (0xCC) in the return sequence and
477 // REX.WB (0x48+register bit) for the call sequence.
478 return pc_[Assembler::kMoveAddressIntoScratchRegisterInstructionLength] !=
483 bool RelocInfo::IsPatchedDebugBreakSlotSequence() {
484 return !Assembler::IsNop(pc());
488 Handle<Object> RelocInfo::code_age_stub_handle(Assembler* origin) {
489 DCHECK(rmode_ == RelocInfo::CODE_AGE_SEQUENCE);
490 DCHECK(*pc_ == kCallOpcode);
491 return origin->code_target_object_handle_at(pc_ + 1);
495 Code* RelocInfo::code_age_stub() {
496 DCHECK(rmode_ == RelocInfo::CODE_AGE_SEQUENCE);
497 DCHECK(*pc_ == kCallOpcode);
498 return Code::GetCodeFromTargetAddress(
499 Assembler::target_address_at(pc_ + 1, host_));
503 void RelocInfo::set_code_age_stub(Code* stub,
504 ICacheFlushMode icache_flush_mode) {
505 DCHECK(*pc_ == kCallOpcode);
506 DCHECK(rmode_ == RelocInfo::CODE_AGE_SEQUENCE);
507 Assembler::set_target_address_at(pc_ + 1, host_, stub->instruction_start(),
512 Address RelocInfo::call_address() {
513 DCHECK((IsJSReturn(rmode()) && IsPatchedReturnSequence()) ||
514 (IsDebugBreakSlot(rmode()) && IsPatchedDebugBreakSlotSequence()));
515 return Memory::Address_at(
516 pc_ + Assembler::kRealPatchReturnSequenceAddressOffset);
520 void RelocInfo::set_call_address(Address target) {
521 DCHECK((IsJSReturn(rmode()) && IsPatchedReturnSequence()) ||
522 (IsDebugBreakSlot(rmode()) && IsPatchedDebugBreakSlotSequence()));
523 Memory::Address_at(pc_ + Assembler::kRealPatchReturnSequenceAddressOffset) =
525 CpuFeatures::FlushICache(
526 pc_ + Assembler::kRealPatchReturnSequenceAddressOffset, sizeof(Address));
527 if (host() != NULL) {
528 Object* target_code = Code::GetCodeFromTargetAddress(target);
529 host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(
530 host(), this, HeapObject::cast(target_code));
535 Object* RelocInfo::call_object() {
536 return *call_object_address();
540 void RelocInfo::set_call_object(Object* target) {
541 *call_object_address() = target;
545 Object** RelocInfo::call_object_address() {
546 DCHECK((IsJSReturn(rmode()) && IsPatchedReturnSequence()) ||
547 (IsDebugBreakSlot(rmode()) && IsPatchedDebugBreakSlotSequence()));
548 return reinterpret_cast<Object**>(
549 pc_ + Assembler::kPatchReturnSequenceAddressOffset);
553 void RelocInfo::Visit(Isolate* isolate, ObjectVisitor* visitor) {
554 RelocInfo::Mode mode = rmode();
555 if (mode == RelocInfo::EMBEDDED_OBJECT) {
556 visitor->VisitEmbeddedPointer(this);
557 CpuFeatures::FlushICache(pc_, sizeof(Address));
558 } else if (RelocInfo::IsCodeTarget(mode)) {
559 visitor->VisitCodeTarget(this);
560 } else if (mode == RelocInfo::CELL) {
561 visitor->VisitCell(this);
562 } else if (mode == RelocInfo::EXTERNAL_REFERENCE) {
563 visitor->VisitExternalReference(this);
564 } else if (mode == RelocInfo::INTERNAL_REFERENCE) {
565 visitor->VisitInternalReference(this);
566 } else if (RelocInfo::IsCodeAgeSequence(mode)) {
567 visitor->VisitCodeAgeSequence(this);
568 } else if (((RelocInfo::IsJSReturn(mode) &&
569 IsPatchedReturnSequence()) ||
570 (RelocInfo::IsDebugBreakSlot(mode) &&
571 IsPatchedDebugBreakSlotSequence())) &&
572 isolate->debug()->has_break_points()) {
573 visitor->VisitDebugTarget(this);
574 } else if (RelocInfo::IsRuntimeEntry(mode)) {
575 visitor->VisitRuntimeEntry(this);
580 template<typename StaticVisitor>
581 void RelocInfo::Visit(Heap* heap) {
582 RelocInfo::Mode mode = rmode();
583 if (mode == RelocInfo::EMBEDDED_OBJECT) {
584 StaticVisitor::VisitEmbeddedPointer(heap, this);
585 CpuFeatures::FlushICache(pc_, sizeof(Address));
586 } else if (RelocInfo::IsCodeTarget(mode)) {
587 StaticVisitor::VisitCodeTarget(heap, this);
588 } else if (mode == RelocInfo::CELL) {
589 StaticVisitor::VisitCell(heap, this);
590 } else if (mode == RelocInfo::EXTERNAL_REFERENCE) {
591 StaticVisitor::VisitExternalReference(this);
592 } else if (mode == RelocInfo::INTERNAL_REFERENCE) {
593 StaticVisitor::VisitInternalReference(this);
594 } else if (RelocInfo::IsCodeAgeSequence(mode)) {
595 StaticVisitor::VisitCodeAgeSequence(heap, this);
596 } else if (heap->isolate()->debug()->has_break_points() &&
597 ((RelocInfo::IsJSReturn(mode) &&
598 IsPatchedReturnSequence()) ||
599 (RelocInfo::IsDebugBreakSlot(mode) &&
600 IsPatchedDebugBreakSlotSequence()))) {
601 StaticVisitor::VisitDebugTarget(heap, this);
602 } else if (RelocInfo::IsRuntimeEntry(mode)) {
603 StaticVisitor::VisitRuntimeEntry(this);
608 // -----------------------------------------------------------------------------
609 // Implementation of Operand
611 void Operand::set_modrm(int mod, Register rm_reg) {
612 DCHECK(is_uint2(mod));
613 buf_[0] = mod << 6 | rm_reg.low_bits();
614 // Set REX.B to the high bit of rm.code().
615 rex_ |= rm_reg.high_bit();
619 void Operand::set_sib(ScaleFactor scale, Register index, Register base) {
621 DCHECK(is_uint2(scale));
622 // Use SIB with no index register only for base rsp or r12. Otherwise we
623 // would skip the SIB byte entirely.
624 DCHECK(!index.is(rsp) || base.is(rsp) || base.is(r12));
625 buf_[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
626 rex_ |= index.high_bit() << 1 | base.high_bit();
630 void Operand::set_disp8(int disp) {
631 DCHECK(is_int8(disp));
632 DCHECK(len_ == 1 || len_ == 2);
633 int8_t* p = reinterpret_cast<int8_t*>(&buf_[len_]);
635 len_ += sizeof(int8_t);
638 void Operand::set_disp32(int disp) {
639 DCHECK(len_ == 1 || len_ == 2);
640 int32_t* p = reinterpret_cast<int32_t*>(&buf_[len_]);
642 len_ += sizeof(int32_t);
645 void Operand::set_disp64(int64_t disp) {
647 int64_t* p = reinterpret_cast<int64_t*>(&buf_[len_]);
649 len_ += sizeof(disp);
651 } } // namespace v8::internal
653 #endif // V8_X64_ASSEMBLER_X64_INL_H_