1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
9 #include "src/base/bits.h"
10 #include "src/macro-assembler.h"
11 #include "src/serialize.h"
16 // -----------------------------------------------------------------------------
17 // Implementation of CpuFeatures
19 void CpuFeatures::ProbeImpl(bool cross_compile) {
21 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
22 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
24 // Only use statically determined features for cross compile (snapshot).
25 if (cross_compile) return;
27 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
28 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
29 // SAHF is not generally available in long mode.
30 if (cpu.has_sahf() && FLAG_enable_sahf) supported_|= 1u << SAHF;
34 void CpuFeatures::PrintTarget() { }
35 void CpuFeatures::PrintFeatures() { }
38 // -----------------------------------------------------------------------------
39 // Implementation of RelocInfo
41 // Patch the code at the current PC with a call to the target address.
42 // Additional guard int3 instructions can be added if required.
43 void RelocInfo::PatchCodeWithCall(Address target, int guard_bytes) {
44 int code_size = Assembler::kCallSequenceLength + guard_bytes;
46 // Create a code patcher.
47 CodePatcher patcher(pc_, code_size);
49 // Add a label for checking the size of the code used for returning.
52 patcher.masm()->bind(&check_codesize);
56 patcher.masm()->movp(kScratchRegister, reinterpret_cast<void*>(target),
57 Assembler::RelocInfoNone());
58 patcher.masm()->call(kScratchRegister);
60 // Check that the size of the code generated is as expected.
61 DCHECK_EQ(Assembler::kCallSequenceLength,
62 patcher.masm()->SizeOfCodeGeneratedSince(&check_codesize));
64 // Add the requested number of int3 instructions after the call.
65 for (int i = 0; i < guard_bytes; i++) {
66 patcher.masm()->int3();
71 void RelocInfo::PatchCode(byte* instructions, int instruction_count) {
72 // Patch the code at the current address with the supplied instructions.
73 for (int i = 0; i < instruction_count; i++) {
74 *(pc_ + i) = *(instructions + i);
77 // Indicate that code has changed.
78 CpuFeatures::FlushICache(pc_, instruction_count);
82 // -----------------------------------------------------------------------------
83 // Register constants.
86 Register::kRegisterCodeByAllocationIndex[kMaxNumAllocatableRegisters] = {
87 // rax, rbx, rdx, rcx, rsi, rdi, r8, r9, r11, r14, r15
88 0, 3, 2, 1, 6, 7, 8, 9, 11, 14, 15
91 const int Register::kAllocationIndexByRegisterCode[kNumRegisters] = {
92 0, 3, 2, 1, -1, -1, 4, 5, 6, 7, -1, 8, -1, -1, 9, 10
96 // -----------------------------------------------------------------------------
97 // Implementation of Operand
99 Operand::Operand(Register base, int32_t disp) : rex_(0) {
101 if (base.is(rsp) || base.is(r12)) {
102 // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
103 set_sib(times_1, rsp, base);
106 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
108 } else if (is_int8(disp)) {
118 Operand::Operand(Register base,
121 int32_t disp) : rex_(0) {
122 DCHECK(!index.is(rsp));
124 set_sib(scale, index, base);
125 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
126 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
127 // possibly set by set_sib.
129 } else if (is_int8(disp)) {
139 Operand::Operand(Register index,
141 int32_t disp) : rex_(0) {
142 DCHECK(!index.is(rsp));
145 set_sib(scale, index, rbp);
150 Operand::Operand(const Operand& operand, int32_t offset) {
151 DCHECK(operand.len_ >= 1);
152 // Operand encodes REX ModR/M [SIB] [Disp].
153 byte modrm = operand.buf_[0];
154 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target).
155 bool has_sib = ((modrm & 0x07) == 0x04);
156 byte mode = modrm & 0xC0;
157 int disp_offset = has_sib ? 2 : 1;
158 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07;
159 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
161 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
162 int32_t disp_value = 0;
163 if (mode == 0x80 || is_baseless) {
164 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
165 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]);
166 } else if (mode == 0x40) {
167 // Mode 1: Byte displacement.
168 disp_value = static_cast<signed char>(operand.buf_[disp_offset]);
171 // Write new operand with same registers, but with modified displacement.
172 DCHECK(offset >= 0 ? disp_value + offset > disp_value
173 : disp_value + offset < disp_value); // No overflow.
174 disp_value += offset;
176 if (!is_int8(disp_value) || is_baseless) {
177 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
178 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80);
179 len_ = disp_offset + 4;
180 Memory::int32_at(&buf_[disp_offset]) = disp_value;
181 } else if (disp_value != 0 || (base_reg == 0x05)) {
182 // Need 8 bits of displacement.
183 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1.
184 len_ = disp_offset + 1;
185 buf_[disp_offset] = static_cast<byte>(disp_value);
187 // Need no displacement.
188 buf_[0] = (modrm & 0x3f); // Mode 0.
192 buf_[1] = operand.buf_[1];
197 bool Operand::AddressUsesRegister(Register reg) const {
198 int code = reg.code();
199 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand.
200 // Start with only low three bits of base register. Initial decoding doesn't
201 // distinguish on the REX.B bit.
202 int base_code = buf_[0] & 0x07;
203 if (base_code == rsp.code()) {
204 // SIB byte present in buf_[1].
205 // Check the index register from the SIB byte + REX.X prefix.
206 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2);
207 // Index code (including REX.X) of 0x04 (rsp) means no index register.
208 if (index_code != rsp.code() && index_code == code) return true;
209 // Add REX.B to get the full base register code.
210 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3);
211 // A base register of 0x05 (rbp) with mod = 0 means no base register.
212 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
213 return code == base_code;
215 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
217 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
218 base_code |= ((rex_ & 0x01) << 3);
219 return code == base_code;
224 // -----------------------------------------------------------------------------
225 // Implementation of Assembler.
227 #ifdef GENERATED_CODE_COVERAGE
228 static void InitCoverageLog();
231 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
232 : AssemblerBase(isolate, buffer, buffer_size),
234 positions_recorder_(this) {
235 // Clear the buffer in debug mode unless it was provided by the
236 // caller in which case we can't be sure it's okay to overwrite
237 // existing code in it.
240 memset(buffer_, 0xCC, buffer_size_); // int3
244 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
247 #ifdef GENERATED_CODE_COVERAGE
253 void Assembler::GetCode(CodeDesc* desc) {
254 // Finalize code (at this point overflow() may be true, but the gap ensures
255 // that we are still not overlapping instructions and relocation info).
256 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
257 // Set up code descriptor.
258 desc->buffer = buffer_;
259 desc->buffer_size = buffer_size_;
260 desc->instr_size = pc_offset();
261 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system.
263 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
268 void Assembler::Align(int m) {
269 DCHECK(base::bits::IsPowerOfTwo32(m));
270 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
275 void Assembler::CodeTargetAlign() {
276 Align(16); // Preferred alignment of jump targets on x64.
280 bool Assembler::IsNop(Address addr) {
282 while (*a == 0x66) a++;
283 if (*a == 0x90) return true;
284 if (a[0] == 0xf && a[1] == 0x1f) return true;
289 void Assembler::bind_to(Label* L, int pos) {
290 DCHECK(!L->is_bound()); // Label may only be bound once.
291 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
292 if (L->is_linked()) {
293 int current = L->pos();
294 int next = long_at(current);
295 while (next != current) {
296 // Relative address, relative to point after address.
297 int imm32 = pos - (current + sizeof(int32_t));
298 long_at_put(current, imm32);
300 next = long_at(next);
302 // Fix up last fixup on linked list.
303 int last_imm32 = pos - (current + sizeof(int32_t));
304 long_at_put(current, last_imm32);
306 while (L->is_near_linked()) {
307 int fixup_pos = L->near_link_pos();
309 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
310 DCHECK(offset_to_next <= 0);
311 int disp = pos - (fixup_pos + sizeof(int8_t));
312 CHECK(is_int8(disp));
313 set_byte_at(fixup_pos, disp);
314 if (offset_to_next < 0) {
315 L->link_to(fixup_pos + offset_to_next, Label::kNear);
324 void Assembler::bind(Label* L) {
325 bind_to(L, pc_offset());
329 void Assembler::GrowBuffer() {
330 DCHECK(buffer_overflow());
331 if (!own_buffer_) FATAL("external code buffer is too small");
333 // Compute new buffer size.
334 CodeDesc desc; // the new buffer
335 desc.buffer_size = 2 * buffer_size_;
337 // Some internal data structures overflow for very large buffers,
338 // they must ensure that kMaximalBufferSize is not too large.
339 if ((desc.buffer_size > kMaximalBufferSize) ||
340 (desc.buffer_size > isolate()->heap()->MaxOldGenerationSize())) {
341 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer");
344 // Set up new buffer.
345 desc.buffer = NewArray<byte>(desc.buffer_size);
346 desc.instr_size = pc_offset();
348 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
350 // Clear the buffer in debug mode. Use 'int3' instructions to make
351 // sure to get into problems if we ever run uninitialized code.
353 memset(desc.buffer, 0xCC, desc.buffer_size);
357 intptr_t pc_delta = desc.buffer - buffer_;
358 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
359 (buffer_ + buffer_size_);
360 MemMove(desc.buffer, buffer_, desc.instr_size);
361 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
365 DeleteArray(buffer_);
366 buffer_ = desc.buffer;
367 buffer_size_ = desc.buffer_size;
369 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
370 reloc_info_writer.last_pc() + pc_delta);
372 // Relocate runtime entries.
373 for (RelocIterator it(desc); !it.done(); it.next()) {
374 RelocInfo::Mode rmode = it.rinfo()->rmode();
375 if (rmode == RelocInfo::INTERNAL_REFERENCE) {
376 intptr_t* p = reinterpret_cast<intptr_t*>(it.rinfo()->pc());
377 if (*p != 0) { // 0 means uninitialized.
383 DCHECK(!buffer_overflow());
387 void Assembler::emit_operand(int code, const Operand& adr) {
388 DCHECK(is_uint3(code));
389 const unsigned length = adr.len_;
392 // Emit updated ModR/M byte containing the given register.
393 DCHECK((adr.buf_[0] & 0x38) == 0);
394 pc_[0] = adr.buf_[0] | code << 3;
396 // Emit the rest of the encoded operand.
397 for (unsigned i = 1; i < length; i++) pc_[i] = adr.buf_[i];
402 // Assembler Instruction implementations.
404 void Assembler::arithmetic_op(byte opcode,
408 EnsureSpace ensure_space(this);
409 emit_rex(reg, op, size);
411 emit_operand(reg, op);
415 void Assembler::arithmetic_op(byte opcode,
419 EnsureSpace ensure_space(this);
420 DCHECK((opcode & 0xC6) == 2);
421 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
422 // Swap reg and rm_reg and change opcode operand order.
423 emit_rex(rm_reg, reg, size);
425 emit_modrm(rm_reg, reg);
427 emit_rex(reg, rm_reg, size);
429 emit_modrm(reg, rm_reg);
434 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
435 EnsureSpace ensure_space(this);
436 DCHECK((opcode & 0xC6) == 2);
437 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
438 // Swap reg and rm_reg and change opcode operand order.
440 emit_optional_rex_32(rm_reg, reg);
442 emit_modrm(rm_reg, reg);
445 emit_optional_rex_32(reg, rm_reg);
447 emit_modrm(reg, rm_reg);
452 void Assembler::arithmetic_op_16(byte opcode,
454 const Operand& rm_reg) {
455 EnsureSpace ensure_space(this);
457 emit_optional_rex_32(reg, rm_reg);
459 emit_operand(reg, rm_reg);
463 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) {
464 EnsureSpace ensure_space(this);
465 if (!reg.is_byte_register()) {
466 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
470 emit_operand(reg, op);
474 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
475 EnsureSpace ensure_space(this);
476 DCHECK((opcode & 0xC6) == 2);
477 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
478 // Swap reg and rm_reg and change opcode operand order.
479 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
480 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
481 emit_rex_32(rm_reg, reg);
484 emit_modrm(rm_reg, reg);
486 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
487 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
488 emit_rex_32(reg, rm_reg);
491 emit_modrm(reg, rm_reg);
496 void Assembler::immediate_arithmetic_op(byte subcode,
500 EnsureSpace ensure_space(this);
502 if (is_int8(src.value_)) {
504 emit_modrm(subcode, dst);
506 } else if (dst.is(rax)) {
507 emit(0x05 | (subcode << 3));
511 emit_modrm(subcode, dst);
516 void Assembler::immediate_arithmetic_op(byte subcode,
520 EnsureSpace ensure_space(this);
522 if (is_int8(src.value_)) {
524 emit_operand(subcode, dst);
528 emit_operand(subcode, dst);
534 void Assembler::immediate_arithmetic_op_16(byte subcode,
537 EnsureSpace ensure_space(this);
538 emit(0x66); // Operand size override prefix.
539 emit_optional_rex_32(dst);
540 if (is_int8(src.value_)) {
542 emit_modrm(subcode, dst);
544 } else if (dst.is(rax)) {
545 emit(0x05 | (subcode << 3));
549 emit_modrm(subcode, dst);
555 void Assembler::immediate_arithmetic_op_16(byte subcode,
558 EnsureSpace ensure_space(this);
559 emit(0x66); // Operand size override prefix.
560 emit_optional_rex_32(dst);
561 if (is_int8(src.value_)) {
563 emit_operand(subcode, dst);
567 emit_operand(subcode, dst);
573 void Assembler::immediate_arithmetic_op_8(byte subcode,
576 EnsureSpace ensure_space(this);
577 emit_optional_rex_32(dst);
578 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
580 emit_operand(subcode, dst);
585 void Assembler::immediate_arithmetic_op_8(byte subcode,
588 EnsureSpace ensure_space(this);
589 if (!dst.is_byte_register()) {
590 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
593 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
595 emit_modrm(subcode, dst);
600 void Assembler::shift(Register dst,
601 Immediate shift_amount,
604 EnsureSpace ensure_space(this);
605 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
606 : is_uint5(shift_amount.value_));
607 if (shift_amount.value_ == 1) {
610 emit_modrm(subcode, dst);
614 emit_modrm(subcode, dst);
615 emit(shift_amount.value_);
620 void Assembler::shift(Register dst, int subcode, int size) {
621 EnsureSpace ensure_space(this);
624 emit_modrm(subcode, dst);
628 void Assembler::bt(const Operand& dst, Register src) {
629 EnsureSpace ensure_space(this);
630 emit_rex_64(src, dst);
633 emit_operand(src, dst);
637 void Assembler::bts(const Operand& dst, Register src) {
638 EnsureSpace ensure_space(this);
639 emit_rex_64(src, dst);
642 emit_operand(src, dst);
646 void Assembler::bsrl(Register dst, Register src) {
647 EnsureSpace ensure_space(this);
648 emit_optional_rex_32(dst, src);
651 emit_modrm(dst, src);
655 void Assembler::call(Label* L) {
656 positions_recorder()->WriteRecordedPositions();
657 EnsureSpace ensure_space(this);
658 // 1110 1000 #32-bit disp.
661 int offset = L->pos() - pc_offset() - sizeof(int32_t);
664 } else if (L->is_linked()) {
666 L->link_to(pc_offset() - sizeof(int32_t));
668 DCHECK(L->is_unused());
669 int32_t current = pc_offset();
676 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
677 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
678 positions_recorder()->WriteRecordedPositions();
679 EnsureSpace ensure_space(this);
680 // 1110 1000 #32-bit disp.
682 emit_runtime_entry(entry, rmode);
686 void Assembler::call(Handle<Code> target,
687 RelocInfo::Mode rmode,
688 TypeFeedbackId ast_id) {
689 positions_recorder()->WriteRecordedPositions();
690 EnsureSpace ensure_space(this);
691 // 1110 1000 #32-bit disp.
693 emit_code_target(target, rmode, ast_id);
697 void Assembler::call(Register adr) {
698 positions_recorder()->WriteRecordedPositions();
699 EnsureSpace ensure_space(this);
700 // Opcode: FF /2 r64.
701 emit_optional_rex_32(adr);
703 emit_modrm(0x2, adr);
707 void Assembler::call(const Operand& op) {
708 positions_recorder()->WriteRecordedPositions();
709 EnsureSpace ensure_space(this);
710 // Opcode: FF /2 m64.
711 emit_optional_rex_32(op);
713 emit_operand(0x2, op);
717 // Calls directly to the given address using a relative offset.
718 // Should only ever be used in Code objects for calls within the
719 // same Code object. Should not be used when generating new code (use labels),
720 // but only when patching existing code.
721 void Assembler::call(Address target) {
722 positions_recorder()->WriteRecordedPositions();
723 EnsureSpace ensure_space(this);
724 // 1110 1000 #32-bit disp.
726 Address source = pc_ + 4;
727 intptr_t displacement = target - source;
728 DCHECK(is_int32(displacement));
729 emitl(static_cast<int32_t>(displacement));
733 void Assembler::clc() {
734 EnsureSpace ensure_space(this);
739 void Assembler::cld() {
740 EnsureSpace ensure_space(this);
745 void Assembler::cdq() {
746 EnsureSpace ensure_space(this);
751 void Assembler::cmovq(Condition cc, Register dst, Register src) {
754 } else if (cc == never) {
757 // No need to check CpuInfo for CMOV support, it's a required part of the
758 // 64-bit architecture.
759 DCHECK(cc >= 0); // Use mov for unconditional moves.
760 EnsureSpace ensure_space(this);
761 // Opcode: REX.W 0f 40 + cc /r.
762 emit_rex_64(dst, src);
765 emit_modrm(dst, src);
769 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) {
772 } else if (cc == never) {
776 EnsureSpace ensure_space(this);
777 // Opcode: REX.W 0f 40 + cc /r.
778 emit_rex_64(dst, src);
781 emit_operand(dst, src);
785 void Assembler::cmovl(Condition cc, Register dst, Register src) {
788 } else if (cc == never) {
792 EnsureSpace ensure_space(this);
793 // Opcode: 0f 40 + cc /r.
794 emit_optional_rex_32(dst, src);
797 emit_modrm(dst, src);
801 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) {
804 } else if (cc == never) {
808 EnsureSpace ensure_space(this);
809 // Opcode: 0f 40 + cc /r.
810 emit_optional_rex_32(dst, src);
813 emit_operand(dst, src);
817 void Assembler::cmpb_al(Immediate imm8) {
818 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
819 EnsureSpace ensure_space(this);
825 void Assembler::cpuid() {
826 EnsureSpace ensure_space(this);
832 void Assembler::cqo() {
833 EnsureSpace ensure_space(this);
839 void Assembler::emit_dec(Register dst, int size) {
840 EnsureSpace ensure_space(this);
843 emit_modrm(0x1, dst);
847 void Assembler::emit_dec(const Operand& dst, int size) {
848 EnsureSpace ensure_space(this);
851 emit_operand(1, dst);
855 void Assembler::decb(Register dst) {
856 EnsureSpace ensure_space(this);
857 if (!dst.is_byte_register()) {
858 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
862 emit_modrm(0x1, dst);
866 void Assembler::decb(const Operand& dst) {
867 EnsureSpace ensure_space(this);
868 emit_optional_rex_32(dst);
870 emit_operand(1, dst);
874 void Assembler::enter(Immediate size) {
875 EnsureSpace ensure_space(this);
877 emitw(size.value_); // 16 bit operand, always.
882 void Assembler::hlt() {
883 EnsureSpace ensure_space(this);
888 void Assembler::emit_idiv(Register src, int size) {
889 EnsureSpace ensure_space(this);
892 emit_modrm(0x7, src);
896 void Assembler::emit_div(Register src, int size) {
897 EnsureSpace ensure_space(this);
900 emit_modrm(0x6, src);
904 void Assembler::emit_imul(Register src, int size) {
905 EnsureSpace ensure_space(this);
908 emit_modrm(0x5, src);
912 void Assembler::emit_imul(Register dst, Register src, int size) {
913 EnsureSpace ensure_space(this);
914 emit_rex(dst, src, size);
917 emit_modrm(dst, src);
921 void Assembler::emit_imul(Register dst, const Operand& src, int size) {
922 EnsureSpace ensure_space(this);
923 emit_rex(dst, src, size);
926 emit_operand(dst, src);
930 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
931 EnsureSpace ensure_space(this);
932 emit_rex(dst, src, size);
933 if (is_int8(imm.value_)) {
935 emit_modrm(dst, src);
939 emit_modrm(dst, src);
945 void Assembler::emit_inc(Register dst, int size) {
946 EnsureSpace ensure_space(this);
949 emit_modrm(0x0, dst);
953 void Assembler::emit_inc(const Operand& dst, int size) {
954 EnsureSpace ensure_space(this);
957 emit_operand(0, dst);
961 void Assembler::int3() {
962 EnsureSpace ensure_space(this);
967 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
971 } else if (cc == never) {
974 EnsureSpace ensure_space(this);
975 DCHECK(is_uint4(cc));
977 const int short_size = 2;
978 const int long_size = 6;
979 int offs = L->pos() - pc_offset();
981 // Determine whether we can use 1-byte offsets for backwards branches,
982 // which have a max range of 128 bytes.
984 // We also need to check predictable_code_size() flag here, because on x64,
985 // when the full code generator recompiles code for debugging, some places
986 // need to be padded out to a certain size. The debugger is keeping track of
987 // how often it did this so that it can adjust return addresses on the
988 // stack, but if the size of jump instructions can also change, that's not
989 // enough and the calculated offsets would be incorrect.
990 if (is_int8(offs - short_size) && !predictable_code_size()) {
991 // 0111 tttn #8-bit disp.
993 emit((offs - short_size) & 0xFF);
995 // 0000 1111 1000 tttn #32-bit disp.
998 emitl(offs - long_size);
1000 } else if (distance == Label::kNear) {
1001 // 0111 tttn #8-bit disp
1004 if (L->is_near_linked()) {
1005 int offset = L->near_link_pos() - pc_offset();
1006 DCHECK(is_int8(offset));
1007 disp = static_cast<byte>(offset & 0xFF);
1009 L->link_to(pc_offset(), Label::kNear);
1011 } else if (L->is_linked()) {
1012 // 0000 1111 1000 tttn #32-bit disp.
1016 L->link_to(pc_offset() - sizeof(int32_t));
1018 DCHECK(L->is_unused());
1021 int32_t current = pc_offset();
1023 L->link_to(current);
1028 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1029 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1030 EnsureSpace ensure_space(this);
1031 DCHECK(is_uint4(cc));
1034 emit_runtime_entry(entry, rmode);
1038 void Assembler::j(Condition cc,
1039 Handle<Code> target,
1040 RelocInfo::Mode rmode) {
1041 EnsureSpace ensure_space(this);
1042 DCHECK(is_uint4(cc));
1043 // 0000 1111 1000 tttn #32-bit disp.
1046 emit_code_target(target, rmode);
1050 void Assembler::jmp(Label* L, Label::Distance distance) {
1051 EnsureSpace ensure_space(this);
1052 const int short_size = sizeof(int8_t);
1053 const int long_size = sizeof(int32_t);
1054 if (L->is_bound()) {
1055 int offs = L->pos() - pc_offset() - 1;
1057 if (is_int8(offs - short_size) && !predictable_code_size()) {
1058 // 1110 1011 #8-bit disp.
1060 emit((offs - short_size) & 0xFF);
1062 // 1110 1001 #32-bit disp.
1064 emitl(offs - long_size);
1066 } else if (distance == Label::kNear) {
1069 if (L->is_near_linked()) {
1070 int offset = L->near_link_pos() - pc_offset();
1071 DCHECK(is_int8(offset));
1072 disp = static_cast<byte>(offset & 0xFF);
1074 L->link_to(pc_offset(), Label::kNear);
1076 } else if (L->is_linked()) {
1077 // 1110 1001 #32-bit disp.
1080 L->link_to(pc_offset() - long_size);
1082 // 1110 1001 #32-bit disp.
1083 DCHECK(L->is_unused());
1085 int32_t current = pc_offset();
1087 L->link_to(current);
1092 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1093 EnsureSpace ensure_space(this);
1094 // 1110 1001 #32-bit disp.
1096 emit_code_target(target, rmode);
1100 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1101 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1102 EnsureSpace ensure_space(this);
1103 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1105 emit_runtime_entry(entry, rmode);
1109 void Assembler::jmp(Register target) {
1110 EnsureSpace ensure_space(this);
1112 emit_optional_rex_32(target);
1114 emit_modrm(0x4, target);
1118 void Assembler::jmp(const Operand& src) {
1119 EnsureSpace ensure_space(this);
1121 emit_optional_rex_32(src);
1123 emit_operand(0x4, src);
1127 void Assembler::emit_lea(Register dst, const Operand& src, int size) {
1128 EnsureSpace ensure_space(this);
1129 emit_rex(dst, src, size);
1131 emit_operand(dst, src);
1135 void Assembler::load_rax(void* value, RelocInfo::Mode mode) {
1136 EnsureSpace ensure_space(this);
1137 if (kPointerSize == kInt64Size) {
1138 emit(0x48); // REX.W
1142 DCHECK(kPointerSize == kInt32Size);
1145 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1146 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1147 // Developer's Manual Volume 2.
1153 void Assembler::load_rax(ExternalReference ref) {
1154 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1158 void Assembler::leave() {
1159 EnsureSpace ensure_space(this);
1164 void Assembler::movb(Register dst, const Operand& src) {
1165 EnsureSpace ensure_space(this);
1166 if (!dst.is_byte_register()) {
1167 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1168 emit_rex_32(dst, src);
1170 emit_optional_rex_32(dst, src);
1173 emit_operand(dst, src);
1177 void Assembler::movb(Register dst, Immediate imm) {
1178 EnsureSpace ensure_space(this);
1179 if (!dst.is_byte_register()) {
1180 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1183 emit(0xB0 + dst.low_bits());
1188 void Assembler::movb(const Operand& dst, Register src) {
1189 EnsureSpace ensure_space(this);
1190 if (!src.is_byte_register()) {
1191 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1192 emit_rex_32(src, dst);
1194 emit_optional_rex_32(src, dst);
1197 emit_operand(src, dst);
1201 void Assembler::movb(const Operand& dst, Immediate imm) {
1202 EnsureSpace ensure_space(this);
1203 emit_optional_rex_32(dst);
1205 emit_operand(0x0, dst);
1206 emit(static_cast<byte>(imm.value_));
1210 void Assembler::movw(Register dst, const Operand& src) {
1211 EnsureSpace ensure_space(this);
1213 emit_optional_rex_32(dst, src);
1215 emit_operand(dst, src);
1219 void Assembler::movw(const Operand& dst, Register src) {
1220 EnsureSpace ensure_space(this);
1222 emit_optional_rex_32(src, dst);
1224 emit_operand(src, dst);
1228 void Assembler::movw(const Operand& dst, Immediate imm) {
1229 EnsureSpace ensure_space(this);
1231 emit_optional_rex_32(dst);
1233 emit_operand(0x0, dst);
1234 emit(static_cast<byte>(imm.value_ & 0xff));
1235 emit(static_cast<byte>(imm.value_ >> 8));
1239 void Assembler::emit_mov(Register dst, const Operand& src, int size) {
1240 EnsureSpace ensure_space(this);
1241 emit_rex(dst, src, size);
1243 emit_operand(dst, src);
1247 void Assembler::emit_mov(Register dst, Register src, int size) {
1248 EnsureSpace ensure_space(this);
1249 if (src.low_bits() == 4) {
1250 emit_rex(src, dst, size);
1252 emit_modrm(src, dst);
1254 emit_rex(dst, src, size);
1256 emit_modrm(dst, src);
1261 void Assembler::emit_mov(const Operand& dst, Register src, int size) {
1262 EnsureSpace ensure_space(this);
1263 emit_rex(src, dst, size);
1265 emit_operand(src, dst);
1269 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1270 EnsureSpace ensure_space(this);
1271 emit_rex(dst, size);
1272 if (size == kInt64Size) {
1274 emit_modrm(0x0, dst);
1276 DCHECK(size == kInt32Size);
1277 emit(0xB8 + dst.low_bits());
1283 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) {
1284 EnsureSpace ensure_space(this);
1285 emit_rex(dst, size);
1287 emit_operand(0x0, dst);
1292 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) {
1293 EnsureSpace ensure_space(this);
1294 emit_rex(dst, kPointerSize);
1295 emit(0xB8 | dst.low_bits());
1296 emitp(value, rmode);
1300 void Assembler::movq(Register dst, int64_t value) {
1301 EnsureSpace ensure_space(this);
1303 emit(0xB8 | dst.low_bits());
1308 void Assembler::movq(Register dst, uint64_t value) {
1309 movq(dst, static_cast<int64_t>(value));
1313 // Loads the ip-relative location of the src label into the target location
1314 // (as a 32-bit offset sign extended to 64-bit).
1315 void Assembler::movl(const Operand& dst, Label* src) {
1316 EnsureSpace ensure_space(this);
1317 emit_optional_rex_32(dst);
1319 emit_operand(0, dst);
1320 if (src->is_bound()) {
1321 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1322 DCHECK(offset <= 0);
1324 } else if (src->is_linked()) {
1326 src->link_to(pc_offset() - sizeof(int32_t));
1328 DCHECK(src->is_unused());
1329 int32_t current = pc_offset();
1331 src->link_to(current);
1336 void Assembler::movsxbl(Register dst, const Operand& src) {
1337 EnsureSpace ensure_space(this);
1338 emit_optional_rex_32(dst, src);
1341 emit_operand(dst, src);
1345 void Assembler::movsxbq(Register dst, const Operand& src) {
1346 EnsureSpace ensure_space(this);
1347 emit_rex_64(dst, src);
1350 emit_operand(dst, src);
1354 void Assembler::movsxwl(Register dst, const Operand& src) {
1355 EnsureSpace ensure_space(this);
1356 emit_optional_rex_32(dst, src);
1359 emit_operand(dst, src);
1363 void Assembler::movsxwq(Register dst, const Operand& src) {
1364 EnsureSpace ensure_space(this);
1365 emit_rex_64(dst, src);
1368 emit_operand(dst, src);
1372 void Assembler::movsxlq(Register dst, Register src) {
1373 EnsureSpace ensure_space(this);
1374 emit_rex_64(dst, src);
1376 emit_modrm(dst, src);
1380 void Assembler::movsxlq(Register dst, const Operand& src) {
1381 EnsureSpace ensure_space(this);
1382 emit_rex_64(dst, src);
1384 emit_operand(dst, src);
1388 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) {
1389 EnsureSpace ensure_space(this);
1390 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1391 // there is no need to make this a 64 bit operation.
1392 emit_optional_rex_32(dst, src);
1395 emit_operand(dst, src);
1399 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1400 EnsureSpace ensure_space(this);
1401 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1402 // there is no need to make this a 64 bit operation.
1403 if (!src.is_byte_register()) {
1404 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1405 emit_rex_32(dst, src);
1407 emit_optional_rex_32(dst, src);
1411 emit_modrm(dst, src);
1415 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) {
1416 EnsureSpace ensure_space(this);
1417 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1418 // there is no need to make this a 64 bit operation.
1419 emit_optional_rex_32(dst, src);
1422 emit_operand(dst, src);
1426 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1427 EnsureSpace ensure_space(this);
1428 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1429 // there is no need to make this a 64 bit operation.
1430 emit_optional_rex_32(dst, src);
1433 emit_modrm(dst, src);
1437 void Assembler::repmovsb() {
1438 EnsureSpace ensure_space(this);
1444 void Assembler::repmovsw() {
1445 EnsureSpace ensure_space(this);
1446 emit(0x66); // Operand size override.
1452 void Assembler::emit_repmovs(int size) {
1453 EnsureSpace ensure_space(this);
1460 void Assembler::mul(Register src) {
1461 EnsureSpace ensure_space(this);
1464 emit_modrm(0x4, src);
1468 void Assembler::emit_neg(Register dst, int size) {
1469 EnsureSpace ensure_space(this);
1470 emit_rex(dst, size);
1472 emit_modrm(0x3, dst);
1476 void Assembler::emit_neg(const Operand& dst, int size) {
1477 EnsureSpace ensure_space(this);
1480 emit_operand(3, dst);
1484 void Assembler::nop() {
1485 EnsureSpace ensure_space(this);
1490 void Assembler::emit_not(Register dst, int size) {
1491 EnsureSpace ensure_space(this);
1492 emit_rex(dst, size);
1494 emit_modrm(0x2, dst);
1498 void Assembler::emit_not(const Operand& dst, int size) {
1499 EnsureSpace ensure_space(this);
1500 emit_rex(dst, size);
1502 emit_operand(2, dst);
1506 void Assembler::Nop(int n) {
1507 // The recommended muti-byte sequences of NOP instructions from the Intel 64
1508 // and IA-32 Architectures Software Developer's Manual.
1510 // Length Assembly Byte Sequence
1511 // 2 bytes 66 NOP 66 90H
1512 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
1513 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
1514 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
1515 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
1516 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
1517 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
1518 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
1521 EnsureSpace ensure_space(this);
1583 void Assembler::popq(Register dst) {
1584 EnsureSpace ensure_space(this);
1585 emit_optional_rex_32(dst);
1586 emit(0x58 | dst.low_bits());
1590 void Assembler::popq(const Operand& dst) {
1591 EnsureSpace ensure_space(this);
1592 emit_optional_rex_32(dst);
1594 emit_operand(0, dst);
1598 void Assembler::popfq() {
1599 EnsureSpace ensure_space(this);
1604 void Assembler::pushq(Register src) {
1605 EnsureSpace ensure_space(this);
1606 emit_optional_rex_32(src);
1607 emit(0x50 | src.low_bits());
1611 void Assembler::pushq(const Operand& src) {
1612 EnsureSpace ensure_space(this);
1613 emit_optional_rex_32(src);
1615 emit_operand(6, src);
1619 void Assembler::pushq(Immediate value) {
1620 EnsureSpace ensure_space(this);
1621 if (is_int8(value.value_)) {
1623 emit(value.value_); // Emit low byte of value.
1626 emitl(value.value_);
1631 void Assembler::pushq_imm32(int32_t imm32) {
1632 EnsureSpace ensure_space(this);
1638 void Assembler::pushfq() {
1639 EnsureSpace ensure_space(this);
1644 void Assembler::ret(int imm16) {
1645 EnsureSpace ensure_space(this);
1646 DCHECK(is_uint16(imm16));
1652 emit((imm16 >> 8) & 0xFF);
1657 void Assembler::setcc(Condition cc, Register reg) {
1658 if (cc > last_condition) {
1659 movb(reg, Immediate(cc == always ? 1 : 0));
1662 EnsureSpace ensure_space(this);
1663 DCHECK(is_uint4(cc));
1664 if (!reg.is_byte_register()) {
1665 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1670 emit_modrm(0x0, reg);
1674 void Assembler::shld(Register dst, Register src) {
1675 EnsureSpace ensure_space(this);
1676 emit_rex_64(src, dst);
1679 emit_modrm(src, dst);
1683 void Assembler::shrd(Register dst, Register src) {
1684 EnsureSpace ensure_space(this);
1685 emit_rex_64(src, dst);
1688 emit_modrm(src, dst);
1692 void Assembler::emit_xchg(Register dst, Register src, int size) {
1693 EnsureSpace ensure_space(this);
1694 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding
1695 Register other = src.is(rax) ? dst : src;
1696 emit_rex(other, size);
1697 emit(0x90 | other.low_bits());
1698 } else if (dst.low_bits() == 4) {
1699 emit_rex(dst, src, size);
1701 emit_modrm(dst, src);
1703 emit_rex(src, dst, size);
1705 emit_modrm(src, dst);
1710 void Assembler::emit_xchg(Register dst, const Operand& src, int size) {
1711 EnsureSpace ensure_space(this);
1712 emit_rex(dst, src, size);
1714 emit_operand(dst, src);
1718 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) {
1719 EnsureSpace ensure_space(this);
1720 if (kPointerSize == kInt64Size) {
1721 emit(0x48); // REX.W
1725 DCHECK(kPointerSize == kInt32Size);
1728 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1729 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1730 // Developer's Manual Volume 2.
1736 void Assembler::store_rax(ExternalReference ref) {
1737 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1741 void Assembler::testb(Register dst, Register src) {
1742 EnsureSpace ensure_space(this);
1743 if (src.low_bits() == 4) {
1744 emit_rex_32(src, dst);
1746 emit_modrm(src, dst);
1748 if (!dst.is_byte_register() || !src.is_byte_register()) {
1749 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1750 emit_rex_32(dst, src);
1753 emit_modrm(dst, src);
1758 void Assembler::testb(Register reg, Immediate mask) {
1759 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1760 EnsureSpace ensure_space(this);
1763 emit(mask.value_); // Low byte emitted.
1765 if (!reg.is_byte_register()) {
1766 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1770 emit_modrm(0x0, reg);
1771 emit(mask.value_); // Low byte emitted.
1776 void Assembler::testb(const Operand& op, Immediate mask) {
1777 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1778 EnsureSpace ensure_space(this);
1779 emit_optional_rex_32(rax, op);
1781 emit_operand(rax, op); // Operation code 0
1782 emit(mask.value_); // Low byte emitted.
1786 void Assembler::testb(const Operand& op, Register reg) {
1787 EnsureSpace ensure_space(this);
1788 if (!reg.is_byte_register()) {
1789 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1790 emit_rex_32(reg, op);
1792 emit_optional_rex_32(reg, op);
1795 emit_operand(reg, op);
1799 void Assembler::emit_test(Register dst, Register src, int size) {
1800 EnsureSpace ensure_space(this);
1801 if (src.low_bits() == 4) {
1802 emit_rex(src, dst, size);
1804 emit_modrm(src, dst);
1806 emit_rex(dst, src, size);
1808 emit_modrm(dst, src);
1813 void Assembler::emit_test(Register reg, Immediate mask, int size) {
1814 // testl with a mask that fits in the low byte is exactly testb.
1815 if (is_uint8(mask.value_)) {
1819 EnsureSpace ensure_space(this);
1821 emit_rex(rax, size);
1825 emit_rex(reg, size);
1827 emit_modrm(0x0, reg);
1833 void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
1834 // testl with a mask that fits in the low byte is exactly testb.
1835 if (is_uint8(mask.value_)) {
1839 EnsureSpace ensure_space(this);
1840 emit_rex(rax, op, size);
1842 emit_operand(rax, op); // Operation code 0
1847 void Assembler::emit_test(const Operand& op, Register reg, int size) {
1848 EnsureSpace ensure_space(this);
1849 emit_rex(reg, op, size);
1851 emit_operand(reg, op);
1855 // FPU instructions.
1858 void Assembler::fld(int i) {
1859 EnsureSpace ensure_space(this);
1860 emit_farith(0xD9, 0xC0, i);
1864 void Assembler::fld1() {
1865 EnsureSpace ensure_space(this);
1871 void Assembler::fldz() {
1872 EnsureSpace ensure_space(this);
1878 void Assembler::fldpi() {
1879 EnsureSpace ensure_space(this);
1885 void Assembler::fldln2() {
1886 EnsureSpace ensure_space(this);
1892 void Assembler::fld_s(const Operand& adr) {
1893 EnsureSpace ensure_space(this);
1894 emit_optional_rex_32(adr);
1896 emit_operand(0, adr);
1900 void Assembler::fld_d(const Operand& adr) {
1901 EnsureSpace ensure_space(this);
1902 emit_optional_rex_32(adr);
1904 emit_operand(0, adr);
1908 void Assembler::fstp_s(const Operand& adr) {
1909 EnsureSpace ensure_space(this);
1910 emit_optional_rex_32(adr);
1912 emit_operand(3, adr);
1916 void Assembler::fstp_d(const Operand& adr) {
1917 EnsureSpace ensure_space(this);
1918 emit_optional_rex_32(adr);
1920 emit_operand(3, adr);
1924 void Assembler::fstp(int index) {
1925 DCHECK(is_uint3(index));
1926 EnsureSpace ensure_space(this);
1927 emit_farith(0xDD, 0xD8, index);
1931 void Assembler::fild_s(const Operand& adr) {
1932 EnsureSpace ensure_space(this);
1933 emit_optional_rex_32(adr);
1935 emit_operand(0, adr);
1939 void Assembler::fild_d(const Operand& adr) {
1940 EnsureSpace ensure_space(this);
1941 emit_optional_rex_32(adr);
1943 emit_operand(5, adr);
1947 void Assembler::fistp_s(const Operand& adr) {
1948 EnsureSpace ensure_space(this);
1949 emit_optional_rex_32(adr);
1951 emit_operand(3, adr);
1955 void Assembler::fisttp_s(const Operand& adr) {
1956 DCHECK(IsEnabled(SSE3));
1957 EnsureSpace ensure_space(this);
1958 emit_optional_rex_32(adr);
1960 emit_operand(1, adr);
1964 void Assembler::fisttp_d(const Operand& adr) {
1965 DCHECK(IsEnabled(SSE3));
1966 EnsureSpace ensure_space(this);
1967 emit_optional_rex_32(adr);
1969 emit_operand(1, adr);
1973 void Assembler::fist_s(const Operand& adr) {
1974 EnsureSpace ensure_space(this);
1975 emit_optional_rex_32(adr);
1977 emit_operand(2, adr);
1981 void Assembler::fistp_d(const Operand& adr) {
1982 EnsureSpace ensure_space(this);
1983 emit_optional_rex_32(adr);
1985 emit_operand(7, adr);
1989 void Assembler::fabs() {
1990 EnsureSpace ensure_space(this);
1996 void Assembler::fchs() {
1997 EnsureSpace ensure_space(this);
2003 void Assembler::fcos() {
2004 EnsureSpace ensure_space(this);
2010 void Assembler::fsin() {
2011 EnsureSpace ensure_space(this);
2017 void Assembler::fptan() {
2018 EnsureSpace ensure_space(this);
2024 void Assembler::fyl2x() {
2025 EnsureSpace ensure_space(this);
2031 void Assembler::f2xm1() {
2032 EnsureSpace ensure_space(this);
2038 void Assembler::fscale() {
2039 EnsureSpace ensure_space(this);
2045 void Assembler::fninit() {
2046 EnsureSpace ensure_space(this);
2052 void Assembler::fadd(int i) {
2053 EnsureSpace ensure_space(this);
2054 emit_farith(0xDC, 0xC0, i);
2058 void Assembler::fsub(int i) {
2059 EnsureSpace ensure_space(this);
2060 emit_farith(0xDC, 0xE8, i);
2064 void Assembler::fisub_s(const Operand& adr) {
2065 EnsureSpace ensure_space(this);
2066 emit_optional_rex_32(adr);
2068 emit_operand(4, adr);
2072 void Assembler::fmul(int i) {
2073 EnsureSpace ensure_space(this);
2074 emit_farith(0xDC, 0xC8, i);
2078 void Assembler::fdiv(int i) {
2079 EnsureSpace ensure_space(this);
2080 emit_farith(0xDC, 0xF8, i);
2084 void Assembler::faddp(int i) {
2085 EnsureSpace ensure_space(this);
2086 emit_farith(0xDE, 0xC0, i);
2090 void Assembler::fsubp(int i) {
2091 EnsureSpace ensure_space(this);
2092 emit_farith(0xDE, 0xE8, i);
2096 void Assembler::fsubrp(int i) {
2097 EnsureSpace ensure_space(this);
2098 emit_farith(0xDE, 0xE0, i);
2102 void Assembler::fmulp(int i) {
2103 EnsureSpace ensure_space(this);
2104 emit_farith(0xDE, 0xC8, i);
2108 void Assembler::fdivp(int i) {
2109 EnsureSpace ensure_space(this);
2110 emit_farith(0xDE, 0xF8, i);
2114 void Assembler::fprem() {
2115 EnsureSpace ensure_space(this);
2121 void Assembler::fprem1() {
2122 EnsureSpace ensure_space(this);
2128 void Assembler::fxch(int i) {
2129 EnsureSpace ensure_space(this);
2130 emit_farith(0xD9, 0xC8, i);
2134 void Assembler::fincstp() {
2135 EnsureSpace ensure_space(this);
2141 void Assembler::ffree(int i) {
2142 EnsureSpace ensure_space(this);
2143 emit_farith(0xDD, 0xC0, i);
2147 void Assembler::ftst() {
2148 EnsureSpace ensure_space(this);
2154 void Assembler::fucomp(int i) {
2155 EnsureSpace ensure_space(this);
2156 emit_farith(0xDD, 0xE8, i);
2160 void Assembler::fucompp() {
2161 EnsureSpace ensure_space(this);
2167 void Assembler::fucomi(int i) {
2168 EnsureSpace ensure_space(this);
2174 void Assembler::fucomip() {
2175 EnsureSpace ensure_space(this);
2181 void Assembler::fcompp() {
2182 EnsureSpace ensure_space(this);
2188 void Assembler::fnstsw_ax() {
2189 EnsureSpace ensure_space(this);
2195 void Assembler::fwait() {
2196 EnsureSpace ensure_space(this);
2201 void Assembler::frndint() {
2202 EnsureSpace ensure_space(this);
2208 void Assembler::fnclex() {
2209 EnsureSpace ensure_space(this);
2215 void Assembler::sahf() {
2216 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2217 // in 64-bit mode. Test CpuID.
2218 DCHECK(IsEnabled(SAHF));
2219 EnsureSpace ensure_space(this);
2224 void Assembler::emit_farith(int b1, int b2, int i) {
2225 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2226 DCHECK(is_uint3(i)); // illegal stack offset
2234 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2235 EnsureSpace ensure_space(this);
2236 emit_optional_rex_32(dst, src);
2239 emit_sse_operand(dst, src);
2243 void Assembler::andps(XMMRegister dst, const Operand& src) {
2244 EnsureSpace ensure_space(this);
2245 emit_optional_rex_32(dst, src);
2248 emit_sse_operand(dst, src);
2252 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2253 EnsureSpace ensure_space(this);
2254 emit_optional_rex_32(dst, src);
2257 emit_sse_operand(dst, src);
2261 void Assembler::orps(XMMRegister dst, const Operand& src) {
2262 EnsureSpace ensure_space(this);
2263 emit_optional_rex_32(dst, src);
2266 emit_sse_operand(dst, src);
2270 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2271 EnsureSpace ensure_space(this);
2272 emit_optional_rex_32(dst, src);
2275 emit_sse_operand(dst, src);
2279 void Assembler::xorps(XMMRegister dst, const Operand& src) {
2280 EnsureSpace ensure_space(this);
2281 emit_optional_rex_32(dst, src);
2284 emit_sse_operand(dst, src);
2288 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2289 EnsureSpace ensure_space(this);
2290 emit_optional_rex_32(dst, src);
2293 emit_sse_operand(dst, src);
2297 void Assembler::addps(XMMRegister dst, const Operand& src) {
2298 EnsureSpace ensure_space(this);
2299 emit_optional_rex_32(dst, src);
2302 emit_sse_operand(dst, src);
2306 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2307 EnsureSpace ensure_space(this);
2308 emit_optional_rex_32(dst, src);
2311 emit_sse_operand(dst, src);
2315 void Assembler::subps(XMMRegister dst, const Operand& src) {
2316 EnsureSpace ensure_space(this);
2317 emit_optional_rex_32(dst, src);
2320 emit_sse_operand(dst, src);
2324 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2325 EnsureSpace ensure_space(this);
2326 emit_optional_rex_32(dst, src);
2329 emit_sse_operand(dst, src);
2333 void Assembler::mulps(XMMRegister dst, const Operand& src) {
2334 EnsureSpace ensure_space(this);
2335 emit_optional_rex_32(dst, src);
2338 emit_sse_operand(dst, src);
2342 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2343 EnsureSpace ensure_space(this);
2344 emit_optional_rex_32(dst, src);
2347 emit_sse_operand(dst, src);
2351 void Assembler::divps(XMMRegister dst, const Operand& src) {
2352 EnsureSpace ensure_space(this);
2353 emit_optional_rex_32(dst, src);
2356 emit_sse_operand(dst, src);
2360 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
2361 EnsureSpace ensure_space(this);
2363 emit_optional_rex_32(dst, src);
2366 emit_sse_operand(dst, src);
2370 void Assembler::addpd(XMMRegister dst, const Operand& src) {
2371 EnsureSpace ensure_space(this);
2373 emit_optional_rex_32(dst, src);
2376 emit_sse_operand(dst, src);
2380 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
2381 EnsureSpace ensure_space(this);
2383 emit_optional_rex_32(dst, src);
2386 emit_sse_operand(dst, src);
2390 void Assembler::subpd(XMMRegister dst, const Operand& src) {
2391 EnsureSpace ensure_space(this);
2393 emit_optional_rex_32(dst, src);
2396 emit_sse_operand(dst, src);
2400 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
2401 EnsureSpace ensure_space(this);
2403 emit_optional_rex_32(dst, src);
2406 emit_sse_operand(dst, src);
2410 void Assembler::mulpd(XMMRegister dst, const Operand& src) {
2411 EnsureSpace ensure_space(this);
2413 emit_optional_rex_32(dst, src);
2416 emit_sse_operand(dst, src);
2420 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
2421 EnsureSpace ensure_space(this);
2423 emit_optional_rex_32(dst, src);
2426 emit_sse_operand(dst, src);
2430 void Assembler::divpd(XMMRegister dst, const Operand& src) {
2431 EnsureSpace ensure_space(this);
2433 emit_optional_rex_32(dst, src);
2436 emit_sse_operand(dst, src);
2440 // SSE 2 operations.
2442 void Assembler::movd(XMMRegister dst, Register src) {
2443 EnsureSpace ensure_space(this);
2445 emit_optional_rex_32(dst, src);
2448 emit_sse_operand(dst, src);
2452 void Assembler::movd(Register dst, XMMRegister src) {
2453 EnsureSpace ensure_space(this);
2455 emit_optional_rex_32(src, dst);
2458 emit_sse_operand(src, dst);
2462 void Assembler::movq(XMMRegister dst, Register src) {
2463 EnsureSpace ensure_space(this);
2465 emit_rex_64(dst, src);
2468 emit_sse_operand(dst, src);
2472 void Assembler::movq(Register dst, XMMRegister src) {
2473 EnsureSpace ensure_space(this);
2475 emit_rex_64(src, dst);
2478 emit_sse_operand(src, dst);
2482 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2483 EnsureSpace ensure_space(this);
2484 if (dst.low_bits() == 4) {
2485 // Avoid unnecessary SIB byte.
2487 emit_optional_rex_32(dst, src);
2490 emit_sse_operand(dst, src);
2493 emit_optional_rex_32(src, dst);
2496 emit_sse_operand(src, dst);
2501 void Assembler::movdqa(const Operand& dst, XMMRegister src) {
2502 EnsureSpace ensure_space(this);
2504 emit_rex_64(src, dst);
2507 emit_sse_operand(src, dst);
2511 void Assembler::movdqa(XMMRegister dst, const Operand& src) {
2512 EnsureSpace ensure_space(this);
2514 emit_rex_64(dst, src);
2517 emit_sse_operand(dst, src);
2521 void Assembler::movdqu(const Operand& dst, XMMRegister src) {
2522 EnsureSpace ensure_space(this);
2524 emit_rex_64(src, dst);
2527 emit_sse_operand(src, dst);
2531 void Assembler::movdqu(XMMRegister dst, const Operand& src) {
2532 EnsureSpace ensure_space(this);
2534 emit_rex_64(dst, src);
2537 emit_sse_operand(dst, src);
2541 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2542 DCHECK(IsEnabled(SSE4_1));
2543 DCHECK(is_uint8(imm8));
2544 EnsureSpace ensure_space(this);
2546 emit_optional_rex_32(src, dst);
2550 emit_sse_operand(src, dst);
2555 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2556 DCHECK(CpuFeatures::IsSupported(SSE4_1));
2557 DCHECK(is_uint8(imm8));
2558 EnsureSpace ensure_space(this);
2560 emit_optional_rex_32(dst, src);
2564 emit_sse_operand(dst, src);
2569 void Assembler::pinsrd(XMMRegister dst, Register src, byte imm8) {
2570 DCHECK(CpuFeatures::IsSupported(SSE4_1));
2571 DCHECK(is_uint8(imm8));
2572 EnsureSpace ensure_space(this);
2574 emit_optional_rex_32(dst, src);
2578 emit_sse_operand(dst, src);
2583 void Assembler::movsd(const Operand& dst, XMMRegister src) {
2584 EnsureSpace ensure_space(this);
2585 emit(0xF2); // double
2586 emit_optional_rex_32(src, dst);
2588 emit(0x11); // store
2589 emit_sse_operand(src, dst);
2593 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2594 EnsureSpace ensure_space(this);
2595 emit(0xF2); // double
2596 emit_optional_rex_32(dst, src);
2599 emit_sse_operand(dst, src);
2603 void Assembler::movsd(XMMRegister dst, const Operand& src) {
2604 EnsureSpace ensure_space(this);
2605 emit(0xF2); // double
2606 emit_optional_rex_32(dst, src);
2609 emit_sse_operand(dst, src);
2613 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2614 EnsureSpace ensure_space(this);
2615 if (src.low_bits() == 4) {
2616 // Try to avoid an unnecessary SIB byte.
2617 emit_optional_rex_32(src, dst);
2620 emit_sse_operand(src, dst);
2622 emit_optional_rex_32(dst, src);
2625 emit_sse_operand(dst, src);
2630 void Assembler::movups(XMMRegister dst, const Operand& src) {
2631 EnsureSpace ensure_space(this);
2632 emit_optional_rex_32(dst, src);
2635 emit_sse_operand(dst, src);
2639 void Assembler::movups(const Operand& dst, XMMRegister src) {
2640 EnsureSpace ensure_space(this);
2641 emit_optional_rex_32(src, dst);
2644 emit_sse_operand(src, dst);
2648 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2649 DCHECK(is_uint8(imm8));
2650 EnsureSpace ensure_space(this);
2651 emit_optional_rex_32(dst, src);
2654 emit_sse_operand(dst, src);
2659 void Assembler::shufpd(XMMRegister dst, XMMRegister src, byte imm8) {
2660 DCHECK(is_uint8(imm8));
2661 EnsureSpace ensure_space(this);
2663 emit_optional_rex_32(dst, src);
2666 emit_sse_operand(dst, src);
2671 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2672 EnsureSpace ensure_space(this);
2673 if (src.low_bits() == 4) {
2674 // Try to avoid an unnecessary SIB byte.
2676 emit_optional_rex_32(src, dst);
2679 emit_sse_operand(src, dst);
2682 emit_optional_rex_32(dst, src);
2685 emit_sse_operand(dst, src);
2690 void Assembler::movss(XMMRegister dst, const Operand& src) {
2691 EnsureSpace ensure_space(this);
2692 emit(0xF3); // single
2693 emit_optional_rex_32(dst, src);
2696 emit_sse_operand(dst, src);
2700 void Assembler::movss(const Operand& src, XMMRegister dst) {
2701 EnsureSpace ensure_space(this);
2702 emit(0xF3); // single
2703 emit_optional_rex_32(dst, src);
2705 emit(0x11); // store
2706 emit_sse_operand(dst, src);
2710 void Assembler::psllq(XMMRegister reg, byte imm8) {
2711 EnsureSpace ensure_space(this);
2715 emit_sse_operand(rsi, reg); // rsi == 6
2720 void Assembler::cvttss2si(Register dst, const Operand& src) {
2721 EnsureSpace ensure_space(this);
2723 emit_optional_rex_32(dst, src);
2726 emit_operand(dst, src);
2730 void Assembler::cvttss2si(Register dst, XMMRegister src) {
2731 EnsureSpace ensure_space(this);
2733 emit_optional_rex_32(dst, src);
2736 emit_sse_operand(dst, src);
2740 void Assembler::cvttsd2si(Register dst, const Operand& src) {
2741 EnsureSpace ensure_space(this);
2743 emit_optional_rex_32(dst, src);
2746 emit_operand(dst, src);
2750 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
2751 EnsureSpace ensure_space(this);
2753 emit_optional_rex_32(dst, src);
2756 emit_sse_operand(dst, src);
2760 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
2761 EnsureSpace ensure_space(this);
2763 emit_rex_64(dst, src);
2766 emit_sse_operand(dst, src);
2770 void Assembler::cvttsd2siq(Register dst, const Operand& src) {
2771 EnsureSpace ensure_space(this);
2773 emit_rex_64(dst, src);
2776 emit_sse_operand(dst, src);
2780 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) {
2781 EnsureSpace ensure_space(this);
2783 emit_optional_rex_32(dst, src);
2786 emit_sse_operand(dst, src);
2790 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
2791 EnsureSpace ensure_space(this);
2793 emit_optional_rex_32(dst, src);
2796 emit_sse_operand(dst, src);
2800 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
2801 EnsureSpace ensure_space(this);
2803 emit_optional_rex_32(dst, src);
2806 emit_sse_operand(dst, src);
2810 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
2811 EnsureSpace ensure_space(this);
2813 emit_rex_64(dst, src);
2816 emit_sse_operand(dst, src);
2820 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
2821 EnsureSpace ensure_space(this);
2823 emit_optional_rex_32(dst, src);
2826 emit_sse_operand(dst, src);
2830 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) {
2831 EnsureSpace ensure_space(this);
2833 emit_optional_rex_32(dst, src);
2836 emit_sse_operand(dst, src);
2840 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
2841 EnsureSpace ensure_space(this);
2843 emit_optional_rex_32(dst, src);
2846 emit_sse_operand(dst, src);
2850 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
2851 EnsureSpace ensure_space(this);
2853 emit_optional_rex_32(dst, src);
2856 emit_sse_operand(dst, src);
2860 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
2861 EnsureSpace ensure_space(this);
2863 emit_rex_64(dst, src);
2866 emit_sse_operand(dst, src);
2870 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
2871 EnsureSpace ensure_space(this);
2873 emit_optional_rex_32(dst, src);
2876 emit_sse_operand(dst, src);
2880 void Assembler::addsd(XMMRegister dst, const Operand& src) {
2881 EnsureSpace ensure_space(this);
2883 emit_optional_rex_32(dst, src);
2886 emit_sse_operand(dst, src);
2890 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2891 EnsureSpace ensure_space(this);
2893 emit_optional_rex_32(dst, src);
2896 emit_sse_operand(dst, src);
2900 void Assembler::mulsd(XMMRegister dst, const Operand& src) {
2901 EnsureSpace ensure_space(this);
2903 emit_optional_rex_32(dst, src);
2906 emit_sse_operand(dst, src);
2910 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
2911 EnsureSpace ensure_space(this);
2913 emit_optional_rex_32(dst, src);
2916 emit_sse_operand(dst, src);
2920 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
2921 EnsureSpace ensure_space(this);
2923 emit_optional_rex_32(dst, src);
2926 emit_sse_operand(dst, src);
2930 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
2931 EnsureSpace ensure_space(this);
2933 emit_optional_rex_32(dst, src);
2936 emit_sse_operand(dst, src);
2940 void Assembler::andpd(XMMRegister dst, const Operand& src) {
2941 EnsureSpace ensure_space(this);
2943 emit_optional_rex_32(dst, src);
2946 emit_sse_operand(dst, src);
2950 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
2951 EnsureSpace ensure_space(this);
2953 emit_optional_rex_32(dst, src);
2956 emit_sse_operand(dst, src);
2960 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
2961 EnsureSpace ensure_space(this);
2963 emit_optional_rex_32(dst, src);
2966 emit_sse_operand(dst, src);
2970 void Assembler::xorpd(XMMRegister dst, const Operand& src) {
2971 EnsureSpace ensure_space(this);
2973 emit_optional_rex_32(dst, src);
2976 emit_sse_operand(dst, src);
2980 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
2981 EnsureSpace ensure_space(this);
2983 emit_optional_rex_32(dst, src);
2986 emit_sse_operand(dst, src);
2990 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) {
2991 EnsureSpace ensure_space(this);
2993 emit_optional_rex_32(dst, src);
2996 emit_sse_operand(dst, src);
3000 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3001 EnsureSpace ensure_space(this);
3003 emit_optional_rex_32(dst, src);
3006 emit_sse_operand(dst, src);
3010 void Assembler::ucomisd(XMMRegister dst, const Operand& src) {
3011 EnsureSpace ensure_space(this);
3013 emit_optional_rex_32(dst, src);
3016 emit_sse_operand(dst, src);
3020 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3021 EnsureSpace ensure_space(this);
3023 emit_optional_rex_32(dst, src);
3026 emit_sse_operand(dst, src);
3027 emit(0x01); // LT == 1
3031 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3032 EnsureSpace ensure_space(this);
3033 emit_optional_rex_32(dst, src);
3036 emit_sse_operand(dst, src);
3041 void Assembler::cmpeqps(XMMRegister dst, XMMRegister src) {
3042 cmpps(dst, src, 0x0);
3046 void Assembler::cmpltps(XMMRegister dst, XMMRegister src) {
3047 cmpps(dst, src, 0x1);
3051 void Assembler::cmpleps(XMMRegister dst, XMMRegister src) {
3052 cmpps(dst, src, 0x2);
3056 void Assembler::cmpneqps(XMMRegister dst, XMMRegister src) {
3057 cmpps(dst, src, 0x4);
3061 void Assembler::cmpnltps(XMMRegister dst, XMMRegister src) {
3062 cmpps(dst, src, 0x5);
3066 void Assembler::cmpnleps(XMMRegister dst, XMMRegister src) {
3067 cmpps(dst, src, 0x6);
3071 void Assembler::pslld(XMMRegister reg, int8_t shift) {
3072 EnsureSpace ensure_space(this);
3074 emit_optional_rex_32(reg);
3077 emit_sse_operand(rsi, reg); // rsi == 6
3082 void Assembler::pslld(XMMRegister dst, XMMRegister src) {
3083 EnsureSpace ensure_space(this);
3085 emit_optional_rex_32(dst, src);
3088 emit_sse_operand(dst, src);
3092 void Assembler::psrld(XMMRegister reg, int8_t shift) {
3093 EnsureSpace ensure_space(this);
3095 emit_optional_rex_32(reg);
3098 emit_sse_operand(rdx, reg); // rdx == 2
3103 void Assembler::psrld(XMMRegister dst, XMMRegister src) {
3104 EnsureSpace ensure_space(this);
3106 emit_optional_rex_32(dst, src);
3109 emit_sse_operand(dst, src);
3113 void Assembler::psrad(XMMRegister reg, int8_t shift) {
3114 EnsureSpace ensure_space(this);
3116 emit_optional_rex_32(reg);
3119 emit_sse_operand(rsp, reg); // rsp == 4
3124 void Assembler::psrad(XMMRegister dst, XMMRegister src) {
3125 EnsureSpace ensure_space(this);
3127 emit_optional_rex_32(dst, src);
3130 emit_sse_operand(dst, src);
3134 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3135 EnsureSpace ensure_space(this);
3137 emit_optional_rex_32(dst, src);
3140 emit_sse_operand(dst, src);
3144 void Assembler::pcmpgtd(XMMRegister dst, XMMRegister src) {
3145 EnsureSpace ensure_space(this);
3147 emit_optional_rex_32(dst, src);
3150 emit_sse_operand(dst, src);
3154 void Assembler::roundsd(XMMRegister dst, XMMRegister src,
3155 Assembler::RoundingMode mode) {
3156 DCHECK(IsEnabled(SSE4_1));
3157 EnsureSpace ensure_space(this);
3159 emit_optional_rex_32(dst, src);
3163 emit_sse_operand(dst, src);
3164 // Mask precision exeption.
3165 emit(static_cast<byte>(mode) | 0x8);
3169 void Assembler::movmskpd(Register dst, XMMRegister src) {
3170 EnsureSpace ensure_space(this);
3172 emit_optional_rex_32(dst, src);
3175 emit_sse_operand(dst, src);
3179 void Assembler::movmskps(Register dst, XMMRegister src) {
3180 EnsureSpace ensure_space(this);
3181 emit_optional_rex_32(dst, src);
3184 emit_sse_operand(dst, src);
3188 void Assembler::minps(XMMRegister dst, XMMRegister src) {
3189 EnsureSpace ensure_space(this);
3190 emit_optional_rex_32(dst, src);
3193 emit_sse_operand(dst, src);
3197 void Assembler::minps(XMMRegister dst, const Operand& src) {
3198 EnsureSpace ensure_space(this);
3199 emit_optional_rex_32(dst, src);
3202 emit_sse_operand(dst, src);
3206 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
3207 EnsureSpace ensure_space(this);
3208 emit_optional_rex_32(dst, src);
3211 emit_sse_operand(dst, src);
3215 void Assembler::maxps(XMMRegister dst, const Operand& src) {
3216 EnsureSpace ensure_space(this);
3217 emit_optional_rex_32(dst, src);
3220 emit_sse_operand(dst, src);
3224 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
3225 EnsureSpace ensure_space(this);
3227 emit_optional_rex_32(dst, src);
3230 emit_sse_operand(dst, src);
3234 void Assembler::minpd(XMMRegister dst, const Operand& src) {
3235 EnsureSpace ensure_space(this);
3237 emit_optional_rex_32(dst, src);
3240 emit_sse_operand(dst, src);
3244 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
3245 EnsureSpace ensure_space(this);
3247 emit_optional_rex_32(dst, src);
3250 emit_sse_operand(dst, src);
3254 void Assembler::maxpd(XMMRegister dst, const Operand& src) {
3255 EnsureSpace ensure_space(this);
3257 emit_optional_rex_32(dst, src);
3260 emit_sse_operand(dst, src);
3264 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3265 EnsureSpace ensure_space(this);
3266 emit_optional_rex_32(dst, src);
3269 emit_sse_operand(dst, src);
3273 void Assembler::rcpps(XMMRegister dst, const Operand& src) {
3274 EnsureSpace ensure_space(this);
3275 emit_optional_rex_32(dst, src);
3278 emit_sse_operand(dst, src);
3282 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) {
3283 EnsureSpace ensure_space(this);
3284 emit_optional_rex_32(dst, src);
3287 emit_sse_operand(dst, src);
3291 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) {
3292 EnsureSpace ensure_space(this);
3293 emit_optional_rex_32(dst, src);
3296 emit_sse_operand(dst, src);
3300 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) {
3301 EnsureSpace ensure_space(this);
3302 emit_optional_rex_32(dst, src);
3305 emit_sse_operand(dst, src);
3309 void Assembler::sqrtps(XMMRegister dst, const Operand& src) {
3310 EnsureSpace ensure_space(this);
3311 emit_optional_rex_32(dst, src);
3314 emit_sse_operand(dst, src);
3318 void Assembler::sqrtpd(XMMRegister dst, XMMRegister src) {
3319 EnsureSpace ensure_space(this);
3321 emit_optional_rex_32(dst, src);
3324 emit_sse_operand(dst, src);
3328 void Assembler::sqrtpd(XMMRegister dst, const Operand& src) {
3329 EnsureSpace ensure_space(this);
3331 emit_optional_rex_32(dst, src);
3334 emit_sse_operand(dst, src);
3338 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
3339 EnsureSpace ensure_space(this);
3340 emit_optional_rex_32(dst, src);
3343 emit_sse_operand(dst, src);
3347 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) {
3348 EnsureSpace ensure_space(this);
3349 emit_optional_rex_32(dst, src);
3352 emit_sse_operand(dst, src);
3356 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
3357 EnsureSpace ensure_space(this);
3359 emit_optional_rex_32(dst, src);
3362 emit_sse_operand(dst, src);
3366 void Assembler::paddd(XMMRegister dst, const Operand& src) {
3367 EnsureSpace ensure_space(this);
3369 emit_optional_rex_32(dst, src);
3372 emit_sse_operand(dst, src);
3376 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
3377 EnsureSpace ensure_space(this);
3379 emit_optional_rex_32(dst, src);
3382 emit_sse_operand(dst, src);
3386 void Assembler::psubd(XMMRegister dst, const Operand& src) {
3387 EnsureSpace ensure_space(this);
3389 emit_optional_rex_32(dst, src);
3392 emit_sse_operand(dst, src);
3396 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
3397 DCHECK(IsEnabled(SSE4_1));
3398 EnsureSpace ensure_space(this);
3400 emit_optional_rex_32(dst, src);
3404 emit_sse_operand(dst, src);
3408 void Assembler::pmulld(XMMRegister dst, const Operand& src) {
3409 EnsureSpace ensure_space(this);
3411 emit_optional_rex_32(dst, src);
3414 emit_sse_operand(dst, src);
3418 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
3419 EnsureSpace ensure_space(this);
3421 emit_optional_rex_32(dst, src);
3424 emit_sse_operand(dst, src);
3428 void Assembler::pmuludq(XMMRegister dst, const Operand& src) {
3429 EnsureSpace ensure_space(this);
3431 emit_optional_rex_32(dst, src);
3434 emit_sse_operand(dst, src);
3438 void Assembler::punpackldq(XMMRegister dst, XMMRegister src) {
3439 EnsureSpace ensure_space(this);
3441 emit_optional_rex_32(dst, src);
3444 emit_sse_operand(dst, src);
3448 void Assembler::punpackldq(XMMRegister dst, const Operand& src) {
3449 EnsureSpace ensure_space(this);
3451 emit_optional_rex_32(dst, src);
3454 emit_sse_operand(dst, src);
3458 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
3459 EnsureSpace ensure_space(this);
3461 emit_optional_rex_32(dst);
3464 emit_sse_operand(dst);
3469 void Assembler::cvtps2dq(XMMRegister dst, XMMRegister src) {
3470 EnsureSpace ensure_space(this);
3472 emit_optional_rex_32(dst, src);
3475 emit_sse_operand(dst, src);
3479 void Assembler::cvtps2dq(XMMRegister dst, const Operand& src) {
3480 EnsureSpace ensure_space(this);
3482 emit_optional_rex_32(dst, src);
3485 emit_sse_operand(dst, src);
3489 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
3490 EnsureSpace ensure_space(this);
3492 emit_optional_rex_32(dst, src);
3495 emit_sse_operand(dst, src);
3500 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) {
3501 Register ireg = { reg.code() };
3502 emit_operand(ireg, adr);
3506 void Assembler::emit_sse_operand(Register reg, const Operand& adr) {
3507 Register ireg = {reg.code()};
3508 emit_operand(ireg, adr);
3512 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
3513 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
3517 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
3518 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
3522 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
3523 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
3527 void Assembler::emit_sse_operand(XMMRegister dst) {
3528 emit(0xD8 | dst.low_bits());
3532 void Assembler::db(uint8_t data) {
3533 EnsureSpace ensure_space(this);
3538 void Assembler::dd(uint32_t data) {
3539 EnsureSpace ensure_space(this);
3544 // Relocation information implementations.
3546 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
3547 DCHECK(!RelocInfo::IsNone(rmode));
3548 // Don't record external references unless the heap will be serialized.
3549 if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
3550 !serializer_enabled() && !emit_debug_code()) {
3552 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) {
3553 // Don't record psuedo relocation info for code age sequence mode.
3556 RelocInfo rinfo(pc_, rmode, data, NULL);
3557 reloc_info_writer.Write(&rinfo);
3561 void Assembler::RecordJSReturn() {
3562 positions_recorder()->WriteRecordedPositions();
3563 EnsureSpace ensure_space(this);
3564 RecordRelocInfo(RelocInfo::JS_RETURN);
3568 void Assembler::RecordDebugBreakSlot() {
3569 positions_recorder()->WriteRecordedPositions();
3570 EnsureSpace ensure_space(this);
3571 RecordRelocInfo(RelocInfo::DEBUG_BREAK_SLOT);
3575 void Assembler::RecordComment(const char* msg, bool force) {
3576 if (FLAG_code_comments || force) {
3577 EnsureSpace ensure_space(this);
3578 RecordRelocInfo(RelocInfo::COMMENT, reinterpret_cast<intptr_t>(msg));
3583 Handle<ConstantPoolArray> Assembler::NewConstantPool(Isolate* isolate) {
3584 // No out-of-line constant pool support.
3585 DCHECK(!FLAG_enable_ool_constant_pool);
3586 return isolate->factory()->empty_constant_pool_array();
3590 void Assembler::PopulateConstantPool(ConstantPoolArray* constant_pool) {
3591 // No out-of-line constant pool support.
3592 DCHECK(!FLAG_enable_ool_constant_pool);
3597 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
3598 1 << RelocInfo::RUNTIME_ENTRY |
3599 1 << RelocInfo::INTERNAL_REFERENCE |
3600 1 << RelocInfo::CODE_AGE_SEQUENCE;
3603 bool RelocInfo::IsCodedSpecially() {
3604 // The deserializer needs to know whether a pointer is specially coded. Being
3605 // specially coded on x64 means that it is a relative 32 bit address, as used
3606 // by branch instructions.
3607 return (1 << rmode_) & kApplyMask;
3611 bool RelocInfo::IsInConstantPool() {
3616 } } // namespace v8::internal
3618 #endif // V8_TARGET_ARCH_X64