src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/scopes.h"
  11 #include "src/x64/assembler-x64.h"
  12 #include "src/x64/macro-assembler-x64.h"
  13
  14 namespace v8 {
  15 namespace internal {
  16 namespace compiler {
  17
  18 #define __ masm()->
  19
  20
  21 #define kScratchDoubleReg xmm0
  22
  23
  24 // Adds X64 specific methods for decoding operands.
  25 class X64OperandConverter : public InstructionOperandConverter {
  26  public:
  27   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  28       : InstructionOperandConverter(gen, instr) {}
  29
  30   Immediate InputImmediate(size_t index) {
  31     return ToImmediate(instr_->InputAt(index));
  32   }
  33
  34   Operand InputOperand(size_t index, int extra = 0) {
  35     return ToOperand(instr_->InputAt(index), extra);
  36   }
  37
  38   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  39
  40   Immediate ToImmediate(InstructionOperand* operand) {
  41     Constant constant = ToConstant(operand);
  42     if (constant.type() == Constant::kFloat64) {
  43       DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
  44       return Immediate(0);
  45     }
  46     return Immediate(constant.ToInt32());
  47   }
  48
  49   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  50     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  51     FrameOffset offset =
  52         linkage()->GetFrameOffset(AllocatedOperand::cast(op)->index(), frame());
  53     return Operand(offset.from_stack_pointer() ? rsp : rbp,
  54                    offset.offset() + extra);
  55   }
  56
  57   static size_t NextOffset(size_t* offset) {
  58     size_t i = *offset;
  59     (*offset)++;
  60     return i;
  61   }
  62
  63   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  64     STATIC_ASSERT(0 == static_cast<int>(times_1));
  65     STATIC_ASSERT(1 == static_cast<int>(times_2));
  66     STATIC_ASSERT(2 == static_cast<int>(times_4));
  67     STATIC_ASSERT(3 == static_cast<int>(times_8));
  68     int scale = static_cast<int>(mode - one);
  69     DCHECK(scale >= 0 && scale < 4);
  70     return static_cast<ScaleFactor>(scale);
  71   }
  72
  73   Operand MemoryOperand(size_t* offset) {
  74     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  75     switch (mode) {
  76       case kMode_MR: {
  77         Register base = InputRegister(NextOffset(offset));
  78         int32_t disp = 0;
  79         return Operand(base, disp);
  80       }
  81       case kMode_MRI: {
  82         Register base = InputRegister(NextOffset(offset));
  83         int32_t disp = InputInt32(NextOffset(offset));
  84         return Operand(base, disp);
  85       }
  86       case kMode_MR1:
  87       case kMode_MR2:
  88       case kMode_MR4:
  89       case kMode_MR8: {
  90         Register base = InputRegister(NextOffset(offset));
  91         Register index = InputRegister(NextOffset(offset));
  92         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  93         int32_t disp = 0;
  94         return Operand(base, index, scale, disp);
  95       }
  96       case kMode_MR1I:
  97       case kMode_MR2I:
  98       case kMode_MR4I:
  99       case kMode_MR8I: {
 100         Register base = InputRegister(NextOffset(offset));
 101         Register index = InputRegister(NextOffset(offset));
 102         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
 103         int32_t disp = InputInt32(NextOffset(offset));
 104         return Operand(base, index, scale, disp);
 105       }
 106       case kMode_M1: {
 107         Register base = InputRegister(NextOffset(offset));
 108         int32_t disp = 0;
 109         return Operand(base, disp);
 110       }
 111       case kMode_M2:
 112         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 113         return Operand(no_reg, 0);
 114       case kMode_M4:
 115       case kMode_M8: {
 116         Register index = InputRegister(NextOffset(offset));
 117         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 118         int32_t disp = 0;
 119         return Operand(index, scale, disp);
 120       }
 121       case kMode_M1I:
 122       case kMode_M2I:
 123       case kMode_M4I:
 124       case kMode_M8I: {
 125         Register index = InputRegister(NextOffset(offset));
 126         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 127         int32_t disp = InputInt32(NextOffset(offset));
 128         return Operand(index, scale, disp);
 129       }
 130       case kMode_None:
 131         UNREACHABLE();
 132         return Operand(no_reg, 0);
 133     }
 134     UNREACHABLE();
 135     return Operand(no_reg, 0);
 136   }
 137
 138   Operand MemoryOperand(size_t first_input = 0) {
 139     return MemoryOperand(&first_input);
 140   }
 141 };
 142
 143
 144 namespace {
 145
 146 bool HasImmediateInput(Instruction* instr, size_t index) {
 147   return instr->InputAt(index)->IsImmediate();
 148 }
 149
 150
 151 class OutOfLineLoadZero final : public OutOfLineCode {
 152  public:
 153   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 154       : OutOfLineCode(gen), result_(result) {}
 155
 156   void Generate() final { __ xorl(result_, result_); }
 157
 158  private:
 159   Register const result_;
 160 };
 161
 162
 163 class OutOfLineLoadNaN final : public OutOfLineCode {
 164  public:
 165   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 166       : OutOfLineCode(gen), result_(result) {}
 167
 168   void Generate() final { __ pcmpeqd(result_, result_); }
 169
 170  private:
 171   XMMRegister const result_;
 172 };
 173
 174
 175 class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
 176  public:
 177   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 178                              XMMRegister input)
 179       : OutOfLineCode(gen), result_(result), input_(input) {}
 180
 181   void Generate() final {
 182     __ subp(rsp, Immediate(kDoubleSize));
 183     __ movsd(MemOperand(rsp, 0), input_);
 184     __ SlowTruncateToI(result_, rsp, 0);
 185     __ addp(rsp, Immediate(kDoubleSize));
 186   }
 187
 188  private:
 189   Register const result_;
 190   XMMRegister const input_;
 191 };
 192
 193 }  // namespace
 194
 195
 196 #define ASSEMBLE_UNOP(asm_instr)         \
 197   do {                                   \
 198     if (instr->Output()->IsRegister()) { \
 199       __ asm_instr(i.OutputRegister());  \
 200     } else {                             \
 201       __ asm_instr(i.OutputOperand());   \
 202     }                                    \
 203   } while (0)
 204
 205
 206 #define ASSEMBLE_BINOP(asm_instr)                              \
 207   do {                                                         \
 208     if (HasImmediateInput(instr, 1)) {                         \
 209       if (instr->InputAt(0)->IsRegister()) {                   \
 210         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 211       } else {                                                 \
 212         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 213       }                                                        \
 214     } else {                                                   \
 215       if (instr->InputAt(1)->IsRegister()) {                   \
 216         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 217       } else {                                                 \
 218         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 219       }                                                        \
 220     }                                                          \
 221   } while (0)
 222
 223
 224 #define ASSEMBLE_MULT(asm_instr)                              \
 225   do {                                                        \
 226     if (HasImmediateInput(instr, 1)) {                        \
 227       if (instr->InputAt(0)->IsRegister()) {                  \
 228         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 229                      i.InputImmediate(1));                    \
 230       } else {                                                \
 231         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 232                      i.InputImmediate(1));                    \
 233       }                                                       \
 234     } else {                                                  \
 235       if (instr->InputAt(1)->IsRegister()) {                  \
 236         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 237       } else {                                                \
 238         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 239       }                                                       \
 240     }                                                         \
 241   } while (0)
 242
 243
 244 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 245   do {                                                                     \
 246     if (HasImmediateInput(instr, 1)) {                                     \
 247       if (instr->Output()->IsRegister()) {                                 \
 248         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 249       } else {                                                             \
 250         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 251       }                                                                    \
 252     } else {                                                               \
 253       if (instr->Output()->IsRegister()) {                                 \
 254         __ asm_instr##_cl(i.OutputRegister());                             \
 255       } else {                                                             \
 256         __ asm_instr##_cl(i.OutputOperand());                              \
 257       }                                                                    \
 258     }                                                                      \
 259   } while (0)
 260
 261
 262 #define ASSEMBLE_MOVX(asm_instr)                            \
 263   do {                                                      \
 264     if (instr->addressing_mode() != kMode_None) {           \
 265       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
 266     } else if (instr->InputAt(0)->IsRegister()) {           \
 267       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
 268     } else {                                                \
 269       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
 270     }                                                       \
 271   } while (0)
 272
 273
 274 #define ASSEMBLE_SSE_BINOP(asm_instr)                                   \
 275   do {                                                                  \
 276     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 277       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 278     } else {                                                            \
 279       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 280     }                                                                   \
 281   } while (0)
 282
 283
 284 #define ASSEMBLE_SSE_UNOP(asm_instr)                                    \
 285   do {                                                                  \
 286     if (instr->InputAt(0)->IsDoubleRegister()) {                        \
 287       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
 288     } else {                                                            \
 289       __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0));        \
 290     }                                                                   \
 291   } while (0)
 292
 293
 294 #define ASSEMBLE_AVX_BINOP(asm_instr)                                  \
 295   do {                                                                 \
 296     CpuFeatureScope avx_scope(masm(), AVX);                            \
 297     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 298       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 299                    i.InputDoubleRegister(1));                          \
 300     } else {                                                           \
 301       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 302                    i.InputOperand(1));                                 \
 303     }                                                                  \
 304   } while (0)
 305
 306
 307 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 308   do {                                                                       \
 309     auto result = i.OutputDoubleRegister();                                  \
 310     auto buffer = i.InputRegister(0);                                        \
 311     auto index1 = i.InputRegister(1);                                        \
 312     auto index2 = i.InputInt32(2);                                           \
 313     OutOfLineCode* ool;                                                      \
 314     if (instr->InputAt(3)->IsRegister()) {                                   \
 315       auto length = i.InputRegister(3);                                      \
 316       DCHECK_EQ(0, index2);                                                  \
 317       __ cmpl(index1, length);                                               \
 318       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 319     } else {                                                                 \
 320       auto length = i.InputInt32(3);                                         \
 321       DCHECK_LE(index2, length);                                             \
 322       __ cmpq(index1, Immediate(length - index2));                           \
 323       class OutOfLineLoadFloat final : public OutOfLineCode {                \
 324        public:                                                               \
 325         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 326                            Register buffer, Register index1, int32_t index2, \
 327                            int32_t length)                                   \
 328             : OutOfLineCode(gen),                                            \
 329               result_(result),                                               \
 330               buffer_(buffer),                                               \
 331               index1_(index1),                                               \
 332               index2_(index2),                                               \
 333               length_(length) {}                                             \
 334                                                                              \
 335         void Generate() final {                                              \
 336           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 337           __ pcmpeqd(result_, result_);                                      \
 338           __ cmpl(kScratchRegister, Immediate(length_));                     \
 339           __ j(above_equal, exit());                                         \
 340           __ asm_instr(result_,                                              \
 341                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 342         }                                                                    \
 343                                                                              \
 344        private:                                                              \
 345         XMMRegister const result_;                                           \
 346         Register const buffer_;                                              \
 347         Register const index1_;                                              \
 348         int32_t const index2_;                                               \
 349         int32_t const length_;                                               \
 350       };                                                                     \
 351       ool = new (zone())                                                     \
 352           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 353     }                                                                        \
 354     __ j(above_equal, ool->entry());                                         \
 355     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 356     __ bind(ool->exit());                                                    \
 357   } while (false)
 358
 359
 360 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 361   do {                                                                         \
 362     auto result = i.OutputRegister();                                          \
 363     auto buffer = i.InputRegister(0);                                          \
 364     auto index1 = i.InputRegister(1);                                          \
 365     auto index2 = i.InputInt32(2);                                             \
 366     OutOfLineCode* ool;                                                        \
 367     if (instr->InputAt(3)->IsRegister()) {                                     \
 368       auto length = i.InputRegister(3);                                        \
 369       DCHECK_EQ(0, index2);                                                    \
 370       __ cmpl(index1, length);                                                 \
 371       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 372     } else {                                                                   \
 373       auto length = i.InputInt32(3);                                           \
 374       DCHECK_LE(index2, length);                                               \
 375       __ cmpq(index1, Immediate(length - index2));                             \
 376       class OutOfLineLoadInteger final : public OutOfLineCode {                \
 377        public:                                                                 \
 378         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 379                              Register buffer, Register index1, int32_t index2, \
 380                              int32_t length)                                   \
 381             : OutOfLineCode(gen),                                              \
 382               result_(result),                                                 \
 383               buffer_(buffer),                                                 \
 384               index1_(index1),                                                 \
 385               index2_(index2),                                                 \
 386               length_(length) {}                                               \
 387                                                                                \
 388         void Generate() final {                                                \
 389           Label oob;                                                           \
 390           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 391           __ cmpl(kScratchRegister, Immediate(length_));                       \
 392           __ j(above_equal, &oob, Label::kNear);                               \
 393           __ asm_instr(result_,                                                \
 394                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 395           __ jmp(exit());                                                      \
 396           __ bind(&oob);                                                       \
 397           __ xorl(result_, result_);                                           \
 398         }                                                                      \
 399                                                                                \
 400        private:                                                                \
 401         Register const result_;                                                \
 402         Register const buffer_;                                                \
 403         Register const index1_;                                                \
 404         int32_t const index2_;                                                 \
 405         int32_t const length_;                                                 \
 406       };                                                                       \
 407       ool = new (zone())                                                       \
 408           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 409     }                                                                          \
 410     __ j(above_equal, ool->entry());                                           \
 411     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 412     __ bind(ool->exit());                                                      \
 413   } while (false)
 414
 415
 416 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 417   do {                                                                       \
 418     auto buffer = i.InputRegister(0);                                        \
 419     auto index1 = i.InputRegister(1);                                        \
 420     auto index2 = i.InputInt32(2);                                           \
 421     auto value = i.InputDoubleRegister(4);                                   \
 422     if (instr->InputAt(3)->IsRegister()) {                                   \
 423       auto length = i.InputRegister(3);                                      \
 424       DCHECK_EQ(0, index2);                                                  \
 425       Label done;                                                            \
 426       __ cmpl(index1, length);                                               \
 427       __ j(above_equal, &done, Label::kNear);                                \
 428       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 429       __ bind(&done);                                                        \
 430     } else {                                                                 \
 431       auto length = i.InputInt32(3);                                         \
 432       DCHECK_LE(index2, length);                                             \
 433       __ cmpq(index1, Immediate(length - index2));                           \
 434       class OutOfLineStoreFloat final : public OutOfLineCode {               \
 435        public:                                                               \
 436         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 437                             Register index1, int32_t index2, int32_t length, \
 438                             XMMRegister value)                               \
 439             : OutOfLineCode(gen),                                            \
 440               buffer_(buffer),                                               \
 441               index1_(index1),                                               \
 442               index2_(index2),                                               \
 443               length_(length),                                               \
 444               value_(value) {}                                               \
 445                                                                              \
 446         void Generate() final {                                              \
 447           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 448           __ cmpl(kScratchRegister, Immediate(length_));                     \
 449           __ j(above_equal, exit());                                         \
 450           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 451                        value_);                                              \
 452         }                                                                    \
 453                                                                              \
 454        private:                                                              \
 455         Register const buffer_;                                              \
 456         Register const index1_;                                              \
 457         int32_t const index2_;                                               \
 458         int32_t const length_;                                               \
 459         XMMRegister const value_;                                            \
 460       };                                                                     \
 461       auto ool = new (zone())                                                \
 462           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 463       __ j(above_equal, ool->entry());                                       \
 464       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 465       __ bind(ool->exit());                                                  \
 466     }                                                                        \
 467   } while (false)
 468
 469
 470 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 471   do {                                                                         \
 472     auto buffer = i.InputRegister(0);                                          \
 473     auto index1 = i.InputRegister(1);                                          \
 474     auto index2 = i.InputInt32(2);                                             \
 475     if (instr->InputAt(3)->IsRegister()) {                                     \
 476       auto length = i.InputRegister(3);                                        \
 477       DCHECK_EQ(0, index2);                                                    \
 478       Label done;                                                              \
 479       __ cmpl(index1, length);                                                 \
 480       __ j(above_equal, &done, Label::kNear);                                  \
 481       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 482       __ bind(&done);                                                          \
 483     } else {                                                                   \
 484       auto length = i.InputInt32(3);                                           \
 485       DCHECK_LE(index2, length);                                               \
 486       __ cmpq(index1, Immediate(length - index2));                             \
 487       class OutOfLineStoreInteger final : public OutOfLineCode {               \
 488        public:                                                                 \
 489         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 490                               Register index1, int32_t index2, int32_t length, \
 491                               Value value)                                     \
 492             : OutOfLineCode(gen),                                              \
 493               buffer_(buffer),                                                 \
 494               index1_(index1),                                                 \
 495               index2_(index2),                                                 \
 496               length_(length),                                                 \
 497               value_(value) {}                                                 \
 498                                                                                \
 499         void Generate() final {                                                \
 500           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 501           __ cmpl(kScratchRegister, Immediate(length_));                       \
 502           __ j(above_equal, exit());                                           \
 503           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 504                        value_);                                                \
 505         }                                                                      \
 506                                                                                \
 507        private:                                                                \
 508         Register const buffer_;                                                \
 509         Register const index1_;                                                \
 510         int32_t const index2_;                                                 \
 511         int32_t const length_;                                                 \
 512         Value const value_;                                                    \
 513       };                                                                       \
 514       auto ool = new (zone())                                                  \
 515           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 516       __ j(above_equal, ool->entry());                                         \
 517       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 518       __ bind(ool->exit());                                                    \
 519     }                                                                          \
 520   } while (false)
 521
 522
 523 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 524   do {                                                           \
 525     if (instr->InputAt(4)->IsRegister()) {                       \
 526       Register value = i.InputRegister(4);                       \
 527       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 528     } else {                                                     \
 529       Immediate value = i.InputImmediate(4);                     \
 530       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 531     }                                                            \
 532   } while (false)
 533
 534
 535 void CodeGenerator::AssembleDeconstructActivationRecord() {
 536   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
 537   int stack_slots = frame()->GetSpillSlotCount();
 538   if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
 539     __ movq(rsp, rbp);
 540     __ popq(rbp);
 541   }
 542 }
 543
 544
 545 // Assembles an instruction after register allocation, producing machine code.
 546 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 547   X64OperandConverter i(this, instr);
 548
 549   switch (ArchOpcodeField::decode(instr->opcode())) {
 550     case kArchCallCodeObject: {
 551       EnsureSpaceForLazyDeopt();
 552       if (HasImmediateInput(instr, 0)) {
 553         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 554         __ Call(code, RelocInfo::CODE_TARGET);
 555       } else {
 556         Register reg = i.InputRegister(0);
 557         int entry = Code::kHeaderSize - kHeapObjectTag;
 558         __ Call(Operand(reg, entry));
 559       }
 560       RecordCallPosition(instr);
 561       break;
 562     }
 563     case kArchTailCallCodeObject: {
 564       AssembleDeconstructActivationRecord();
 565       if (HasImmediateInput(instr, 0)) {
 566         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 567         __ jmp(code, RelocInfo::CODE_TARGET);
 568       } else {
 569         Register reg = i.InputRegister(0);
 570         __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
 571         __ jmp(reg);
 572       }
 573       break;
 574     }
 575     case kArchCallJSFunction: {
 576       EnsureSpaceForLazyDeopt();
 577       Register func = i.InputRegister(0);
 578       if (FLAG_debug_code) {
 579         // Check the function's context matches the context argument.
 580         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 581         __ Assert(equal, kWrongFunctionContext);
 582       }
 583       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 584       RecordCallPosition(instr);
 585       break;
 586     }
 587     case kArchTailCallJSFunction: {
 588       Register func = i.InputRegister(0);
 589       if (FLAG_debug_code) {
 590         // Check the function's context matches the context argument.
 591         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 592         __ Assert(equal, kWrongFunctionContext);
 593       }
 594       AssembleDeconstructActivationRecord();
 595       __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
 596       break;
 597     }
 598     case kArchPrepareCallCFunction: {
 599       int const num_parameters = MiscField::decode(instr->opcode());
 600       __ PrepareCallCFunction(num_parameters);
 601       break;
 602     }
 603     case kArchCallCFunction: {
 604       int const num_parameters = MiscField::decode(instr->opcode());
 605       if (HasImmediateInput(instr, 0)) {
 606         ExternalReference ref = i.InputExternalReference(0);
 607         __ CallCFunction(ref, num_parameters);
 608       } else {
 609         Register func = i.InputRegister(0);
 610         __ CallCFunction(func, num_parameters);
 611       }
 612       break;
 613     }
 614     case kArchJmp:
 615       AssembleArchJump(i.InputRpo(0));
 616       break;
 617     case kArchLookupSwitch:
 618       AssembleArchLookupSwitch(instr);
 619       break;
 620     case kArchTableSwitch:
 621       AssembleArchTableSwitch(instr);
 622       break;
 623     case kArchNop:
 624       // don't emit code for nops.
 625       break;
 626     case kArchDeoptimize: {
 627       int deopt_state_id =
 628           BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
 629       AssembleDeoptimizerCall(deopt_state_id, Deoptimizer::EAGER);
 630       break;
 631     }
 632     case kArchRet:
 633       AssembleReturn();
 634       break;
 635     case kArchStackPointer:
 636       __ movq(i.OutputRegister(), rsp);
 637       break;
 638     case kArchFramePointer:
 639       __ movq(i.OutputRegister(), rbp);
 640       break;
 641     case kArchTruncateDoubleToI: {
 642       auto result = i.OutputRegister();
 643       auto input = i.InputDoubleRegister(0);
 644       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 645       __ cvttsd2siq(result, input);
 646       __ cmpq(result, Immediate(1));
 647       __ j(overflow, ool->entry());
 648       __ bind(ool->exit());
 649       break;
 650     }
 651     case kX64Add32:
 652       ASSEMBLE_BINOP(addl);
 653       break;
 654     case kX64Add:
 655       ASSEMBLE_BINOP(addq);
 656       break;
 657     case kX64Sub32:
 658       ASSEMBLE_BINOP(subl);
 659       break;
 660     case kX64Sub:
 661       ASSEMBLE_BINOP(subq);
 662       break;
 663     case kX64And32:
 664       ASSEMBLE_BINOP(andl);
 665       break;
 666     case kX64And:
 667       ASSEMBLE_BINOP(andq);
 668       break;
 669     case kX64Cmp32:
 670       ASSEMBLE_BINOP(cmpl);
 671       break;
 672     case kX64Cmp:
 673       ASSEMBLE_BINOP(cmpq);
 674       break;
 675     case kX64Test32:
 676       ASSEMBLE_BINOP(testl);
 677       break;
 678     case kX64Test:
 679       ASSEMBLE_BINOP(testq);
 680       break;
 681     case kX64Imul32:
 682       ASSEMBLE_MULT(imull);
 683       break;
 684     case kX64Imul:
 685       ASSEMBLE_MULT(imulq);
 686       break;
 687     case kX64ImulHigh32:
 688       if (instr->InputAt(1)->IsRegister()) {
 689         __ imull(i.InputRegister(1));
 690       } else {
 691         __ imull(i.InputOperand(1));
 692       }
 693       break;
 694     case kX64UmulHigh32:
 695       if (instr->InputAt(1)->IsRegister()) {
 696         __ mull(i.InputRegister(1));
 697       } else {
 698         __ mull(i.InputOperand(1));
 699       }
 700       break;
 701     case kX64Idiv32:
 702       __ cdq();
 703       __ idivl(i.InputRegister(1));
 704       break;
 705     case kX64Idiv:
 706       __ cqo();
 707       __ idivq(i.InputRegister(1));
 708       break;
 709     case kX64Udiv32:
 710       __ xorl(rdx, rdx);
 711       __ divl(i.InputRegister(1));
 712       break;
 713     case kX64Udiv:
 714       __ xorq(rdx, rdx);
 715       __ divq(i.InputRegister(1));
 716       break;
 717     case kX64Not:
 718       ASSEMBLE_UNOP(notq);
 719       break;
 720     case kX64Not32:
 721       ASSEMBLE_UNOP(notl);
 722       break;
 723     case kX64Neg:
 724       ASSEMBLE_UNOP(negq);
 725       break;
 726     case kX64Neg32:
 727       ASSEMBLE_UNOP(negl);
 728       break;
 729     case kX64Or32:
 730       ASSEMBLE_BINOP(orl);
 731       break;
 732     case kX64Or:
 733       ASSEMBLE_BINOP(orq);
 734       break;
 735     case kX64Xor32:
 736       ASSEMBLE_BINOP(xorl);
 737       break;
 738     case kX64Xor:
 739       ASSEMBLE_BINOP(xorq);
 740       break;
 741     case kX64Shl32:
 742       ASSEMBLE_SHIFT(shll, 5);
 743       break;
 744     case kX64Shl:
 745       ASSEMBLE_SHIFT(shlq, 6);
 746       break;
 747     case kX64Shr32:
 748       ASSEMBLE_SHIFT(shrl, 5);
 749       break;
 750     case kX64Shr:
 751       ASSEMBLE_SHIFT(shrq, 6);
 752       break;
 753     case kX64Sar32:
 754       ASSEMBLE_SHIFT(sarl, 5);
 755       break;
 756     case kX64Sar:
 757       ASSEMBLE_SHIFT(sarq, 6);
 758       break;
 759     case kX64Ror32:
 760       ASSEMBLE_SHIFT(rorl, 5);
 761       break;
 762     case kX64Ror:
 763       ASSEMBLE_SHIFT(rorq, 6);
 764       break;
 765     case kX64Lzcnt32:
 766       if (instr->InputAt(0)->IsRegister()) {
 767         __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
 768       } else {
 769         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
 770       }
 771       break;
 772     case kSSEFloat32Cmp:
 773       ASSEMBLE_SSE_BINOP(ucomiss);
 774       break;
 775     case kSSEFloat32Add:
 776       ASSEMBLE_SSE_BINOP(addss);
 777       break;
 778     case kSSEFloat32Sub:
 779       ASSEMBLE_SSE_BINOP(subss);
 780       break;
 781     case kSSEFloat32Mul:
 782       ASSEMBLE_SSE_BINOP(mulss);
 783       break;
 784     case kSSEFloat32Div:
 785       ASSEMBLE_SSE_BINOP(divss);
 786       // Don't delete this mov. It may improve performance on some CPUs,
 787       // when there is a (v)mulss depending on the result.
 788       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 789       break;
 790     case kSSEFloat32Abs: {
 791       // TODO(bmeurer): Use RIP relative 128-bit constants.
 792       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 793       __ psrlq(kScratchDoubleReg, 33);
 794       __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
 795       break;
 796     }
 797     case kSSEFloat32Neg: {
 798       // TODO(bmeurer): Use RIP relative 128-bit constants.
 799       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 800       __ psllq(kScratchDoubleReg, 31);
 801       __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
 802       break;
 803     }
 804     case kSSEFloat32Sqrt:
 805       ASSEMBLE_SSE_UNOP(sqrtss);
 806       break;
 807     case kSSEFloat32Max:
 808       ASSEMBLE_SSE_BINOP(maxss);
 809       break;
 810     case kSSEFloat32Min:
 811       ASSEMBLE_SSE_BINOP(minss);
 812       break;
 813     case kSSEFloat32ToFloat64:
 814       ASSEMBLE_SSE_UNOP(cvtss2sd);
 815       break;
 816     case kSSEFloat64Cmp:
 817       ASSEMBLE_SSE_BINOP(ucomisd);
 818       break;
 819     case kSSEFloat64Add:
 820       ASSEMBLE_SSE_BINOP(addsd);
 821       break;
 822     case kSSEFloat64Sub:
 823       ASSEMBLE_SSE_BINOP(subsd);
 824       break;
 825     case kSSEFloat64Mul:
 826       ASSEMBLE_SSE_BINOP(mulsd);
 827       break;
 828     case kSSEFloat64Div:
 829       ASSEMBLE_SSE_BINOP(divsd);
 830       // Don't delete this mov. It may improve performance on some CPUs,
 831       // when there is a (v)mulsd depending on the result.
 832       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 833       break;
 834     case kSSEFloat64Mod: {
 835       __ subq(rsp, Immediate(kDoubleSize));
 836       // Move values to st(0) and st(1).
 837       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 838       __ fld_d(Operand(rsp, 0));
 839       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 840       __ fld_d(Operand(rsp, 0));
 841       // Loop while fprem isn't done.
 842       Label mod_loop;
 843       __ bind(&mod_loop);
 844       // This instructions traps on all kinds inputs, but we are assuming the
 845       // floating point control word is set to ignore them all.
 846       __ fprem();
 847       // The following 2 instruction implicitly use rax.
 848       __ fnstsw_ax();
 849       if (CpuFeatures::IsSupported(SAHF)) {
 850         CpuFeatureScope sahf_scope(masm(), SAHF);
 851         __ sahf();
 852       } else {
 853         __ shrl(rax, Immediate(8));
 854         __ andl(rax, Immediate(0xFF));
 855         __ pushq(rax);
 856         __ popfq();
 857       }
 858       __ j(parity_even, &mod_loop);
 859       // Move output to stack and clean up.
 860       __ fstp(1);
 861       __ fstp_d(Operand(rsp, 0));
 862       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 863       __ addq(rsp, Immediate(kDoubleSize));
 864       break;
 865     }
 866     case kSSEFloat64Max:
 867       ASSEMBLE_SSE_BINOP(maxsd);
 868       break;
 869     case kSSEFloat64Min:
 870       ASSEMBLE_SSE_BINOP(minsd);
 871       break;
 872     case kSSEFloat64Abs: {
 873       // TODO(bmeurer): Use RIP relative 128-bit constants.
 874       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 875       __ psrlq(kScratchDoubleReg, 1);
 876       __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 877       break;
 878     }
 879     case kSSEFloat64Neg: {
 880       // TODO(bmeurer): Use RIP relative 128-bit constants.
 881       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 882       __ psllq(kScratchDoubleReg, 63);
 883       __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 884       break;
 885     }
 886     case kSSEFloat64Sqrt:
 887       ASSEMBLE_SSE_UNOP(sqrtsd);
 888       break;
 889     case kSSEFloat64Round: {
 890       CpuFeatureScope sse_scope(masm(), SSE4_1);
 891       RoundingMode const mode =
 892           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
 893       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
 894       break;
 895     }
 896     case kSSEFloat64ToFloat32:
 897       ASSEMBLE_SSE_UNOP(cvtsd2ss);
 898       break;
 899     case kSSEFloat64ToInt32:
 900       if (instr->InputAt(0)->IsDoubleRegister()) {
 901         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 902       } else {
 903         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 904       }
 905       break;
 906     case kSSEFloat64ToUint32: {
 907       if (instr->InputAt(0)->IsDoubleRegister()) {
 908         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 909       } else {
 910         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 911       }
 912       __ AssertZeroExtended(i.OutputRegister());
 913       break;
 914     }
 915     case kSSEInt32ToFloat64:
 916       if (instr->InputAt(0)->IsRegister()) {
 917         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 918       } else {
 919         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 920       }
 921       break;
 922     case kSSEUint32ToFloat64:
 923       if (instr->InputAt(0)->IsRegister()) {
 924         __ movl(kScratchRegister, i.InputRegister(0));
 925       } else {
 926         __ movl(kScratchRegister, i.InputOperand(0));
 927       }
 928       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 929       break;
 930     case kSSEFloat64ExtractLowWord32:
 931       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 932         __ movl(i.OutputRegister(), i.InputOperand(0));
 933       } else {
 934         __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
 935       }
 936       break;
 937     case kSSEFloat64ExtractHighWord32:
 938       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 939         __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
 940       } else {
 941         __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
 942       }
 943       break;
 944     case kSSEFloat64InsertLowWord32:
 945       if (instr->InputAt(1)->IsRegister()) {
 946         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
 947       } else {
 948         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
 949       }
 950       break;
 951     case kSSEFloat64InsertHighWord32:
 952       if (instr->InputAt(1)->IsRegister()) {
 953         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
 954       } else {
 955         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
 956       }
 957       break;
 958     case kSSEFloat64LoadLowWord32:
 959       if (instr->InputAt(0)->IsRegister()) {
 960         __ movd(i.OutputDoubleRegister(), i.InputRegister(0));
 961       } else {
 962         __ movd(i.OutputDoubleRegister(), i.InputOperand(0));
 963       }
 964       break;
 965     case kAVXFloat32Cmp: {
 966       CpuFeatureScope avx_scope(masm(), AVX);
 967       if (instr->InputAt(1)->IsDoubleRegister()) {
 968         __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
 969       } else {
 970         __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
 971       }
 972       break;
 973     }
 974     case kAVXFloat32Add:
 975       ASSEMBLE_AVX_BINOP(vaddss);
 976       break;
 977     case kAVXFloat32Sub:
 978       ASSEMBLE_AVX_BINOP(vsubss);
 979       break;
 980     case kAVXFloat32Mul:
 981       ASSEMBLE_AVX_BINOP(vmulss);
 982       break;
 983     case kAVXFloat32Div:
 984       ASSEMBLE_AVX_BINOP(vdivss);
 985       // Don't delete this mov. It may improve performance on some CPUs,
 986       // when there is a (v)mulss depending on the result.
 987       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 988       break;
 989     case kAVXFloat32Max:
 990       ASSEMBLE_AVX_BINOP(vmaxss);
 991       break;
 992     case kAVXFloat32Min:
 993       ASSEMBLE_AVX_BINOP(vminss);
 994       break;
 995     case kAVXFloat64Cmp: {
 996       CpuFeatureScope avx_scope(masm(), AVX);
 997       if (instr->InputAt(1)->IsDoubleRegister()) {
 998         __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
 999       } else {
1000         __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1001       }
1002       break;
1003     }
1004     case kAVXFloat64Add:
1005       ASSEMBLE_AVX_BINOP(vaddsd);
1006       break;
1007     case kAVXFloat64Sub:
1008       ASSEMBLE_AVX_BINOP(vsubsd);
1009       break;
1010     case kAVXFloat64Mul:
1011       ASSEMBLE_AVX_BINOP(vmulsd);
1012       break;
1013     case kAVXFloat64Div:
1014       ASSEMBLE_AVX_BINOP(vdivsd);
1015       // Don't delete this mov. It may improve performance on some CPUs,
1016       // when there is a (v)mulsd depending on the result.
1017       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1018       break;
1019     case kAVXFloat64Max:
1020       ASSEMBLE_AVX_BINOP(vmaxsd);
1021       break;
1022     case kAVXFloat64Min:
1023       ASSEMBLE_AVX_BINOP(vminsd);
1024       break;
1025     case kAVXFloat32Abs: {
1026       // TODO(bmeurer): Use RIP relative 128-bit constants.
1027       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1028       __ psrlq(kScratchDoubleReg, 33);
1029       CpuFeatureScope avx_scope(masm(), AVX);
1030       if (instr->InputAt(0)->IsDoubleRegister()) {
1031         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1032                   i.InputDoubleRegister(0));
1033       } else {
1034         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1035                   i.InputOperand(0));
1036       }
1037       break;
1038     }
1039     case kAVXFloat32Neg: {
1040       // TODO(bmeurer): Use RIP relative 128-bit constants.
1041       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1042       __ psllq(kScratchDoubleReg, 31);
1043       CpuFeatureScope avx_scope(masm(), AVX);
1044       if (instr->InputAt(0)->IsDoubleRegister()) {
1045         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1046                   i.InputDoubleRegister(0));
1047       } else {
1048         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1049                   i.InputOperand(0));
1050       }
1051       break;
1052     }
1053     case kAVXFloat64Abs: {
1054       // TODO(bmeurer): Use RIP relative 128-bit constants.
1055       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1056       __ psrlq(kScratchDoubleReg, 1);
1057       CpuFeatureScope avx_scope(masm(), AVX);
1058       if (instr->InputAt(0)->IsDoubleRegister()) {
1059         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1060                   i.InputDoubleRegister(0));
1061       } else {
1062         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1063                   i.InputOperand(0));
1064       }
1065       break;
1066     }
1067     case kAVXFloat64Neg: {
1068       // TODO(bmeurer): Use RIP relative 128-bit constants.
1069       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1070       __ psllq(kScratchDoubleReg, 63);
1071       CpuFeatureScope avx_scope(masm(), AVX);
1072       if (instr->InputAt(0)->IsDoubleRegister()) {
1073         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1074                   i.InputDoubleRegister(0));
1075       } else {
1076         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1077                   i.InputOperand(0));
1078       }
1079       break;
1080     }
1081     case kX64Movsxbl:
1082       ASSEMBLE_MOVX(movsxbl);
1083       __ AssertZeroExtended(i.OutputRegister());
1084       break;
1085     case kX64Movzxbl:
1086       ASSEMBLE_MOVX(movzxbl);
1087       __ AssertZeroExtended(i.OutputRegister());
1088       break;
1089     case kX64Movb: {
1090       size_t index = 0;
1091       Operand operand = i.MemoryOperand(&index);
1092       if (HasImmediateInput(instr, index)) {
1093         __ movb(operand, Immediate(i.InputInt8(index)));
1094       } else {
1095         __ movb(operand, i.InputRegister(index));
1096       }
1097       break;
1098     }
1099     case kX64Movsxwl:
1100       ASSEMBLE_MOVX(movsxwl);
1101       __ AssertZeroExtended(i.OutputRegister());
1102       break;
1103     case kX64Movzxwl:
1104       ASSEMBLE_MOVX(movzxwl);
1105       __ AssertZeroExtended(i.OutputRegister());
1106       break;
1107     case kX64Movw: {
1108       size_t index = 0;
1109       Operand operand = i.MemoryOperand(&index);
1110       if (HasImmediateInput(instr, index)) {
1111         __ movw(operand, Immediate(i.InputInt16(index)));
1112       } else {
1113         __ movw(operand, i.InputRegister(index));
1114       }
1115       break;
1116     }
1117     case kX64Movl:
1118       if (instr->HasOutput()) {
1119         if (instr->addressing_mode() == kMode_None) {
1120           if (instr->InputAt(0)->IsRegister()) {
1121             __ movl(i.OutputRegister(), i.InputRegister(0));
1122           } else {
1123             __ movl(i.OutputRegister(), i.InputOperand(0));
1124           }
1125         } else {
1126           __ movl(i.OutputRegister(), i.MemoryOperand());
1127         }
1128         __ AssertZeroExtended(i.OutputRegister());
1129       } else {
1130         size_t index = 0;
1131         Operand operand = i.MemoryOperand(&index);
1132         if (HasImmediateInput(instr, index)) {
1133           __ movl(operand, i.InputImmediate(index));
1134         } else {
1135           __ movl(operand, i.InputRegister(index));
1136         }
1137       }
1138       break;
1139     case kX64Movsxlq:
1140       ASSEMBLE_MOVX(movsxlq);
1141       break;
1142     case kX64Movq:
1143       if (instr->HasOutput()) {
1144         __ movq(i.OutputRegister(), i.MemoryOperand());
1145       } else {
1146         size_t index = 0;
1147         Operand operand = i.MemoryOperand(&index);
1148         if (HasImmediateInput(instr, index)) {
1149           __ movq(operand, i.InputImmediate(index));
1150         } else {
1151           __ movq(operand, i.InputRegister(index));
1152         }
1153       }
1154       break;
1155     case kX64Movss:
1156       if (instr->HasOutput()) {
1157         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1158       } else {
1159         size_t index = 0;
1160         Operand operand = i.MemoryOperand(&index);
1161         __ movss(operand, i.InputDoubleRegister(index));
1162       }
1163       break;
1164     case kX64Movsd:
1165       if (instr->HasOutput()) {
1166         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1167       } else {
1168         size_t index = 0;
1169         Operand operand = i.MemoryOperand(&index);
1170         __ movsd(operand, i.InputDoubleRegister(index));
1171       }
1172       break;
1173     case kX64Lea32: {
1174       AddressingMode mode = AddressingModeField::decode(instr->opcode());
1175       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
1176       // and addressing mode just happens to work out. The "addl"/"subl" forms
1177       // in these cases are faster based on measurements.
1178       if (i.InputRegister(0).is(i.OutputRegister())) {
1179         if (mode == kMode_MRI) {
1180           int32_t constant_summand = i.InputInt32(1);
1181           if (constant_summand > 0) {
1182             __ addl(i.OutputRegister(), Immediate(constant_summand));
1183           } else if (constant_summand < 0) {
1184             __ subl(i.OutputRegister(), Immediate(-constant_summand));
1185           }
1186         } else if (mode == kMode_MR1) {
1187           if (i.InputRegister(1).is(i.OutputRegister())) {
1188             __ shll(i.OutputRegister(), Immediate(1));
1189           } else {
1190             __ leal(i.OutputRegister(), i.MemoryOperand());
1191           }
1192         } else if (mode == kMode_M2) {
1193           __ shll(i.OutputRegister(), Immediate(1));
1194         } else if (mode == kMode_M4) {
1195           __ shll(i.OutputRegister(), Immediate(2));
1196         } else if (mode == kMode_M8) {
1197           __ shll(i.OutputRegister(), Immediate(3));
1198         } else {
1199           __ leal(i.OutputRegister(), i.MemoryOperand());
1200         }
1201       } else {
1202         __ leal(i.OutputRegister(), i.MemoryOperand());
1203       }
1204       __ AssertZeroExtended(i.OutputRegister());
1205       break;
1206     }
1207     case kX64Lea:
1208       __ leaq(i.OutputRegister(), i.MemoryOperand());
1209       break;
1210     case kX64Dec32:
1211       __ decl(i.OutputRegister());
1212       break;
1213     case kX64Inc32:
1214       __ incl(i.OutputRegister());
1215       break;
1216     case kX64Push:
1217       if (HasImmediateInput(instr, 0)) {
1218         __ pushq(i.InputImmediate(0));
1219       } else {
1220         if (instr->InputAt(0)->IsRegister()) {
1221           __ pushq(i.InputRegister(0));
1222         } else if (instr->InputAt(0)->IsDoubleRegister()) {
1223           // TODO(titzer): use another machine instruction?
1224           __ subq(rsp, Immediate(kDoubleSize));
1225           __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1226         } else {
1227           __ pushq(i.InputOperand(0));
1228         }
1229       }
1230       break;
1231     case kX64Poke: {
1232       int const slot = MiscField::decode(instr->opcode());
1233       if (HasImmediateInput(instr, 0)) {
1234         __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
1235       } else {
1236         __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
1237       }
1238       break;
1239     }
1240     case kX64StoreWriteBarrier: {
1241       Register object = i.InputRegister(0);
1242       Register value = i.InputRegister(2);
1243       SaveFPRegsMode mode =
1244           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
1245       if (HasImmediateInput(instr, 1)) {
1246         int index = i.InputInt32(1);
1247         Register scratch = i.TempRegister(1);
1248         __ movq(Operand(object, index), value);
1249         __ RecordWriteContextSlot(object, index, value, scratch, mode);
1250       } else {
1251         Register index = i.InputRegister(1);
1252         __ movq(Operand(object, index, times_1, 0), value);
1253         __ leaq(index, Operand(object, index, times_1, 0));
1254         __ RecordWrite(object, index, value, mode);
1255       }
1256       break;
1257     }
1258     case kCheckedLoadInt8:
1259       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
1260       break;
1261     case kCheckedLoadUint8:
1262       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
1263       break;
1264     case kCheckedLoadInt16:
1265       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
1266       break;
1267     case kCheckedLoadUint16:
1268       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
1269       break;
1270     case kCheckedLoadWord32:
1271       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
1272       break;
1273     case kCheckedLoadFloat32:
1274       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
1275       break;
1276     case kCheckedLoadFloat64:
1277       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
1278       break;
1279     case kCheckedStoreWord8:
1280       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
1281       break;
1282     case kCheckedStoreWord16:
1283       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
1284       break;
1285     case kCheckedStoreWord32:
1286       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1287       break;
1288     case kCheckedStoreFloat32:
1289       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1290       break;
1291     case kCheckedStoreFloat64:
1292       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1293       break;
1294     case kX64StackCheck:
1295       __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
1296       break;
1297   }
1298 }  // NOLINT(readability/fn_size)
1299
1300
1301 // Assembles branches after this instruction.
1302 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1303   X64OperandConverter i(this, instr);
1304   Label::Distance flabel_distance =
1305       branch->fallthru ? Label::kNear : Label::kFar;
1306   Label* tlabel = branch->true_label;
1307   Label* flabel = branch->false_label;
1308   switch (branch->condition) {
1309     case kUnorderedEqual:
1310       __ j(parity_even, flabel, flabel_distance);
1311     // Fall through.
1312     case kEqual:
1313       __ j(equal, tlabel);
1314       break;
1315     case kUnorderedNotEqual:
1316       __ j(parity_even, tlabel);
1317     // Fall through.
1318     case kNotEqual:
1319       __ j(not_equal, tlabel);
1320       break;
1321     case kSignedLessThan:
1322       __ j(less, tlabel);
1323       break;
1324     case kSignedGreaterThanOrEqual:
1325       __ j(greater_equal, tlabel);
1326       break;
1327     case kSignedLessThanOrEqual:
1328       __ j(less_equal, tlabel);
1329       break;
1330     case kSignedGreaterThan:
1331       __ j(greater, tlabel);
1332       break;
1333     case kUnsignedLessThan:
1334       __ j(below, tlabel);
1335       break;
1336     case kUnsignedGreaterThanOrEqual:
1337       __ j(above_equal, tlabel);
1338       break;
1339     case kUnsignedLessThanOrEqual:
1340       __ j(below_equal, tlabel);
1341       break;
1342     case kUnsignedGreaterThan:
1343       __ j(above, tlabel);
1344       break;
1345     case kOverflow:
1346       __ j(overflow, tlabel);
1347       break;
1348     case kNotOverflow:
1349       __ j(no_overflow, tlabel);
1350       break;
1351   }
1352   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1353 }
1354
1355
1356 void CodeGenerator::AssembleArchJump(RpoNumber target) {
1357   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1358 }
1359
1360
1361 // Assembles boolean materializations after this instruction.
1362 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1363                                         FlagsCondition condition) {
1364   X64OperandConverter i(this, instr);
1365   Label done;
1366
1367   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1368   // last output of the instruction.
1369   Label check;
1370   DCHECK_NE(0u, instr->OutputCount());
1371   Register reg = i.OutputRegister(instr->OutputCount() - 1);
1372   Condition cc = no_condition;
1373   switch (condition) {
1374     case kUnorderedEqual:
1375       __ j(parity_odd, &check, Label::kNear);
1376       __ movl(reg, Immediate(0));
1377       __ jmp(&done, Label::kNear);
1378     // Fall through.
1379     case kEqual:
1380       cc = equal;
1381       break;
1382     case kUnorderedNotEqual:
1383       __ j(parity_odd, &check, Label::kNear);
1384       __ movl(reg, Immediate(1));
1385       __ jmp(&done, Label::kNear);
1386     // Fall through.
1387     case kNotEqual:
1388       cc = not_equal;
1389       break;
1390     case kSignedLessThan:
1391       cc = less;
1392       break;
1393     case kSignedGreaterThanOrEqual:
1394       cc = greater_equal;
1395       break;
1396     case kSignedLessThanOrEqual:
1397       cc = less_equal;
1398       break;
1399     case kSignedGreaterThan:
1400       cc = greater;
1401       break;
1402     case kUnsignedLessThan:
1403       cc = below;
1404       break;
1405     case kUnsignedGreaterThanOrEqual:
1406       cc = above_equal;
1407       break;
1408     case kUnsignedLessThanOrEqual:
1409       cc = below_equal;
1410       break;
1411     case kUnsignedGreaterThan:
1412       cc = above;
1413       break;
1414     case kOverflow:
1415       cc = overflow;
1416       break;
1417     case kNotOverflow:
1418       cc = no_overflow;
1419       break;
1420   }
1421   __ bind(&check);
1422   __ setcc(cc, reg);
1423   __ movzxbl(reg, reg);
1424   __ bind(&done);
1425 }
1426
1427
1428 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1429   X64OperandConverter i(this, instr);
1430   Register input = i.InputRegister(0);
1431   for (size_t index = 2; index < instr->InputCount(); index += 2) {
1432     __ cmpl(input, Immediate(i.InputInt32(index + 0)));
1433     __ j(equal, GetLabel(i.InputRpo(index + 1)));
1434   }
1435   AssembleArchJump(i.InputRpo(1));
1436 }
1437
1438
1439 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1440   X64OperandConverter i(this, instr);
1441   Register input = i.InputRegister(0);
1442   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
1443   Label** cases = zone()->NewArray<Label*>(case_count);
1444   for (int32_t index = 0; index < case_count; ++index) {
1445     cases[index] = GetLabel(i.InputRpo(index + 2));
1446   }
1447   Label* const table = AddJumpTable(cases, case_count);
1448   __ cmpl(input, Immediate(case_count));
1449   __ j(above_equal, GetLabel(i.InputRpo(1)));
1450   __ leaq(kScratchRegister, Operand(table));
1451   __ jmp(Operand(kScratchRegister, input, times_8, 0));
1452 }
1453
1454
1455 void CodeGenerator::AssembleDeoptimizerCall(
1456     int deoptimization_id, Deoptimizer::BailoutType bailout_type) {
1457   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1458       isolate(), deoptimization_id, bailout_type);
1459   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1460 }
1461
1462
1463 void CodeGenerator::AssemblePrologue() {
1464   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1465   int stack_slots = frame()->GetSpillSlotCount();
1466   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1467     __ pushq(rbp);
1468     __ movq(rbp, rsp);
1469     int register_save_area_size = 0;
1470     const RegList saves = descriptor->CalleeSavedRegisters();
1471     if (saves != 0) {  // Save callee-saved registers.
1472       for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1473         if (!((1 << i) & saves)) continue;
1474         __ pushq(Register::from_code(i));
1475         register_save_area_size += kPointerSize;
1476       }
1477     }
1478     const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
1479     if (saves_fp != 0) {  // Save callee-saved XMM registers.
1480       const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
1481       const int stack_size = saves_fp_count * 16;
1482       // Adjust the stack pointer.
1483       __ subp(rsp, Immediate(stack_size));
1484       // Store the registers on the stack.
1485       int slot_idx = 0;
1486       for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
1487         if (!((1 << i) & saves_fp)) continue;
1488         __ movdqu(Operand(rsp, 16 * slot_idx), XMMRegister::from_code(i));
1489         slot_idx++;
1490       }
1491       register_save_area_size += stack_size;
1492     }
1493     if (register_save_area_size > 0) {
1494       frame()->SetRegisterSaveAreaSize(register_save_area_size);
1495     }
1496   } else if (descriptor->IsJSFunctionCall()) {
1497     CompilationInfo* info = this->info();
1498     __ Prologue(info->IsCodePreAgingActive());
1499     frame()->SetRegisterSaveAreaSize(
1500         StandardFrameConstants::kFixedFrameSizeFromFp);
1501   } else if (needs_frame_) {
1502     __ StubPrologue();
1503     frame()->SetRegisterSaveAreaSize(
1504         StandardFrameConstants::kFixedFrameSizeFromFp);
1505   }
1506
1507   if (info()->is_osr()) {
1508     // TurboFan OSR-compiled functions cannot be entered directly.
1509     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1510
1511     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1512     // frame is still on the stack. Optimized code uses OSR values directly from
1513     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1514     // remaining stack slots.
1515     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1516     osr_pc_offset_ = __ pc_offset();
1517     // TODO(titzer): cannot address target function == local #-1
1518     __ movq(rdi, Operand(rbp, JavaScriptFrameConstants::kFunctionOffset));
1519     DCHECK(stack_slots >= frame()->GetOsrStackSlotCount());
1520     stack_slots -= frame()->GetOsrStackSlotCount();
1521   }
1522
1523   if (stack_slots > 0) {
1524     __ subq(rsp, Immediate(stack_slots * kPointerSize));
1525   }
1526 }
1527
1528
1529 void CodeGenerator::AssembleReturn() {
1530   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1531   int stack_slots = frame()->GetSpillSlotCount();
1532   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1533     if (frame()->GetRegisterSaveAreaSize() > 0) {
1534       // Remove this frame's spill slots first.
1535       if (stack_slots > 0) {
1536         __ addq(rsp, Immediate(stack_slots * kPointerSize));
1537       }
1538       // Restore registers.
1539       const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
1540       if (saves_fp != 0) {
1541         const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
1542         const int stack_size = saves_fp_count * 16;
1543         // Load the registers from the stack.
1544         int slot_idx = 0;
1545         for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
1546           if (!((1 << i) & saves_fp)) continue;
1547           __ movdqu(XMMRegister::from_code(i), Operand(rsp, 16 * slot_idx));
1548           slot_idx++;
1549         }
1550         // Adjust the stack pointer.
1551         __ addp(rsp, Immediate(stack_size));
1552       }
1553       const RegList saves = descriptor->CalleeSavedRegisters();
1554       if (saves != 0) {
1555         for (int i = 0; i < Register::kNumRegisters; i++) {
1556           if (!((1 << i) & saves)) continue;
1557           __ popq(Register::from_code(i));
1558         }
1559       }
1560       __ popq(rbp);  // Pop caller's frame pointer.
1561     } else {
1562       // No saved registers.
1563       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1564       __ popq(rbp);       // Pop caller's frame pointer.
1565     }
1566   } else if (descriptor->IsJSFunctionCall() || needs_frame_) {
1567     // Canonicalize JSFunction return sites for now.
1568     if (return_label_.is_bound()) {
1569       __ jmp(&return_label_);
1570       return;
1571     } else {
1572       __ bind(&return_label_);
1573       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1574       __ popq(rbp);       // Pop caller's frame pointer.
1575     }
1576   }
1577   size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
1578   // Might need rcx for scratch if pop_size is too big.
1579   DCHECK_EQ(0, descriptor->CalleeSavedRegisters() & rcx.bit());
1580   __ Ret(static_cast<int>(pop_size), rcx);
1581 }
1582
1583
1584 void CodeGenerator::AssembleMove(InstructionOperand* source,
1585                                  InstructionOperand* destination) {
1586   X64OperandConverter g(this, NULL);
1587   // Dispatch on the source and destination operand kinds.  Not all
1588   // combinations are possible.
1589   if (source->IsRegister()) {
1590     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1591     Register src = g.ToRegister(source);
1592     if (destination->IsRegister()) {
1593       __ movq(g.ToRegister(destination), src);
1594     } else {
1595       __ movq(g.ToOperand(destination), src);
1596     }
1597   } else if (source->IsStackSlot()) {
1598     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1599     Operand src = g.ToOperand(source);
1600     if (destination->IsRegister()) {
1601       Register dst = g.ToRegister(destination);
1602       __ movq(dst, src);
1603     } else {
1604       // Spill on demand to use a temporary register for memory-to-memory
1605       // moves.
1606       Register tmp = kScratchRegister;
1607       Operand dst = g.ToOperand(destination);
1608       __ movq(tmp, src);
1609       __ movq(dst, tmp);
1610     }
1611   } else if (source->IsConstant()) {
1612     ConstantOperand* constant_source = ConstantOperand::cast(source);
1613     Constant src = g.ToConstant(constant_source);
1614     if (destination->IsRegister() || destination->IsStackSlot()) {
1615       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1616                                                : kScratchRegister;
1617       switch (src.type()) {
1618         case Constant::kInt32:
1619           // TODO(dcarney): don't need scratch in this case.
1620           __ Set(dst, src.ToInt32());
1621           break;
1622         case Constant::kInt64:
1623           __ Set(dst, src.ToInt64());
1624           break;
1625         case Constant::kFloat32:
1626           __ Move(dst,
1627                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1628           break;
1629         case Constant::kFloat64:
1630           __ Move(dst,
1631                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1632           break;
1633         case Constant::kExternalReference:
1634           __ Move(dst, src.ToExternalReference());
1635           break;
1636         case Constant::kHeapObject: {
1637           Handle<HeapObject> src_object = src.ToHeapObject();
1638           Heap::RootListIndex index;
1639           int offset;
1640           if (IsMaterializableFromFrame(src_object, &offset)) {
1641             __ movp(dst, Operand(rbp, offset));
1642           } else if (IsMaterializableFromRoot(src_object, &index)) {
1643             __ LoadRoot(dst, index);
1644           } else {
1645             __ Move(dst, src_object);
1646           }
1647           break;
1648         }
1649         case Constant::kRpoNumber:
1650           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1651           break;
1652       }
1653       if (destination->IsStackSlot()) {
1654         __ movq(g.ToOperand(destination), kScratchRegister);
1655       }
1656     } else if (src.type() == Constant::kFloat32) {
1657       // TODO(turbofan): Can we do better here?
1658       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1659       if (destination->IsDoubleRegister()) {
1660         __ Move(g.ToDoubleRegister(destination), src_const);
1661       } else {
1662         DCHECK(destination->IsDoubleStackSlot());
1663         Operand dst = g.ToOperand(destination);
1664         __ movl(dst, Immediate(src_const));
1665       }
1666     } else {
1667       DCHECK_EQ(Constant::kFloat64, src.type());
1668       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1669       if (destination->IsDoubleRegister()) {
1670         __ Move(g.ToDoubleRegister(destination), src_const);
1671       } else {
1672         DCHECK(destination->IsDoubleStackSlot());
1673         __ movq(kScratchRegister, src_const);
1674         __ movq(g.ToOperand(destination), kScratchRegister);
1675       }
1676     }
1677   } else if (source->IsDoubleRegister()) {
1678     XMMRegister src = g.ToDoubleRegister(source);
1679     if (destination->IsDoubleRegister()) {
1680       XMMRegister dst = g.ToDoubleRegister(destination);
1681       __ movaps(dst, src);
1682     } else {
1683       DCHECK(destination->IsDoubleStackSlot());
1684       Operand dst = g.ToOperand(destination);
1685       __ movsd(dst, src);
1686     }
1687   } else if (source->IsDoubleStackSlot()) {
1688     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1689     Operand src = g.ToOperand(source);
1690     if (destination->IsDoubleRegister()) {
1691       XMMRegister dst = g.ToDoubleRegister(destination);
1692       __ movsd(dst, src);
1693     } else {
1694       // We rely on having xmm0 available as a fixed scratch register.
1695       Operand dst = g.ToOperand(destination);
1696       __ movsd(xmm0, src);
1697       __ movsd(dst, xmm0);
1698     }
1699   } else {
1700     UNREACHABLE();
1701   }
1702 }
1703
1704
1705 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1706                                  InstructionOperand* destination) {
1707   X64OperandConverter g(this, NULL);
1708   // Dispatch on the source and destination operand kinds.  Not all
1709   // combinations are possible.
1710   if (source->IsRegister() && destination->IsRegister()) {
1711     // Register-register.
1712     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1713   } else if (source->IsRegister() && destination->IsStackSlot()) {
1714     Register src = g.ToRegister(source);
1715     Operand dst = g.ToOperand(destination);
1716     __ xchgq(src, dst);
1717   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1718              (source->IsDoubleStackSlot() &&
1719               destination->IsDoubleStackSlot())) {
1720     // Memory-memory.
1721     Register tmp = kScratchRegister;
1722     Operand src = g.ToOperand(source);
1723     Operand dst = g.ToOperand(destination);
1724     __ movq(tmp, dst);
1725     __ xchgq(tmp, src);
1726     __ movq(dst, tmp);
1727   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1728     // XMM register-register swap. We rely on having xmm0
1729     // available as a fixed scratch register.
1730     XMMRegister src = g.ToDoubleRegister(source);
1731     XMMRegister dst = g.ToDoubleRegister(destination);
1732     __ movaps(xmm0, src);
1733     __ movaps(src, dst);
1734     __ movaps(dst, xmm0);
1735   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1736     // XMM register-memory swap.  We rely on having xmm0
1737     // available as a fixed scratch register.
1738     XMMRegister src = g.ToDoubleRegister(source);
1739     Operand dst = g.ToOperand(destination);
1740     __ movsd(xmm0, src);
1741     __ movsd(src, dst);
1742     __ movsd(dst, xmm0);
1743   } else {
1744     // No other combinations are possible.
1745     UNREACHABLE();
1746   }
1747 }
1748
1749
1750 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1751   for (size_t index = 0; index < target_count; ++index) {
1752     __ dq(targets[index]);
1753   }
1754 }
1755
1756
1757 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1758
1759
1760 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1761   int space_needed = Deoptimizer::patch_size();
1762   if (!info()->IsStub()) {
1763     // Ensure that we have enough space after the previous lazy-bailout
1764     // instruction for patching the code here.
1765     int current_pc = masm()->pc_offset();
1766     if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1767       int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1768       __ Nop(padding_size);
1769     }
1770   }
1771 }
1772
1773 #undef __
1774
1775 }  // namespace internal
1776 }  // namespace compiler
1777 }  // namespace v8