deps/v8/src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/scopes.h"
  11 #include "src/x64/assembler-x64.h"
  12 #include "src/x64/macro-assembler-x64.h"
  13
  14 namespace v8 {
  15 namespace internal {
  16 namespace compiler {
  17
  18 #define __ masm()->
  19
  20
  21 // Adds X64 specific methods for decoding operands.
  22 class X64OperandConverter : public InstructionOperandConverter {
  23  public:
  24   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  25       : InstructionOperandConverter(gen, instr) {}
  26
  27   Immediate InputImmediate(int index) {
  28     return ToImmediate(instr_->InputAt(index));
  29   }
  30
  31   Operand InputOperand(int index) { return ToOperand(instr_->InputAt(index)); }
  32
  33   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  34
  35   Immediate ToImmediate(InstructionOperand* operand) {
  36     return Immediate(ToConstant(operand).ToInt32());
  37   }
  38
  39   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  40     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  41     // The linkage computes where all spill slots are located.
  42     FrameOffset offset = linkage()->GetFrameOffset(op->index(), frame(), extra);
  43     return Operand(offset.from_stack_pointer() ? rsp : rbp, offset.offset());
  44   }
  45
  46   static int NextOffset(int* offset) {
  47     int i = *offset;
  48     (*offset)++;
  49     return i;
  50   }
  51
  52   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  53     STATIC_ASSERT(0 == static_cast<int>(times_1));
  54     STATIC_ASSERT(1 == static_cast<int>(times_2));
  55     STATIC_ASSERT(2 == static_cast<int>(times_4));
  56     STATIC_ASSERT(3 == static_cast<int>(times_8));
  57     int scale = static_cast<int>(mode - one);
  58     DCHECK(scale >= 0 && scale < 4);
  59     return static_cast<ScaleFactor>(scale);
  60   }
  61
  62   Operand MemoryOperand(int* offset) {
  63     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  64     switch (mode) {
  65       case kMode_MR: {
  66         Register base = InputRegister(NextOffset(offset));
  67         int32_t disp = 0;
  68         return Operand(base, disp);
  69       }
  70       case kMode_MRI: {
  71         Register base = InputRegister(NextOffset(offset));
  72         int32_t disp = InputInt32(NextOffset(offset));
  73         return Operand(base, disp);
  74       }
  75       case kMode_MR1:
  76       case kMode_MR2:
  77       case kMode_MR4:
  78       case kMode_MR8: {
  79         Register base = InputRegister(NextOffset(offset));
  80         Register index = InputRegister(NextOffset(offset));
  81         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  82         int32_t disp = 0;
  83         return Operand(base, index, scale, disp);
  84       }
  85       case kMode_MR1I:
  86       case kMode_MR2I:
  87       case kMode_MR4I:
  88       case kMode_MR8I: {
  89         Register base = InputRegister(NextOffset(offset));
  90         Register index = InputRegister(NextOffset(offset));
  91         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
  92         int32_t disp = InputInt32(NextOffset(offset));
  93         return Operand(base, index, scale, disp);
  94       }
  95       case kMode_M1: {
  96         Register base = InputRegister(NextOffset(offset));
  97         int32_t disp = 0;
  98         return Operand(base, disp);
  99       }
 100       case kMode_M2:
 101         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 102         return Operand(no_reg, 0);
 103       case kMode_M4:
 104       case kMode_M8: {
 105         Register index = InputRegister(NextOffset(offset));
 106         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 107         int32_t disp = 0;
 108         return Operand(index, scale, disp);
 109       }
 110       case kMode_M1I:
 111       case kMode_M2I:
 112       case kMode_M4I:
 113       case kMode_M8I: {
 114         Register index = InputRegister(NextOffset(offset));
 115         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 116         int32_t disp = InputInt32(NextOffset(offset));
 117         return Operand(index, scale, disp);
 118       }
 119       case kMode_None:
 120         UNREACHABLE();
 121         return Operand(no_reg, 0);
 122     }
 123     UNREACHABLE();
 124     return Operand(no_reg, 0);
 125   }
 126
 127   Operand MemoryOperand(int first_input = 0) {
 128     return MemoryOperand(&first_input);
 129   }
 130 };
 131
 132
 133 namespace {
 134
 135 bool HasImmediateInput(Instruction* instr, int index) {
 136   return instr->InputAt(index)->IsImmediate();
 137 }
 138
 139
 140 class OutOfLineLoadZero FINAL : public OutOfLineCode {
 141  public:
 142   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 143       : OutOfLineCode(gen), result_(result) {}
 144
 145   void Generate() FINAL { __ xorl(result_, result_); }
 146
 147  private:
 148   Register const result_;
 149 };
 150
 151
 152 class OutOfLineLoadNaN FINAL : public OutOfLineCode {
 153  public:
 154   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 155       : OutOfLineCode(gen), result_(result) {}
 156
 157   void Generate() FINAL { __ pcmpeqd(result_, result_); }
 158
 159  private:
 160   XMMRegister const result_;
 161 };
 162
 163
 164 class OutOfLineTruncateDoubleToI FINAL : public OutOfLineCode {
 165  public:
 166   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 167                              XMMRegister input)
 168       : OutOfLineCode(gen), result_(result), input_(input) {}
 169
 170   void Generate() FINAL {
 171     __ subp(rsp, Immediate(kDoubleSize));
 172     __ movsd(MemOperand(rsp, 0), input_);
 173     __ SlowTruncateToI(result_, rsp, 0);
 174     __ addp(rsp, Immediate(kDoubleSize));
 175   }
 176
 177  private:
 178   Register const result_;
 179   XMMRegister const input_;
 180 };
 181
 182 }  // namespace
 183
 184
 185 #define ASSEMBLE_UNOP(asm_instr)         \
 186   do {                                   \
 187     if (instr->Output()->IsRegister()) { \
 188       __ asm_instr(i.OutputRegister());  \
 189     } else {                             \
 190       __ asm_instr(i.OutputOperand());   \
 191     }                                    \
 192   } while (0)
 193
 194
 195 #define ASSEMBLE_BINOP(asm_instr)                              \
 196   do {                                                         \
 197     if (HasImmediateInput(instr, 1)) {                         \
 198       if (instr->InputAt(0)->IsRegister()) {                   \
 199         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 200       } else {                                                 \
 201         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 202       }                                                        \
 203     } else {                                                   \
 204       if (instr->InputAt(1)->IsRegister()) {                   \
 205         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 206       } else {                                                 \
 207         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 208       }                                                        \
 209     }                                                          \
 210   } while (0)
 211
 212
 213 #define ASSEMBLE_MULT(asm_instr)                              \
 214   do {                                                        \
 215     if (HasImmediateInput(instr, 1)) {                        \
 216       if (instr->InputAt(0)->IsRegister()) {                  \
 217         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 218                      i.InputImmediate(1));                    \
 219       } else {                                                \
 220         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 221                      i.InputImmediate(1));                    \
 222       }                                                       \
 223     } else {                                                  \
 224       if (instr->InputAt(1)->IsRegister()) {                  \
 225         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 226       } else {                                                \
 227         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 228       }                                                       \
 229     }                                                         \
 230   } while (0)
 231
 232
 233 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 234   do {                                                                     \
 235     if (HasImmediateInput(instr, 1)) {                                     \
 236       if (instr->Output()->IsRegister()) {                                 \
 237         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 238       } else {                                                             \
 239         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 240       }                                                                    \
 241     } else {                                                               \
 242       if (instr->Output()->IsRegister()) {                                 \
 243         __ asm_instr##_cl(i.OutputRegister());                             \
 244       } else {                                                             \
 245         __ asm_instr##_cl(i.OutputOperand());                              \
 246       }                                                                    \
 247     }                                                                      \
 248   } while (0)
 249
 250
 251 #define ASSEMBLE_MOVX(asm_instr)                            \
 252   do {                                                      \
 253     if (instr->addressing_mode() != kMode_None) {           \
 254       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
 255     } else if (instr->InputAt(0)->IsRegister()) {           \
 256       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
 257     } else {                                                \
 258       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
 259     }                                                       \
 260   } while (0)
 261
 262
 263 #define ASSEMBLE_DOUBLE_BINOP(asm_instr)                                \
 264   do {                                                                  \
 265     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 266       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 267     } else {                                                            \
 268       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 269     }                                                                   \
 270   } while (0)
 271
 272
 273 #define ASSEMBLE_AVX_DOUBLE_BINOP(asm_instr)                           \
 274   do {                                                                 \
 275     CpuFeatureScope avx_scope(masm(), AVX);                            \
 276     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 277       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 278                    i.InputDoubleRegister(1));                          \
 279     } else {                                                           \
 280       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 281                    i.InputOperand(1));                                 \
 282     }                                                                  \
 283   } while (0)
 284
 285
 286 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 287   do {                                                                       \
 288     auto result = i.OutputDoubleRegister();                                  \
 289     auto buffer = i.InputRegister(0);                                        \
 290     auto index1 = i.InputRegister(1);                                        \
 291     auto index2 = i.InputInt32(2);                                           \
 292     OutOfLineCode* ool;                                                      \
 293     if (instr->InputAt(3)->IsRegister()) {                                   \
 294       auto length = i.InputRegister(3);                                      \
 295       DCHECK_EQ(0, index2);                                                  \
 296       __ cmpl(index1, length);                                               \
 297       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 298     } else {                                                                 \
 299       auto length = i.InputInt32(3);                                         \
 300       DCHECK_LE(index2, length);                                             \
 301       __ cmpq(index1, Immediate(length - index2));                           \
 302       class OutOfLineLoadFloat FINAL : public OutOfLineCode {                \
 303        public:                                                               \
 304         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 305                            Register buffer, Register index1, int32_t index2, \
 306                            int32_t length)                                   \
 307             : OutOfLineCode(gen),                                            \
 308               result_(result),                                               \
 309               buffer_(buffer),                                               \
 310               index1_(index1),                                               \
 311               index2_(index2),                                               \
 312               length_(length) {}                                             \
 313                                                                              \
 314         void Generate() FINAL {                                              \
 315           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 316           __ pcmpeqd(result_, result_);                                      \
 317           __ cmpl(kScratchRegister, Immediate(length_));                     \
 318           __ j(above_equal, exit());                                         \
 319           __ asm_instr(result_,                                              \
 320                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 321         }                                                                    \
 322                                                                              \
 323        private:                                                              \
 324         XMMRegister const result_;                                           \
 325         Register const buffer_;                                              \
 326         Register const index1_;                                              \
 327         int32_t const index2_;                                               \
 328         int32_t const length_;                                               \
 329       };                                                                     \
 330       ool = new (zone())                                                     \
 331           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 332     }                                                                        \
 333     __ j(above_equal, ool->entry());                                         \
 334     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 335     __ bind(ool->exit());                                                    \
 336   } while (false)
 337
 338
 339 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 340   do {                                                                         \
 341     auto result = i.OutputRegister();                                          \
 342     auto buffer = i.InputRegister(0);                                          \
 343     auto index1 = i.InputRegister(1);                                          \
 344     auto index2 = i.InputInt32(2);                                             \
 345     OutOfLineCode* ool;                                                        \
 346     if (instr->InputAt(3)->IsRegister()) {                                     \
 347       auto length = i.InputRegister(3);                                        \
 348       DCHECK_EQ(0, index2);                                                    \
 349       __ cmpl(index1, length);                                                 \
 350       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 351     } else {                                                                   \
 352       auto length = i.InputInt32(3);                                           \
 353       DCHECK_LE(index2, length);                                               \
 354       __ cmpq(index1, Immediate(length - index2));                             \
 355       class OutOfLineLoadInteger FINAL : public OutOfLineCode {                \
 356        public:                                                                 \
 357         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 358                              Register buffer, Register index1, int32_t index2, \
 359                              int32_t length)                                   \
 360             : OutOfLineCode(gen),                                              \
 361               result_(result),                                                 \
 362               buffer_(buffer),                                                 \
 363               index1_(index1),                                                 \
 364               index2_(index2),                                                 \
 365               length_(length) {}                                               \
 366                                                                                \
 367         void Generate() FINAL {                                                \
 368           Label oob;                                                           \
 369           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 370           __ cmpl(kScratchRegister, Immediate(length_));                       \
 371           __ j(above_equal, &oob, Label::kNear);                               \
 372           __ asm_instr(result_,                                                \
 373                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 374           __ jmp(exit());                                                      \
 375           __ bind(&oob);                                                       \
 376           __ xorl(result_, result_);                                           \
 377         }                                                                      \
 378                                                                                \
 379        private:                                                                \
 380         Register const result_;                                                \
 381         Register const buffer_;                                                \
 382         Register const index1_;                                                \
 383         int32_t const index2_;                                                 \
 384         int32_t const length_;                                                 \
 385       };                                                                       \
 386       ool = new (zone())                                                       \
 387           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 388     }                                                                          \
 389     __ j(above_equal, ool->entry());                                           \
 390     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 391     __ bind(ool->exit());                                                      \
 392   } while (false)
 393
 394
 395 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 396   do {                                                                       \
 397     auto buffer = i.InputRegister(0);                                        \
 398     auto index1 = i.InputRegister(1);                                        \
 399     auto index2 = i.InputInt32(2);                                           \
 400     auto value = i.InputDoubleRegister(4);                                   \
 401     if (instr->InputAt(3)->IsRegister()) {                                   \
 402       auto length = i.InputRegister(3);                                      \
 403       DCHECK_EQ(0, index2);                                                  \
 404       Label done;                                                            \
 405       __ cmpl(index1, length);                                               \
 406       __ j(above_equal, &done, Label::kNear);                                \
 407       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 408       __ bind(&done);                                                        \
 409     } else {                                                                 \
 410       auto length = i.InputInt32(3);                                         \
 411       DCHECK_LE(index2, length);                                             \
 412       __ cmpq(index1, Immediate(length - index2));                           \
 413       class OutOfLineStoreFloat FINAL : public OutOfLineCode {               \
 414        public:                                                               \
 415         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 416                             Register index1, int32_t index2, int32_t length, \
 417                             XMMRegister value)                               \
 418             : OutOfLineCode(gen),                                            \
 419               buffer_(buffer),                                               \
 420               index1_(index1),                                               \
 421               index2_(index2),                                               \
 422               length_(length),                                               \
 423               value_(value) {}                                               \
 424                                                                              \
 425         void Generate() FINAL {                                              \
 426           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 427           __ cmpl(kScratchRegister, Immediate(length_));                     \
 428           __ j(above_equal, exit());                                         \
 429           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 430                        value_);                                              \
 431         }                                                                    \
 432                                                                              \
 433        private:                                                              \
 434         Register const buffer_;                                              \
 435         Register const index1_;                                              \
 436         int32_t const index2_;                                               \
 437         int32_t const length_;                                               \
 438         XMMRegister const value_;                                            \
 439       };                                                                     \
 440       auto ool = new (zone())                                                \
 441           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 442       __ j(above_equal, ool->entry());                                       \
 443       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 444       __ bind(ool->exit());                                                  \
 445     }                                                                        \
 446   } while (false)
 447
 448
 449 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 450   do {                                                                         \
 451     auto buffer = i.InputRegister(0);                                          \
 452     auto index1 = i.InputRegister(1);                                          \
 453     auto index2 = i.InputInt32(2);                                             \
 454     if (instr->InputAt(3)->IsRegister()) {                                     \
 455       auto length = i.InputRegister(3);                                        \
 456       DCHECK_EQ(0, index2);                                                    \
 457       Label done;                                                              \
 458       __ cmpl(index1, length);                                                 \
 459       __ j(above_equal, &done, Label::kNear);                                  \
 460       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 461       __ bind(&done);                                                          \
 462     } else {                                                                   \
 463       auto length = i.InputInt32(3);                                           \
 464       DCHECK_LE(index2, length);                                               \
 465       __ cmpq(index1, Immediate(length - index2));                             \
 466       class OutOfLineStoreInteger FINAL : public OutOfLineCode {               \
 467        public:                                                                 \
 468         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 469                               Register index1, int32_t index2, int32_t length, \
 470                               Value value)                                     \
 471             : OutOfLineCode(gen),                                              \
 472               buffer_(buffer),                                                 \
 473               index1_(index1),                                                 \
 474               index2_(index2),                                                 \
 475               length_(length),                                                 \
 476               value_(value) {}                                                 \
 477                                                                                \
 478         void Generate() FINAL {                                                \
 479           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 480           __ cmpl(kScratchRegister, Immediate(length_));                       \
 481           __ j(above_equal, exit());                                           \
 482           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 483                        value_);                                                \
 484         }                                                                      \
 485                                                                                \
 486        private:                                                                \
 487         Register const buffer_;                                                \
 488         Register const index1_;                                                \
 489         int32_t const index2_;                                                 \
 490         int32_t const length_;                                                 \
 491         Value const value_;                                                    \
 492       };                                                                       \
 493       auto ool = new (zone())                                                  \
 494           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 495       __ j(above_equal, ool->entry());                                         \
 496       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 497       __ bind(ool->exit());                                                    \
 498     }                                                                          \
 499   } while (false)
 500
 501
 502 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 503   do {                                                           \
 504     if (instr->InputAt(4)->IsRegister()) {                       \
 505       Register value = i.InputRegister(4);                       \
 506       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 507     } else {                                                     \
 508       Immediate value = i.InputImmediate(4);                     \
 509       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 510     }                                                            \
 511   } while (false)
 512
 513
 514 // Assembles an instruction after register allocation, producing machine code.
 515 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 516   X64OperandConverter i(this, instr);
 517
 518   switch (ArchOpcodeField::decode(instr->opcode())) {
 519     case kArchCallCodeObject: {
 520       EnsureSpaceForLazyDeopt();
 521       if (HasImmediateInput(instr, 0)) {
 522         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 523         __ Call(code, RelocInfo::CODE_TARGET);
 524       } else {
 525         Register reg = i.InputRegister(0);
 526         int entry = Code::kHeaderSize - kHeapObjectTag;
 527         __ Call(Operand(reg, entry));
 528       }
 529       AddSafepointAndDeopt(instr);
 530       break;
 531     }
 532     case kArchCallJSFunction: {
 533       EnsureSpaceForLazyDeopt();
 534       Register func = i.InputRegister(0);
 535       if (FLAG_debug_code) {
 536         // Check the function's context matches the context argument.
 537         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 538         __ Assert(equal, kWrongFunctionContext);
 539       }
 540       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 541       AddSafepointAndDeopt(instr);
 542       break;
 543     }
 544     case kArchJmp:
 545       AssembleArchJump(i.InputRpo(0));
 546       break;
 547     case kArchLookupSwitch:
 548       AssembleArchLookupSwitch(instr);
 549       break;
 550     case kArchTableSwitch:
 551       AssembleArchTableSwitch(instr);
 552       break;
 553     case kArchNop:
 554       // don't emit code for nops.
 555       break;
 556     case kArchRet:
 557       AssembleReturn();
 558       break;
 559     case kArchStackPointer:
 560       __ movq(i.OutputRegister(), rsp);
 561       break;
 562     case kArchTruncateDoubleToI: {
 563       auto result = i.OutputRegister();
 564       auto input = i.InputDoubleRegister(0);
 565       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 566       __ cvttsd2siq(result, input);
 567       __ cmpq(result, Immediate(1));
 568       __ j(overflow, ool->entry());
 569       __ bind(ool->exit());
 570       break;
 571     }
 572     case kX64Add32:
 573       ASSEMBLE_BINOP(addl);
 574       break;
 575     case kX64Add:
 576       ASSEMBLE_BINOP(addq);
 577       break;
 578     case kX64Sub32:
 579       ASSEMBLE_BINOP(subl);
 580       break;
 581     case kX64Sub:
 582       ASSEMBLE_BINOP(subq);
 583       break;
 584     case kX64And32:
 585       ASSEMBLE_BINOP(andl);
 586       break;
 587     case kX64And:
 588       ASSEMBLE_BINOP(andq);
 589       break;
 590     case kX64Cmp32:
 591       ASSEMBLE_BINOP(cmpl);
 592       break;
 593     case kX64Cmp:
 594       ASSEMBLE_BINOP(cmpq);
 595       break;
 596     case kX64Test32:
 597       ASSEMBLE_BINOP(testl);
 598       break;
 599     case kX64Test:
 600       ASSEMBLE_BINOP(testq);
 601       break;
 602     case kX64Imul32:
 603       ASSEMBLE_MULT(imull);
 604       break;
 605     case kX64Imul:
 606       ASSEMBLE_MULT(imulq);
 607       break;
 608     case kX64ImulHigh32:
 609       if (instr->InputAt(1)->IsRegister()) {
 610         __ imull(i.InputRegister(1));
 611       } else {
 612         __ imull(i.InputOperand(1));
 613       }
 614       break;
 615     case kX64UmulHigh32:
 616       if (instr->InputAt(1)->IsRegister()) {
 617         __ mull(i.InputRegister(1));
 618       } else {
 619         __ mull(i.InputOperand(1));
 620       }
 621       break;
 622     case kX64Idiv32:
 623       __ cdq();
 624       __ idivl(i.InputRegister(1));
 625       break;
 626     case kX64Idiv:
 627       __ cqo();
 628       __ idivq(i.InputRegister(1));
 629       break;
 630     case kX64Udiv32:
 631       __ xorl(rdx, rdx);
 632       __ divl(i.InputRegister(1));
 633       break;
 634     case kX64Udiv:
 635       __ xorq(rdx, rdx);
 636       __ divq(i.InputRegister(1));
 637       break;
 638     case kX64Not:
 639       ASSEMBLE_UNOP(notq);
 640       break;
 641     case kX64Not32:
 642       ASSEMBLE_UNOP(notl);
 643       break;
 644     case kX64Neg:
 645       ASSEMBLE_UNOP(negq);
 646       break;
 647     case kX64Neg32:
 648       ASSEMBLE_UNOP(negl);
 649       break;
 650     case kX64Or32:
 651       ASSEMBLE_BINOP(orl);
 652       break;
 653     case kX64Or:
 654       ASSEMBLE_BINOP(orq);
 655       break;
 656     case kX64Xor32:
 657       ASSEMBLE_BINOP(xorl);
 658       break;
 659     case kX64Xor:
 660       ASSEMBLE_BINOP(xorq);
 661       break;
 662     case kX64Shl32:
 663       ASSEMBLE_SHIFT(shll, 5);
 664       break;
 665     case kX64Shl:
 666       ASSEMBLE_SHIFT(shlq, 6);
 667       break;
 668     case kX64Shr32:
 669       ASSEMBLE_SHIFT(shrl, 5);
 670       break;
 671     case kX64Shr:
 672       ASSEMBLE_SHIFT(shrq, 6);
 673       break;
 674     case kX64Sar32:
 675       ASSEMBLE_SHIFT(sarl, 5);
 676       break;
 677     case kX64Sar:
 678       ASSEMBLE_SHIFT(sarq, 6);
 679       break;
 680     case kX64Ror32:
 681       ASSEMBLE_SHIFT(rorl, 5);
 682       break;
 683     case kX64Ror:
 684       ASSEMBLE_SHIFT(rorq, 6);
 685       break;
 686     case kSSEFloat64Cmp:
 687       ASSEMBLE_DOUBLE_BINOP(ucomisd);
 688       break;
 689     case kSSEFloat64Add:
 690       ASSEMBLE_DOUBLE_BINOP(addsd);
 691       break;
 692     case kSSEFloat64Sub:
 693       ASSEMBLE_DOUBLE_BINOP(subsd);
 694       break;
 695     case kSSEFloat64Mul:
 696       ASSEMBLE_DOUBLE_BINOP(mulsd);
 697       break;
 698     case kSSEFloat64Div:
 699       ASSEMBLE_DOUBLE_BINOP(divsd);
 700       break;
 701     case kSSEFloat64Mod: {
 702       __ subq(rsp, Immediate(kDoubleSize));
 703       // Move values to st(0) and st(1).
 704       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 705       __ fld_d(Operand(rsp, 0));
 706       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 707       __ fld_d(Operand(rsp, 0));
 708       // Loop while fprem isn't done.
 709       Label mod_loop;
 710       __ bind(&mod_loop);
 711       // This instructions traps on all kinds inputs, but we are assuming the
 712       // floating point control word is set to ignore them all.
 713       __ fprem();
 714       // The following 2 instruction implicitly use rax.
 715       __ fnstsw_ax();
 716       if (CpuFeatures::IsSupported(SAHF)) {
 717         CpuFeatureScope sahf_scope(masm(), SAHF);
 718         __ sahf();
 719       } else {
 720         __ shrl(rax, Immediate(8));
 721         __ andl(rax, Immediate(0xFF));
 722         __ pushq(rax);
 723         __ popfq();
 724       }
 725       __ j(parity_even, &mod_loop);
 726       // Move output to stack and clean up.
 727       __ fstp(1);
 728       __ fstp_d(Operand(rsp, 0));
 729       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 730       __ addq(rsp, Immediate(kDoubleSize));
 731       break;
 732     }
 733     case kSSEFloat64Sqrt:
 734       if (instr->InputAt(0)->IsDoubleRegister()) {
 735         __ sqrtsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 736       } else {
 737         __ sqrtsd(i.OutputDoubleRegister(), i.InputOperand(0));
 738       }
 739       break;
 740     case kSSEFloat64Floor: {
 741       CpuFeatureScope sse_scope(masm(), SSE4_1);
 742       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 743                  v8::internal::Assembler::kRoundDown);
 744       break;
 745     }
 746     case kSSEFloat64Ceil: {
 747       CpuFeatureScope sse_scope(masm(), SSE4_1);
 748       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 749                  v8::internal::Assembler::kRoundUp);
 750       break;
 751     }
 752     case kSSEFloat64RoundTruncate: {
 753       CpuFeatureScope sse_scope(masm(), SSE4_1);
 754       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 755                  v8::internal::Assembler::kRoundToZero);
 756       break;
 757     }
 758     case kSSECvtss2sd:
 759       if (instr->InputAt(0)->IsDoubleRegister()) {
 760         __ cvtss2sd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 761       } else {
 762         __ cvtss2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 763       }
 764       break;
 765     case kSSECvtsd2ss:
 766       if (instr->InputAt(0)->IsDoubleRegister()) {
 767         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 768       } else {
 769         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputOperand(0));
 770       }
 771       break;
 772     case kSSEFloat64ToInt32:
 773       if (instr->InputAt(0)->IsDoubleRegister()) {
 774         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 775       } else {
 776         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 777       }
 778       break;
 779     case kSSEFloat64ToUint32: {
 780       if (instr->InputAt(0)->IsDoubleRegister()) {
 781         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 782       } else {
 783         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 784       }
 785       __ AssertZeroExtended(i.OutputRegister());
 786       break;
 787     }
 788     case kSSEInt32ToFloat64:
 789       if (instr->InputAt(0)->IsRegister()) {
 790         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 791       } else {
 792         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 793       }
 794       break;
 795     case kSSEUint32ToFloat64:
 796       if (instr->InputAt(0)->IsRegister()) {
 797         __ movl(kScratchRegister, i.InputRegister(0));
 798       } else {
 799         __ movl(kScratchRegister, i.InputOperand(0));
 800       }
 801       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 802       break;
 803     case kAVXFloat64Add:
 804       ASSEMBLE_AVX_DOUBLE_BINOP(vaddsd);
 805       break;
 806     case kAVXFloat64Sub:
 807       ASSEMBLE_AVX_DOUBLE_BINOP(vsubsd);
 808       break;
 809     case kAVXFloat64Mul:
 810       ASSEMBLE_AVX_DOUBLE_BINOP(vmulsd);
 811       break;
 812     case kAVXFloat64Div:
 813       ASSEMBLE_AVX_DOUBLE_BINOP(vdivsd);
 814       break;
 815     case kX64Movsxbl:
 816       ASSEMBLE_MOVX(movsxbl);
 817       __ AssertZeroExtended(i.OutputRegister());
 818       break;
 819     case kX64Movzxbl:
 820       ASSEMBLE_MOVX(movzxbl);
 821       __ AssertZeroExtended(i.OutputRegister());
 822       break;
 823     case kX64Movb: {
 824       int index = 0;
 825       Operand operand = i.MemoryOperand(&index);
 826       if (HasImmediateInput(instr, index)) {
 827         __ movb(operand, Immediate(i.InputInt8(index)));
 828       } else {
 829         __ movb(operand, i.InputRegister(index));
 830       }
 831       break;
 832     }
 833     case kX64Movsxwl:
 834       ASSEMBLE_MOVX(movsxwl);
 835       __ AssertZeroExtended(i.OutputRegister());
 836       break;
 837     case kX64Movzxwl:
 838       ASSEMBLE_MOVX(movzxwl);
 839       __ AssertZeroExtended(i.OutputRegister());
 840       break;
 841     case kX64Movw: {
 842       int index = 0;
 843       Operand operand = i.MemoryOperand(&index);
 844       if (HasImmediateInput(instr, index)) {
 845         __ movw(operand, Immediate(i.InputInt16(index)));
 846       } else {
 847         __ movw(operand, i.InputRegister(index));
 848       }
 849       break;
 850     }
 851     case kX64Movl:
 852       if (instr->HasOutput()) {
 853         if (instr->addressing_mode() == kMode_None) {
 854           if (instr->InputAt(0)->IsRegister()) {
 855             __ movl(i.OutputRegister(), i.InputRegister(0));
 856           } else {
 857             __ movl(i.OutputRegister(), i.InputOperand(0));
 858           }
 859         } else {
 860           __ movl(i.OutputRegister(), i.MemoryOperand());
 861         }
 862         __ AssertZeroExtended(i.OutputRegister());
 863       } else {
 864         int index = 0;
 865         Operand operand = i.MemoryOperand(&index);
 866         if (HasImmediateInput(instr, index)) {
 867           __ movl(operand, i.InputImmediate(index));
 868         } else {
 869           __ movl(operand, i.InputRegister(index));
 870         }
 871       }
 872       break;
 873     case kX64Movsxlq:
 874       ASSEMBLE_MOVX(movsxlq);
 875       break;
 876     case kX64Movq:
 877       if (instr->HasOutput()) {
 878         __ movq(i.OutputRegister(), i.MemoryOperand());
 879       } else {
 880         int index = 0;
 881         Operand operand = i.MemoryOperand(&index);
 882         if (HasImmediateInput(instr, index)) {
 883           __ movq(operand, i.InputImmediate(index));
 884         } else {
 885           __ movq(operand, i.InputRegister(index));
 886         }
 887       }
 888       break;
 889     case kX64Movss:
 890       if (instr->HasOutput()) {
 891         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
 892       } else {
 893         int index = 0;
 894         Operand operand = i.MemoryOperand(&index);
 895         __ movss(operand, i.InputDoubleRegister(index));
 896       }
 897       break;
 898     case kX64Movsd:
 899       if (instr->HasOutput()) {
 900         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
 901       } else {
 902         int index = 0;
 903         Operand operand = i.MemoryOperand(&index);
 904         __ movsd(operand, i.InputDoubleRegister(index));
 905       }
 906       break;
 907     case kX64Lea32: {
 908       AddressingMode mode = AddressingModeField::decode(instr->opcode());
 909       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
 910       // and addressing mode just happens to work out. The "addl"/"subl" forms
 911       // in these cases are faster based on measurements.
 912       if (i.InputRegister(0).is(i.OutputRegister())) {
 913         if (mode == kMode_MRI) {
 914           int32_t constant_summand = i.InputInt32(1);
 915           if (constant_summand > 0) {
 916             __ addl(i.OutputRegister(), Immediate(constant_summand));
 917           } else if (constant_summand < 0) {
 918             __ subl(i.OutputRegister(), Immediate(-constant_summand));
 919           }
 920         } else if (mode == kMode_MR1) {
 921           if (i.InputRegister(1).is(i.OutputRegister())) {
 922             __ shll(i.OutputRegister(), Immediate(1));
 923           } else {
 924             __ leal(i.OutputRegister(), i.MemoryOperand());
 925           }
 926         } else if (mode == kMode_M2) {
 927           __ shll(i.OutputRegister(), Immediate(1));
 928         } else if (mode == kMode_M4) {
 929           __ shll(i.OutputRegister(), Immediate(2));
 930         } else if (mode == kMode_M8) {
 931           __ shll(i.OutputRegister(), Immediate(3));
 932         } else {
 933           __ leal(i.OutputRegister(), i.MemoryOperand());
 934         }
 935       } else {
 936         __ leal(i.OutputRegister(), i.MemoryOperand());
 937       }
 938       __ AssertZeroExtended(i.OutputRegister());
 939       break;
 940     }
 941     case kX64Lea:
 942       __ leaq(i.OutputRegister(), i.MemoryOperand());
 943       break;
 944     case kX64Dec32:
 945       __ decl(i.OutputRegister());
 946       break;
 947     case kX64Inc32:
 948       __ incl(i.OutputRegister());
 949       break;
 950     case kX64Push:
 951       if (HasImmediateInput(instr, 0)) {
 952         __ pushq(i.InputImmediate(0));
 953       } else {
 954         if (instr->InputAt(0)->IsRegister()) {
 955           __ pushq(i.InputRegister(0));
 956         } else {
 957           __ pushq(i.InputOperand(0));
 958         }
 959       }
 960       break;
 961     case kX64StoreWriteBarrier: {
 962       Register object = i.InputRegister(0);
 963       Register index = i.InputRegister(1);
 964       Register value = i.InputRegister(2);
 965       __ movq(Operand(object, index, times_1, 0), value);
 966       __ leaq(index, Operand(object, index, times_1, 0));
 967       SaveFPRegsMode mode =
 968           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
 969       __ RecordWrite(object, index, value, mode);
 970       break;
 971     }
 972     case kCheckedLoadInt8:
 973       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
 974       break;
 975     case kCheckedLoadUint8:
 976       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
 977       break;
 978     case kCheckedLoadInt16:
 979       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
 980       break;
 981     case kCheckedLoadUint16:
 982       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
 983       break;
 984     case kCheckedLoadWord32:
 985       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
 986       break;
 987     case kCheckedLoadFloat32:
 988       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
 989       break;
 990     case kCheckedLoadFloat64:
 991       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
 992       break;
 993     case kCheckedStoreWord8:
 994       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
 995       break;
 996     case kCheckedStoreWord16:
 997       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
 998       break;
 999     case kCheckedStoreWord32:
1000       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1001       break;
1002     case kCheckedStoreFloat32:
1003       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1004       break;
1005     case kCheckedStoreFloat64:
1006       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1007       break;
1008   }
1009 }
1010
1011
1012 // Assembles branches after this instruction.
1013 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1014   X64OperandConverter i(this, instr);
1015   Label::Distance flabel_distance =
1016       branch->fallthru ? Label::kNear : Label::kFar;
1017   Label* tlabel = branch->true_label;
1018   Label* flabel = branch->false_label;
1019   switch (branch->condition) {
1020     case kUnorderedEqual:
1021       __ j(parity_even, flabel, flabel_distance);
1022     // Fall through.
1023     case kEqual:
1024       __ j(equal, tlabel);
1025       break;
1026     case kUnorderedNotEqual:
1027       __ j(parity_even, tlabel);
1028     // Fall through.
1029     case kNotEqual:
1030       __ j(not_equal, tlabel);
1031       break;
1032     case kSignedLessThan:
1033       __ j(less, tlabel);
1034       break;
1035     case kSignedGreaterThanOrEqual:
1036       __ j(greater_equal, tlabel);
1037       break;
1038     case kSignedLessThanOrEqual:
1039       __ j(less_equal, tlabel);
1040       break;
1041     case kSignedGreaterThan:
1042       __ j(greater, tlabel);
1043       break;
1044     case kUnsignedLessThan:
1045       __ j(below, tlabel);
1046       break;
1047     case kUnsignedGreaterThanOrEqual:
1048       __ j(above_equal, tlabel);
1049       break;
1050     case kUnsignedLessThanOrEqual:
1051       __ j(below_equal, tlabel);
1052       break;
1053     case kUnsignedGreaterThan:
1054       __ j(above, tlabel);
1055       break;
1056     case kOverflow:
1057       __ j(overflow, tlabel);
1058       break;
1059     case kNotOverflow:
1060       __ j(no_overflow, tlabel);
1061       break;
1062   }
1063   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1064 }
1065
1066
1067 void CodeGenerator::AssembleArchJump(BasicBlock::RpoNumber target) {
1068   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1069 }
1070
1071
1072 // Assembles boolean materializations after this instruction.
1073 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1074                                         FlagsCondition condition) {
1075   X64OperandConverter i(this, instr);
1076   Label done;
1077
1078   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1079   // last output of the instruction.
1080   Label check;
1081   DCHECK_NE(0, static_cast<int>(instr->OutputCount()));
1082   Register reg = i.OutputRegister(static_cast<int>(instr->OutputCount() - 1));
1083   Condition cc = no_condition;
1084   switch (condition) {
1085     case kUnorderedEqual:
1086       __ j(parity_odd, &check, Label::kNear);
1087       __ movl(reg, Immediate(0));
1088       __ jmp(&done, Label::kNear);
1089     // Fall through.
1090     case kEqual:
1091       cc = equal;
1092       break;
1093     case kUnorderedNotEqual:
1094       __ j(parity_odd, &check, Label::kNear);
1095       __ movl(reg, Immediate(1));
1096       __ jmp(&done, Label::kNear);
1097     // Fall through.
1098     case kNotEqual:
1099       cc = not_equal;
1100       break;
1101     case kSignedLessThan:
1102       cc = less;
1103       break;
1104     case kSignedGreaterThanOrEqual:
1105       cc = greater_equal;
1106       break;
1107     case kSignedLessThanOrEqual:
1108       cc = less_equal;
1109       break;
1110     case kSignedGreaterThan:
1111       cc = greater;
1112       break;
1113     case kUnsignedLessThan:
1114       cc = below;
1115       break;
1116     case kUnsignedGreaterThanOrEqual:
1117       cc = above_equal;
1118       break;
1119     case kUnsignedLessThanOrEqual:
1120       cc = below_equal;
1121       break;
1122     case kUnsignedGreaterThan:
1123       cc = above;
1124       break;
1125     case kOverflow:
1126       cc = overflow;
1127       break;
1128     case kNotOverflow:
1129       cc = no_overflow;
1130       break;
1131   }
1132   __ bind(&check);
1133   __ setcc(cc, reg);
1134   __ movzxbl(reg, reg);
1135   __ bind(&done);
1136 }
1137
1138
1139 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1140   X64OperandConverter i(this, instr);
1141   Register input = i.InputRegister(0);
1142   for (size_t index = 2; index < instr->InputCount(); index += 2) {
1143     __ cmpl(input, Immediate(i.InputInt32(static_cast<int>(index + 0))));
1144     __ j(equal, GetLabel(i.InputRpo(static_cast<int>(index + 1))));
1145   }
1146   AssembleArchJump(i.InputRpo(1));
1147 }
1148
1149
1150 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1151   X64OperandConverter i(this, instr);
1152   Register input = i.InputRegister(0);
1153   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
1154   Label** cases = zone()->NewArray<Label*>(case_count);
1155   for (int32_t index = 0; index < case_count; ++index) {
1156     cases[index] = GetLabel(i.InputRpo(index + 2));
1157   }
1158   Label* const table = AddJumpTable(cases, case_count);
1159   __ cmpl(input, Immediate(case_count));
1160   __ j(above_equal, GetLabel(i.InputRpo(1)));
1161   __ leaq(kScratchRegister, Operand(table));
1162   __ jmp(Operand(kScratchRegister, input, times_8, 0));
1163 }
1164
1165
1166 void CodeGenerator::AssembleDeoptimizerCall(int deoptimization_id) {
1167   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1168       isolate(), deoptimization_id, Deoptimizer::LAZY);
1169   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1170 }
1171
1172
1173 void CodeGenerator::AssemblePrologue() {
1174   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1175   int stack_slots = frame()->GetSpillSlotCount();
1176   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1177     __ pushq(rbp);
1178     __ movq(rbp, rsp);
1179     const RegList saves = descriptor->CalleeSavedRegisters();
1180     if (saves != 0) {  // Save callee-saved registers.
1181       int register_save_area_size = 0;
1182       for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1183         if (!((1 << i) & saves)) continue;
1184         __ pushq(Register::from_code(i));
1185         register_save_area_size += kPointerSize;
1186       }
1187       frame()->SetRegisterSaveAreaSize(register_save_area_size);
1188     }
1189   } else if (descriptor->IsJSFunctionCall()) {
1190     CompilationInfo* info = this->info();
1191     __ Prologue(info->IsCodePreAgingActive());
1192     frame()->SetRegisterSaveAreaSize(
1193         StandardFrameConstants::kFixedFrameSizeFromFp);
1194   } else if (stack_slots > 0) {
1195     __ StubPrologue();
1196     frame()->SetRegisterSaveAreaSize(
1197         StandardFrameConstants::kFixedFrameSizeFromFp);
1198   }
1199
1200   if (info()->is_osr()) {
1201     // TurboFan OSR-compiled functions cannot be entered directly.
1202     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1203
1204     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1205     // frame is still on the stack. Optimized code uses OSR values directly from
1206     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1207     // remaining stack slots.
1208     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1209     osr_pc_offset_ = __ pc_offset();
1210     DCHECK(stack_slots >= frame()->GetOsrStackSlotCount());
1211     stack_slots -= frame()->GetOsrStackSlotCount();
1212   }
1213
1214   if (stack_slots > 0) {
1215     __ subq(rsp, Immediate(stack_slots * kPointerSize));
1216   }
1217 }
1218
1219
1220 void CodeGenerator::AssembleReturn() {
1221   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1222   int stack_slots = frame()->GetSpillSlotCount();
1223   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1224     if (frame()->GetRegisterSaveAreaSize() > 0) {
1225       // Remove this frame's spill slots first.
1226       if (stack_slots > 0) {
1227         __ addq(rsp, Immediate(stack_slots * kPointerSize));
1228       }
1229       const RegList saves = descriptor->CalleeSavedRegisters();
1230       // Restore registers.
1231       if (saves != 0) {
1232         for (int i = 0; i < Register::kNumRegisters; i++) {
1233           if (!((1 << i) & saves)) continue;
1234           __ popq(Register::from_code(i));
1235         }
1236       }
1237       __ popq(rbp);  // Pop caller's frame pointer.
1238       __ ret(0);
1239     } else {
1240       // No saved registers.
1241       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1242       __ popq(rbp);       // Pop caller's frame pointer.
1243       __ ret(0);
1244     }
1245   } else if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
1246     __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1247     __ popq(rbp);       // Pop caller's frame pointer.
1248     int pop_count = descriptor->IsJSFunctionCall()
1249                         ? static_cast<int>(descriptor->JSParameterCount())
1250                         : 0;
1251     __ ret(pop_count * kPointerSize);
1252   } else {
1253     __ ret(0);
1254   }
1255 }
1256
1257
1258 void CodeGenerator::AssembleMove(InstructionOperand* source,
1259                                  InstructionOperand* destination) {
1260   X64OperandConverter g(this, NULL);
1261   // Dispatch on the source and destination operand kinds.  Not all
1262   // combinations are possible.
1263   if (source->IsRegister()) {
1264     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1265     Register src = g.ToRegister(source);
1266     if (destination->IsRegister()) {
1267       __ movq(g.ToRegister(destination), src);
1268     } else {
1269       __ movq(g.ToOperand(destination), src);
1270     }
1271   } else if (source->IsStackSlot()) {
1272     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1273     Operand src = g.ToOperand(source);
1274     if (destination->IsRegister()) {
1275       Register dst = g.ToRegister(destination);
1276       __ movq(dst, src);
1277     } else {
1278       // Spill on demand to use a temporary register for memory-to-memory
1279       // moves.
1280       Register tmp = kScratchRegister;
1281       Operand dst = g.ToOperand(destination);
1282       __ movq(tmp, src);
1283       __ movq(dst, tmp);
1284     }
1285   } else if (source->IsConstant()) {
1286     ConstantOperand* constant_source = ConstantOperand::cast(source);
1287     Constant src = g.ToConstant(constant_source);
1288     if (destination->IsRegister() || destination->IsStackSlot()) {
1289       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1290                                                : kScratchRegister;
1291       switch (src.type()) {
1292         case Constant::kInt32:
1293           // TODO(dcarney): don't need scratch in this case.
1294           __ Set(dst, src.ToInt32());
1295           break;
1296         case Constant::kInt64:
1297           __ Set(dst, src.ToInt64());
1298           break;
1299         case Constant::kFloat32:
1300           __ Move(dst,
1301                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1302           break;
1303         case Constant::kFloat64:
1304           __ Move(dst,
1305                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1306           break;
1307         case Constant::kExternalReference:
1308           __ Move(dst, src.ToExternalReference());
1309           break;
1310         case Constant::kHeapObject:
1311           __ Move(dst, src.ToHeapObject());
1312           break;
1313         case Constant::kRpoNumber:
1314           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1315           break;
1316       }
1317       if (destination->IsStackSlot()) {
1318         __ movq(g.ToOperand(destination), kScratchRegister);
1319       }
1320     } else if (src.type() == Constant::kFloat32) {
1321       // TODO(turbofan): Can we do better here?
1322       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1323       if (destination->IsDoubleRegister()) {
1324         __ Move(g.ToDoubleRegister(destination), src_const);
1325       } else {
1326         DCHECK(destination->IsDoubleStackSlot());
1327         Operand dst = g.ToOperand(destination);
1328         __ movl(dst, Immediate(src_const));
1329       }
1330     } else {
1331       DCHECK_EQ(Constant::kFloat64, src.type());
1332       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1333       if (destination->IsDoubleRegister()) {
1334         __ Move(g.ToDoubleRegister(destination), src_const);
1335       } else {
1336         DCHECK(destination->IsDoubleStackSlot());
1337         __ movq(kScratchRegister, src_const);
1338         __ movq(g.ToOperand(destination), kScratchRegister);
1339       }
1340     }
1341   } else if (source->IsDoubleRegister()) {
1342     XMMRegister src = g.ToDoubleRegister(source);
1343     if (destination->IsDoubleRegister()) {
1344       XMMRegister dst = g.ToDoubleRegister(destination);
1345       __ movsd(dst, src);
1346     } else {
1347       DCHECK(destination->IsDoubleStackSlot());
1348       Operand dst = g.ToOperand(destination);
1349       __ movsd(dst, src);
1350     }
1351   } else if (source->IsDoubleStackSlot()) {
1352     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1353     Operand src = g.ToOperand(source);
1354     if (destination->IsDoubleRegister()) {
1355       XMMRegister dst = g.ToDoubleRegister(destination);
1356       __ movsd(dst, src);
1357     } else {
1358       // We rely on having xmm0 available as a fixed scratch register.
1359       Operand dst = g.ToOperand(destination);
1360       __ movsd(xmm0, src);
1361       __ movsd(dst, xmm0);
1362     }
1363   } else {
1364     UNREACHABLE();
1365   }
1366 }
1367
1368
1369 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1370                                  InstructionOperand* destination) {
1371   X64OperandConverter g(this, NULL);
1372   // Dispatch on the source and destination operand kinds.  Not all
1373   // combinations are possible.
1374   if (source->IsRegister() && destination->IsRegister()) {
1375     // Register-register.
1376     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1377   } else if (source->IsRegister() && destination->IsStackSlot()) {
1378     Register src = g.ToRegister(source);
1379     Operand dst = g.ToOperand(destination);
1380     __ xchgq(src, dst);
1381   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1382              (source->IsDoubleStackSlot() &&
1383               destination->IsDoubleStackSlot())) {
1384     // Memory-memory.
1385     Register tmp = kScratchRegister;
1386     Operand src = g.ToOperand(source);
1387     Operand dst = g.ToOperand(destination);
1388     __ movq(tmp, dst);
1389     __ xchgq(tmp, src);
1390     __ movq(dst, tmp);
1391   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1392     // XMM register-register swap. We rely on having xmm0
1393     // available as a fixed scratch register.
1394     XMMRegister src = g.ToDoubleRegister(source);
1395     XMMRegister dst = g.ToDoubleRegister(destination);
1396     __ movsd(xmm0, src);
1397     __ movsd(src, dst);
1398     __ movsd(dst, xmm0);
1399   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1400     // XMM register-memory swap.  We rely on having xmm0
1401     // available as a fixed scratch register.
1402     XMMRegister src = g.ToDoubleRegister(source);
1403     Operand dst = g.ToOperand(destination);
1404     __ movsd(xmm0, src);
1405     __ movsd(src, dst);
1406     __ movsd(dst, xmm0);
1407   } else {
1408     // No other combinations are possible.
1409     UNREACHABLE();
1410   }
1411 }
1412
1413
1414 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1415   for (size_t index = 0; index < target_count; ++index) {
1416     __ dq(targets[index]);
1417   }
1418 }
1419
1420
1421 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1422
1423
1424 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1425   int space_needed = Deoptimizer::patch_size();
1426   if (!info()->IsStub()) {
1427     // Ensure that we have enough space after the previous lazy-bailout
1428     // instruction for patching the code here.
1429     int current_pc = masm()->pc_offset();
1430     if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1431       int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1432       __ Nop(padding_size);
1433     }
1434   }
1435   MarkLazyDeoptSite();
1436 }
1437
1438 #undef __
1439
1440 }  // namespace internal
1441 }  // namespace compiler
1442 }  // namespace v8