src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/scopes.h"
  11 #include "src/x64/assembler-x64.h"
  12 #include "src/x64/macro-assembler-x64.h"
  13
  14 namespace v8 {
  15 namespace internal {
  16 namespace compiler {
  17
  18 #define __ masm()->
  19
  20
  21 // Adds X64 specific methods for decoding operands.
  22 class X64OperandConverter : public InstructionOperandConverter {
  23  public:
  24   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  25       : InstructionOperandConverter(gen, instr) {}
  26
  27   Immediate InputImmediate(int index) {
  28     return ToImmediate(instr_->InputAt(index));
  29   }
  30
  31   Operand InputOperand(int index) { return ToOperand(instr_->InputAt(index)); }
  32
  33   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  34
  35   Immediate ToImmediate(InstructionOperand* operand) {
  36     return Immediate(ToConstant(operand).ToInt32());
  37   }
  38
  39   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  40     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  41     // The linkage computes where all spill slots are located.
  42     FrameOffset offset = linkage()->GetFrameOffset(op->index(), frame(), extra);
  43     return Operand(offset.from_stack_pointer() ? rsp : rbp, offset.offset());
  44   }
  45
  46   static int NextOffset(int* offset) {
  47     int i = *offset;
  48     (*offset)++;
  49     return i;
  50   }
  51
  52   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  53     STATIC_ASSERT(0 == static_cast<int>(times_1));
  54     STATIC_ASSERT(1 == static_cast<int>(times_2));
  55     STATIC_ASSERT(2 == static_cast<int>(times_4));
  56     STATIC_ASSERT(3 == static_cast<int>(times_8));
  57     int scale = static_cast<int>(mode - one);
  58     DCHECK(scale >= 0 && scale < 4);
  59     return static_cast<ScaleFactor>(scale);
  60   }
  61
  62   Operand MemoryOperand(int* offset) {
  63     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  64     switch (mode) {
  65       case kMode_MR: {
  66         Register base = InputRegister(NextOffset(offset));
  67         int32_t disp = 0;
  68         return Operand(base, disp);
  69       }
  70       case kMode_MRI: {
  71         Register base = InputRegister(NextOffset(offset));
  72         int32_t disp = InputInt32(NextOffset(offset));
  73         return Operand(base, disp);
  74       }
  75       case kMode_MR1:
  76       case kMode_MR2:
  77       case kMode_MR4:
  78       case kMode_MR8: {
  79         Register base = InputRegister(NextOffset(offset));
  80         Register index = InputRegister(NextOffset(offset));
  81         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  82         int32_t disp = 0;
  83         return Operand(base, index, scale, disp);
  84       }
  85       case kMode_MR1I:
  86       case kMode_MR2I:
  87       case kMode_MR4I:
  88       case kMode_MR8I: {
  89         Register base = InputRegister(NextOffset(offset));
  90         Register index = InputRegister(NextOffset(offset));
  91         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
  92         int32_t disp = InputInt32(NextOffset(offset));
  93         return Operand(base, index, scale, disp);
  94       }
  95       case kMode_M1: {
  96         Register base = InputRegister(NextOffset(offset));
  97         int32_t disp = 0;
  98         return Operand(base, disp);
  99       }
 100       case kMode_M2:
 101         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 102         return Operand(no_reg, 0);
 103       case kMode_M4:
 104       case kMode_M8: {
 105         Register index = InputRegister(NextOffset(offset));
 106         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 107         int32_t disp = 0;
 108         return Operand(index, scale, disp);
 109       }
 110       case kMode_M1I:
 111       case kMode_M2I:
 112       case kMode_M4I:
 113       case kMode_M8I: {
 114         Register index = InputRegister(NextOffset(offset));
 115         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 116         int32_t disp = InputInt32(NextOffset(offset));
 117         return Operand(index, scale, disp);
 118       }
 119       case kMode_None:
 120         UNREACHABLE();
 121         return Operand(no_reg, 0);
 122     }
 123     UNREACHABLE();
 124     return Operand(no_reg, 0);
 125   }
 126
 127   Operand MemoryOperand(int first_input = 0) {
 128     return MemoryOperand(&first_input);
 129   }
 130 };
 131
 132
 133 namespace {
 134
 135 bool HasImmediateInput(Instruction* instr, int index) {
 136   return instr->InputAt(index)->IsImmediate();
 137 }
 138
 139
 140 class OutOfLineLoadZero FINAL : public OutOfLineCode {
 141  public:
 142   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 143       : OutOfLineCode(gen), result_(result) {}
 144
 145   void Generate() FINAL { __ xorl(result_, result_); }
 146
 147  private:
 148   Register const result_;
 149 };
 150
 151
 152 class OutOfLineLoadNaN FINAL : public OutOfLineCode {
 153  public:
 154   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 155       : OutOfLineCode(gen), result_(result) {}
 156
 157   void Generate() FINAL { __ pcmpeqd(result_, result_); }
 158
 159  private:
 160   XMMRegister const result_;
 161 };
 162
 163
 164 class OutOfLineTruncateDoubleToI FINAL : public OutOfLineCode {
 165  public:
 166   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 167                              XMMRegister input)
 168       : OutOfLineCode(gen), result_(result), input_(input) {}
 169
 170   void Generate() FINAL {
 171     __ subp(rsp, Immediate(kDoubleSize));
 172     __ movsd(MemOperand(rsp, 0), input_);
 173     __ SlowTruncateToI(result_, rsp, 0);
 174     __ addp(rsp, Immediate(kDoubleSize));
 175   }
 176
 177  private:
 178   Register const result_;
 179   XMMRegister const input_;
 180 };
 181
 182 }  // namespace
 183
 184
 185 #define ASSEMBLE_UNOP(asm_instr)         \
 186   do {                                   \
 187     if (instr->Output()->IsRegister()) { \
 188       __ asm_instr(i.OutputRegister());  \
 189     } else {                             \
 190       __ asm_instr(i.OutputOperand());   \
 191     }                                    \
 192   } while (0)
 193
 194
 195 #define ASSEMBLE_BINOP(asm_instr)                              \
 196   do {                                                         \
 197     if (HasImmediateInput(instr, 1)) {                         \
 198       if (instr->InputAt(0)->IsRegister()) {                   \
 199         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 200       } else {                                                 \
 201         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 202       }                                                        \
 203     } else {                                                   \
 204       if (instr->InputAt(1)->IsRegister()) {                   \
 205         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 206       } else {                                                 \
 207         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 208       }                                                        \
 209     }                                                          \
 210   } while (0)
 211
 212
 213 #define ASSEMBLE_MULT(asm_instr)                              \
 214   do {                                                        \
 215     if (HasImmediateInput(instr, 1)) {                        \
 216       if (instr->InputAt(0)->IsRegister()) {                  \
 217         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 218                      i.InputImmediate(1));                    \
 219       } else {                                                \
 220         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 221                      i.InputImmediate(1));                    \
 222       }                                                       \
 223     } else {                                                  \
 224       if (instr->InputAt(1)->IsRegister()) {                  \
 225         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 226       } else {                                                \
 227         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 228       }                                                       \
 229     }                                                         \
 230   } while (0)
 231
 232
 233 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 234   do {                                                                     \
 235     if (HasImmediateInput(instr, 1)) {                                     \
 236       if (instr->Output()->IsRegister()) {                                 \
 237         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 238       } else {                                                             \
 239         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 240       }                                                                    \
 241     } else {                                                               \
 242       if (instr->Output()->IsRegister()) {                                 \
 243         __ asm_instr##_cl(i.OutputRegister());                             \
 244       } else {                                                             \
 245         __ asm_instr##_cl(i.OutputOperand());                              \
 246       }                                                                    \
 247     }                                                                      \
 248   } while (0)
 249
 250
 251 #define ASSEMBLE_DOUBLE_BINOP(asm_instr)                                \
 252   do {                                                                  \
 253     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 254       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 255     } else {                                                            \
 256       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 257     }                                                                   \
 258   } while (0)
 259
 260
 261 #define ASSEMBLE_AVX_DOUBLE_BINOP(asm_instr)                           \
 262   do {                                                                 \
 263     CpuFeatureScope avx_scope(masm(), AVX);                            \
 264     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 265       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 266                    i.InputDoubleRegister(1));                          \
 267     } else {                                                           \
 268       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 269                    i.InputOperand(1));                                 \
 270     }                                                                  \
 271   } while (0)
 272
 273
 274 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 275   do {                                                                       \
 276     auto result = i.OutputDoubleRegister();                                  \
 277     auto buffer = i.InputRegister(0);                                        \
 278     auto index1 = i.InputRegister(1);                                        \
 279     auto index2 = i.InputInt32(2);                                           \
 280     OutOfLineCode* ool;                                                      \
 281     if (instr->InputAt(3)->IsRegister()) {                                   \
 282       auto length = i.InputRegister(3);                                      \
 283       DCHECK_EQ(0, index2);                                                  \
 284       __ cmpl(index1, length);                                               \
 285       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 286     } else {                                                                 \
 287       auto length = i.InputInt32(3);                                         \
 288       DCHECK_LE(index2, length);                                             \
 289       __ cmpq(index1, Immediate(length - index2));                           \
 290       class OutOfLineLoadFloat FINAL : public OutOfLineCode {                \
 291        public:                                                               \
 292         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 293                            Register buffer, Register index1, int32_t index2, \
 294                            int32_t length)                                   \
 295             : OutOfLineCode(gen),                                            \
 296               result_(result),                                               \
 297               buffer_(buffer),                                               \
 298               index1_(index1),                                               \
 299               index2_(index2),                                               \
 300               length_(length) {}                                             \
 301                                                                              \
 302         void Generate() FINAL {                                              \
 303           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 304           __ pcmpeqd(result_, result_);                                      \
 305           __ cmpl(kScratchRegister, Immediate(length_));                     \
 306           __ j(above_equal, exit());                                         \
 307           __ asm_instr(result_,                                              \
 308                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 309         }                                                                    \
 310                                                                              \
 311        private:                                                              \
 312         XMMRegister const result_;                                           \
 313         Register const buffer_;                                              \
 314         Register const index1_;                                              \
 315         int32_t const index2_;                                               \
 316         int32_t const length_;                                               \
 317       };                                                                     \
 318       ool = new (zone())                                                     \
 319           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 320     }                                                                        \
 321     __ j(above_equal, ool->entry());                                         \
 322     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 323     __ bind(ool->exit());                                                    \
 324   } while (false)
 325
 326
 327 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 328   do {                                                                         \
 329     auto result = i.OutputRegister();                                          \
 330     auto buffer = i.InputRegister(0);                                          \
 331     auto index1 = i.InputRegister(1);                                          \
 332     auto index2 = i.InputInt32(2);                                             \
 333     OutOfLineCode* ool;                                                        \
 334     if (instr->InputAt(3)->IsRegister()) {                                     \
 335       auto length = i.InputRegister(3);                                        \
 336       DCHECK_EQ(0, index2);                                                    \
 337       __ cmpl(index1, length);                                                 \
 338       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 339     } else {                                                                   \
 340       auto length = i.InputInt32(3);                                           \
 341       DCHECK_LE(index2, length);                                               \
 342       __ cmpq(index1, Immediate(length - index2));                             \
 343       class OutOfLineLoadInteger FINAL : public OutOfLineCode {                \
 344        public:                                                                 \
 345         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 346                              Register buffer, Register index1, int32_t index2, \
 347                              int32_t length)                                   \
 348             : OutOfLineCode(gen),                                              \
 349               result_(result),                                                 \
 350               buffer_(buffer),                                                 \
 351               index1_(index1),                                                 \
 352               index2_(index2),                                                 \
 353               length_(length) {}                                               \
 354                                                                                \
 355         void Generate() FINAL {                                                \
 356           Label oob;                                                           \
 357           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 358           __ cmpl(kScratchRegister, Immediate(length_));                       \
 359           __ j(above_equal, &oob, Label::kNear);                               \
 360           __ asm_instr(result_,                                                \
 361                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 362           __ jmp(exit());                                                      \
 363           __ bind(&oob);                                                       \
 364           __ xorl(result_, result_);                                           \
 365         }                                                                      \
 366                                                                                \
 367        private:                                                                \
 368         Register const result_;                                                \
 369         Register const buffer_;                                                \
 370         Register const index1_;                                                \
 371         int32_t const index2_;                                                 \
 372         int32_t const length_;                                                 \
 373       };                                                                       \
 374       ool = new (zone())                                                       \
 375           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 376     }                                                                          \
 377     __ j(above_equal, ool->entry());                                           \
 378     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 379     __ bind(ool->exit());                                                      \
 380   } while (false)
 381
 382
 383 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 384   do {                                                                       \
 385     auto buffer = i.InputRegister(0);                                        \
 386     auto index1 = i.InputRegister(1);                                        \
 387     auto index2 = i.InputInt32(2);                                           \
 388     auto value = i.InputDoubleRegister(4);                                   \
 389     if (instr->InputAt(3)->IsRegister()) {                                   \
 390       auto length = i.InputRegister(3);                                      \
 391       DCHECK_EQ(0, index2);                                                  \
 392       Label done;                                                            \
 393       __ cmpl(index1, length);                                               \
 394       __ j(above_equal, &done, Label::kNear);                                \
 395       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 396       __ bind(&done);                                                        \
 397     } else {                                                                 \
 398       auto length = i.InputInt32(3);                                         \
 399       DCHECK_LE(index2, length);                                             \
 400       __ cmpq(index1, Immediate(length - index2));                           \
 401       class OutOfLineStoreFloat FINAL : public OutOfLineCode {               \
 402        public:                                                               \
 403         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 404                             Register index1, int32_t index2, int32_t length, \
 405                             XMMRegister value)                               \
 406             : OutOfLineCode(gen),                                            \
 407               buffer_(buffer),                                               \
 408               index1_(index1),                                               \
 409               index2_(index2),                                               \
 410               length_(length),                                               \
 411               value_(value) {}                                               \
 412                                                                              \
 413         void Generate() FINAL {                                              \
 414           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 415           __ cmpl(kScratchRegister, Immediate(length_));                     \
 416           __ j(above_equal, exit());                                         \
 417           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 418                        value_);                                              \
 419         }                                                                    \
 420                                                                              \
 421        private:                                                              \
 422         Register const buffer_;                                              \
 423         Register const index1_;                                              \
 424         int32_t const index2_;                                               \
 425         int32_t const length_;                                               \
 426         XMMRegister const value_;                                            \
 427       };                                                                     \
 428       auto ool = new (zone())                                                \
 429           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 430       __ j(above_equal, ool->entry());                                       \
 431       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 432       __ bind(ool->exit());                                                  \
 433     }                                                                        \
 434   } while (false)
 435
 436
 437 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 438   do {                                                                         \
 439     auto buffer = i.InputRegister(0);                                          \
 440     auto index1 = i.InputRegister(1);                                          \
 441     auto index2 = i.InputInt32(2);                                             \
 442     if (instr->InputAt(3)->IsRegister()) {                                     \
 443       auto length = i.InputRegister(3);                                        \
 444       DCHECK_EQ(0, index2);                                                    \
 445       Label done;                                                              \
 446       __ cmpl(index1, length);                                                 \
 447       __ j(above_equal, &done, Label::kNear);                                  \
 448       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 449       __ bind(&done);                                                          \
 450     } else {                                                                   \
 451       auto length = i.InputInt32(3);                                           \
 452       DCHECK_LE(index2, length);                                               \
 453       __ cmpq(index1, Immediate(length - index2));                             \
 454       class OutOfLineStoreInteger FINAL : public OutOfLineCode {               \
 455        public:                                                                 \
 456         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 457                               Register index1, int32_t index2, int32_t length, \
 458                               Value value)                                     \
 459             : OutOfLineCode(gen),                                              \
 460               buffer_(buffer),                                                 \
 461               index1_(index1),                                                 \
 462               index2_(index2),                                                 \
 463               length_(length),                                                 \
 464               value_(value) {}                                                 \
 465                                                                                \
 466         void Generate() FINAL {                                                \
 467           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 468           __ cmpl(kScratchRegister, Immediate(length_));                       \
 469           __ j(above_equal, exit());                                           \
 470           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 471                        value_);                                                \
 472         }                                                                      \
 473                                                                                \
 474        private:                                                                \
 475         Register const buffer_;                                                \
 476         Register const index1_;                                                \
 477         int32_t const index2_;                                                 \
 478         int32_t const length_;                                                 \
 479         Value const value_;                                                    \
 480       };                                                                       \
 481       auto ool = new (zone())                                                  \
 482           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 483       __ j(above_equal, ool->entry());                                         \
 484       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 485       __ bind(ool->exit());                                                    \
 486     }                                                                          \
 487   } while (false)
 488
 489
 490 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 491   do {                                                           \
 492     if (instr->InputAt(4)->IsRegister()) {                       \
 493       Register value = i.InputRegister(4);                       \
 494       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 495     } else {                                                     \
 496       Immediate value = i.InputImmediate(4);                     \
 497       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 498     }                                                            \
 499   } while (false)
 500
 501
 502 // Assembles an instruction after register allocation, producing machine code.
 503 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 504   X64OperandConverter i(this, instr);
 505
 506   switch (ArchOpcodeField::decode(instr->opcode())) {
 507     case kArchCallCodeObject: {
 508       EnsureSpaceForLazyDeopt();
 509       if (HasImmediateInput(instr, 0)) {
 510         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 511         __ Call(code, RelocInfo::CODE_TARGET);
 512       } else {
 513         Register reg = i.InputRegister(0);
 514         int entry = Code::kHeaderSize - kHeapObjectTag;
 515         __ Call(Operand(reg, entry));
 516       }
 517       AddSafepointAndDeopt(instr);
 518       break;
 519     }
 520     case kArchCallJSFunction: {
 521       EnsureSpaceForLazyDeopt();
 522       Register func = i.InputRegister(0);
 523       if (FLAG_debug_code) {
 524         // Check the function's context matches the context argument.
 525         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 526         __ Assert(equal, kWrongFunctionContext);
 527       }
 528       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 529       AddSafepointAndDeopt(instr);
 530       break;
 531     }
 532     case kArchJmp:
 533       AssembleArchJump(i.InputRpo(0));
 534       break;
 535     case kArchSwitch:
 536       AssembleArchSwitch(instr);
 537       break;
 538     case kArchNop:
 539       // don't emit code for nops.
 540       break;
 541     case kArchRet:
 542       AssembleReturn();
 543       break;
 544     case kArchStackPointer:
 545       __ movq(i.OutputRegister(), rsp);
 546       break;
 547     case kArchTruncateDoubleToI: {
 548       auto result = i.OutputRegister();
 549       auto input = i.InputDoubleRegister(0);
 550       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 551       __ cvttsd2siq(result, input);
 552       __ cmpq(result, Immediate(1));
 553       __ j(overflow, ool->entry());
 554       __ bind(ool->exit());
 555       break;
 556     }
 557     case kX64Add32:
 558       ASSEMBLE_BINOP(addl);
 559       break;
 560     case kX64Add:
 561       ASSEMBLE_BINOP(addq);
 562       break;
 563     case kX64Sub32:
 564       ASSEMBLE_BINOP(subl);
 565       break;
 566     case kX64Sub:
 567       ASSEMBLE_BINOP(subq);
 568       break;
 569     case kX64And32:
 570       ASSEMBLE_BINOP(andl);
 571       break;
 572     case kX64And:
 573       ASSEMBLE_BINOP(andq);
 574       break;
 575     case kX64Cmp32:
 576       ASSEMBLE_BINOP(cmpl);
 577       break;
 578     case kX64Cmp:
 579       ASSEMBLE_BINOP(cmpq);
 580       break;
 581     case kX64Test32:
 582       ASSEMBLE_BINOP(testl);
 583       break;
 584     case kX64Test:
 585       ASSEMBLE_BINOP(testq);
 586       break;
 587     case kX64Imul32:
 588       ASSEMBLE_MULT(imull);
 589       break;
 590     case kX64Imul:
 591       ASSEMBLE_MULT(imulq);
 592       break;
 593     case kX64ImulHigh32:
 594       if (instr->InputAt(1)->IsRegister()) {
 595         __ imull(i.InputRegister(1));
 596       } else {
 597         __ imull(i.InputOperand(1));
 598       }
 599       break;
 600     case kX64UmulHigh32:
 601       if (instr->InputAt(1)->IsRegister()) {
 602         __ mull(i.InputRegister(1));
 603       } else {
 604         __ mull(i.InputOperand(1));
 605       }
 606       break;
 607     case kX64Idiv32:
 608       __ cdq();
 609       __ idivl(i.InputRegister(1));
 610       break;
 611     case kX64Idiv:
 612       __ cqo();
 613       __ idivq(i.InputRegister(1));
 614       break;
 615     case kX64Udiv32:
 616       __ xorl(rdx, rdx);
 617       __ divl(i.InputRegister(1));
 618       break;
 619     case kX64Udiv:
 620       __ xorq(rdx, rdx);
 621       __ divq(i.InputRegister(1));
 622       break;
 623     case kX64Not:
 624       ASSEMBLE_UNOP(notq);
 625       break;
 626     case kX64Not32:
 627       ASSEMBLE_UNOP(notl);
 628       break;
 629     case kX64Neg:
 630       ASSEMBLE_UNOP(negq);
 631       break;
 632     case kX64Neg32:
 633       ASSEMBLE_UNOP(negl);
 634       break;
 635     case kX64Or32:
 636       ASSEMBLE_BINOP(orl);
 637       break;
 638     case kX64Or:
 639       ASSEMBLE_BINOP(orq);
 640       break;
 641     case kX64Xor32:
 642       ASSEMBLE_BINOP(xorl);
 643       break;
 644     case kX64Xor:
 645       ASSEMBLE_BINOP(xorq);
 646       break;
 647     case kX64Shl32:
 648       ASSEMBLE_SHIFT(shll, 5);
 649       break;
 650     case kX64Shl:
 651       ASSEMBLE_SHIFT(shlq, 6);
 652       break;
 653     case kX64Shr32:
 654       ASSEMBLE_SHIFT(shrl, 5);
 655       break;
 656     case kX64Shr:
 657       ASSEMBLE_SHIFT(shrq, 6);
 658       break;
 659     case kX64Sar32:
 660       ASSEMBLE_SHIFT(sarl, 5);
 661       break;
 662     case kX64Sar:
 663       ASSEMBLE_SHIFT(sarq, 6);
 664       break;
 665     case kX64Ror32:
 666       ASSEMBLE_SHIFT(rorl, 5);
 667       break;
 668     case kX64Ror:
 669       ASSEMBLE_SHIFT(rorq, 6);
 670       break;
 671     case kSSEFloat64Cmp:
 672       ASSEMBLE_DOUBLE_BINOP(ucomisd);
 673       break;
 674     case kSSEFloat64Add:
 675       ASSEMBLE_DOUBLE_BINOP(addsd);
 676       break;
 677     case kSSEFloat64Sub:
 678       ASSEMBLE_DOUBLE_BINOP(subsd);
 679       break;
 680     case kSSEFloat64Mul:
 681       ASSEMBLE_DOUBLE_BINOP(mulsd);
 682       break;
 683     case kSSEFloat64Div:
 684       ASSEMBLE_DOUBLE_BINOP(divsd);
 685       break;
 686     case kSSEFloat64Mod: {
 687       __ subq(rsp, Immediate(kDoubleSize));
 688       // Move values to st(0) and st(1).
 689       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 690       __ fld_d(Operand(rsp, 0));
 691       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 692       __ fld_d(Operand(rsp, 0));
 693       // Loop while fprem isn't done.
 694       Label mod_loop;
 695       __ bind(&mod_loop);
 696       // This instructions traps on all kinds inputs, but we are assuming the
 697       // floating point control word is set to ignore them all.
 698       __ fprem();
 699       // The following 2 instruction implicitly use rax.
 700       __ fnstsw_ax();
 701       if (CpuFeatures::IsSupported(SAHF)) {
 702         CpuFeatureScope sahf_scope(masm(), SAHF);
 703         __ sahf();
 704       } else {
 705         __ shrl(rax, Immediate(8));
 706         __ andl(rax, Immediate(0xFF));
 707         __ pushq(rax);
 708         __ popfq();
 709       }
 710       __ j(parity_even, &mod_loop);
 711       // Move output to stack and clean up.
 712       __ fstp(1);
 713       __ fstp_d(Operand(rsp, 0));
 714       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 715       __ addq(rsp, Immediate(kDoubleSize));
 716       break;
 717     }
 718     case kSSEFloat64Sqrt:
 719       if (instr->InputAt(0)->IsDoubleRegister()) {
 720         __ sqrtsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 721       } else {
 722         __ sqrtsd(i.OutputDoubleRegister(), i.InputOperand(0));
 723       }
 724       break;
 725     case kSSEFloat64Floor: {
 726       CpuFeatureScope sse_scope(masm(), SSE4_1);
 727       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 728                  v8::internal::Assembler::kRoundDown);
 729       break;
 730     }
 731     case kSSEFloat64Ceil: {
 732       CpuFeatureScope sse_scope(masm(), SSE4_1);
 733       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 734                  v8::internal::Assembler::kRoundUp);
 735       break;
 736     }
 737     case kSSEFloat64RoundTruncate: {
 738       CpuFeatureScope sse_scope(masm(), SSE4_1);
 739       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
 740                  v8::internal::Assembler::kRoundToZero);
 741       break;
 742     }
 743     case kSSECvtss2sd:
 744       if (instr->InputAt(0)->IsDoubleRegister()) {
 745         __ cvtss2sd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 746       } else {
 747         __ cvtss2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 748       }
 749       break;
 750     case kSSECvtsd2ss:
 751       if (instr->InputAt(0)->IsDoubleRegister()) {
 752         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 753       } else {
 754         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputOperand(0));
 755       }
 756       break;
 757     case kSSEFloat64ToInt32:
 758       if (instr->InputAt(0)->IsDoubleRegister()) {
 759         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 760       } else {
 761         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 762       }
 763       break;
 764     case kSSEFloat64ToUint32: {
 765       if (instr->InputAt(0)->IsDoubleRegister()) {
 766         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 767       } else {
 768         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 769       }
 770       __ AssertZeroExtended(i.OutputRegister());
 771       break;
 772     }
 773     case kSSEInt32ToFloat64:
 774       if (instr->InputAt(0)->IsRegister()) {
 775         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 776       } else {
 777         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 778       }
 779       break;
 780     case kSSEUint32ToFloat64:
 781       if (instr->InputAt(0)->IsRegister()) {
 782         __ movl(kScratchRegister, i.InputRegister(0));
 783       } else {
 784         __ movl(kScratchRegister, i.InputOperand(0));
 785       }
 786       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 787       break;
 788     case kAVXFloat64Add:
 789       ASSEMBLE_AVX_DOUBLE_BINOP(vaddsd);
 790       break;
 791     case kAVXFloat64Sub:
 792       ASSEMBLE_AVX_DOUBLE_BINOP(vsubsd);
 793       break;
 794     case kAVXFloat64Mul:
 795       ASSEMBLE_AVX_DOUBLE_BINOP(vmulsd);
 796       break;
 797     case kAVXFloat64Div:
 798       ASSEMBLE_AVX_DOUBLE_BINOP(vdivsd);
 799       break;
 800     case kX64Movsxbl:
 801       if (instr->addressing_mode() != kMode_None) {
 802         __ movsxbl(i.OutputRegister(), i.MemoryOperand());
 803       } else if (instr->InputAt(0)->IsRegister()) {
 804         __ movsxbl(i.OutputRegister(), i.InputRegister(0));
 805       } else {
 806         __ movsxbl(i.OutputRegister(), i.InputOperand(0));
 807       }
 808       __ AssertZeroExtended(i.OutputRegister());
 809       break;
 810     case kX64Movzxbl:
 811       __ movzxbl(i.OutputRegister(), i.MemoryOperand());
 812       break;
 813     case kX64Movb: {
 814       int index = 0;
 815       Operand operand = i.MemoryOperand(&index);
 816       if (HasImmediateInput(instr, index)) {
 817         __ movb(operand, Immediate(i.InputInt8(index)));
 818       } else {
 819         __ movb(operand, i.InputRegister(index));
 820       }
 821       break;
 822     }
 823     case kX64Movsxwl:
 824       if (instr->addressing_mode() != kMode_None) {
 825         __ movsxwl(i.OutputRegister(), i.MemoryOperand());
 826       } else if (instr->InputAt(0)->IsRegister()) {
 827         __ movsxwl(i.OutputRegister(), i.InputRegister(0));
 828       } else {
 829         __ movsxwl(i.OutputRegister(), i.InputOperand(0));
 830       }
 831       __ AssertZeroExtended(i.OutputRegister());
 832       break;
 833     case kX64Movzxwl:
 834       __ movzxwl(i.OutputRegister(), i.MemoryOperand());
 835       __ AssertZeroExtended(i.OutputRegister());
 836       break;
 837     case kX64Movw: {
 838       int index = 0;
 839       Operand operand = i.MemoryOperand(&index);
 840       if (HasImmediateInput(instr, index)) {
 841         __ movw(operand, Immediate(i.InputInt16(index)));
 842       } else {
 843         __ movw(operand, i.InputRegister(index));
 844       }
 845       break;
 846     }
 847     case kX64Movl:
 848       if (instr->HasOutput()) {
 849         if (instr->addressing_mode() == kMode_None) {
 850           if (instr->InputAt(0)->IsRegister()) {
 851             __ movl(i.OutputRegister(), i.InputRegister(0));
 852           } else {
 853             __ movl(i.OutputRegister(), i.InputOperand(0));
 854           }
 855         } else {
 856           __ movl(i.OutputRegister(), i.MemoryOperand());
 857         }
 858         __ AssertZeroExtended(i.OutputRegister());
 859       } else {
 860         int index = 0;
 861         Operand operand = i.MemoryOperand(&index);
 862         if (HasImmediateInput(instr, index)) {
 863           __ movl(operand, i.InputImmediate(index));
 864         } else {
 865           __ movl(operand, i.InputRegister(index));
 866         }
 867       }
 868       break;
 869     case kX64Movsxlq: {
 870       if (instr->InputAt(0)->IsRegister()) {
 871         __ movsxlq(i.OutputRegister(), i.InputRegister(0));
 872       } else {
 873         __ movsxlq(i.OutputRegister(), i.InputOperand(0));
 874       }
 875       break;
 876     }
 877     case kX64Movq:
 878       if (instr->HasOutput()) {
 879         __ movq(i.OutputRegister(), i.MemoryOperand());
 880       } else {
 881         int index = 0;
 882         Operand operand = i.MemoryOperand(&index);
 883         if (HasImmediateInput(instr, index)) {
 884           __ movq(operand, i.InputImmediate(index));
 885         } else {
 886           __ movq(operand, i.InputRegister(index));
 887         }
 888       }
 889       break;
 890     case kX64Movss:
 891       if (instr->HasOutput()) {
 892         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
 893       } else {
 894         int index = 0;
 895         Operand operand = i.MemoryOperand(&index);
 896         __ movss(operand, i.InputDoubleRegister(index));
 897       }
 898       break;
 899     case kX64Movsd:
 900       if (instr->HasOutput()) {
 901         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
 902       } else {
 903         int index = 0;
 904         Operand operand = i.MemoryOperand(&index);
 905         __ movsd(operand, i.InputDoubleRegister(index));
 906       }
 907       break;
 908     case kX64Lea32: {
 909       AddressingMode mode = AddressingModeField::decode(instr->opcode());
 910       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
 911       // and addressing mode just happens to work out. The "addl"/"subl" forms
 912       // in these cases are faster based on measurements.
 913       if (i.InputRegister(0).is(i.OutputRegister())) {
 914         if (mode == kMode_MRI) {
 915           int32_t constant_summand = i.InputInt32(1);
 916           if (constant_summand > 0) {
 917             __ addl(i.OutputRegister(), Immediate(constant_summand));
 918           } else if (constant_summand < 0) {
 919             __ subl(i.OutputRegister(), Immediate(-constant_summand));
 920           }
 921         } else if (mode == kMode_MR1) {
 922           if (i.InputRegister(1).is(i.OutputRegister())) {
 923             __ shll(i.OutputRegister(), Immediate(1));
 924           } else {
 925             __ leal(i.OutputRegister(), i.MemoryOperand());
 926           }
 927         } else if (mode == kMode_M2) {
 928           __ shll(i.OutputRegister(), Immediate(1));
 929         } else if (mode == kMode_M4) {
 930           __ shll(i.OutputRegister(), Immediate(2));
 931         } else if (mode == kMode_M8) {
 932           __ shll(i.OutputRegister(), Immediate(3));
 933         } else {
 934           __ leal(i.OutputRegister(), i.MemoryOperand());
 935         }
 936       } else {
 937         __ leal(i.OutputRegister(), i.MemoryOperand());
 938       }
 939       __ AssertZeroExtended(i.OutputRegister());
 940       break;
 941     }
 942     case kX64Lea:
 943       __ leaq(i.OutputRegister(), i.MemoryOperand());
 944       break;
 945     case kX64Dec32:
 946       __ decl(i.OutputRegister());
 947       break;
 948     case kX64Inc32:
 949       __ incl(i.OutputRegister());
 950       break;
 951     case kX64Push:
 952       if (HasImmediateInput(instr, 0)) {
 953         __ pushq(i.InputImmediate(0));
 954       } else {
 955         if (instr->InputAt(0)->IsRegister()) {
 956           __ pushq(i.InputRegister(0));
 957         } else {
 958           __ pushq(i.InputOperand(0));
 959         }
 960       }
 961       break;
 962     case kX64StoreWriteBarrier: {
 963       Register object = i.InputRegister(0);
 964       Register index = i.InputRegister(1);
 965       Register value = i.InputRegister(2);
 966       __ movq(Operand(object, index, times_1, 0), value);
 967       __ leaq(index, Operand(object, index, times_1, 0));
 968       SaveFPRegsMode mode =
 969           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
 970       __ RecordWrite(object, index, value, mode);
 971       break;
 972     }
 973     case kCheckedLoadInt8:
 974       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
 975       break;
 976     case kCheckedLoadUint8:
 977       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
 978       break;
 979     case kCheckedLoadInt16:
 980       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
 981       break;
 982     case kCheckedLoadUint16:
 983       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
 984       break;
 985     case kCheckedLoadWord32:
 986       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
 987       break;
 988     case kCheckedLoadFloat32:
 989       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
 990       break;
 991     case kCheckedLoadFloat64:
 992       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
 993       break;
 994     case kCheckedStoreWord8:
 995       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
 996       break;
 997     case kCheckedStoreWord16:
 998       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
 999       break;
1000     case kCheckedStoreWord32:
1001       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1002       break;
1003     case kCheckedStoreFloat32:
1004       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1005       break;
1006     case kCheckedStoreFloat64:
1007       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1008       break;
1009   }
1010 }
1011
1012
1013 // Assembles branches after this instruction.
1014 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1015   X64OperandConverter i(this, instr);
1016   Label::Distance flabel_distance =
1017       branch->fallthru ? Label::kNear : Label::kFar;
1018   Label* tlabel = branch->true_label;
1019   Label* flabel = branch->false_label;
1020   switch (branch->condition) {
1021     case kUnorderedEqual:
1022       __ j(parity_even, flabel, flabel_distance);
1023     // Fall through.
1024     case kEqual:
1025       __ j(equal, tlabel);
1026       break;
1027     case kUnorderedNotEqual:
1028       __ j(parity_even, tlabel);
1029     // Fall through.
1030     case kNotEqual:
1031       __ j(not_equal, tlabel);
1032       break;
1033     case kSignedLessThan:
1034       __ j(less, tlabel);
1035       break;
1036     case kSignedGreaterThanOrEqual:
1037       __ j(greater_equal, tlabel);
1038       break;
1039     case kSignedLessThanOrEqual:
1040       __ j(less_equal, tlabel);
1041       break;
1042     case kSignedGreaterThan:
1043       __ j(greater, tlabel);
1044       break;
1045     case kUnsignedLessThan:
1046       __ j(below, tlabel);
1047       break;
1048     case kUnsignedGreaterThanOrEqual:
1049       __ j(above_equal, tlabel);
1050       break;
1051     case kUnsignedLessThanOrEqual:
1052       __ j(below_equal, tlabel);
1053       break;
1054     case kUnsignedGreaterThan:
1055       __ j(above, tlabel);
1056       break;
1057     case kOverflow:
1058       __ j(overflow, tlabel);
1059       break;
1060     case kNotOverflow:
1061       __ j(no_overflow, tlabel);
1062       break;
1063   }
1064   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1065 }
1066
1067
1068 void CodeGenerator::AssembleArchJump(BasicBlock::RpoNumber target) {
1069   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1070 }
1071
1072
1073 void CodeGenerator::AssembleArchSwitch(Instruction* instr) {
1074   X64OperandConverter i(this, instr);
1075   size_t const label_count = instr->InputCount() - 1;
1076   Label** labels = zone()->NewArray<Label*>(static_cast<int>(label_count));
1077   for (size_t index = 0; index < label_count; ++index) {
1078     labels[index] = GetLabel(i.InputRpo(static_cast<int>(index + 1)));
1079   }
1080   Label* const table = AddJumpTable(labels, label_count);
1081   __ leaq(kScratchRegister, Operand(table));
1082   __ jmp(Operand(kScratchRegister, i.InputRegister(0), times_8, 0));
1083 }
1084
1085
1086 // Assembles boolean materializations after this instruction.
1087 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1088                                         FlagsCondition condition) {
1089   X64OperandConverter i(this, instr);
1090   Label done;
1091
1092   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1093   // last output of the instruction.
1094   Label check;
1095   DCHECK_NE(0, static_cast<int>(instr->OutputCount()));
1096   Register reg = i.OutputRegister(static_cast<int>(instr->OutputCount() - 1));
1097   Condition cc = no_condition;
1098   switch (condition) {
1099     case kUnorderedEqual:
1100       __ j(parity_odd, &check, Label::kNear);
1101       __ movl(reg, Immediate(0));
1102       __ jmp(&done, Label::kNear);
1103     // Fall through.
1104     case kEqual:
1105       cc = equal;
1106       break;
1107     case kUnorderedNotEqual:
1108       __ j(parity_odd, &check, Label::kNear);
1109       __ movl(reg, Immediate(1));
1110       __ jmp(&done, Label::kNear);
1111     // Fall through.
1112     case kNotEqual:
1113       cc = not_equal;
1114       break;
1115     case kSignedLessThan:
1116       cc = less;
1117       break;
1118     case kSignedGreaterThanOrEqual:
1119       cc = greater_equal;
1120       break;
1121     case kSignedLessThanOrEqual:
1122       cc = less_equal;
1123       break;
1124     case kSignedGreaterThan:
1125       cc = greater;
1126       break;
1127     case kUnsignedLessThan:
1128       cc = below;
1129       break;
1130     case kUnsignedGreaterThanOrEqual:
1131       cc = above_equal;
1132       break;
1133     case kUnsignedLessThanOrEqual:
1134       cc = below_equal;
1135       break;
1136     case kUnsignedGreaterThan:
1137       cc = above;
1138       break;
1139     case kOverflow:
1140       cc = overflow;
1141       break;
1142     case kNotOverflow:
1143       cc = no_overflow;
1144       break;
1145   }
1146   __ bind(&check);
1147   __ setcc(cc, reg);
1148   __ movzxbl(reg, reg);
1149   __ bind(&done);
1150 }
1151
1152
1153 void CodeGenerator::AssembleDeoptimizerCall(int deoptimization_id) {
1154   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1155       isolate(), deoptimization_id, Deoptimizer::LAZY);
1156   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1157 }
1158
1159
1160 void CodeGenerator::AssemblePrologue() {
1161   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1162   int stack_slots = frame()->GetSpillSlotCount();
1163   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1164     __ pushq(rbp);
1165     __ movq(rbp, rsp);
1166     const RegList saves = descriptor->CalleeSavedRegisters();
1167     if (saves != 0) {  // Save callee-saved registers.
1168       int register_save_area_size = 0;
1169       for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1170         if (!((1 << i) & saves)) continue;
1171         __ pushq(Register::from_code(i));
1172         register_save_area_size += kPointerSize;
1173       }
1174       frame()->SetRegisterSaveAreaSize(register_save_area_size);
1175     }
1176   } else if (descriptor->IsJSFunctionCall()) {
1177     CompilationInfo* info = this->info();
1178     __ Prologue(info->IsCodePreAgingActive());
1179     frame()->SetRegisterSaveAreaSize(
1180         StandardFrameConstants::kFixedFrameSizeFromFp);
1181   } else {
1182     __ StubPrologue();
1183     frame()->SetRegisterSaveAreaSize(
1184         StandardFrameConstants::kFixedFrameSizeFromFp);
1185   }
1186
1187   if (info()->is_osr()) {
1188     // TurboFan OSR-compiled functions cannot be entered directly.
1189     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1190
1191     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1192     // frame is still on the stack. Optimized code uses OSR values directly from
1193     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1194     // remaining stack slots.
1195     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1196     osr_pc_offset_ = __ pc_offset();
1197     DCHECK(stack_slots >= frame()->GetOsrStackSlotCount());
1198     stack_slots -= frame()->GetOsrStackSlotCount();
1199   }
1200
1201   if (stack_slots > 0) {
1202     __ subq(rsp, Immediate(stack_slots * kPointerSize));
1203   }
1204 }
1205
1206
1207 void CodeGenerator::AssembleReturn() {
1208   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1209   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1210     if (frame()->GetRegisterSaveAreaSize() > 0) {
1211       // Remove this frame's spill slots first.
1212       int stack_slots = frame()->GetSpillSlotCount();
1213       if (stack_slots > 0) {
1214         __ addq(rsp, Immediate(stack_slots * kPointerSize));
1215       }
1216       const RegList saves = descriptor->CalleeSavedRegisters();
1217       // Restore registers.
1218       if (saves != 0) {
1219         for (int i = 0; i < Register::kNumRegisters; i++) {
1220           if (!((1 << i) & saves)) continue;
1221           __ popq(Register::from_code(i));
1222         }
1223       }
1224       __ popq(rbp);  // Pop caller's frame pointer.
1225       __ ret(0);
1226     } else {
1227       // No saved registers.
1228       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1229       __ popq(rbp);       // Pop caller's frame pointer.
1230       __ ret(0);
1231     }
1232   } else {
1233     __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1234     __ popq(rbp);       // Pop caller's frame pointer.
1235     int pop_count = descriptor->IsJSFunctionCall()
1236                         ? static_cast<int>(descriptor->JSParameterCount())
1237                         : 0;
1238     __ ret(pop_count * kPointerSize);
1239   }
1240 }
1241
1242
1243 void CodeGenerator::AssembleMove(InstructionOperand* source,
1244                                  InstructionOperand* destination) {
1245   X64OperandConverter g(this, NULL);
1246   // Dispatch on the source and destination operand kinds.  Not all
1247   // combinations are possible.
1248   if (source->IsRegister()) {
1249     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1250     Register src = g.ToRegister(source);
1251     if (destination->IsRegister()) {
1252       __ movq(g.ToRegister(destination), src);
1253     } else {
1254       __ movq(g.ToOperand(destination), src);
1255     }
1256   } else if (source->IsStackSlot()) {
1257     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1258     Operand src = g.ToOperand(source);
1259     if (destination->IsRegister()) {
1260       Register dst = g.ToRegister(destination);
1261       __ movq(dst, src);
1262     } else {
1263       // Spill on demand to use a temporary register for memory-to-memory
1264       // moves.
1265       Register tmp = kScratchRegister;
1266       Operand dst = g.ToOperand(destination);
1267       __ movq(tmp, src);
1268       __ movq(dst, tmp);
1269     }
1270   } else if (source->IsConstant()) {
1271     ConstantOperand* constant_source = ConstantOperand::cast(source);
1272     Constant src = g.ToConstant(constant_source);
1273     if (destination->IsRegister() || destination->IsStackSlot()) {
1274       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1275                                                : kScratchRegister;
1276       switch (src.type()) {
1277         case Constant::kInt32:
1278           // TODO(dcarney): don't need scratch in this case.
1279           __ Set(dst, src.ToInt32());
1280           break;
1281         case Constant::kInt64:
1282           __ Set(dst, src.ToInt64());
1283           break;
1284         case Constant::kFloat32:
1285           __ Move(dst,
1286                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1287           break;
1288         case Constant::kFloat64:
1289           __ Move(dst,
1290                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1291           break;
1292         case Constant::kExternalReference:
1293           __ Move(dst, src.ToExternalReference());
1294           break;
1295         case Constant::kHeapObject:
1296           __ Move(dst, src.ToHeapObject());
1297           break;
1298         case Constant::kRpoNumber:
1299           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1300           break;
1301       }
1302       if (destination->IsStackSlot()) {
1303         __ movq(g.ToOperand(destination), kScratchRegister);
1304       }
1305     } else if (src.type() == Constant::kFloat32) {
1306       // TODO(turbofan): Can we do better here?
1307       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1308       if (destination->IsDoubleRegister()) {
1309         __ Move(g.ToDoubleRegister(destination), src_const);
1310       } else {
1311         DCHECK(destination->IsDoubleStackSlot());
1312         Operand dst = g.ToOperand(destination);
1313         __ movl(dst, Immediate(src_const));
1314       }
1315     } else {
1316       DCHECK_EQ(Constant::kFloat64, src.type());
1317       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1318       if (destination->IsDoubleRegister()) {
1319         __ Move(g.ToDoubleRegister(destination), src_const);
1320       } else {
1321         DCHECK(destination->IsDoubleStackSlot());
1322         __ movq(kScratchRegister, src_const);
1323         __ movq(g.ToOperand(destination), kScratchRegister);
1324       }
1325     }
1326   } else if (source->IsDoubleRegister()) {
1327     XMMRegister src = g.ToDoubleRegister(source);
1328     if (destination->IsDoubleRegister()) {
1329       XMMRegister dst = g.ToDoubleRegister(destination);
1330       __ movsd(dst, src);
1331     } else {
1332       DCHECK(destination->IsDoubleStackSlot());
1333       Operand dst = g.ToOperand(destination);
1334       __ movsd(dst, src);
1335     }
1336   } else if (source->IsDoubleStackSlot()) {
1337     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1338     Operand src = g.ToOperand(source);
1339     if (destination->IsDoubleRegister()) {
1340       XMMRegister dst = g.ToDoubleRegister(destination);
1341       __ movsd(dst, src);
1342     } else {
1343       // We rely on having xmm0 available as a fixed scratch register.
1344       Operand dst = g.ToOperand(destination);
1345       __ movsd(xmm0, src);
1346       __ movsd(dst, xmm0);
1347     }
1348   } else {
1349     UNREACHABLE();
1350   }
1351 }
1352
1353
1354 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1355                                  InstructionOperand* destination) {
1356   X64OperandConverter g(this, NULL);
1357   // Dispatch on the source and destination operand kinds.  Not all
1358   // combinations are possible.
1359   if (source->IsRegister() && destination->IsRegister()) {
1360     // Register-register.
1361     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1362   } else if (source->IsRegister() && destination->IsStackSlot()) {
1363     Register src = g.ToRegister(source);
1364     Operand dst = g.ToOperand(destination);
1365     __ xchgq(src, dst);
1366   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1367              (source->IsDoubleStackSlot() &&
1368               destination->IsDoubleStackSlot())) {
1369     // Memory-memory.
1370     Register tmp = kScratchRegister;
1371     Operand src = g.ToOperand(source);
1372     Operand dst = g.ToOperand(destination);
1373     __ movq(tmp, dst);
1374     __ xchgq(tmp, src);
1375     __ movq(dst, tmp);
1376   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1377     // XMM register-register swap. We rely on having xmm0
1378     // available as a fixed scratch register.
1379     XMMRegister src = g.ToDoubleRegister(source);
1380     XMMRegister dst = g.ToDoubleRegister(destination);
1381     __ movsd(xmm0, src);
1382     __ movsd(src, dst);
1383     __ movsd(dst, xmm0);
1384   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1385     // XMM register-memory swap.  We rely on having xmm0
1386     // available as a fixed scratch register.
1387     XMMRegister src = g.ToDoubleRegister(source);
1388     Operand dst = g.ToOperand(destination);
1389     __ movsd(xmm0, src);
1390     __ movsd(src, dst);
1391     __ movsd(dst, xmm0);
1392   } else {
1393     // No other combinations are possible.
1394     UNREACHABLE();
1395   }
1396 }
1397
1398
1399 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1400   for (size_t index = 0; index < target_count; ++index) {
1401     __ dq(targets[index]);
1402   }
1403 }
1404
1405
1406 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1407
1408
1409 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1410   int space_needed = Deoptimizer::patch_size();
1411   if (!info()->IsStub()) {
1412     // Ensure that we have enough space after the previous lazy-bailout
1413     // instruction for patching the code here.
1414     int current_pc = masm()->pc_offset();
1415     if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1416       int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1417       __ Nop(padding_size);
1418     }
1419   }
1420   MarkLazyDeoptSite();
1421 }
1422
1423 #undef __
1424
1425 }  // namespace internal
1426 }  // namespace compiler
1427 }  // namespace v8