deps/v8/src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/scopes.h"
  11 #include "src/x64/assembler-x64.h"
  12 #include "src/x64/macro-assembler-x64.h"
  13
  14 namespace v8 {
  15 namespace internal {
  16 namespace compiler {
  17
  18 #define __ masm()->
  19
  20
  21 #define kScratchDoubleReg xmm0
  22
  23
  24 // Adds X64 specific methods for decoding operands.
  25 class X64OperandConverter : public InstructionOperandConverter {
  26  public:
  27   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  28       : InstructionOperandConverter(gen, instr) {}
  29
  30   Immediate InputImmediate(size_t index) {
  31     return ToImmediate(instr_->InputAt(index));
  32   }
  33
  34   Operand InputOperand(size_t index, int extra = 0) {
  35     return ToOperand(instr_->InputAt(index), extra);
  36   }
  37
  38   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  39
  40   Immediate ToImmediate(InstructionOperand* operand) {
  41     return Immediate(ToConstant(operand).ToInt32());
  42   }
  43
  44   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  45     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  46     // The linkage computes where all spill slots are located.
  47     FrameOffset offset = linkage()->GetFrameOffset(op->index(), frame(), extra);
  48     return Operand(offset.from_stack_pointer() ? rsp : rbp, offset.offset());
  49   }
  50
  51   static size_t NextOffset(size_t* offset) {
  52     size_t i = *offset;
  53     (*offset)++;
  54     return i;
  55   }
  56
  57   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  58     STATIC_ASSERT(0 == static_cast<int>(times_1));
  59     STATIC_ASSERT(1 == static_cast<int>(times_2));
  60     STATIC_ASSERT(2 == static_cast<int>(times_4));
  61     STATIC_ASSERT(3 == static_cast<int>(times_8));
  62     int scale = static_cast<int>(mode - one);
  63     DCHECK(scale >= 0 && scale < 4);
  64     return static_cast<ScaleFactor>(scale);
  65   }
  66
  67   Operand MemoryOperand(size_t* offset) {
  68     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  69     switch (mode) {
  70       case kMode_MR: {
  71         Register base = InputRegister(NextOffset(offset));
  72         int32_t disp = 0;
  73         return Operand(base, disp);
  74       }
  75       case kMode_MRI: {
  76         Register base = InputRegister(NextOffset(offset));
  77         int32_t disp = InputInt32(NextOffset(offset));
  78         return Operand(base, disp);
  79       }
  80       case kMode_MR1:
  81       case kMode_MR2:
  82       case kMode_MR4:
  83       case kMode_MR8: {
  84         Register base = InputRegister(NextOffset(offset));
  85         Register index = InputRegister(NextOffset(offset));
  86         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  87         int32_t disp = 0;
  88         return Operand(base, index, scale, disp);
  89       }
  90       case kMode_MR1I:
  91       case kMode_MR2I:
  92       case kMode_MR4I:
  93       case kMode_MR8I: {
  94         Register base = InputRegister(NextOffset(offset));
  95         Register index = InputRegister(NextOffset(offset));
  96         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
  97         int32_t disp = InputInt32(NextOffset(offset));
  98         return Operand(base, index, scale, disp);
  99       }
 100       case kMode_M1: {
 101         Register base = InputRegister(NextOffset(offset));
 102         int32_t disp = 0;
 103         return Operand(base, disp);
 104       }
 105       case kMode_M2:
 106         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 107         return Operand(no_reg, 0);
 108       case kMode_M4:
 109       case kMode_M8: {
 110         Register index = InputRegister(NextOffset(offset));
 111         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 112         int32_t disp = 0;
 113         return Operand(index, scale, disp);
 114       }
 115       case kMode_M1I:
 116       case kMode_M2I:
 117       case kMode_M4I:
 118       case kMode_M8I: {
 119         Register index = InputRegister(NextOffset(offset));
 120         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 121         int32_t disp = InputInt32(NextOffset(offset));
 122         return Operand(index, scale, disp);
 123       }
 124       case kMode_None:
 125         UNREACHABLE();
 126         return Operand(no_reg, 0);
 127     }
 128     UNREACHABLE();
 129     return Operand(no_reg, 0);
 130   }
 131
 132   Operand MemoryOperand(size_t first_input = 0) {
 133     return MemoryOperand(&first_input);
 134   }
 135 };
 136
 137
 138 namespace {
 139
 140 bool HasImmediateInput(Instruction* instr, size_t index) {
 141   return instr->InputAt(index)->IsImmediate();
 142 }
 143
 144
 145 class OutOfLineLoadZero FINAL : public OutOfLineCode {
 146  public:
 147   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 148       : OutOfLineCode(gen), result_(result) {}
 149
 150   void Generate() FINAL { __ xorl(result_, result_); }
 151
 152  private:
 153   Register const result_;
 154 };
 155
 156
 157 class OutOfLineLoadNaN FINAL : public OutOfLineCode {
 158  public:
 159   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 160       : OutOfLineCode(gen), result_(result) {}
 161
 162   void Generate() FINAL { __ pcmpeqd(result_, result_); }
 163
 164  private:
 165   XMMRegister const result_;
 166 };
 167
 168
 169 class OutOfLineTruncateDoubleToI FINAL : public OutOfLineCode {
 170  public:
 171   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 172                              XMMRegister input)
 173       : OutOfLineCode(gen), result_(result), input_(input) {}
 174
 175   void Generate() FINAL {
 176     __ subp(rsp, Immediate(kDoubleSize));
 177     __ movsd(MemOperand(rsp, 0), input_);
 178     __ SlowTruncateToI(result_, rsp, 0);
 179     __ addp(rsp, Immediate(kDoubleSize));
 180   }
 181
 182  private:
 183   Register const result_;
 184   XMMRegister const input_;
 185 };
 186
 187 }  // namespace
 188
 189
 190 #define ASSEMBLE_UNOP(asm_instr)         \
 191   do {                                   \
 192     if (instr->Output()->IsRegister()) { \
 193       __ asm_instr(i.OutputRegister());  \
 194     } else {                             \
 195       __ asm_instr(i.OutputOperand());   \
 196     }                                    \
 197   } while (0)
 198
 199
 200 #define ASSEMBLE_BINOP(asm_instr)                              \
 201   do {                                                         \
 202     if (HasImmediateInput(instr, 1)) {                         \
 203       if (instr->InputAt(0)->IsRegister()) {                   \
 204         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 205       } else {                                                 \
 206         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 207       }                                                        \
 208     } else {                                                   \
 209       if (instr->InputAt(1)->IsRegister()) {                   \
 210         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 211       } else {                                                 \
 212         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 213       }                                                        \
 214     }                                                          \
 215   } while (0)
 216
 217
 218 #define ASSEMBLE_MULT(asm_instr)                              \
 219   do {                                                        \
 220     if (HasImmediateInput(instr, 1)) {                        \
 221       if (instr->InputAt(0)->IsRegister()) {                  \
 222         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 223                      i.InputImmediate(1));                    \
 224       } else {                                                \
 225         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 226                      i.InputImmediate(1));                    \
 227       }                                                       \
 228     } else {                                                  \
 229       if (instr->InputAt(1)->IsRegister()) {                  \
 230         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 231       } else {                                                \
 232         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 233       }                                                       \
 234     }                                                         \
 235   } while (0)
 236
 237
 238 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 239   do {                                                                     \
 240     if (HasImmediateInput(instr, 1)) {                                     \
 241       if (instr->Output()->IsRegister()) {                                 \
 242         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 243       } else {                                                             \
 244         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 245       }                                                                    \
 246     } else {                                                               \
 247       if (instr->Output()->IsRegister()) {                                 \
 248         __ asm_instr##_cl(i.OutputRegister());                             \
 249       } else {                                                             \
 250         __ asm_instr##_cl(i.OutputOperand());                              \
 251       }                                                                    \
 252     }                                                                      \
 253   } while (0)
 254
 255
 256 #define ASSEMBLE_MOVX(asm_instr)                            \
 257   do {                                                      \
 258     if (instr->addressing_mode() != kMode_None) {           \
 259       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
 260     } else if (instr->InputAt(0)->IsRegister()) {           \
 261       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
 262     } else {                                                \
 263       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
 264     }                                                       \
 265   } while (0)
 266
 267
 268 #define ASSEMBLE_DOUBLE_BINOP(asm_instr)                                \
 269   do {                                                                  \
 270     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 271       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 272     } else {                                                            \
 273       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 274     }                                                                   \
 275   } while (0)
 276
 277
 278 #define ASSEMBLE_AVX_DOUBLE_BINOP(asm_instr)                           \
 279   do {                                                                 \
 280     CpuFeatureScope avx_scope(masm(), AVX);                            \
 281     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 282       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 283                    i.InputDoubleRegister(1));                          \
 284     } else {                                                           \
 285       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 286                    i.InputOperand(1));                                 \
 287     }                                                                  \
 288   } while (0)
 289
 290
 291 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 292   do {                                                                       \
 293     auto result = i.OutputDoubleRegister();                                  \
 294     auto buffer = i.InputRegister(0);                                        \
 295     auto index1 = i.InputRegister(1);                                        \
 296     auto index2 = i.InputInt32(2);                                           \
 297     OutOfLineCode* ool;                                                      \
 298     if (instr->InputAt(3)->IsRegister()) {                                   \
 299       auto length = i.InputRegister(3);                                      \
 300       DCHECK_EQ(0, index2);                                                  \
 301       __ cmpl(index1, length);                                               \
 302       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 303     } else {                                                                 \
 304       auto length = i.InputInt32(3);                                         \
 305       DCHECK_LE(index2, length);                                             \
 306       __ cmpq(index1, Immediate(length - index2));                           \
 307       class OutOfLineLoadFloat FINAL : public OutOfLineCode {                \
 308        public:                                                               \
 309         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 310                            Register buffer, Register index1, int32_t index2, \
 311                            int32_t length)                                   \
 312             : OutOfLineCode(gen),                                            \
 313               result_(result),                                               \
 314               buffer_(buffer),                                               \
 315               index1_(index1),                                               \
 316               index2_(index2),                                               \
 317               length_(length) {}                                             \
 318                                                                              \
 319         void Generate() FINAL {                                              \
 320           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 321           __ pcmpeqd(result_, result_);                                      \
 322           __ cmpl(kScratchRegister, Immediate(length_));                     \
 323           __ j(above_equal, exit());                                         \
 324           __ asm_instr(result_,                                              \
 325                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 326         }                                                                    \
 327                                                                              \
 328        private:                                                              \
 329         XMMRegister const result_;                                           \
 330         Register const buffer_;                                              \
 331         Register const index1_;                                              \
 332         int32_t const index2_;                                               \
 333         int32_t const length_;                                               \
 334       };                                                                     \
 335       ool = new (zone())                                                     \
 336           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 337     }                                                                        \
 338     __ j(above_equal, ool->entry());                                         \
 339     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 340     __ bind(ool->exit());                                                    \
 341   } while (false)
 342
 343
 344 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 345   do {                                                                         \
 346     auto result = i.OutputRegister();                                          \
 347     auto buffer = i.InputRegister(0);                                          \
 348     auto index1 = i.InputRegister(1);                                          \
 349     auto index2 = i.InputInt32(2);                                             \
 350     OutOfLineCode* ool;                                                        \
 351     if (instr->InputAt(3)->IsRegister()) {                                     \
 352       auto length = i.InputRegister(3);                                        \
 353       DCHECK_EQ(0, index2);                                                    \
 354       __ cmpl(index1, length);                                                 \
 355       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 356     } else {                                                                   \
 357       auto length = i.InputInt32(3);                                           \
 358       DCHECK_LE(index2, length);                                               \
 359       __ cmpq(index1, Immediate(length - index2));                             \
 360       class OutOfLineLoadInteger FINAL : public OutOfLineCode {                \
 361        public:                                                                 \
 362         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 363                              Register buffer, Register index1, int32_t index2, \
 364                              int32_t length)                                   \
 365             : OutOfLineCode(gen),                                              \
 366               result_(result),                                                 \
 367               buffer_(buffer),                                                 \
 368               index1_(index1),                                                 \
 369               index2_(index2),                                                 \
 370               length_(length) {}                                               \
 371                                                                                \
 372         void Generate() FINAL {                                                \
 373           Label oob;                                                           \
 374           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 375           __ cmpl(kScratchRegister, Immediate(length_));                       \
 376           __ j(above_equal, &oob, Label::kNear);                               \
 377           __ asm_instr(result_,                                                \
 378                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 379           __ jmp(exit());                                                      \
 380           __ bind(&oob);                                                       \
 381           __ xorl(result_, result_);                                           \
 382         }                                                                      \
 383                                                                                \
 384        private:                                                                \
 385         Register const result_;                                                \
 386         Register const buffer_;                                                \
 387         Register const index1_;                                                \
 388         int32_t const index2_;                                                 \
 389         int32_t const length_;                                                 \
 390       };                                                                       \
 391       ool = new (zone())                                                       \
 392           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 393     }                                                                          \
 394     __ j(above_equal, ool->entry());                                           \
 395     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 396     __ bind(ool->exit());                                                      \
 397   } while (false)
 398
 399
 400 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 401   do {                                                                       \
 402     auto buffer = i.InputRegister(0);                                        \
 403     auto index1 = i.InputRegister(1);                                        \
 404     auto index2 = i.InputInt32(2);                                           \
 405     auto value = i.InputDoubleRegister(4);                                   \
 406     if (instr->InputAt(3)->IsRegister()) {                                   \
 407       auto length = i.InputRegister(3);                                      \
 408       DCHECK_EQ(0, index2);                                                  \
 409       Label done;                                                            \
 410       __ cmpl(index1, length);                                               \
 411       __ j(above_equal, &done, Label::kNear);                                \
 412       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 413       __ bind(&done);                                                        \
 414     } else {                                                                 \
 415       auto length = i.InputInt32(3);                                         \
 416       DCHECK_LE(index2, length);                                             \
 417       __ cmpq(index1, Immediate(length - index2));                           \
 418       class OutOfLineStoreFloat FINAL : public OutOfLineCode {               \
 419        public:                                                               \
 420         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 421                             Register index1, int32_t index2, int32_t length, \
 422                             XMMRegister value)                               \
 423             : OutOfLineCode(gen),                                            \
 424               buffer_(buffer),                                               \
 425               index1_(index1),                                               \
 426               index2_(index2),                                               \
 427               length_(length),                                               \
 428               value_(value) {}                                               \
 429                                                                              \
 430         void Generate() FINAL {                                              \
 431           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 432           __ cmpl(kScratchRegister, Immediate(length_));                     \
 433           __ j(above_equal, exit());                                         \
 434           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 435                        value_);                                              \
 436         }                                                                    \
 437                                                                              \
 438        private:                                                              \
 439         Register const buffer_;                                              \
 440         Register const index1_;                                              \
 441         int32_t const index2_;                                               \
 442         int32_t const length_;                                               \
 443         XMMRegister const value_;                                            \
 444       };                                                                     \
 445       auto ool = new (zone())                                                \
 446           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 447       __ j(above_equal, ool->entry());                                       \
 448       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 449       __ bind(ool->exit());                                                  \
 450     }                                                                        \
 451   } while (false)
 452
 453
 454 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 455   do {                                                                         \
 456     auto buffer = i.InputRegister(0);                                          \
 457     auto index1 = i.InputRegister(1);                                          \
 458     auto index2 = i.InputInt32(2);                                             \
 459     if (instr->InputAt(3)->IsRegister()) {                                     \
 460       auto length = i.InputRegister(3);                                        \
 461       DCHECK_EQ(0, index2);                                                    \
 462       Label done;                                                              \
 463       __ cmpl(index1, length);                                                 \
 464       __ j(above_equal, &done, Label::kNear);                                  \
 465       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 466       __ bind(&done);                                                          \
 467     } else {                                                                   \
 468       auto length = i.InputInt32(3);                                           \
 469       DCHECK_LE(index2, length);                                               \
 470       __ cmpq(index1, Immediate(length - index2));                             \
 471       class OutOfLineStoreInteger FINAL : public OutOfLineCode {               \
 472        public:                                                                 \
 473         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 474                               Register index1, int32_t index2, int32_t length, \
 475                               Value value)                                     \
 476             : OutOfLineCode(gen),                                              \
 477               buffer_(buffer),                                                 \
 478               index1_(index1),                                                 \
 479               index2_(index2),                                                 \
 480               length_(length),                                                 \
 481               value_(value) {}                                                 \
 482                                                                                \
 483         void Generate() FINAL {                                                \
 484           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 485           __ cmpl(kScratchRegister, Immediate(length_));                       \
 486           __ j(above_equal, exit());                                           \
 487           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 488                        value_);                                                \
 489         }                                                                      \
 490                                                                                \
 491        private:                                                                \
 492         Register const buffer_;                                                \
 493         Register const index1_;                                                \
 494         int32_t const index2_;                                                 \
 495         int32_t const length_;                                                 \
 496         Value const value_;                                                    \
 497       };                                                                       \
 498       auto ool = new (zone())                                                  \
 499           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 500       __ j(above_equal, ool->entry());                                         \
 501       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 502       __ bind(ool->exit());                                                    \
 503     }                                                                          \
 504   } while (false)
 505
 506
 507 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 508   do {                                                           \
 509     if (instr->InputAt(4)->IsRegister()) {                       \
 510       Register value = i.InputRegister(4);                       \
 511       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 512     } else {                                                     \
 513       Immediate value = i.InputImmediate(4);                     \
 514       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 515     }                                                            \
 516   } while (false)
 517
 518
 519 // Assembles an instruction after register allocation, producing machine code.
 520 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 521   X64OperandConverter i(this, instr);
 522
 523   switch (ArchOpcodeField::decode(instr->opcode())) {
 524     case kArchCallCodeObject: {
 525       EnsureSpaceForLazyDeopt();
 526       if (HasImmediateInput(instr, 0)) {
 527         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 528         __ Call(code, RelocInfo::CODE_TARGET);
 529       } else {
 530         Register reg = i.InputRegister(0);
 531         int entry = Code::kHeaderSize - kHeapObjectTag;
 532         __ Call(Operand(reg, entry));
 533       }
 534       RecordCallPosition(instr);
 535       break;
 536     }
 537     case kArchCallJSFunction: {
 538       EnsureSpaceForLazyDeopt();
 539       Register func = i.InputRegister(0);
 540       if (FLAG_debug_code) {
 541         // Check the function's context matches the context argument.
 542         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 543         __ Assert(equal, kWrongFunctionContext);
 544       }
 545       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 546       RecordCallPosition(instr);
 547       break;
 548     }
 549     case kArchJmp:
 550       AssembleArchJump(i.InputRpo(0));
 551       break;
 552     case kArchLookupSwitch:
 553       AssembleArchLookupSwitch(instr);
 554       break;
 555     case kArchTableSwitch:
 556       AssembleArchTableSwitch(instr);
 557       break;
 558     case kArchNop:
 559       // don't emit code for nops.
 560       break;
 561     case kArchDeoptimize: {
 562       int deopt_state_id =
 563           BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
 564       AssembleDeoptimizerCall(deopt_state_id, Deoptimizer::EAGER);
 565       break;
 566     }
 567     case kArchRet:
 568       AssembleReturn();
 569       break;
 570     case kArchStackPointer:
 571       __ movq(i.OutputRegister(), rsp);
 572       break;
 573     case kArchTruncateDoubleToI: {
 574       auto result = i.OutputRegister();
 575       auto input = i.InputDoubleRegister(0);
 576       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 577       __ cvttsd2siq(result, input);
 578       __ cmpq(result, Immediate(1));
 579       __ j(overflow, ool->entry());
 580       __ bind(ool->exit());
 581       break;
 582     }
 583     case kX64Add32:
 584       ASSEMBLE_BINOP(addl);
 585       break;
 586     case kX64Add:
 587       ASSEMBLE_BINOP(addq);
 588       break;
 589     case kX64Sub32:
 590       ASSEMBLE_BINOP(subl);
 591       break;
 592     case kX64Sub:
 593       ASSEMBLE_BINOP(subq);
 594       break;
 595     case kX64And32:
 596       ASSEMBLE_BINOP(andl);
 597       break;
 598     case kX64And:
 599       ASSEMBLE_BINOP(andq);
 600       break;
 601     case kX64Cmp32:
 602       ASSEMBLE_BINOP(cmpl);
 603       break;
 604     case kX64Cmp:
 605       ASSEMBLE_BINOP(cmpq);
 606       break;
 607     case kX64Test32:
 608       ASSEMBLE_BINOP(testl);
 609       break;
 610     case kX64Test:
 611       ASSEMBLE_BINOP(testq);
 612       break;
 613     case kX64Imul32:
 614       ASSEMBLE_MULT(imull);
 615       break;
 616     case kX64Imul:
 617       ASSEMBLE_MULT(imulq);
 618       break;
 619     case kX64ImulHigh32:
 620       if (instr->InputAt(1)->IsRegister()) {
 621         __ imull(i.InputRegister(1));
 622       } else {
 623         __ imull(i.InputOperand(1));
 624       }
 625       break;
 626     case kX64UmulHigh32:
 627       if (instr->InputAt(1)->IsRegister()) {
 628         __ mull(i.InputRegister(1));
 629       } else {
 630         __ mull(i.InputOperand(1));
 631       }
 632       break;
 633     case kX64Idiv32:
 634       __ cdq();
 635       __ idivl(i.InputRegister(1));
 636       break;
 637     case kX64Idiv:
 638       __ cqo();
 639       __ idivq(i.InputRegister(1));
 640       break;
 641     case kX64Udiv32:
 642       __ xorl(rdx, rdx);
 643       __ divl(i.InputRegister(1));
 644       break;
 645     case kX64Udiv:
 646       __ xorq(rdx, rdx);
 647       __ divq(i.InputRegister(1));
 648       break;
 649     case kX64Not:
 650       ASSEMBLE_UNOP(notq);
 651       break;
 652     case kX64Not32:
 653       ASSEMBLE_UNOP(notl);
 654       break;
 655     case kX64Neg:
 656       ASSEMBLE_UNOP(negq);
 657       break;
 658     case kX64Neg32:
 659       ASSEMBLE_UNOP(negl);
 660       break;
 661     case kX64Or32:
 662       ASSEMBLE_BINOP(orl);
 663       break;
 664     case kX64Or:
 665       ASSEMBLE_BINOP(orq);
 666       break;
 667     case kX64Xor32:
 668       ASSEMBLE_BINOP(xorl);
 669       break;
 670     case kX64Xor:
 671       ASSEMBLE_BINOP(xorq);
 672       break;
 673     case kX64Shl32:
 674       ASSEMBLE_SHIFT(shll, 5);
 675       break;
 676     case kX64Shl:
 677       ASSEMBLE_SHIFT(shlq, 6);
 678       break;
 679     case kX64Shr32:
 680       ASSEMBLE_SHIFT(shrl, 5);
 681       break;
 682     case kX64Shr:
 683       ASSEMBLE_SHIFT(shrq, 6);
 684       break;
 685     case kX64Sar32:
 686       ASSEMBLE_SHIFT(sarl, 5);
 687       break;
 688     case kX64Sar:
 689       ASSEMBLE_SHIFT(sarq, 6);
 690       break;
 691     case kX64Ror32:
 692       ASSEMBLE_SHIFT(rorl, 5);
 693       break;
 694     case kX64Ror:
 695       ASSEMBLE_SHIFT(rorq, 6);
 696       break;
 697     case kX64Lzcnt32:
 698       if (instr->InputAt(0)->IsRegister()) {
 699         __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
 700       } else {
 701         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
 702       }
 703       break;
 704     case kSSEFloat64Cmp:
 705       ASSEMBLE_DOUBLE_BINOP(ucomisd);
 706       break;
 707     case kSSEFloat64Add:
 708       ASSEMBLE_DOUBLE_BINOP(addsd);
 709       break;
 710     case kSSEFloat64Sub:
 711       ASSEMBLE_DOUBLE_BINOP(subsd);
 712       break;
 713     case kSSEFloat64Mul:
 714       ASSEMBLE_DOUBLE_BINOP(mulsd);
 715       break;
 716     case kSSEFloat64Div:
 717       ASSEMBLE_DOUBLE_BINOP(divsd);
 718       break;
 719     case kSSEFloat64Mod: {
 720       __ subq(rsp, Immediate(kDoubleSize));
 721       // Move values to st(0) and st(1).
 722       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 723       __ fld_d(Operand(rsp, 0));
 724       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 725       __ fld_d(Operand(rsp, 0));
 726       // Loop while fprem isn't done.
 727       Label mod_loop;
 728       __ bind(&mod_loop);
 729       // This instructions traps on all kinds inputs, but we are assuming the
 730       // floating point control word is set to ignore them all.
 731       __ fprem();
 732       // The following 2 instruction implicitly use rax.
 733       __ fnstsw_ax();
 734       if (CpuFeatures::IsSupported(SAHF)) {
 735         CpuFeatureScope sahf_scope(masm(), SAHF);
 736         __ sahf();
 737       } else {
 738         __ shrl(rax, Immediate(8));
 739         __ andl(rax, Immediate(0xFF));
 740         __ pushq(rax);
 741         __ popfq();
 742       }
 743       __ j(parity_even, &mod_loop);
 744       // Move output to stack and clean up.
 745       __ fstp(1);
 746       __ fstp_d(Operand(rsp, 0));
 747       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 748       __ addq(rsp, Immediate(kDoubleSize));
 749       break;
 750     }
 751     case kSSEFloat64Max:
 752       ASSEMBLE_DOUBLE_BINOP(maxsd);
 753       break;
 754     case kSSEFloat64Min:
 755       ASSEMBLE_DOUBLE_BINOP(minsd);
 756       break;
 757     case kSSEFloat64Sqrt:
 758       if (instr->InputAt(0)->IsDoubleRegister()) {
 759         __ sqrtsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 760       } else {
 761         __ sqrtsd(i.OutputDoubleRegister(), i.InputOperand(0));
 762       }
 763       break;
 764     case kSSEFloat64Round: {
 765       CpuFeatureScope sse_scope(masm(), SSE4_1);
 766       RoundingMode const mode =
 767           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
 768       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
 769       break;
 770     }
 771     case kSSECvtss2sd:
 772       if (instr->InputAt(0)->IsDoubleRegister()) {
 773         __ cvtss2sd(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 774       } else {
 775         __ cvtss2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 776       }
 777       break;
 778     case kSSECvtsd2ss:
 779       if (instr->InputAt(0)->IsDoubleRegister()) {
 780         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
 781       } else {
 782         __ cvtsd2ss(i.OutputDoubleRegister(), i.InputOperand(0));
 783       }
 784       break;
 785     case kSSEFloat64ToInt32:
 786       if (instr->InputAt(0)->IsDoubleRegister()) {
 787         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 788       } else {
 789         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 790       }
 791       break;
 792     case kSSEFloat64ToUint32: {
 793       if (instr->InputAt(0)->IsDoubleRegister()) {
 794         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 795       } else {
 796         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 797       }
 798       __ AssertZeroExtended(i.OutputRegister());
 799       break;
 800     }
 801     case kSSEInt32ToFloat64:
 802       if (instr->InputAt(0)->IsRegister()) {
 803         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 804       } else {
 805         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 806       }
 807       break;
 808     case kSSEUint32ToFloat64:
 809       if (instr->InputAt(0)->IsRegister()) {
 810         __ movl(kScratchRegister, i.InputRegister(0));
 811       } else {
 812         __ movl(kScratchRegister, i.InputOperand(0));
 813       }
 814       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 815       break;
 816     case kSSEFloat64ExtractLowWord32:
 817       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 818         __ movl(i.OutputRegister(), i.InputOperand(0));
 819       } else {
 820         __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
 821       }
 822       break;
 823     case kSSEFloat64ExtractHighWord32:
 824       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 825         __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
 826       } else {
 827         __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
 828       }
 829       break;
 830     case kSSEFloat64InsertLowWord32:
 831       if (instr->InputAt(1)->IsRegister()) {
 832         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
 833       } else {
 834         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
 835       }
 836       break;
 837     case kSSEFloat64InsertHighWord32:
 838       if (instr->InputAt(1)->IsRegister()) {
 839         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
 840       } else {
 841         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
 842       }
 843       break;
 844     case kSSEFloat64LoadLowWord32:
 845       if (instr->InputAt(0)->IsRegister()) {
 846         __ movd(i.OutputDoubleRegister(), i.InputRegister(0));
 847       } else {
 848         __ movd(i.OutputDoubleRegister(), i.InputOperand(0));
 849       }
 850       break;
 851     case kAVXFloat64Add:
 852       ASSEMBLE_AVX_DOUBLE_BINOP(vaddsd);
 853       break;
 854     case kAVXFloat64Sub:
 855       ASSEMBLE_AVX_DOUBLE_BINOP(vsubsd);
 856       break;
 857     case kAVXFloat64Mul:
 858       ASSEMBLE_AVX_DOUBLE_BINOP(vmulsd);
 859       break;
 860     case kAVXFloat64Div:
 861       ASSEMBLE_AVX_DOUBLE_BINOP(vdivsd);
 862       break;
 863     case kAVXFloat64Max:
 864       ASSEMBLE_AVX_DOUBLE_BINOP(vmaxsd);
 865       break;
 866     case kAVXFloat64Min:
 867       ASSEMBLE_AVX_DOUBLE_BINOP(vminsd);
 868       break;
 869     case kX64Movsxbl:
 870       ASSEMBLE_MOVX(movsxbl);
 871       __ AssertZeroExtended(i.OutputRegister());
 872       break;
 873     case kX64Movzxbl:
 874       ASSEMBLE_MOVX(movzxbl);
 875       __ AssertZeroExtended(i.OutputRegister());
 876       break;
 877     case kX64Movb: {
 878       size_t index = 0;
 879       Operand operand = i.MemoryOperand(&index);
 880       if (HasImmediateInput(instr, index)) {
 881         __ movb(operand, Immediate(i.InputInt8(index)));
 882       } else {
 883         __ movb(operand, i.InputRegister(index));
 884       }
 885       break;
 886     }
 887     case kX64Movsxwl:
 888       ASSEMBLE_MOVX(movsxwl);
 889       __ AssertZeroExtended(i.OutputRegister());
 890       break;
 891     case kX64Movzxwl:
 892       ASSEMBLE_MOVX(movzxwl);
 893       __ AssertZeroExtended(i.OutputRegister());
 894       break;
 895     case kX64Movw: {
 896       size_t index = 0;
 897       Operand operand = i.MemoryOperand(&index);
 898       if (HasImmediateInput(instr, index)) {
 899         __ movw(operand, Immediate(i.InputInt16(index)));
 900       } else {
 901         __ movw(operand, i.InputRegister(index));
 902       }
 903       break;
 904     }
 905     case kX64Movl:
 906       if (instr->HasOutput()) {
 907         if (instr->addressing_mode() == kMode_None) {
 908           if (instr->InputAt(0)->IsRegister()) {
 909             __ movl(i.OutputRegister(), i.InputRegister(0));
 910           } else {
 911             __ movl(i.OutputRegister(), i.InputOperand(0));
 912           }
 913         } else {
 914           __ movl(i.OutputRegister(), i.MemoryOperand());
 915         }
 916         __ AssertZeroExtended(i.OutputRegister());
 917       } else {
 918         size_t index = 0;
 919         Operand operand = i.MemoryOperand(&index);
 920         if (HasImmediateInput(instr, index)) {
 921           __ movl(operand, i.InputImmediate(index));
 922         } else {
 923           __ movl(operand, i.InputRegister(index));
 924         }
 925       }
 926       break;
 927     case kX64Movsxlq:
 928       ASSEMBLE_MOVX(movsxlq);
 929       break;
 930     case kX64Movq:
 931       if (instr->HasOutput()) {
 932         __ movq(i.OutputRegister(), i.MemoryOperand());
 933       } else {
 934         size_t index = 0;
 935         Operand operand = i.MemoryOperand(&index);
 936         if (HasImmediateInput(instr, index)) {
 937           __ movq(operand, i.InputImmediate(index));
 938         } else {
 939           __ movq(operand, i.InputRegister(index));
 940         }
 941       }
 942       break;
 943     case kX64Movss:
 944       if (instr->HasOutput()) {
 945         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
 946       } else {
 947         size_t index = 0;
 948         Operand operand = i.MemoryOperand(&index);
 949         __ movss(operand, i.InputDoubleRegister(index));
 950       }
 951       break;
 952     case kX64Movsd:
 953       if (instr->HasOutput()) {
 954         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
 955       } else {
 956         size_t index = 0;
 957         Operand operand = i.MemoryOperand(&index);
 958         __ movsd(operand, i.InputDoubleRegister(index));
 959       }
 960       break;
 961     case kX64Lea32: {
 962       AddressingMode mode = AddressingModeField::decode(instr->opcode());
 963       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
 964       // and addressing mode just happens to work out. The "addl"/"subl" forms
 965       // in these cases are faster based on measurements.
 966       if (i.InputRegister(0).is(i.OutputRegister())) {
 967         if (mode == kMode_MRI) {
 968           int32_t constant_summand = i.InputInt32(1);
 969           if (constant_summand > 0) {
 970             __ addl(i.OutputRegister(), Immediate(constant_summand));
 971           } else if (constant_summand < 0) {
 972             __ subl(i.OutputRegister(), Immediate(-constant_summand));
 973           }
 974         } else if (mode == kMode_MR1) {
 975           if (i.InputRegister(1).is(i.OutputRegister())) {
 976             __ shll(i.OutputRegister(), Immediate(1));
 977           } else {
 978             __ leal(i.OutputRegister(), i.MemoryOperand());
 979           }
 980         } else if (mode == kMode_M2) {
 981           __ shll(i.OutputRegister(), Immediate(1));
 982         } else if (mode == kMode_M4) {
 983           __ shll(i.OutputRegister(), Immediate(2));
 984         } else if (mode == kMode_M8) {
 985           __ shll(i.OutputRegister(), Immediate(3));
 986         } else {
 987           __ leal(i.OutputRegister(), i.MemoryOperand());
 988         }
 989       } else {
 990         __ leal(i.OutputRegister(), i.MemoryOperand());
 991       }
 992       __ AssertZeroExtended(i.OutputRegister());
 993       break;
 994     }
 995     case kX64Lea:
 996       __ leaq(i.OutputRegister(), i.MemoryOperand());
 997       break;
 998     case kX64Dec32:
 999       __ decl(i.OutputRegister());
1000       break;
1001     case kX64Inc32:
1002       __ incl(i.OutputRegister());
1003       break;
1004     case kX64Push:
1005       if (HasImmediateInput(instr, 0)) {
1006         __ pushq(i.InputImmediate(0));
1007       } else {
1008         if (instr->InputAt(0)->IsRegister()) {
1009           __ pushq(i.InputRegister(0));
1010         } else {
1011           __ pushq(i.InputOperand(0));
1012         }
1013       }
1014       break;
1015     case kX64StoreWriteBarrier: {
1016       Register object = i.InputRegister(0);
1017       Register index = i.InputRegister(1);
1018       Register value = i.InputRegister(2);
1019       __ movq(Operand(object, index, times_1, 0), value);
1020       __ leaq(index, Operand(object, index, times_1, 0));
1021       SaveFPRegsMode mode =
1022           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
1023       __ RecordWrite(object, index, value, mode);
1024       break;
1025     }
1026     case kCheckedLoadInt8:
1027       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
1028       break;
1029     case kCheckedLoadUint8:
1030       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
1031       break;
1032     case kCheckedLoadInt16:
1033       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
1034       break;
1035     case kCheckedLoadUint16:
1036       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
1037       break;
1038     case kCheckedLoadWord32:
1039       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
1040       break;
1041     case kCheckedLoadFloat32:
1042       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
1043       break;
1044     case kCheckedLoadFloat64:
1045       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
1046       break;
1047     case kCheckedStoreWord8:
1048       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
1049       break;
1050     case kCheckedStoreWord16:
1051       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
1052       break;
1053     case kCheckedStoreWord32:
1054       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1055       break;
1056     case kCheckedStoreFloat32:
1057       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1058       break;
1059     case kCheckedStoreFloat64:
1060       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1061       break;
1062     case kX64StackCheck:
1063       __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
1064       break;
1065   }
1066 }  // NOLINT(readability/fn_size)
1067
1068
1069 // Assembles branches after this instruction.
1070 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1071   X64OperandConverter i(this, instr);
1072   Label::Distance flabel_distance =
1073       branch->fallthru ? Label::kNear : Label::kFar;
1074   Label* tlabel = branch->true_label;
1075   Label* flabel = branch->false_label;
1076   switch (branch->condition) {
1077     case kUnorderedEqual:
1078       __ j(parity_even, flabel, flabel_distance);
1079     // Fall through.
1080     case kEqual:
1081       __ j(equal, tlabel);
1082       break;
1083     case kUnorderedNotEqual:
1084       __ j(parity_even, tlabel);
1085     // Fall through.
1086     case kNotEqual:
1087       __ j(not_equal, tlabel);
1088       break;
1089     case kSignedLessThan:
1090       __ j(less, tlabel);
1091       break;
1092     case kSignedGreaterThanOrEqual:
1093       __ j(greater_equal, tlabel);
1094       break;
1095     case kSignedLessThanOrEqual:
1096       __ j(less_equal, tlabel);
1097       break;
1098     case kSignedGreaterThan:
1099       __ j(greater, tlabel);
1100       break;
1101     case kUnsignedLessThan:
1102       __ j(below, tlabel);
1103       break;
1104     case kUnsignedGreaterThanOrEqual:
1105       __ j(above_equal, tlabel);
1106       break;
1107     case kUnsignedLessThanOrEqual:
1108       __ j(below_equal, tlabel);
1109       break;
1110     case kUnsignedGreaterThan:
1111       __ j(above, tlabel);
1112       break;
1113     case kOverflow:
1114       __ j(overflow, tlabel);
1115       break;
1116     case kNotOverflow:
1117       __ j(no_overflow, tlabel);
1118       break;
1119   }
1120   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1121 }
1122
1123
1124 void CodeGenerator::AssembleArchJump(RpoNumber target) {
1125   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1126 }
1127
1128
1129 // Assembles boolean materializations after this instruction.
1130 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1131                                         FlagsCondition condition) {
1132   X64OperandConverter i(this, instr);
1133   Label done;
1134
1135   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1136   // last output of the instruction.
1137   Label check;
1138   DCHECK_NE(0u, instr->OutputCount());
1139   Register reg = i.OutputRegister(instr->OutputCount() - 1);
1140   Condition cc = no_condition;
1141   switch (condition) {
1142     case kUnorderedEqual:
1143       __ j(parity_odd, &check, Label::kNear);
1144       __ movl(reg, Immediate(0));
1145       __ jmp(&done, Label::kNear);
1146     // Fall through.
1147     case kEqual:
1148       cc = equal;
1149       break;
1150     case kUnorderedNotEqual:
1151       __ j(parity_odd, &check, Label::kNear);
1152       __ movl(reg, Immediate(1));
1153       __ jmp(&done, Label::kNear);
1154     // Fall through.
1155     case kNotEqual:
1156       cc = not_equal;
1157       break;
1158     case kSignedLessThan:
1159       cc = less;
1160       break;
1161     case kSignedGreaterThanOrEqual:
1162       cc = greater_equal;
1163       break;
1164     case kSignedLessThanOrEqual:
1165       cc = less_equal;
1166       break;
1167     case kSignedGreaterThan:
1168       cc = greater;
1169       break;
1170     case kUnsignedLessThan:
1171       cc = below;
1172       break;
1173     case kUnsignedGreaterThanOrEqual:
1174       cc = above_equal;
1175       break;
1176     case kUnsignedLessThanOrEqual:
1177       cc = below_equal;
1178       break;
1179     case kUnsignedGreaterThan:
1180       cc = above;
1181       break;
1182     case kOverflow:
1183       cc = overflow;
1184       break;
1185     case kNotOverflow:
1186       cc = no_overflow;
1187       break;
1188   }
1189   __ bind(&check);
1190   __ setcc(cc, reg);
1191   __ movzxbl(reg, reg);
1192   __ bind(&done);
1193 }
1194
1195
1196 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1197   X64OperandConverter i(this, instr);
1198   Register input = i.InputRegister(0);
1199   for (size_t index = 2; index < instr->InputCount(); index += 2) {
1200     __ cmpl(input, Immediate(i.InputInt32(index + 0)));
1201     __ j(equal, GetLabel(i.InputRpo(index + 1)));
1202   }
1203   AssembleArchJump(i.InputRpo(1));
1204 }
1205
1206
1207 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1208   X64OperandConverter i(this, instr);
1209   Register input = i.InputRegister(0);
1210   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
1211   Label** cases = zone()->NewArray<Label*>(case_count);
1212   for (int32_t index = 0; index < case_count; ++index) {
1213     cases[index] = GetLabel(i.InputRpo(index + 2));
1214   }
1215   Label* const table = AddJumpTable(cases, case_count);
1216   __ cmpl(input, Immediate(case_count));
1217   __ j(above_equal, GetLabel(i.InputRpo(1)));
1218   __ leaq(kScratchRegister, Operand(table));
1219   __ jmp(Operand(kScratchRegister, input, times_8, 0));
1220 }
1221
1222
1223 void CodeGenerator::AssembleDeoptimizerCall(
1224     int deoptimization_id, Deoptimizer::BailoutType bailout_type) {
1225   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1226       isolate(), deoptimization_id, bailout_type);
1227   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1228 }
1229
1230
1231 void CodeGenerator::AssemblePrologue() {
1232   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1233   int stack_slots = frame()->GetSpillSlotCount();
1234   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1235     __ pushq(rbp);
1236     __ movq(rbp, rsp);
1237     const RegList saves = descriptor->CalleeSavedRegisters();
1238     if (saves != 0) {  // Save callee-saved registers.
1239       int register_save_area_size = 0;
1240       for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1241         if (!((1 << i) & saves)) continue;
1242         __ pushq(Register::from_code(i));
1243         register_save_area_size += kPointerSize;
1244       }
1245       frame()->SetRegisterSaveAreaSize(register_save_area_size);
1246     }
1247   } else if (descriptor->IsJSFunctionCall()) {
1248     CompilationInfo* info = this->info();
1249     __ Prologue(info->IsCodePreAgingActive());
1250     frame()->SetRegisterSaveAreaSize(
1251         StandardFrameConstants::kFixedFrameSizeFromFp);
1252   } else if (stack_slots > 0) {
1253     __ StubPrologue();
1254     frame()->SetRegisterSaveAreaSize(
1255         StandardFrameConstants::kFixedFrameSizeFromFp);
1256   }
1257
1258   if (info()->is_osr()) {
1259     // TurboFan OSR-compiled functions cannot be entered directly.
1260     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1261
1262     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1263     // frame is still on the stack. Optimized code uses OSR values directly from
1264     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1265     // remaining stack slots.
1266     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1267     osr_pc_offset_ = __ pc_offset();
1268     // TODO(titzer): cannot address target function == local #-1
1269     __ movq(rdi, Operand(rbp, JavaScriptFrameConstants::kFunctionOffset));
1270     DCHECK(stack_slots >= frame()->GetOsrStackSlotCount());
1271     stack_slots -= frame()->GetOsrStackSlotCount();
1272   }
1273
1274   if (stack_slots > 0) {
1275     __ subq(rsp, Immediate(stack_slots * kPointerSize));
1276   }
1277 }
1278
1279
1280 void CodeGenerator::AssembleReturn() {
1281   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1282   int stack_slots = frame()->GetSpillSlotCount();
1283   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1284     if (frame()->GetRegisterSaveAreaSize() > 0) {
1285       // Remove this frame's spill slots first.
1286       if (stack_slots > 0) {
1287         __ addq(rsp, Immediate(stack_slots * kPointerSize));
1288       }
1289       const RegList saves = descriptor->CalleeSavedRegisters();
1290       // Restore registers.
1291       if (saves != 0) {
1292         for (int i = 0; i < Register::kNumRegisters; i++) {
1293           if (!((1 << i) & saves)) continue;
1294           __ popq(Register::from_code(i));
1295         }
1296       }
1297       __ popq(rbp);  // Pop caller's frame pointer.
1298       __ ret(0);
1299     } else {
1300       // No saved registers.
1301       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1302       __ popq(rbp);       // Pop caller's frame pointer.
1303       __ ret(0);
1304     }
1305   } else if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
1306     __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1307     __ popq(rbp);       // Pop caller's frame pointer.
1308     int pop_count = descriptor->IsJSFunctionCall()
1309                         ? static_cast<int>(descriptor->JSParameterCount())
1310                         : 0;
1311     __ ret(pop_count * kPointerSize);
1312   } else {
1313     __ ret(0);
1314   }
1315 }
1316
1317
1318 void CodeGenerator::AssembleMove(InstructionOperand* source,
1319                                  InstructionOperand* destination) {
1320   X64OperandConverter g(this, NULL);
1321   // Dispatch on the source and destination operand kinds.  Not all
1322   // combinations are possible.
1323   if (source->IsRegister()) {
1324     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1325     Register src = g.ToRegister(source);
1326     if (destination->IsRegister()) {
1327       __ movq(g.ToRegister(destination), src);
1328     } else {
1329       __ movq(g.ToOperand(destination), src);
1330     }
1331   } else if (source->IsStackSlot()) {
1332     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1333     Operand src = g.ToOperand(source);
1334     if (destination->IsRegister()) {
1335       Register dst = g.ToRegister(destination);
1336       __ movq(dst, src);
1337     } else {
1338       // Spill on demand to use a temporary register for memory-to-memory
1339       // moves.
1340       Register tmp = kScratchRegister;
1341       Operand dst = g.ToOperand(destination);
1342       __ movq(tmp, src);
1343       __ movq(dst, tmp);
1344     }
1345   } else if (source->IsConstant()) {
1346     ConstantOperand* constant_source = ConstantOperand::cast(source);
1347     Constant src = g.ToConstant(constant_source);
1348     if (destination->IsRegister() || destination->IsStackSlot()) {
1349       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1350                                                : kScratchRegister;
1351       switch (src.type()) {
1352         case Constant::kInt32:
1353           // TODO(dcarney): don't need scratch in this case.
1354           __ Set(dst, src.ToInt32());
1355           break;
1356         case Constant::kInt64:
1357           __ Set(dst, src.ToInt64());
1358           break;
1359         case Constant::kFloat32:
1360           __ Move(dst,
1361                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1362           break;
1363         case Constant::kFloat64:
1364           __ Move(dst,
1365                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1366           break;
1367         case Constant::kExternalReference:
1368           __ Move(dst, src.ToExternalReference());
1369           break;
1370         case Constant::kHeapObject: {
1371           Handle<HeapObject> src_object = src.ToHeapObject();
1372           if (info()->IsOptimizing() &&
1373               src_object.is_identical_to(info()->context())) {
1374             // Loading the context from the frame is way cheaper than
1375             // materializing the actual context heap object address.
1376             __ movp(dst, Operand(rbp, StandardFrameConstants::kContextOffset));
1377           } else {
1378             __ Move(dst, src_object);
1379           }
1380           break;
1381         }
1382         case Constant::kRpoNumber:
1383           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1384           break;
1385       }
1386       if (destination->IsStackSlot()) {
1387         __ movq(g.ToOperand(destination), kScratchRegister);
1388       }
1389     } else if (src.type() == Constant::kFloat32) {
1390       // TODO(turbofan): Can we do better here?
1391       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1392       if (destination->IsDoubleRegister()) {
1393         __ Move(g.ToDoubleRegister(destination), src_const);
1394       } else {
1395         DCHECK(destination->IsDoubleStackSlot());
1396         Operand dst = g.ToOperand(destination);
1397         __ movl(dst, Immediate(src_const));
1398       }
1399     } else {
1400       DCHECK_EQ(Constant::kFloat64, src.type());
1401       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1402       if (destination->IsDoubleRegister()) {
1403         __ Move(g.ToDoubleRegister(destination), src_const);
1404       } else {
1405         DCHECK(destination->IsDoubleStackSlot());
1406         __ movq(kScratchRegister, src_const);
1407         __ movq(g.ToOperand(destination), kScratchRegister);
1408       }
1409     }
1410   } else if (source->IsDoubleRegister()) {
1411     XMMRegister src = g.ToDoubleRegister(source);
1412     if (destination->IsDoubleRegister()) {
1413       XMMRegister dst = g.ToDoubleRegister(destination);
1414       __ movaps(dst, src);
1415     } else {
1416       DCHECK(destination->IsDoubleStackSlot());
1417       Operand dst = g.ToOperand(destination);
1418       __ movsd(dst, src);
1419     }
1420   } else if (source->IsDoubleStackSlot()) {
1421     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1422     Operand src = g.ToOperand(source);
1423     if (destination->IsDoubleRegister()) {
1424       XMMRegister dst = g.ToDoubleRegister(destination);
1425       __ movsd(dst, src);
1426     } else {
1427       // We rely on having xmm0 available as a fixed scratch register.
1428       Operand dst = g.ToOperand(destination);
1429       __ movsd(xmm0, src);
1430       __ movsd(dst, xmm0);
1431     }
1432   } else {
1433     UNREACHABLE();
1434   }
1435 }
1436
1437
1438 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1439                                  InstructionOperand* destination) {
1440   X64OperandConverter g(this, NULL);
1441   // Dispatch on the source and destination operand kinds.  Not all
1442   // combinations are possible.
1443   if (source->IsRegister() && destination->IsRegister()) {
1444     // Register-register.
1445     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1446   } else if (source->IsRegister() && destination->IsStackSlot()) {
1447     Register src = g.ToRegister(source);
1448     Operand dst = g.ToOperand(destination);
1449     __ xchgq(src, dst);
1450   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1451              (source->IsDoubleStackSlot() &&
1452               destination->IsDoubleStackSlot())) {
1453     // Memory-memory.
1454     Register tmp = kScratchRegister;
1455     Operand src = g.ToOperand(source);
1456     Operand dst = g.ToOperand(destination);
1457     __ movq(tmp, dst);
1458     __ xchgq(tmp, src);
1459     __ movq(dst, tmp);
1460   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1461     // XMM register-register swap. We rely on having xmm0
1462     // available as a fixed scratch register.
1463     XMMRegister src = g.ToDoubleRegister(source);
1464     XMMRegister dst = g.ToDoubleRegister(destination);
1465     __ movaps(xmm0, src);
1466     __ movaps(src, dst);
1467     __ movaps(dst, xmm0);
1468   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1469     // XMM register-memory swap.  We rely on having xmm0
1470     // available as a fixed scratch register.
1471     XMMRegister src = g.ToDoubleRegister(source);
1472     Operand dst = g.ToOperand(destination);
1473     __ movsd(xmm0, src);
1474     __ movsd(src, dst);
1475     __ movsd(dst, xmm0);
1476   } else {
1477     // No other combinations are possible.
1478     UNREACHABLE();
1479   }
1480 }
1481
1482
1483 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1484   for (size_t index = 0; index < target_count; ++index) {
1485     __ dq(targets[index]);
1486   }
1487 }
1488
1489
1490 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1491
1492
1493 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1494   int space_needed = Deoptimizer::patch_size();
1495   if (!info()->IsStub()) {
1496     // Ensure that we have enough space after the previous lazy-bailout
1497     // instruction for patching the code here.
1498     int current_pc = masm()->pc_offset();
1499     if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1500       int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1501       __ Nop(padding_size);
1502     }
1503   }
1504   MarkLazyDeoptSite();
1505 }
1506
1507 #undef __
1508
1509 }  // namespace internal
1510 }  // namespace compiler
1511 }  // namespace v8