src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/scopes.h"
  11 #include "src/x64/assembler-x64.h"
  12 #include "src/x64/macro-assembler-x64.h"
  13
  14 namespace v8 {
  15 namespace internal {
  16 namespace compiler {
  17
  18 #define __ masm()->
  19
  20
  21 #define kScratchDoubleReg xmm0
  22
  23
  24 // Adds X64 specific methods for decoding operands.
  25 class X64OperandConverter : public InstructionOperandConverter {
  26  public:
  27   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  28       : InstructionOperandConverter(gen, instr) {}
  29
  30   Immediate InputImmediate(size_t index) {
  31     return ToImmediate(instr_->InputAt(index));
  32   }
  33
  34   Operand InputOperand(size_t index, int extra = 0) {
  35     return ToOperand(instr_->InputAt(index), extra);
  36   }
  37
  38   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  39
  40   Immediate ToImmediate(InstructionOperand* operand) {
  41     return Immediate(ToConstant(operand).ToInt32());
  42   }
  43
  44   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  45     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  46     // The linkage computes where all spill slots are located.
  47     FrameOffset offset = linkage()->GetFrameOffset(
  48         AllocatedOperand::cast(op)->index(), frame(), extra);
  49     return Operand(offset.from_stack_pointer() ? rsp : rbp, offset.offset());
  50   }
  51
  52   static size_t NextOffset(size_t* offset) {
  53     size_t i = *offset;
  54     (*offset)++;
  55     return i;
  56   }
  57
  58   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  59     STATIC_ASSERT(0 == static_cast<int>(times_1));
  60     STATIC_ASSERT(1 == static_cast<int>(times_2));
  61     STATIC_ASSERT(2 == static_cast<int>(times_4));
  62     STATIC_ASSERT(3 == static_cast<int>(times_8));
  63     int scale = static_cast<int>(mode - one);
  64     DCHECK(scale >= 0 && scale < 4);
  65     return static_cast<ScaleFactor>(scale);
  66   }
  67
  68   Operand MemoryOperand(size_t* offset) {
  69     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  70     switch (mode) {
  71       case kMode_MR: {
  72         Register base = InputRegister(NextOffset(offset));
  73         int32_t disp = 0;
  74         return Operand(base, disp);
  75       }
  76       case kMode_MRI: {
  77         Register base = InputRegister(NextOffset(offset));
  78         int32_t disp = InputInt32(NextOffset(offset));
  79         return Operand(base, disp);
  80       }
  81       case kMode_MR1:
  82       case kMode_MR2:
  83       case kMode_MR4:
  84       case kMode_MR8: {
  85         Register base = InputRegister(NextOffset(offset));
  86         Register index = InputRegister(NextOffset(offset));
  87         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  88         int32_t disp = 0;
  89         return Operand(base, index, scale, disp);
  90       }
  91       case kMode_MR1I:
  92       case kMode_MR2I:
  93       case kMode_MR4I:
  94       case kMode_MR8I: {
  95         Register base = InputRegister(NextOffset(offset));
  96         Register index = InputRegister(NextOffset(offset));
  97         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
  98         int32_t disp = InputInt32(NextOffset(offset));
  99         return Operand(base, index, scale, disp);
 100       }
 101       case kMode_M1: {
 102         Register base = InputRegister(NextOffset(offset));
 103         int32_t disp = 0;
 104         return Operand(base, disp);
 105       }
 106       case kMode_M2:
 107         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 108         return Operand(no_reg, 0);
 109       case kMode_M4:
 110       case kMode_M8: {
 111         Register index = InputRegister(NextOffset(offset));
 112         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 113         int32_t disp = 0;
 114         return Operand(index, scale, disp);
 115       }
 116       case kMode_M1I:
 117       case kMode_M2I:
 118       case kMode_M4I:
 119       case kMode_M8I: {
 120         Register index = InputRegister(NextOffset(offset));
 121         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 122         int32_t disp = InputInt32(NextOffset(offset));
 123         return Operand(index, scale, disp);
 124       }
 125       case kMode_None:
 126         UNREACHABLE();
 127         return Operand(no_reg, 0);
 128     }
 129     UNREACHABLE();
 130     return Operand(no_reg, 0);
 131   }
 132
 133   Operand MemoryOperand(size_t first_input = 0) {
 134     return MemoryOperand(&first_input);
 135   }
 136 };
 137
 138
 139 namespace {
 140
 141 bool HasImmediateInput(Instruction* instr, size_t index) {
 142   return instr->InputAt(index)->IsImmediate();
 143 }
 144
 145
 146 class OutOfLineLoadZero FINAL : public OutOfLineCode {
 147  public:
 148   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 149       : OutOfLineCode(gen), result_(result) {}
 150
 151   void Generate() FINAL { __ xorl(result_, result_); }
 152
 153  private:
 154   Register const result_;
 155 };
 156
 157
 158 class OutOfLineLoadNaN FINAL : public OutOfLineCode {
 159  public:
 160   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 161       : OutOfLineCode(gen), result_(result) {}
 162
 163   void Generate() FINAL { __ pcmpeqd(result_, result_); }
 164
 165  private:
 166   XMMRegister const result_;
 167 };
 168
 169
 170 class OutOfLineTruncateDoubleToI FINAL : public OutOfLineCode {
 171  public:
 172   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 173                              XMMRegister input)
 174       : OutOfLineCode(gen), result_(result), input_(input) {}
 175
 176   void Generate() FINAL {
 177     __ subp(rsp, Immediate(kDoubleSize));
 178     __ movsd(MemOperand(rsp, 0), input_);
 179     __ SlowTruncateToI(result_, rsp, 0);
 180     __ addp(rsp, Immediate(kDoubleSize));
 181   }
 182
 183  private:
 184   Register const result_;
 185   XMMRegister const input_;
 186 };
 187
 188 }  // namespace
 189
 190
 191 #define ASSEMBLE_UNOP(asm_instr)         \
 192   do {                                   \
 193     if (instr->Output()->IsRegister()) { \
 194       __ asm_instr(i.OutputRegister());  \
 195     } else {                             \
 196       __ asm_instr(i.OutputOperand());   \
 197     }                                    \
 198   } while (0)
 199
 200
 201 #define ASSEMBLE_BINOP(asm_instr)                              \
 202   do {                                                         \
 203     if (HasImmediateInput(instr, 1)) {                         \
 204       if (instr->InputAt(0)->IsRegister()) {                   \
 205         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 206       } else {                                                 \
 207         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 208       }                                                        \
 209     } else {                                                   \
 210       if (instr->InputAt(1)->IsRegister()) {                   \
 211         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 212       } else {                                                 \
 213         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 214       }                                                        \
 215     }                                                          \
 216   } while (0)
 217
 218
 219 #define ASSEMBLE_MULT(asm_instr)                              \
 220   do {                                                        \
 221     if (HasImmediateInput(instr, 1)) {                        \
 222       if (instr->InputAt(0)->IsRegister()) {                  \
 223         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 224                      i.InputImmediate(1));                    \
 225       } else {                                                \
 226         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 227                      i.InputImmediate(1));                    \
 228       }                                                       \
 229     } else {                                                  \
 230       if (instr->InputAt(1)->IsRegister()) {                  \
 231         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 232       } else {                                                \
 233         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 234       }                                                       \
 235     }                                                         \
 236   } while (0)
 237
 238
 239 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 240   do {                                                                     \
 241     if (HasImmediateInput(instr, 1)) {                                     \
 242       if (instr->Output()->IsRegister()) {                                 \
 243         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 244       } else {                                                             \
 245         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 246       }                                                                    \
 247     } else {                                                               \
 248       if (instr->Output()->IsRegister()) {                                 \
 249         __ asm_instr##_cl(i.OutputRegister());                             \
 250       } else {                                                             \
 251         __ asm_instr##_cl(i.OutputOperand());                              \
 252       }                                                                    \
 253     }                                                                      \
 254   } while (0)
 255
 256
 257 #define ASSEMBLE_MOVX(asm_instr)                            \
 258   do {                                                      \
 259     if (instr->addressing_mode() != kMode_None) {           \
 260       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
 261     } else if (instr->InputAt(0)->IsRegister()) {           \
 262       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
 263     } else {                                                \
 264       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
 265     }                                                       \
 266   } while (0)
 267
 268
 269 #define ASSEMBLE_SSE_BINOP(asm_instr)                                   \
 270   do {                                                                  \
 271     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 272       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 273     } else {                                                            \
 274       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 275     }                                                                   \
 276   } while (0)
 277
 278
 279 #define ASSEMBLE_SSE_UNOP(asm_instr)                                    \
 280   do {                                                                  \
 281     if (instr->InputAt(0)->IsDoubleRegister()) {                        \
 282       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
 283     } else {                                                            \
 284       __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0));        \
 285     }                                                                   \
 286   } while (0)
 287
 288
 289 #define ASSEMBLE_AVX_BINOP(asm_instr)                                  \
 290   do {                                                                 \
 291     CpuFeatureScope avx_scope(masm(), AVX);                            \
 292     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 293       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 294                    i.InputDoubleRegister(1));                          \
 295     } else {                                                           \
 296       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 297                    i.InputOperand(1));                                 \
 298     }                                                                  \
 299   } while (0)
 300
 301
 302 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 303   do {                                                                       \
 304     auto result = i.OutputDoubleRegister();                                  \
 305     auto buffer = i.InputRegister(0);                                        \
 306     auto index1 = i.InputRegister(1);                                        \
 307     auto index2 = i.InputInt32(2);                                           \
 308     OutOfLineCode* ool;                                                      \
 309     if (instr->InputAt(3)->IsRegister()) {                                   \
 310       auto length = i.InputRegister(3);                                      \
 311       DCHECK_EQ(0, index2);                                                  \
 312       __ cmpl(index1, length);                                               \
 313       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 314     } else {                                                                 \
 315       auto length = i.InputInt32(3);                                         \
 316       DCHECK_LE(index2, length);                                             \
 317       __ cmpq(index1, Immediate(length - index2));                           \
 318       class OutOfLineLoadFloat FINAL : public OutOfLineCode {                \
 319        public:                                                               \
 320         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 321                            Register buffer, Register index1, int32_t index2, \
 322                            int32_t length)                                   \
 323             : OutOfLineCode(gen),                                            \
 324               result_(result),                                               \
 325               buffer_(buffer),                                               \
 326               index1_(index1),                                               \
 327               index2_(index2),                                               \
 328               length_(length) {}                                             \
 329                                                                              \
 330         void Generate() FINAL {                                              \
 331           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 332           __ pcmpeqd(result_, result_);                                      \
 333           __ cmpl(kScratchRegister, Immediate(length_));                     \
 334           __ j(above_equal, exit());                                         \
 335           __ asm_instr(result_,                                              \
 336                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 337         }                                                                    \
 338                                                                              \
 339        private:                                                              \
 340         XMMRegister const result_;                                           \
 341         Register const buffer_;                                              \
 342         Register const index1_;                                              \
 343         int32_t const index2_;                                               \
 344         int32_t const length_;                                               \
 345       };                                                                     \
 346       ool = new (zone())                                                     \
 347           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 348     }                                                                        \
 349     __ j(above_equal, ool->entry());                                         \
 350     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 351     __ bind(ool->exit());                                                    \
 352   } while (false)
 353
 354
 355 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 356   do {                                                                         \
 357     auto result = i.OutputRegister();                                          \
 358     auto buffer = i.InputRegister(0);                                          \
 359     auto index1 = i.InputRegister(1);                                          \
 360     auto index2 = i.InputInt32(2);                                             \
 361     OutOfLineCode* ool;                                                        \
 362     if (instr->InputAt(3)->IsRegister()) {                                     \
 363       auto length = i.InputRegister(3);                                        \
 364       DCHECK_EQ(0, index2);                                                    \
 365       __ cmpl(index1, length);                                                 \
 366       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 367     } else {                                                                   \
 368       auto length = i.InputInt32(3);                                           \
 369       DCHECK_LE(index2, length);                                               \
 370       __ cmpq(index1, Immediate(length - index2));                             \
 371       class OutOfLineLoadInteger FINAL : public OutOfLineCode {                \
 372        public:                                                                 \
 373         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 374                              Register buffer, Register index1, int32_t index2, \
 375                              int32_t length)                                   \
 376             : OutOfLineCode(gen),                                              \
 377               result_(result),                                                 \
 378               buffer_(buffer),                                                 \
 379               index1_(index1),                                                 \
 380               index2_(index2),                                                 \
 381               length_(length) {}                                               \
 382                                                                                \
 383         void Generate() FINAL {                                                \
 384           Label oob;                                                           \
 385           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 386           __ cmpl(kScratchRegister, Immediate(length_));                       \
 387           __ j(above_equal, &oob, Label::kNear);                               \
 388           __ asm_instr(result_,                                                \
 389                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 390           __ jmp(exit());                                                      \
 391           __ bind(&oob);                                                       \
 392           __ xorl(result_, result_);                                           \
 393         }                                                                      \
 394                                                                                \
 395        private:                                                                \
 396         Register const result_;                                                \
 397         Register const buffer_;                                                \
 398         Register const index1_;                                                \
 399         int32_t const index2_;                                                 \
 400         int32_t const length_;                                                 \
 401       };                                                                       \
 402       ool = new (zone())                                                       \
 403           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 404     }                                                                          \
 405     __ j(above_equal, ool->entry());                                           \
 406     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 407     __ bind(ool->exit());                                                      \
 408   } while (false)
 409
 410
 411 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 412   do {                                                                       \
 413     auto buffer = i.InputRegister(0);                                        \
 414     auto index1 = i.InputRegister(1);                                        \
 415     auto index2 = i.InputInt32(2);                                           \
 416     auto value = i.InputDoubleRegister(4);                                   \
 417     if (instr->InputAt(3)->IsRegister()) {                                   \
 418       auto length = i.InputRegister(3);                                      \
 419       DCHECK_EQ(0, index2);                                                  \
 420       Label done;                                                            \
 421       __ cmpl(index1, length);                                               \
 422       __ j(above_equal, &done, Label::kNear);                                \
 423       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 424       __ bind(&done);                                                        \
 425     } else {                                                                 \
 426       auto length = i.InputInt32(3);                                         \
 427       DCHECK_LE(index2, length);                                             \
 428       __ cmpq(index1, Immediate(length - index2));                           \
 429       class OutOfLineStoreFloat FINAL : public OutOfLineCode {               \
 430        public:                                                               \
 431         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 432                             Register index1, int32_t index2, int32_t length, \
 433                             XMMRegister value)                               \
 434             : OutOfLineCode(gen),                                            \
 435               buffer_(buffer),                                               \
 436               index1_(index1),                                               \
 437               index2_(index2),                                               \
 438               length_(length),                                               \
 439               value_(value) {}                                               \
 440                                                                              \
 441         void Generate() FINAL {                                              \
 442           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 443           __ cmpl(kScratchRegister, Immediate(length_));                     \
 444           __ j(above_equal, exit());                                         \
 445           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 446                        value_);                                              \
 447         }                                                                    \
 448                                                                              \
 449        private:                                                              \
 450         Register const buffer_;                                              \
 451         Register const index1_;                                              \
 452         int32_t const index2_;                                               \
 453         int32_t const length_;                                               \
 454         XMMRegister const value_;                                            \
 455       };                                                                     \
 456       auto ool = new (zone())                                                \
 457           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 458       __ j(above_equal, ool->entry());                                       \
 459       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 460       __ bind(ool->exit());                                                  \
 461     }                                                                        \
 462   } while (false)
 463
 464
 465 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 466   do {                                                                         \
 467     auto buffer = i.InputRegister(0);                                          \
 468     auto index1 = i.InputRegister(1);                                          \
 469     auto index2 = i.InputInt32(2);                                             \
 470     if (instr->InputAt(3)->IsRegister()) {                                     \
 471       auto length = i.InputRegister(3);                                        \
 472       DCHECK_EQ(0, index2);                                                    \
 473       Label done;                                                              \
 474       __ cmpl(index1, length);                                                 \
 475       __ j(above_equal, &done, Label::kNear);                                  \
 476       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 477       __ bind(&done);                                                          \
 478     } else {                                                                   \
 479       auto length = i.InputInt32(3);                                           \
 480       DCHECK_LE(index2, length);                                               \
 481       __ cmpq(index1, Immediate(length - index2));                             \
 482       class OutOfLineStoreInteger FINAL : public OutOfLineCode {               \
 483        public:                                                                 \
 484         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 485                               Register index1, int32_t index2, int32_t length, \
 486                               Value value)                                     \
 487             : OutOfLineCode(gen),                                              \
 488               buffer_(buffer),                                                 \
 489               index1_(index1),                                                 \
 490               index2_(index2),                                                 \
 491               length_(length),                                                 \
 492               value_(value) {}                                                 \
 493                                                                                \
 494         void Generate() FINAL {                                                \
 495           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 496           __ cmpl(kScratchRegister, Immediate(length_));                       \
 497           __ j(above_equal, exit());                                           \
 498           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 499                        value_);                                                \
 500         }                                                                      \
 501                                                                                \
 502        private:                                                                \
 503         Register const buffer_;                                                \
 504         Register const index1_;                                                \
 505         int32_t const index2_;                                                 \
 506         int32_t const length_;                                                 \
 507         Value const value_;                                                    \
 508       };                                                                       \
 509       auto ool = new (zone())                                                  \
 510           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 511       __ j(above_equal, ool->entry());                                         \
 512       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 513       __ bind(ool->exit());                                                    \
 514     }                                                                          \
 515   } while (false)
 516
 517
 518 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 519   do {                                                           \
 520     if (instr->InputAt(4)->IsRegister()) {                       \
 521       Register value = i.InputRegister(4);                       \
 522       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 523     } else {                                                     \
 524       Immediate value = i.InputImmediate(4);                     \
 525       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 526     }                                                            \
 527   } while (false)
 528
 529
 530 // Assembles an instruction after register allocation, producing machine code.
 531 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 532   X64OperandConverter i(this, instr);
 533
 534   switch (ArchOpcodeField::decode(instr->opcode())) {
 535     case kArchCallCodeObject: {
 536       EnsureSpaceForLazyDeopt();
 537       if (HasImmediateInput(instr, 0)) {
 538         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 539         __ Call(code, RelocInfo::CODE_TARGET);
 540       } else {
 541         Register reg = i.InputRegister(0);
 542         int entry = Code::kHeaderSize - kHeapObjectTag;
 543         __ Call(Operand(reg, entry));
 544       }
 545       RecordCallPosition(instr);
 546       break;
 547     }
 548     case kArchCallJSFunction: {
 549       EnsureSpaceForLazyDeopt();
 550       Register func = i.InputRegister(0);
 551       if (FLAG_debug_code) {
 552         // Check the function's context matches the context argument.
 553         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 554         __ Assert(equal, kWrongFunctionContext);
 555       }
 556       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 557       RecordCallPosition(instr);
 558       break;
 559     }
 560     case kArchJmp:
 561       AssembleArchJump(i.InputRpo(0));
 562       break;
 563     case kArchLookupSwitch:
 564       AssembleArchLookupSwitch(instr);
 565       break;
 566     case kArchTableSwitch:
 567       AssembleArchTableSwitch(instr);
 568       break;
 569     case kArchNop:
 570       // don't emit code for nops.
 571       break;
 572     case kArchDeoptimize: {
 573       int deopt_state_id =
 574           BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
 575       AssembleDeoptimizerCall(deopt_state_id, Deoptimizer::EAGER);
 576       break;
 577     }
 578     case kArchRet:
 579       AssembleReturn();
 580       break;
 581     case kArchStackPointer:
 582       __ movq(i.OutputRegister(), rsp);
 583       break;
 584     case kArchTruncateDoubleToI: {
 585       auto result = i.OutputRegister();
 586       auto input = i.InputDoubleRegister(0);
 587       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 588       __ cvttsd2siq(result, input);
 589       __ cmpq(result, Immediate(1));
 590       __ j(overflow, ool->entry());
 591       __ bind(ool->exit());
 592       break;
 593     }
 594     case kX64Add32:
 595       ASSEMBLE_BINOP(addl);
 596       break;
 597     case kX64Add:
 598       ASSEMBLE_BINOP(addq);
 599       break;
 600     case kX64Sub32:
 601       ASSEMBLE_BINOP(subl);
 602       break;
 603     case kX64Sub:
 604       ASSEMBLE_BINOP(subq);
 605       break;
 606     case kX64And32:
 607       ASSEMBLE_BINOP(andl);
 608       break;
 609     case kX64And:
 610       ASSEMBLE_BINOP(andq);
 611       break;
 612     case kX64Cmp32:
 613       ASSEMBLE_BINOP(cmpl);
 614       break;
 615     case kX64Cmp:
 616       ASSEMBLE_BINOP(cmpq);
 617       break;
 618     case kX64Test32:
 619       ASSEMBLE_BINOP(testl);
 620       break;
 621     case kX64Test:
 622       ASSEMBLE_BINOP(testq);
 623       break;
 624     case kX64Imul32:
 625       ASSEMBLE_MULT(imull);
 626       break;
 627     case kX64Imul:
 628       ASSEMBLE_MULT(imulq);
 629       break;
 630     case kX64ImulHigh32:
 631       if (instr->InputAt(1)->IsRegister()) {
 632         __ imull(i.InputRegister(1));
 633       } else {
 634         __ imull(i.InputOperand(1));
 635       }
 636       break;
 637     case kX64UmulHigh32:
 638       if (instr->InputAt(1)->IsRegister()) {
 639         __ mull(i.InputRegister(1));
 640       } else {
 641         __ mull(i.InputOperand(1));
 642       }
 643       break;
 644     case kX64Idiv32:
 645       __ cdq();
 646       __ idivl(i.InputRegister(1));
 647       break;
 648     case kX64Idiv:
 649       __ cqo();
 650       __ idivq(i.InputRegister(1));
 651       break;
 652     case kX64Udiv32:
 653       __ xorl(rdx, rdx);
 654       __ divl(i.InputRegister(1));
 655       break;
 656     case kX64Udiv:
 657       __ xorq(rdx, rdx);
 658       __ divq(i.InputRegister(1));
 659       break;
 660     case kX64Not:
 661       ASSEMBLE_UNOP(notq);
 662       break;
 663     case kX64Not32:
 664       ASSEMBLE_UNOP(notl);
 665       break;
 666     case kX64Neg:
 667       ASSEMBLE_UNOP(negq);
 668       break;
 669     case kX64Neg32:
 670       ASSEMBLE_UNOP(negl);
 671       break;
 672     case kX64Or32:
 673       ASSEMBLE_BINOP(orl);
 674       break;
 675     case kX64Or:
 676       ASSEMBLE_BINOP(orq);
 677       break;
 678     case kX64Xor32:
 679       ASSEMBLE_BINOP(xorl);
 680       break;
 681     case kX64Xor:
 682       ASSEMBLE_BINOP(xorq);
 683       break;
 684     case kX64Shl32:
 685       ASSEMBLE_SHIFT(shll, 5);
 686       break;
 687     case kX64Shl:
 688       ASSEMBLE_SHIFT(shlq, 6);
 689       break;
 690     case kX64Shr32:
 691       ASSEMBLE_SHIFT(shrl, 5);
 692       break;
 693     case kX64Shr:
 694       ASSEMBLE_SHIFT(shrq, 6);
 695       break;
 696     case kX64Sar32:
 697       ASSEMBLE_SHIFT(sarl, 5);
 698       break;
 699     case kX64Sar:
 700       ASSEMBLE_SHIFT(sarq, 6);
 701       break;
 702     case kX64Ror32:
 703       ASSEMBLE_SHIFT(rorl, 5);
 704       break;
 705     case kX64Ror:
 706       ASSEMBLE_SHIFT(rorq, 6);
 707       break;
 708     case kX64Lzcnt32:
 709       if (instr->InputAt(0)->IsRegister()) {
 710         __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
 711       } else {
 712         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
 713       }
 714       break;
 715     case kSSEFloat32Cmp:
 716       ASSEMBLE_SSE_BINOP(ucomiss);
 717       break;
 718     case kSSEFloat32Add:
 719       ASSEMBLE_SSE_BINOP(addss);
 720       break;
 721     case kSSEFloat32Sub:
 722       ASSEMBLE_SSE_BINOP(subss);
 723       break;
 724     case kSSEFloat32Mul:
 725       ASSEMBLE_SSE_BINOP(mulss);
 726       break;
 727     case kSSEFloat32Div:
 728       ASSEMBLE_SSE_BINOP(divss);
 729       // Don't delete this mov. It may improve performance on some CPUs,
 730       // when there is a (v)mulss depending on the result.
 731       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 732       break;
 733     case kSSEFloat32Abs: {
 734       // TODO(bmeurer): Use RIP relative 128-bit constants.
 735       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 736       __ psrlq(kScratchDoubleReg, 33);
 737       __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
 738       break;
 739     }
 740     case kSSEFloat32Neg: {
 741       // TODO(bmeurer): Use RIP relative 128-bit constants.
 742       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 743       __ psllq(kScratchDoubleReg, 31);
 744       __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
 745       break;
 746     }
 747     case kSSEFloat32Sqrt:
 748       ASSEMBLE_SSE_UNOP(sqrtss);
 749       break;
 750     case kSSEFloat32Max:
 751       ASSEMBLE_SSE_BINOP(maxss);
 752       break;
 753     case kSSEFloat32Min:
 754       ASSEMBLE_SSE_BINOP(minss);
 755       break;
 756     case kSSEFloat32ToFloat64:
 757       ASSEMBLE_SSE_UNOP(cvtss2sd);
 758       break;
 759     case kSSEFloat64Cmp:
 760       ASSEMBLE_SSE_BINOP(ucomisd);
 761       break;
 762     case kSSEFloat64Add:
 763       ASSEMBLE_SSE_BINOP(addsd);
 764       break;
 765     case kSSEFloat64Sub:
 766       ASSEMBLE_SSE_BINOP(subsd);
 767       break;
 768     case kSSEFloat64Mul:
 769       ASSEMBLE_SSE_BINOP(mulsd);
 770       break;
 771     case kSSEFloat64Div:
 772       ASSEMBLE_SSE_BINOP(divsd);
 773       // Don't delete this mov. It may improve performance on some CPUs,
 774       // when there is a (v)mulsd depending on the result.
 775       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 776       break;
 777     case kSSEFloat64Mod: {
 778       __ subq(rsp, Immediate(kDoubleSize));
 779       // Move values to st(0) and st(1).
 780       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 781       __ fld_d(Operand(rsp, 0));
 782       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 783       __ fld_d(Operand(rsp, 0));
 784       // Loop while fprem isn't done.
 785       Label mod_loop;
 786       __ bind(&mod_loop);
 787       // This instructions traps on all kinds inputs, but we are assuming the
 788       // floating point control word is set to ignore them all.
 789       __ fprem();
 790       // The following 2 instruction implicitly use rax.
 791       __ fnstsw_ax();
 792       if (CpuFeatures::IsSupported(SAHF)) {
 793         CpuFeatureScope sahf_scope(masm(), SAHF);
 794         __ sahf();
 795       } else {
 796         __ shrl(rax, Immediate(8));
 797         __ andl(rax, Immediate(0xFF));
 798         __ pushq(rax);
 799         __ popfq();
 800       }
 801       __ j(parity_even, &mod_loop);
 802       // Move output to stack and clean up.
 803       __ fstp(1);
 804       __ fstp_d(Operand(rsp, 0));
 805       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 806       __ addq(rsp, Immediate(kDoubleSize));
 807       break;
 808     }
 809     case kSSEFloat64Max:
 810       ASSEMBLE_SSE_BINOP(maxsd);
 811       break;
 812     case kSSEFloat64Min:
 813       ASSEMBLE_SSE_BINOP(minsd);
 814       break;
 815     case kSSEFloat64Abs: {
 816       // TODO(bmeurer): Use RIP relative 128-bit constants.
 817       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 818       __ psrlq(kScratchDoubleReg, 1);
 819       __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 820       break;
 821     }
 822     case kSSEFloat64Neg: {
 823       // TODO(bmeurer): Use RIP relative 128-bit constants.
 824       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 825       __ psllq(kScratchDoubleReg, 63);
 826       __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 827       break;
 828     }
 829     case kSSEFloat64Sqrt:
 830       ASSEMBLE_SSE_UNOP(sqrtsd);
 831       break;
 832     case kSSEFloat64Round: {
 833       CpuFeatureScope sse_scope(masm(), SSE4_1);
 834       RoundingMode const mode =
 835           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
 836       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
 837       break;
 838     }
 839     case kSSEFloat64ToFloat32:
 840       ASSEMBLE_SSE_UNOP(cvtsd2ss);
 841       break;
 842     case kSSEFloat64ToInt32:
 843       if (instr->InputAt(0)->IsDoubleRegister()) {
 844         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 845       } else {
 846         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 847       }
 848       break;
 849     case kSSEFloat64ToUint32: {
 850       if (instr->InputAt(0)->IsDoubleRegister()) {
 851         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 852       } else {
 853         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 854       }
 855       __ AssertZeroExtended(i.OutputRegister());
 856       break;
 857     }
 858     case kSSEInt32ToFloat64:
 859       if (instr->InputAt(0)->IsRegister()) {
 860         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 861       } else {
 862         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 863       }
 864       break;
 865     case kSSEUint32ToFloat64:
 866       if (instr->InputAt(0)->IsRegister()) {
 867         __ movl(kScratchRegister, i.InputRegister(0));
 868       } else {
 869         __ movl(kScratchRegister, i.InputOperand(0));
 870       }
 871       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 872       break;
 873     case kSSEFloat64ExtractLowWord32:
 874       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 875         __ movl(i.OutputRegister(), i.InputOperand(0));
 876       } else {
 877         __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
 878       }
 879       break;
 880     case kSSEFloat64ExtractHighWord32:
 881       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 882         __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
 883       } else {
 884         __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
 885       }
 886       break;
 887     case kSSEFloat64InsertLowWord32:
 888       if (instr->InputAt(1)->IsRegister()) {
 889         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
 890       } else {
 891         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
 892       }
 893       break;
 894     case kSSEFloat64InsertHighWord32:
 895       if (instr->InputAt(1)->IsRegister()) {
 896         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
 897       } else {
 898         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
 899       }
 900       break;
 901     case kSSEFloat64LoadLowWord32:
 902       if (instr->InputAt(0)->IsRegister()) {
 903         __ movd(i.OutputDoubleRegister(), i.InputRegister(0));
 904       } else {
 905         __ movd(i.OutputDoubleRegister(), i.InputOperand(0));
 906       }
 907       break;
 908     case kAVXFloat32Cmp: {
 909       CpuFeatureScope avx_scope(masm(), AVX);
 910       if (instr->InputAt(1)->IsDoubleRegister()) {
 911         __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
 912       } else {
 913         __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
 914       }
 915       break;
 916     }
 917     case kAVXFloat32Add:
 918       ASSEMBLE_AVX_BINOP(vaddss);
 919       break;
 920     case kAVXFloat32Sub:
 921       ASSEMBLE_AVX_BINOP(vsubss);
 922       break;
 923     case kAVXFloat32Mul:
 924       ASSEMBLE_AVX_BINOP(vmulss);
 925       break;
 926     case kAVXFloat32Div:
 927       ASSEMBLE_AVX_BINOP(vdivss);
 928       // Don't delete this mov. It may improve performance on some CPUs,
 929       // when there is a (v)mulss depending on the result.
 930       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 931       break;
 932     case kAVXFloat32Max:
 933       ASSEMBLE_AVX_BINOP(vmaxss);
 934       break;
 935     case kAVXFloat32Min:
 936       ASSEMBLE_AVX_BINOP(vminss);
 937       break;
 938     case kAVXFloat64Cmp: {
 939       CpuFeatureScope avx_scope(masm(), AVX);
 940       if (instr->InputAt(1)->IsDoubleRegister()) {
 941         __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
 942       } else {
 943         __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
 944       }
 945       break;
 946     }
 947     case kAVXFloat64Add:
 948       ASSEMBLE_AVX_BINOP(vaddsd);
 949       break;
 950     case kAVXFloat64Sub:
 951       ASSEMBLE_AVX_BINOP(vsubsd);
 952       break;
 953     case kAVXFloat64Mul:
 954       ASSEMBLE_AVX_BINOP(vmulsd);
 955       break;
 956     case kAVXFloat64Div:
 957       ASSEMBLE_AVX_BINOP(vdivsd);
 958       // Don't delete this mov. It may improve performance on some CPUs,
 959       // when there is a (v)mulsd depending on the result.
 960       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 961       break;
 962     case kAVXFloat64Max:
 963       ASSEMBLE_AVX_BINOP(vmaxsd);
 964       break;
 965     case kAVXFloat64Min:
 966       ASSEMBLE_AVX_BINOP(vminsd);
 967       break;
 968     case kAVXFloat32Abs: {
 969       // TODO(bmeurer): Use RIP relative 128-bit constants.
 970       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 971       __ psrlq(kScratchDoubleReg, 33);
 972       CpuFeatureScope avx_scope(masm(), AVX);
 973       if (instr->InputAt(0)->IsDoubleRegister()) {
 974         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
 975                   i.InputDoubleRegister(0));
 976       } else {
 977         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
 978                   i.InputOperand(0));
 979       }
 980       break;
 981     }
 982     case kAVXFloat32Neg: {
 983       // TODO(bmeurer): Use RIP relative 128-bit constants.
 984       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 985       __ psllq(kScratchDoubleReg, 31);
 986       CpuFeatureScope avx_scope(masm(), AVX);
 987       if (instr->InputAt(0)->IsDoubleRegister()) {
 988         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
 989                   i.InputDoubleRegister(0));
 990       } else {
 991         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
 992                   i.InputOperand(0));
 993       }
 994       break;
 995     }
 996     case kAVXFloat64Abs: {
 997       // TODO(bmeurer): Use RIP relative 128-bit constants.
 998       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 999       __ psrlq(kScratchDoubleReg, 1);
1000       CpuFeatureScope avx_scope(masm(), AVX);
1001       if (instr->InputAt(0)->IsDoubleRegister()) {
1002         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1003                   i.InputDoubleRegister(0));
1004       } else {
1005         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1006                   i.InputOperand(0));
1007       }
1008       break;
1009     }
1010     case kAVXFloat64Neg: {
1011       // TODO(bmeurer): Use RIP relative 128-bit constants.
1012       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1013       __ psllq(kScratchDoubleReg, 63);
1014       CpuFeatureScope avx_scope(masm(), AVX);
1015       if (instr->InputAt(0)->IsDoubleRegister()) {
1016         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1017                   i.InputDoubleRegister(0));
1018       } else {
1019         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1020                   i.InputOperand(0));
1021       }
1022       break;
1023     }
1024     case kX64Movsxbl:
1025       ASSEMBLE_MOVX(movsxbl);
1026       __ AssertZeroExtended(i.OutputRegister());
1027       break;
1028     case kX64Movzxbl:
1029       ASSEMBLE_MOVX(movzxbl);
1030       __ AssertZeroExtended(i.OutputRegister());
1031       break;
1032     case kX64Movb: {
1033       size_t index = 0;
1034       Operand operand = i.MemoryOperand(&index);
1035       if (HasImmediateInput(instr, index)) {
1036         __ movb(operand, Immediate(i.InputInt8(index)));
1037       } else {
1038         __ movb(operand, i.InputRegister(index));
1039       }
1040       break;
1041     }
1042     case kX64Movsxwl:
1043       ASSEMBLE_MOVX(movsxwl);
1044       __ AssertZeroExtended(i.OutputRegister());
1045       break;
1046     case kX64Movzxwl:
1047       ASSEMBLE_MOVX(movzxwl);
1048       __ AssertZeroExtended(i.OutputRegister());
1049       break;
1050     case kX64Movw: {
1051       size_t index = 0;
1052       Operand operand = i.MemoryOperand(&index);
1053       if (HasImmediateInput(instr, index)) {
1054         __ movw(operand, Immediate(i.InputInt16(index)));
1055       } else {
1056         __ movw(operand, i.InputRegister(index));
1057       }
1058       break;
1059     }
1060     case kX64Movl:
1061       if (instr->HasOutput()) {
1062         if (instr->addressing_mode() == kMode_None) {
1063           if (instr->InputAt(0)->IsRegister()) {
1064             __ movl(i.OutputRegister(), i.InputRegister(0));
1065           } else {
1066             __ movl(i.OutputRegister(), i.InputOperand(0));
1067           }
1068         } else {
1069           __ movl(i.OutputRegister(), i.MemoryOperand());
1070         }
1071         __ AssertZeroExtended(i.OutputRegister());
1072       } else {
1073         size_t index = 0;
1074         Operand operand = i.MemoryOperand(&index);
1075         if (HasImmediateInput(instr, index)) {
1076           __ movl(operand, i.InputImmediate(index));
1077         } else {
1078           __ movl(operand, i.InputRegister(index));
1079         }
1080       }
1081       break;
1082     case kX64Movsxlq:
1083       ASSEMBLE_MOVX(movsxlq);
1084       break;
1085     case kX64Movq:
1086       if (instr->HasOutput()) {
1087         __ movq(i.OutputRegister(), i.MemoryOperand());
1088       } else {
1089         size_t index = 0;
1090         Operand operand = i.MemoryOperand(&index);
1091         if (HasImmediateInput(instr, index)) {
1092           __ movq(operand, i.InputImmediate(index));
1093         } else {
1094           __ movq(operand, i.InputRegister(index));
1095         }
1096       }
1097       break;
1098     case kX64Movss:
1099       if (instr->HasOutput()) {
1100         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1101       } else {
1102         size_t index = 0;
1103         Operand operand = i.MemoryOperand(&index);
1104         __ movss(operand, i.InputDoubleRegister(index));
1105       }
1106       break;
1107     case kX64Movsd:
1108       if (instr->HasOutput()) {
1109         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1110       } else {
1111         size_t index = 0;
1112         Operand operand = i.MemoryOperand(&index);
1113         __ movsd(operand, i.InputDoubleRegister(index));
1114       }
1115       break;
1116     case kX64Lea32: {
1117       AddressingMode mode = AddressingModeField::decode(instr->opcode());
1118       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
1119       // and addressing mode just happens to work out. The "addl"/"subl" forms
1120       // in these cases are faster based on measurements.
1121       if (i.InputRegister(0).is(i.OutputRegister())) {
1122         if (mode == kMode_MRI) {
1123           int32_t constant_summand = i.InputInt32(1);
1124           if (constant_summand > 0) {
1125             __ addl(i.OutputRegister(), Immediate(constant_summand));
1126           } else if (constant_summand < 0) {
1127             __ subl(i.OutputRegister(), Immediate(-constant_summand));
1128           }
1129         } else if (mode == kMode_MR1) {
1130           if (i.InputRegister(1).is(i.OutputRegister())) {
1131             __ shll(i.OutputRegister(), Immediate(1));
1132           } else {
1133             __ leal(i.OutputRegister(), i.MemoryOperand());
1134           }
1135         } else if (mode == kMode_M2) {
1136           __ shll(i.OutputRegister(), Immediate(1));
1137         } else if (mode == kMode_M4) {
1138           __ shll(i.OutputRegister(), Immediate(2));
1139         } else if (mode == kMode_M8) {
1140           __ shll(i.OutputRegister(), Immediate(3));
1141         } else {
1142           __ leal(i.OutputRegister(), i.MemoryOperand());
1143         }
1144       } else {
1145         __ leal(i.OutputRegister(), i.MemoryOperand());
1146       }
1147       __ AssertZeroExtended(i.OutputRegister());
1148       break;
1149     }
1150     case kX64Lea:
1151       __ leaq(i.OutputRegister(), i.MemoryOperand());
1152       break;
1153     case kX64Dec32:
1154       __ decl(i.OutputRegister());
1155       break;
1156     case kX64Inc32:
1157       __ incl(i.OutputRegister());
1158       break;
1159     case kX64Push:
1160       if (HasImmediateInput(instr, 0)) {
1161         __ pushq(i.InputImmediate(0));
1162       } else {
1163         if (instr->InputAt(0)->IsRegister()) {
1164           __ pushq(i.InputRegister(0));
1165         } else {
1166           __ pushq(i.InputOperand(0));
1167         }
1168       }
1169       break;
1170     case kX64StoreWriteBarrier: {
1171       Register object = i.InputRegister(0);
1172       Register value = i.InputRegister(2);
1173       SaveFPRegsMode mode =
1174           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
1175       if (HasImmediateInput(instr, 1)) {
1176         int index = i.InputInt32(1);
1177         Register scratch = i.TempRegister(1);
1178         __ movq(Operand(object, index), value);
1179         __ RecordWriteContextSlot(object, index, value, scratch, mode);
1180       } else {
1181         Register index = i.InputRegister(1);
1182         __ movq(Operand(object, index, times_1, 0), value);
1183         __ leaq(index, Operand(object, index, times_1, 0));
1184         __ RecordWrite(object, index, value, mode);
1185       }
1186       break;
1187     }
1188     case kCheckedLoadInt8:
1189       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
1190       break;
1191     case kCheckedLoadUint8:
1192       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
1193       break;
1194     case kCheckedLoadInt16:
1195       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
1196       break;
1197     case kCheckedLoadUint16:
1198       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
1199       break;
1200     case kCheckedLoadWord32:
1201       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
1202       break;
1203     case kCheckedLoadFloat32:
1204       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
1205       break;
1206     case kCheckedLoadFloat64:
1207       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
1208       break;
1209     case kCheckedStoreWord8:
1210       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
1211       break;
1212     case kCheckedStoreWord16:
1213       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
1214       break;
1215     case kCheckedStoreWord32:
1216       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1217       break;
1218     case kCheckedStoreFloat32:
1219       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1220       break;
1221     case kCheckedStoreFloat64:
1222       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1223       break;
1224     case kX64StackCheck:
1225       __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
1226       break;
1227   }
1228 }  // NOLINT(readability/fn_size)
1229
1230
1231 // Assembles branches after this instruction.
1232 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1233   X64OperandConverter i(this, instr);
1234   Label::Distance flabel_distance =
1235       branch->fallthru ? Label::kNear : Label::kFar;
1236   Label* tlabel = branch->true_label;
1237   Label* flabel = branch->false_label;
1238   switch (branch->condition) {
1239     case kUnorderedEqual:
1240       __ j(parity_even, flabel, flabel_distance);
1241     // Fall through.
1242     case kEqual:
1243       __ j(equal, tlabel);
1244       break;
1245     case kUnorderedNotEqual:
1246       __ j(parity_even, tlabel);
1247     // Fall through.
1248     case kNotEqual:
1249       __ j(not_equal, tlabel);
1250       break;
1251     case kSignedLessThan:
1252       __ j(less, tlabel);
1253       break;
1254     case kSignedGreaterThanOrEqual:
1255       __ j(greater_equal, tlabel);
1256       break;
1257     case kSignedLessThanOrEqual:
1258       __ j(less_equal, tlabel);
1259       break;
1260     case kSignedGreaterThan:
1261       __ j(greater, tlabel);
1262       break;
1263     case kUnsignedLessThan:
1264       __ j(below, tlabel);
1265       break;
1266     case kUnsignedGreaterThanOrEqual:
1267       __ j(above_equal, tlabel);
1268       break;
1269     case kUnsignedLessThanOrEqual:
1270       __ j(below_equal, tlabel);
1271       break;
1272     case kUnsignedGreaterThan:
1273       __ j(above, tlabel);
1274       break;
1275     case kOverflow:
1276       __ j(overflow, tlabel);
1277       break;
1278     case kNotOverflow:
1279       __ j(no_overflow, tlabel);
1280       break;
1281   }
1282   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1283 }
1284
1285
1286 void CodeGenerator::AssembleArchJump(RpoNumber target) {
1287   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1288 }
1289
1290
1291 // Assembles boolean materializations after this instruction.
1292 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1293                                         FlagsCondition condition) {
1294   X64OperandConverter i(this, instr);
1295   Label done;
1296
1297   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1298   // last output of the instruction.
1299   Label check;
1300   DCHECK_NE(0u, instr->OutputCount());
1301   Register reg = i.OutputRegister(instr->OutputCount() - 1);
1302   Condition cc = no_condition;
1303   switch (condition) {
1304     case kUnorderedEqual:
1305       __ j(parity_odd, &check, Label::kNear);
1306       __ movl(reg, Immediate(0));
1307       __ jmp(&done, Label::kNear);
1308     // Fall through.
1309     case kEqual:
1310       cc = equal;
1311       break;
1312     case kUnorderedNotEqual:
1313       __ j(parity_odd, &check, Label::kNear);
1314       __ movl(reg, Immediate(1));
1315       __ jmp(&done, Label::kNear);
1316     // Fall through.
1317     case kNotEqual:
1318       cc = not_equal;
1319       break;
1320     case kSignedLessThan:
1321       cc = less;
1322       break;
1323     case kSignedGreaterThanOrEqual:
1324       cc = greater_equal;
1325       break;
1326     case kSignedLessThanOrEqual:
1327       cc = less_equal;
1328       break;
1329     case kSignedGreaterThan:
1330       cc = greater;
1331       break;
1332     case kUnsignedLessThan:
1333       cc = below;
1334       break;
1335     case kUnsignedGreaterThanOrEqual:
1336       cc = above_equal;
1337       break;
1338     case kUnsignedLessThanOrEqual:
1339       cc = below_equal;
1340       break;
1341     case kUnsignedGreaterThan:
1342       cc = above;
1343       break;
1344     case kOverflow:
1345       cc = overflow;
1346       break;
1347     case kNotOverflow:
1348       cc = no_overflow;
1349       break;
1350   }
1351   __ bind(&check);
1352   __ setcc(cc, reg);
1353   __ movzxbl(reg, reg);
1354   __ bind(&done);
1355 }
1356
1357
1358 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1359   X64OperandConverter i(this, instr);
1360   Register input = i.InputRegister(0);
1361   for (size_t index = 2; index < instr->InputCount(); index += 2) {
1362     __ cmpl(input, Immediate(i.InputInt32(index + 0)));
1363     __ j(equal, GetLabel(i.InputRpo(index + 1)));
1364   }
1365   AssembleArchJump(i.InputRpo(1));
1366 }
1367
1368
1369 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1370   X64OperandConverter i(this, instr);
1371   Register input = i.InputRegister(0);
1372   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
1373   Label** cases = zone()->NewArray<Label*>(case_count);
1374   for (int32_t index = 0; index < case_count; ++index) {
1375     cases[index] = GetLabel(i.InputRpo(index + 2));
1376   }
1377   Label* const table = AddJumpTable(cases, case_count);
1378   __ cmpl(input, Immediate(case_count));
1379   __ j(above_equal, GetLabel(i.InputRpo(1)));
1380   __ leaq(kScratchRegister, Operand(table));
1381   __ jmp(Operand(kScratchRegister, input, times_8, 0));
1382 }
1383
1384
1385 void CodeGenerator::AssembleDeoptimizerCall(
1386     int deoptimization_id, Deoptimizer::BailoutType bailout_type) {
1387   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1388       isolate(), deoptimization_id, bailout_type);
1389   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1390 }
1391
1392
1393 void CodeGenerator::AssemblePrologue() {
1394   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1395   int stack_slots = frame()->GetSpillSlotCount();
1396   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1397     __ pushq(rbp);
1398     __ movq(rbp, rsp);
1399     const RegList saves = descriptor->CalleeSavedRegisters();
1400     if (saves != 0) {  // Save callee-saved registers.
1401       int register_save_area_size = 0;
1402       for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1403         if (!((1 << i) & saves)) continue;
1404         __ pushq(Register::from_code(i));
1405         register_save_area_size += kPointerSize;
1406       }
1407       frame()->SetRegisterSaveAreaSize(register_save_area_size);
1408     }
1409   } else if (descriptor->IsJSFunctionCall()) {
1410     CompilationInfo* info = this->info();
1411     __ Prologue(info->IsCodePreAgingActive());
1412     frame()->SetRegisterSaveAreaSize(
1413         StandardFrameConstants::kFixedFrameSizeFromFp);
1414   } else if (stack_slots > 0) {
1415     __ StubPrologue();
1416     frame()->SetRegisterSaveAreaSize(
1417         StandardFrameConstants::kFixedFrameSizeFromFp);
1418   }
1419
1420   if (info()->is_osr()) {
1421     // TurboFan OSR-compiled functions cannot be entered directly.
1422     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1423
1424     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1425     // frame is still on the stack. Optimized code uses OSR values directly from
1426     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1427     // remaining stack slots.
1428     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1429     osr_pc_offset_ = __ pc_offset();
1430     // TODO(titzer): cannot address target function == local #-1
1431     __ movq(rdi, Operand(rbp, JavaScriptFrameConstants::kFunctionOffset));
1432     DCHECK(stack_slots >= frame()->GetOsrStackSlotCount());
1433     stack_slots -= frame()->GetOsrStackSlotCount();
1434   }
1435
1436   if (stack_slots > 0) {
1437     __ subq(rsp, Immediate(stack_slots * kPointerSize));
1438   }
1439 }
1440
1441
1442 void CodeGenerator::AssembleReturn() {
1443   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1444   int stack_slots = frame()->GetSpillSlotCount();
1445   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1446     if (frame()->GetRegisterSaveAreaSize() > 0) {
1447       // Remove this frame's spill slots first.
1448       if (stack_slots > 0) {
1449         __ addq(rsp, Immediate(stack_slots * kPointerSize));
1450       }
1451       const RegList saves = descriptor->CalleeSavedRegisters();
1452       // Restore registers.
1453       if (saves != 0) {
1454         for (int i = 0; i < Register::kNumRegisters; i++) {
1455           if (!((1 << i) & saves)) continue;
1456           __ popq(Register::from_code(i));
1457         }
1458       }
1459       __ popq(rbp);  // Pop caller's frame pointer.
1460       __ ret(0);
1461     } else {
1462       // No saved registers.
1463       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1464       __ popq(rbp);       // Pop caller's frame pointer.
1465       __ ret(0);
1466     }
1467   } else if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
1468     __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1469     __ popq(rbp);       // Pop caller's frame pointer.
1470     int pop_count = descriptor->IsJSFunctionCall()
1471                         ? static_cast<int>(descriptor->JSParameterCount())
1472                         : 0;
1473     __ ret(pop_count * kPointerSize);
1474   } else {
1475     __ ret(0);
1476   }
1477 }
1478
1479
1480 void CodeGenerator::AssembleMove(InstructionOperand* source,
1481                                  InstructionOperand* destination) {
1482   X64OperandConverter g(this, NULL);
1483   // Dispatch on the source and destination operand kinds.  Not all
1484   // combinations are possible.
1485   if (source->IsRegister()) {
1486     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1487     Register src = g.ToRegister(source);
1488     if (destination->IsRegister()) {
1489       __ movq(g.ToRegister(destination), src);
1490     } else {
1491       __ movq(g.ToOperand(destination), src);
1492     }
1493   } else if (source->IsStackSlot()) {
1494     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1495     Operand src = g.ToOperand(source);
1496     if (destination->IsRegister()) {
1497       Register dst = g.ToRegister(destination);
1498       __ movq(dst, src);
1499     } else {
1500       // Spill on demand to use a temporary register for memory-to-memory
1501       // moves.
1502       Register tmp = kScratchRegister;
1503       Operand dst = g.ToOperand(destination);
1504       __ movq(tmp, src);
1505       __ movq(dst, tmp);
1506     }
1507   } else if (source->IsConstant()) {
1508     ConstantOperand* constant_source = ConstantOperand::cast(source);
1509     Constant src = g.ToConstant(constant_source);
1510     if (destination->IsRegister() || destination->IsStackSlot()) {
1511       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1512                                                : kScratchRegister;
1513       switch (src.type()) {
1514         case Constant::kInt32:
1515           // TODO(dcarney): don't need scratch in this case.
1516           __ Set(dst, src.ToInt32());
1517           break;
1518         case Constant::kInt64:
1519           __ Set(dst, src.ToInt64());
1520           break;
1521         case Constant::kFloat32:
1522           __ Move(dst,
1523                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1524           break;
1525         case Constant::kFloat64:
1526           __ Move(dst,
1527                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1528           break;
1529         case Constant::kExternalReference:
1530           __ Move(dst, src.ToExternalReference());
1531           break;
1532         case Constant::kHeapObject: {
1533           Handle<HeapObject> src_object = src.ToHeapObject();
1534           if (info()->IsOptimizing() &&
1535               src_object.is_identical_to(info()->context())) {
1536             // Loading the context from the frame is way cheaper than
1537             // materializing the actual context heap object address.
1538             __ movp(dst, Operand(rbp, StandardFrameConstants::kContextOffset));
1539           } else if (info()->IsOptimizing() &&
1540                      src_object.is_identical_to(info()->closure())) {
1541             // Loading the JSFunction from the frame is way cheaper than
1542             // materializing the actual JSFunction heap object address.
1543             __ movp(dst,
1544                     Operand(rbp, JavaScriptFrameConstants::kFunctionOffset));
1545           } else {
1546             __ Move(dst, src_object);
1547           }
1548           break;
1549         }
1550         case Constant::kRpoNumber:
1551           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1552           break;
1553       }
1554       if (destination->IsStackSlot()) {
1555         __ movq(g.ToOperand(destination), kScratchRegister);
1556       }
1557     } else if (src.type() == Constant::kFloat32) {
1558       // TODO(turbofan): Can we do better here?
1559       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1560       if (destination->IsDoubleRegister()) {
1561         __ Move(g.ToDoubleRegister(destination), src_const);
1562       } else {
1563         DCHECK(destination->IsDoubleStackSlot());
1564         Operand dst = g.ToOperand(destination);
1565         __ movl(dst, Immediate(src_const));
1566       }
1567     } else {
1568       DCHECK_EQ(Constant::kFloat64, src.type());
1569       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1570       if (destination->IsDoubleRegister()) {
1571         __ Move(g.ToDoubleRegister(destination), src_const);
1572       } else {
1573         DCHECK(destination->IsDoubleStackSlot());
1574         __ movq(kScratchRegister, src_const);
1575         __ movq(g.ToOperand(destination), kScratchRegister);
1576       }
1577     }
1578   } else if (source->IsDoubleRegister()) {
1579     XMMRegister src = g.ToDoubleRegister(source);
1580     if (destination->IsDoubleRegister()) {
1581       XMMRegister dst = g.ToDoubleRegister(destination);
1582       __ movaps(dst, src);
1583     } else {
1584       DCHECK(destination->IsDoubleStackSlot());
1585       Operand dst = g.ToOperand(destination);
1586       __ movsd(dst, src);
1587     }
1588   } else if (source->IsDoubleStackSlot()) {
1589     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1590     Operand src = g.ToOperand(source);
1591     if (destination->IsDoubleRegister()) {
1592       XMMRegister dst = g.ToDoubleRegister(destination);
1593       __ movsd(dst, src);
1594     } else {
1595       // We rely on having xmm0 available as a fixed scratch register.
1596       Operand dst = g.ToOperand(destination);
1597       __ movsd(xmm0, src);
1598       __ movsd(dst, xmm0);
1599     }
1600   } else {
1601     UNREACHABLE();
1602   }
1603 }
1604
1605
1606 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1607                                  InstructionOperand* destination) {
1608   X64OperandConverter g(this, NULL);
1609   // Dispatch on the source and destination operand kinds.  Not all
1610   // combinations are possible.
1611   if (source->IsRegister() && destination->IsRegister()) {
1612     // Register-register.
1613     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1614   } else if (source->IsRegister() && destination->IsStackSlot()) {
1615     Register src = g.ToRegister(source);
1616     Operand dst = g.ToOperand(destination);
1617     __ xchgq(src, dst);
1618   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1619              (source->IsDoubleStackSlot() &&
1620               destination->IsDoubleStackSlot())) {
1621     // Memory-memory.
1622     Register tmp = kScratchRegister;
1623     Operand src = g.ToOperand(source);
1624     Operand dst = g.ToOperand(destination);
1625     __ movq(tmp, dst);
1626     __ xchgq(tmp, src);
1627     __ movq(dst, tmp);
1628   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1629     // XMM register-register swap. We rely on having xmm0
1630     // available as a fixed scratch register.
1631     XMMRegister src = g.ToDoubleRegister(source);
1632     XMMRegister dst = g.ToDoubleRegister(destination);
1633     __ movaps(xmm0, src);
1634     __ movaps(src, dst);
1635     __ movaps(dst, xmm0);
1636   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1637     // XMM register-memory swap.  We rely on having xmm0
1638     // available as a fixed scratch register.
1639     XMMRegister src = g.ToDoubleRegister(source);
1640     Operand dst = g.ToOperand(destination);
1641     __ movsd(xmm0, src);
1642     __ movsd(src, dst);
1643     __ movsd(dst, xmm0);
1644   } else {
1645     // No other combinations are possible.
1646     UNREACHABLE();
1647   }
1648 }
1649
1650
1651 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1652   for (size_t index = 0; index < target_count; ++index) {
1653     __ dq(targets[index]);
1654   }
1655 }
1656
1657
1658 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1659
1660
1661 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1662   int space_needed = Deoptimizer::patch_size();
1663   if (!info()->IsStub()) {
1664     // Ensure that we have enough space after the previous lazy-bailout
1665     // instruction for patching the code here.
1666     int current_pc = masm()->pc_offset();
1667     if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1668       int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1669       __ Nop(padding_size);
1670     }
1671   }
1672   MarkLazyDeoptSite();
1673 }
1674
1675 #undef __
1676
1677 }  // namespace internal
1678 }  // namespace compiler
1679 }  // namespace v8