src/compiler/x64/code-generator-x64.cc

   1 // Copyright 2013 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "src/compiler/code-generator.h"
   6
   7 #include "src/compiler/code-generator-impl.h"
   8 #include "src/compiler/gap-resolver.h"
   9 #include "src/compiler/node-matchers.h"
  10 #include "src/compiler/osr.h"
  11 #include "src/scopes.h"
  12 #include "src/x64/assembler-x64.h"
  13 #include "src/x64/macro-assembler-x64.h"
  14
  15 namespace v8 {
  16 namespace internal {
  17 namespace compiler {
  18
  19 #define __ masm()->
  20
  21
  22 #define kScratchDoubleReg xmm0
  23
  24
  25 // Adds X64 specific methods for decoding operands.
  26 class X64OperandConverter : public InstructionOperandConverter {
  27  public:
  28   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
  29       : InstructionOperandConverter(gen, instr) {}
  30
  31   Immediate InputImmediate(size_t index) {
  32     return ToImmediate(instr_->InputAt(index));
  33   }
  34
  35   Operand InputOperand(size_t index, int extra = 0) {
  36     return ToOperand(instr_->InputAt(index), extra);
  37   }
  38
  39   Operand OutputOperand() { return ToOperand(instr_->Output()); }
  40
  41   Immediate ToImmediate(InstructionOperand* operand) {
  42     Constant constant = ToConstant(operand);
  43     if (constant.type() == Constant::kFloat64) {
  44       DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
  45       return Immediate(0);
  46     }
  47     return Immediate(constant.ToInt32());
  48   }
  49
  50   Operand ToOperand(InstructionOperand* op, int extra = 0) {
  51     DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
  52     FrameOffset offset =
  53         linkage()->GetFrameOffset(AllocatedOperand::cast(op)->index(), frame());
  54     return Operand(offset.from_stack_pointer() ? rsp : rbp,
  55                    offset.offset() + extra);
  56   }
  57
  58   static size_t NextOffset(size_t* offset) {
  59     size_t i = *offset;
  60     (*offset)++;
  61     return i;
  62   }
  63
  64   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
  65     STATIC_ASSERT(0 == static_cast<int>(times_1));
  66     STATIC_ASSERT(1 == static_cast<int>(times_2));
  67     STATIC_ASSERT(2 == static_cast<int>(times_4));
  68     STATIC_ASSERT(3 == static_cast<int>(times_8));
  69     int scale = static_cast<int>(mode - one);
  70     DCHECK(scale >= 0 && scale < 4);
  71     return static_cast<ScaleFactor>(scale);
  72   }
  73
  74   Operand MemoryOperand(size_t* offset) {
  75     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
  76     switch (mode) {
  77       case kMode_MR: {
  78         Register base = InputRegister(NextOffset(offset));
  79         int32_t disp = 0;
  80         return Operand(base, disp);
  81       }
  82       case kMode_MRI: {
  83         Register base = InputRegister(NextOffset(offset));
  84         int32_t disp = InputInt32(NextOffset(offset));
  85         return Operand(base, disp);
  86       }
  87       case kMode_MR1:
  88       case kMode_MR2:
  89       case kMode_MR4:
  90       case kMode_MR8: {
  91         Register base = InputRegister(NextOffset(offset));
  92         Register index = InputRegister(NextOffset(offset));
  93         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
  94         int32_t disp = 0;
  95         return Operand(base, index, scale, disp);
  96       }
  97       case kMode_MR1I:
  98       case kMode_MR2I:
  99       case kMode_MR4I:
 100       case kMode_MR8I: {
 101         Register base = InputRegister(NextOffset(offset));
 102         Register index = InputRegister(NextOffset(offset));
 103         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
 104         int32_t disp = InputInt32(NextOffset(offset));
 105         return Operand(base, index, scale, disp);
 106       }
 107       case kMode_M1: {
 108         Register base = InputRegister(NextOffset(offset));
 109         int32_t disp = 0;
 110         return Operand(base, disp);
 111       }
 112       case kMode_M2:
 113         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
 114         return Operand(no_reg, 0);
 115       case kMode_M4:
 116       case kMode_M8: {
 117         Register index = InputRegister(NextOffset(offset));
 118         ScaleFactor scale = ScaleFor(kMode_M1, mode);
 119         int32_t disp = 0;
 120         return Operand(index, scale, disp);
 121       }
 122       case kMode_M1I:
 123       case kMode_M2I:
 124       case kMode_M4I:
 125       case kMode_M8I: {
 126         Register index = InputRegister(NextOffset(offset));
 127         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
 128         int32_t disp = InputInt32(NextOffset(offset));
 129         return Operand(index, scale, disp);
 130       }
 131       case kMode_None:
 132         UNREACHABLE();
 133         return Operand(no_reg, 0);
 134     }
 135     UNREACHABLE();
 136     return Operand(no_reg, 0);
 137   }
 138
 139   Operand MemoryOperand(size_t first_input = 0) {
 140     return MemoryOperand(&first_input);
 141   }
 142 };
 143
 144
 145 namespace {
 146
 147 bool HasImmediateInput(Instruction* instr, size_t index) {
 148   return instr->InputAt(index)->IsImmediate();
 149 }
 150
 151
 152 class OutOfLineLoadZero final : public OutOfLineCode {
 153  public:
 154   OutOfLineLoadZero(CodeGenerator* gen, Register result)
 155       : OutOfLineCode(gen), result_(result) {}
 156
 157   void Generate() final { __ xorl(result_, result_); }
 158
 159  private:
 160   Register const result_;
 161 };
 162
 163
 164 class OutOfLineLoadNaN final : public OutOfLineCode {
 165  public:
 166   OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result)
 167       : OutOfLineCode(gen), result_(result) {}
 168
 169   void Generate() final { __ pcmpeqd(result_, result_); }
 170
 171  private:
 172   XMMRegister const result_;
 173 };
 174
 175
 176 class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
 177  public:
 178   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
 179                              XMMRegister input)
 180       : OutOfLineCode(gen), result_(result), input_(input) {}
 181
 182   void Generate() final {
 183     __ subp(rsp, Immediate(kDoubleSize));
 184     __ movsd(MemOperand(rsp, 0), input_);
 185     __ SlowTruncateToI(result_, rsp, 0);
 186     __ addp(rsp, Immediate(kDoubleSize));
 187   }
 188
 189  private:
 190   Register const result_;
 191   XMMRegister const input_;
 192 };
 193
 194 }  // namespace
 195
 196
 197 #define ASSEMBLE_UNOP(asm_instr)         \
 198   do {                                   \
 199     if (instr->Output()->IsRegister()) { \
 200       __ asm_instr(i.OutputRegister());  \
 201     } else {                             \
 202       __ asm_instr(i.OutputOperand());   \
 203     }                                    \
 204   } while (0)
 205
 206
 207 #define ASSEMBLE_BINOP(asm_instr)                              \
 208   do {                                                         \
 209     if (HasImmediateInput(instr, 1)) {                         \
 210       if (instr->InputAt(0)->IsRegister()) {                   \
 211         __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
 212       } else {                                                 \
 213         __ asm_instr(i.InputOperand(0), i.InputImmediate(1));  \
 214       }                                                        \
 215     } else {                                                   \
 216       if (instr->InputAt(1)->IsRegister()) {                   \
 217         __ asm_instr(i.InputRegister(0), i.InputRegister(1));  \
 218       } else {                                                 \
 219         __ asm_instr(i.InputRegister(0), i.InputOperand(1));   \
 220       }                                                        \
 221     }                                                          \
 222   } while (0)
 223
 224
 225 #define ASSEMBLE_MULT(asm_instr)                              \
 226   do {                                                        \
 227     if (HasImmediateInput(instr, 1)) {                        \
 228       if (instr->InputAt(0)->IsRegister()) {                  \
 229         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
 230                      i.InputImmediate(1));                    \
 231       } else {                                                \
 232         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
 233                      i.InputImmediate(1));                    \
 234       }                                                       \
 235     } else {                                                  \
 236       if (instr->InputAt(1)->IsRegister()) {                  \
 237         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
 238       } else {                                                \
 239         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
 240       }                                                       \
 241     }                                                         \
 242   } while (0)
 243
 244
 245 #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
 246   do {                                                                     \
 247     if (HasImmediateInput(instr, 1)) {                                     \
 248       if (instr->Output()->IsRegister()) {                                 \
 249         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
 250       } else {                                                             \
 251         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
 252       }                                                                    \
 253     } else {                                                               \
 254       if (instr->Output()->IsRegister()) {                                 \
 255         __ asm_instr##_cl(i.OutputRegister());                             \
 256       } else {                                                             \
 257         __ asm_instr##_cl(i.OutputOperand());                              \
 258       }                                                                    \
 259     }                                                                      \
 260   } while (0)
 261
 262
 263 #define ASSEMBLE_MOVX(asm_instr)                            \
 264   do {                                                      \
 265     if (instr->addressing_mode() != kMode_None) {           \
 266       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
 267     } else if (instr->InputAt(0)->IsRegister()) {           \
 268       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
 269     } else {                                                \
 270       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
 271     }                                                       \
 272   } while (0)
 273
 274
 275 #define ASSEMBLE_SSE_BINOP(asm_instr)                                   \
 276   do {                                                                  \
 277     if (instr->InputAt(1)->IsDoubleRegister()) {                        \
 278       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
 279     } else {                                                            \
 280       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
 281     }                                                                   \
 282   } while (0)
 283
 284
 285 #define ASSEMBLE_SSE_UNOP(asm_instr)                                    \
 286   do {                                                                  \
 287     if (instr->InputAt(0)->IsDoubleRegister()) {                        \
 288       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
 289     } else {                                                            \
 290       __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0));        \
 291     }                                                                   \
 292   } while (0)
 293
 294
 295 #define ASSEMBLE_AVX_BINOP(asm_instr)                                  \
 296   do {                                                                 \
 297     CpuFeatureScope avx_scope(masm(), AVX);                            \
 298     if (instr->InputAt(1)->IsDoubleRegister()) {                       \
 299       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 300                    i.InputDoubleRegister(1));                          \
 301     } else {                                                           \
 302       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
 303                    i.InputOperand(1));                                 \
 304     }                                                                  \
 305   } while (0)
 306
 307
 308 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr)                               \
 309   do {                                                                       \
 310     auto result = i.OutputDoubleRegister();                                  \
 311     auto buffer = i.InputRegister(0);                                        \
 312     auto index1 = i.InputRegister(1);                                        \
 313     auto index2 = i.InputInt32(2);                                           \
 314     OutOfLineCode* ool;                                                      \
 315     if (instr->InputAt(3)->IsRegister()) {                                   \
 316       auto length = i.InputRegister(3);                                      \
 317       DCHECK_EQ(0, index2);                                                  \
 318       __ cmpl(index1, length);                                               \
 319       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
 320     } else {                                                                 \
 321       auto length = i.InputInt32(3);                                         \
 322       DCHECK_LE(index2, length);                                             \
 323       __ cmpq(index1, Immediate(length - index2));                           \
 324       class OutOfLineLoadFloat final : public OutOfLineCode {                \
 325        public:                                                               \
 326         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
 327                            Register buffer, Register index1, int32_t index2, \
 328                            int32_t length)                                   \
 329             : OutOfLineCode(gen),                                            \
 330               result_(result),                                               \
 331               buffer_(buffer),                                               \
 332               index1_(index1),                                               \
 333               index2_(index2),                                               \
 334               length_(length) {}                                             \
 335                                                                              \
 336         void Generate() final {                                              \
 337           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 338           __ pcmpeqd(result_, result_);                                      \
 339           __ cmpl(kScratchRegister, Immediate(length_));                     \
 340           __ j(above_equal, exit());                                         \
 341           __ asm_instr(result_,                                              \
 342                        Operand(buffer_, kScratchRegister, times_1, 0));      \
 343         }                                                                    \
 344                                                                              \
 345        private:                                                              \
 346         XMMRegister const result_;                                           \
 347         Register const buffer_;                                              \
 348         Register const index1_;                                              \
 349         int32_t const index2_;                                               \
 350         int32_t const length_;                                               \
 351       };                                                                     \
 352       ool = new (zone())                                                     \
 353           OutOfLineLoadFloat(this, result, buffer, index1, index2, length);  \
 354     }                                                                        \
 355     __ j(above_equal, ool->entry());                                         \
 356     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
 357     __ bind(ool->exit());                                                    \
 358   } while (false)
 359
 360
 361 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
 362   do {                                                                         \
 363     auto result = i.OutputRegister();                                          \
 364     auto buffer = i.InputRegister(0);                                          \
 365     auto index1 = i.InputRegister(1);                                          \
 366     auto index2 = i.InputInt32(2);                                             \
 367     OutOfLineCode* ool;                                                        \
 368     if (instr->InputAt(3)->IsRegister()) {                                     \
 369       auto length = i.InputRegister(3);                                        \
 370       DCHECK_EQ(0, index2);                                                    \
 371       __ cmpl(index1, length);                                                 \
 372       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
 373     } else {                                                                   \
 374       auto length = i.InputInt32(3);                                           \
 375       DCHECK_LE(index2, length);                                               \
 376       __ cmpq(index1, Immediate(length - index2));                             \
 377       class OutOfLineLoadInteger final : public OutOfLineCode {                \
 378        public:                                                                 \
 379         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
 380                              Register buffer, Register index1, int32_t index2, \
 381                              int32_t length)                                   \
 382             : OutOfLineCode(gen),                                              \
 383               result_(result),                                                 \
 384               buffer_(buffer),                                                 \
 385               index1_(index1),                                                 \
 386               index2_(index2),                                                 \
 387               length_(length) {}                                               \
 388                                                                                \
 389         void Generate() final {                                                \
 390           Label oob;                                                           \
 391           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 392           __ cmpl(kScratchRegister, Immediate(length_));                       \
 393           __ j(above_equal, &oob, Label::kNear);                               \
 394           __ asm_instr(result_,                                                \
 395                        Operand(buffer_, kScratchRegister, times_1, 0));        \
 396           __ jmp(exit());                                                      \
 397           __ bind(&oob);                                                       \
 398           __ xorl(result_, result_);                                           \
 399         }                                                                      \
 400                                                                                \
 401        private:                                                                \
 402         Register const result_;                                                \
 403         Register const buffer_;                                                \
 404         Register const index1_;                                                \
 405         int32_t const index2_;                                                 \
 406         int32_t const length_;                                                 \
 407       };                                                                       \
 408       ool = new (zone())                                                       \
 409           OutOfLineLoadInteger(this, result, buffer, index1, index2, length);  \
 410     }                                                                          \
 411     __ j(above_equal, ool->entry());                                           \
 412     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
 413     __ bind(ool->exit());                                                      \
 414   } while (false)
 415
 416
 417 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
 418   do {                                                                       \
 419     auto buffer = i.InputRegister(0);                                        \
 420     auto index1 = i.InputRegister(1);                                        \
 421     auto index2 = i.InputInt32(2);                                           \
 422     auto value = i.InputDoubleRegister(4);                                   \
 423     if (instr->InputAt(3)->IsRegister()) {                                   \
 424       auto length = i.InputRegister(3);                                      \
 425       DCHECK_EQ(0, index2);                                                  \
 426       Label done;                                                            \
 427       __ cmpl(index1, length);                                               \
 428       __ j(above_equal, &done, Label::kNear);                                \
 429       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 430       __ bind(&done);                                                        \
 431     } else {                                                                 \
 432       auto length = i.InputInt32(3);                                         \
 433       DCHECK_LE(index2, length);                                             \
 434       __ cmpq(index1, Immediate(length - index2));                           \
 435       class OutOfLineStoreFloat final : public OutOfLineCode {               \
 436        public:                                                               \
 437         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
 438                             Register index1, int32_t index2, int32_t length, \
 439                             XMMRegister value)                               \
 440             : OutOfLineCode(gen),                                            \
 441               buffer_(buffer),                                               \
 442               index1_(index1),                                               \
 443               index2_(index2),                                               \
 444               length_(length),                                               \
 445               value_(value) {}                                               \
 446                                                                              \
 447         void Generate() final {                                              \
 448           __ leal(kScratchRegister, Operand(index1_, index2_));              \
 449           __ cmpl(kScratchRegister, Immediate(length_));                     \
 450           __ j(above_equal, exit());                                         \
 451           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
 452                        value_);                                              \
 453         }                                                                    \
 454                                                                              \
 455        private:                                                              \
 456         Register const buffer_;                                              \
 457         Register const index1_;                                              \
 458         int32_t const index2_;                                               \
 459         int32_t const length_;                                               \
 460         XMMRegister const value_;                                            \
 461       };                                                                     \
 462       auto ool = new (zone())                                                \
 463           OutOfLineStoreFloat(this, buffer, index1, index2, length, value);  \
 464       __ j(above_equal, ool->entry());                                       \
 465       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
 466       __ bind(ool->exit());                                                  \
 467     }                                                                        \
 468   } while (false)
 469
 470
 471 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
 472   do {                                                                         \
 473     auto buffer = i.InputRegister(0);                                          \
 474     auto index1 = i.InputRegister(1);                                          \
 475     auto index2 = i.InputInt32(2);                                             \
 476     if (instr->InputAt(3)->IsRegister()) {                                     \
 477       auto length = i.InputRegister(3);                                        \
 478       DCHECK_EQ(0, index2);                                                    \
 479       Label done;                                                              \
 480       __ cmpl(index1, length);                                                 \
 481       __ j(above_equal, &done, Label::kNear);                                  \
 482       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 483       __ bind(&done);                                                          \
 484     } else {                                                                   \
 485       auto length = i.InputInt32(3);                                           \
 486       DCHECK_LE(index2, length);                                               \
 487       __ cmpq(index1, Immediate(length - index2));                             \
 488       class OutOfLineStoreInteger final : public OutOfLineCode {               \
 489        public:                                                                 \
 490         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
 491                               Register index1, int32_t index2, int32_t length, \
 492                               Value value)                                     \
 493             : OutOfLineCode(gen),                                              \
 494               buffer_(buffer),                                                 \
 495               index1_(index1),                                                 \
 496               index2_(index2),                                                 \
 497               length_(length),                                                 \
 498               value_(value) {}                                                 \
 499                                                                                \
 500         void Generate() final {                                                \
 501           __ leal(kScratchRegister, Operand(index1_, index2_));                \
 502           __ cmpl(kScratchRegister, Immediate(length_));                       \
 503           __ j(above_equal, exit());                                           \
 504           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
 505                        value_);                                                \
 506         }                                                                      \
 507                                                                                \
 508        private:                                                                \
 509         Register const buffer_;                                                \
 510         Register const index1_;                                                \
 511         int32_t const index2_;                                                 \
 512         int32_t const length_;                                                 \
 513         Value const value_;                                                    \
 514       };                                                                       \
 515       auto ool = new (zone())                                                  \
 516           OutOfLineStoreInteger(this, buffer, index1, index2, length, value);  \
 517       __ j(above_equal, ool->entry());                                         \
 518       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
 519       __ bind(ool->exit());                                                    \
 520     }                                                                          \
 521   } while (false)
 522
 523
 524 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
 525   do {                                                           \
 526     if (instr->InputAt(4)->IsRegister()) {                       \
 527       Register value = i.InputRegister(4);                       \
 528       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
 529     } else {                                                     \
 530       Immediate value = i.InputImmediate(4);                     \
 531       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
 532     }                                                            \
 533   } while (false)
 534
 535
 536 void CodeGenerator::AssembleDeconstructActivationRecord() {
 537   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
 538   int stack_slots = frame()->GetSpillSlotCount();
 539   if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
 540     __ movq(rsp, rbp);
 541     __ popq(rbp);
 542   }
 543 }
 544
 545
 546 // Assembles an instruction after register allocation, producing machine code.
 547 void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
 548   X64OperandConverter i(this, instr);
 549
 550   switch (ArchOpcodeField::decode(instr->opcode())) {
 551     case kArchCallCodeObject: {
 552       EnsureSpaceForLazyDeopt();
 553       if (HasImmediateInput(instr, 0)) {
 554         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 555         __ Call(code, RelocInfo::CODE_TARGET);
 556       } else {
 557         Register reg = i.InputRegister(0);
 558         int entry = Code::kHeaderSize - kHeapObjectTag;
 559         __ Call(Operand(reg, entry));
 560       }
 561       RecordCallPosition(instr);
 562       break;
 563     }
 564     case kArchTailCallCodeObject: {
 565       AssembleDeconstructActivationRecord();
 566       if (HasImmediateInput(instr, 0)) {
 567         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
 568         __ jmp(code, RelocInfo::CODE_TARGET);
 569       } else {
 570         Register reg = i.InputRegister(0);
 571         __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
 572         __ jmp(reg);
 573       }
 574       break;
 575     }
 576     case kArchCallJSFunction: {
 577       EnsureSpaceForLazyDeopt();
 578       Register func = i.InputRegister(0);
 579       if (FLAG_debug_code) {
 580         // Check the function's context matches the context argument.
 581         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 582         __ Assert(equal, kWrongFunctionContext);
 583       }
 584       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
 585       RecordCallPosition(instr);
 586       break;
 587     }
 588     case kArchTailCallJSFunction: {
 589       Register func = i.InputRegister(0);
 590       if (FLAG_debug_code) {
 591         // Check the function's context matches the context argument.
 592         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
 593         __ Assert(equal, kWrongFunctionContext);
 594       }
 595       AssembleDeconstructActivationRecord();
 596       __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
 597       break;
 598     }
 599     case kArchPrepareCallCFunction: {
 600       int const num_parameters = MiscField::decode(instr->opcode());
 601       __ PrepareCallCFunction(num_parameters);
 602       break;
 603     }
 604     case kArchCallCFunction: {
 605       int const num_parameters = MiscField::decode(instr->opcode());
 606       if (HasImmediateInput(instr, 0)) {
 607         ExternalReference ref = i.InputExternalReference(0);
 608         __ CallCFunction(ref, num_parameters);
 609       } else {
 610         Register func = i.InputRegister(0);
 611         __ CallCFunction(func, num_parameters);
 612       }
 613       break;
 614     }
 615     case kArchJmp:
 616       AssembleArchJump(i.InputRpo(0));
 617       break;
 618     case kArchLookupSwitch:
 619       AssembleArchLookupSwitch(instr);
 620       break;
 621     case kArchTableSwitch:
 622       AssembleArchTableSwitch(instr);
 623       break;
 624     case kArchNop:
 625       // don't emit code for nops.
 626       break;
 627     case kArchDeoptimize: {
 628       int deopt_state_id =
 629           BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
 630       AssembleDeoptimizerCall(deopt_state_id, Deoptimizer::EAGER);
 631       break;
 632     }
 633     case kArchRet:
 634       AssembleReturn();
 635       break;
 636     case kArchStackPointer:
 637       __ movq(i.OutputRegister(), rsp);
 638       break;
 639     case kArchFramePointer:
 640       __ movq(i.OutputRegister(), rbp);
 641       break;
 642     case kArchTruncateDoubleToI: {
 643       auto result = i.OutputRegister();
 644       auto input = i.InputDoubleRegister(0);
 645       auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
 646       __ cvttsd2siq(result, input);
 647       __ cmpq(result, Immediate(1));
 648       __ j(overflow, ool->entry());
 649       __ bind(ool->exit());
 650       break;
 651     }
 652     case kX64Add32:
 653       ASSEMBLE_BINOP(addl);
 654       break;
 655     case kX64Add:
 656       ASSEMBLE_BINOP(addq);
 657       break;
 658     case kX64Sub32:
 659       ASSEMBLE_BINOP(subl);
 660       break;
 661     case kX64Sub:
 662       ASSEMBLE_BINOP(subq);
 663       break;
 664     case kX64And32:
 665       ASSEMBLE_BINOP(andl);
 666       break;
 667     case kX64And:
 668       ASSEMBLE_BINOP(andq);
 669       break;
 670     case kX64Cmp32:
 671       ASSEMBLE_BINOP(cmpl);
 672       break;
 673     case kX64Cmp:
 674       ASSEMBLE_BINOP(cmpq);
 675       break;
 676     case kX64Test32:
 677       ASSEMBLE_BINOP(testl);
 678       break;
 679     case kX64Test:
 680       ASSEMBLE_BINOP(testq);
 681       break;
 682     case kX64Imul32:
 683       ASSEMBLE_MULT(imull);
 684       break;
 685     case kX64Imul:
 686       ASSEMBLE_MULT(imulq);
 687       break;
 688     case kX64ImulHigh32:
 689       if (instr->InputAt(1)->IsRegister()) {
 690         __ imull(i.InputRegister(1));
 691       } else {
 692         __ imull(i.InputOperand(1));
 693       }
 694       break;
 695     case kX64UmulHigh32:
 696       if (instr->InputAt(1)->IsRegister()) {
 697         __ mull(i.InputRegister(1));
 698       } else {
 699         __ mull(i.InputOperand(1));
 700       }
 701       break;
 702     case kX64Idiv32:
 703       __ cdq();
 704       __ idivl(i.InputRegister(1));
 705       break;
 706     case kX64Idiv:
 707       __ cqo();
 708       __ idivq(i.InputRegister(1));
 709       break;
 710     case kX64Udiv32:
 711       __ xorl(rdx, rdx);
 712       __ divl(i.InputRegister(1));
 713       break;
 714     case kX64Udiv:
 715       __ xorq(rdx, rdx);
 716       __ divq(i.InputRegister(1));
 717       break;
 718     case kX64Not:
 719       ASSEMBLE_UNOP(notq);
 720       break;
 721     case kX64Not32:
 722       ASSEMBLE_UNOP(notl);
 723       break;
 724     case kX64Neg:
 725       ASSEMBLE_UNOP(negq);
 726       break;
 727     case kX64Neg32:
 728       ASSEMBLE_UNOP(negl);
 729       break;
 730     case kX64Or32:
 731       ASSEMBLE_BINOP(orl);
 732       break;
 733     case kX64Or:
 734       ASSEMBLE_BINOP(orq);
 735       break;
 736     case kX64Xor32:
 737       ASSEMBLE_BINOP(xorl);
 738       break;
 739     case kX64Xor:
 740       ASSEMBLE_BINOP(xorq);
 741       break;
 742     case kX64Shl32:
 743       ASSEMBLE_SHIFT(shll, 5);
 744       break;
 745     case kX64Shl:
 746       ASSEMBLE_SHIFT(shlq, 6);
 747       break;
 748     case kX64Shr32:
 749       ASSEMBLE_SHIFT(shrl, 5);
 750       break;
 751     case kX64Shr:
 752       ASSEMBLE_SHIFT(shrq, 6);
 753       break;
 754     case kX64Sar32:
 755       ASSEMBLE_SHIFT(sarl, 5);
 756       break;
 757     case kX64Sar:
 758       ASSEMBLE_SHIFT(sarq, 6);
 759       break;
 760     case kX64Ror32:
 761       ASSEMBLE_SHIFT(rorl, 5);
 762       break;
 763     case kX64Ror:
 764       ASSEMBLE_SHIFT(rorq, 6);
 765       break;
 766     case kX64Lzcnt32:
 767       if (instr->InputAt(0)->IsRegister()) {
 768         __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
 769       } else {
 770         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
 771       }
 772       break;
 773     case kSSEFloat32Cmp:
 774       ASSEMBLE_SSE_BINOP(ucomiss);
 775       break;
 776     case kSSEFloat32Add:
 777       ASSEMBLE_SSE_BINOP(addss);
 778       break;
 779     case kSSEFloat32Sub:
 780       ASSEMBLE_SSE_BINOP(subss);
 781       break;
 782     case kSSEFloat32Mul:
 783       ASSEMBLE_SSE_BINOP(mulss);
 784       break;
 785     case kSSEFloat32Div:
 786       ASSEMBLE_SSE_BINOP(divss);
 787       // Don't delete this mov. It may improve performance on some CPUs,
 788       // when there is a (v)mulss depending on the result.
 789       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 790       break;
 791     case kSSEFloat32Abs: {
 792       // TODO(bmeurer): Use RIP relative 128-bit constants.
 793       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 794       __ psrlq(kScratchDoubleReg, 33);
 795       __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
 796       break;
 797     }
 798     case kSSEFloat32Neg: {
 799       // TODO(bmeurer): Use RIP relative 128-bit constants.
 800       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 801       __ psllq(kScratchDoubleReg, 31);
 802       __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
 803       break;
 804     }
 805     case kSSEFloat32Sqrt:
 806       ASSEMBLE_SSE_UNOP(sqrtss);
 807       break;
 808     case kSSEFloat32Max:
 809       ASSEMBLE_SSE_BINOP(maxss);
 810       break;
 811     case kSSEFloat32Min:
 812       ASSEMBLE_SSE_BINOP(minss);
 813       break;
 814     case kSSEFloat32ToFloat64:
 815       ASSEMBLE_SSE_UNOP(cvtss2sd);
 816       break;
 817     case kSSEFloat64Cmp:
 818       ASSEMBLE_SSE_BINOP(ucomisd);
 819       break;
 820     case kSSEFloat64Add:
 821       ASSEMBLE_SSE_BINOP(addsd);
 822       break;
 823     case kSSEFloat64Sub:
 824       ASSEMBLE_SSE_BINOP(subsd);
 825       break;
 826     case kSSEFloat64Mul:
 827       ASSEMBLE_SSE_BINOP(mulsd);
 828       break;
 829     case kSSEFloat64Div:
 830       ASSEMBLE_SSE_BINOP(divsd);
 831       // Don't delete this mov. It may improve performance on some CPUs,
 832       // when there is a (v)mulsd depending on the result.
 833       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 834       break;
 835     case kSSEFloat64Mod: {
 836       __ subq(rsp, Immediate(kDoubleSize));
 837       // Move values to st(0) and st(1).
 838       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
 839       __ fld_d(Operand(rsp, 0));
 840       __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
 841       __ fld_d(Operand(rsp, 0));
 842       // Loop while fprem isn't done.
 843       Label mod_loop;
 844       __ bind(&mod_loop);
 845       // This instructions traps on all kinds inputs, but we are assuming the
 846       // floating point control word is set to ignore them all.
 847       __ fprem();
 848       // The following 2 instruction implicitly use rax.
 849       __ fnstsw_ax();
 850       if (CpuFeatures::IsSupported(SAHF)) {
 851         CpuFeatureScope sahf_scope(masm(), SAHF);
 852         __ sahf();
 853       } else {
 854         __ shrl(rax, Immediate(8));
 855         __ andl(rax, Immediate(0xFF));
 856         __ pushq(rax);
 857         __ popfq();
 858       }
 859       __ j(parity_even, &mod_loop);
 860       // Move output to stack and clean up.
 861       __ fstp(1);
 862       __ fstp_d(Operand(rsp, 0));
 863       __ movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
 864       __ addq(rsp, Immediate(kDoubleSize));
 865       break;
 866     }
 867     case kSSEFloat64Max:
 868       ASSEMBLE_SSE_BINOP(maxsd);
 869       break;
 870     case kSSEFloat64Min:
 871       ASSEMBLE_SSE_BINOP(minsd);
 872       break;
 873     case kSSEFloat64Abs: {
 874       // TODO(bmeurer): Use RIP relative 128-bit constants.
 875       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 876       __ psrlq(kScratchDoubleReg, 1);
 877       __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 878       break;
 879     }
 880     case kSSEFloat64Neg: {
 881       // TODO(bmeurer): Use RIP relative 128-bit constants.
 882       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
 883       __ psllq(kScratchDoubleReg, 63);
 884       __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
 885       break;
 886     }
 887     case kSSEFloat64Sqrt:
 888       ASSEMBLE_SSE_UNOP(sqrtsd);
 889       break;
 890     case kSSEFloat64Round: {
 891       CpuFeatureScope sse_scope(masm(), SSE4_1);
 892       RoundingMode const mode =
 893           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
 894       __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
 895       break;
 896     }
 897     case kSSEFloat64ToFloat32:
 898       ASSEMBLE_SSE_UNOP(cvtsd2ss);
 899       break;
 900     case kSSEFloat64ToInt32:
 901       if (instr->InputAt(0)->IsDoubleRegister()) {
 902         __ cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
 903       } else {
 904         __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
 905       }
 906       break;
 907     case kSSEFloat64ToUint32: {
 908       if (instr->InputAt(0)->IsDoubleRegister()) {
 909         __ cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
 910       } else {
 911         __ cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
 912       }
 913       __ AssertZeroExtended(i.OutputRegister());
 914       break;
 915     }
 916     case kSSEInt32ToFloat64:
 917       if (instr->InputAt(0)->IsRegister()) {
 918         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
 919       } else {
 920         __ cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
 921       }
 922       break;
 923     case kSSEUint32ToFloat64:
 924       if (instr->InputAt(0)->IsRegister()) {
 925         __ movl(kScratchRegister, i.InputRegister(0));
 926       } else {
 927         __ movl(kScratchRegister, i.InputOperand(0));
 928       }
 929       __ cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
 930       break;
 931     case kSSEFloat64ExtractLowWord32:
 932       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 933         __ movl(i.OutputRegister(), i.InputOperand(0));
 934       } else {
 935         __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
 936       }
 937       break;
 938     case kSSEFloat64ExtractHighWord32:
 939       if (instr->InputAt(0)->IsDoubleStackSlot()) {
 940         __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
 941       } else {
 942         __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
 943       }
 944       break;
 945     case kSSEFloat64InsertLowWord32:
 946       if (instr->InputAt(1)->IsRegister()) {
 947         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
 948       } else {
 949         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
 950       }
 951       break;
 952     case kSSEFloat64InsertHighWord32:
 953       if (instr->InputAt(1)->IsRegister()) {
 954         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
 955       } else {
 956         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
 957       }
 958       break;
 959     case kSSEFloat64LoadLowWord32:
 960       if (instr->InputAt(0)->IsRegister()) {
 961         __ movd(i.OutputDoubleRegister(), i.InputRegister(0));
 962       } else {
 963         __ movd(i.OutputDoubleRegister(), i.InputOperand(0));
 964       }
 965       break;
 966     case kAVXFloat32Cmp: {
 967       CpuFeatureScope avx_scope(masm(), AVX);
 968       if (instr->InputAt(1)->IsDoubleRegister()) {
 969         __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
 970       } else {
 971         __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
 972       }
 973       break;
 974     }
 975     case kAVXFloat32Add:
 976       ASSEMBLE_AVX_BINOP(vaddss);
 977       break;
 978     case kAVXFloat32Sub:
 979       ASSEMBLE_AVX_BINOP(vsubss);
 980       break;
 981     case kAVXFloat32Mul:
 982       ASSEMBLE_AVX_BINOP(vmulss);
 983       break;
 984     case kAVXFloat32Div:
 985       ASSEMBLE_AVX_BINOP(vdivss);
 986       // Don't delete this mov. It may improve performance on some CPUs,
 987       // when there is a (v)mulss depending on the result.
 988       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
 989       break;
 990     case kAVXFloat32Max:
 991       ASSEMBLE_AVX_BINOP(vmaxss);
 992       break;
 993     case kAVXFloat32Min:
 994       ASSEMBLE_AVX_BINOP(vminss);
 995       break;
 996     case kAVXFloat64Cmp: {
 997       CpuFeatureScope avx_scope(masm(), AVX);
 998       if (instr->InputAt(1)->IsDoubleRegister()) {
 999         __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1000       } else {
1001         __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1002       }
1003       break;
1004     }
1005     case kAVXFloat64Add:
1006       ASSEMBLE_AVX_BINOP(vaddsd);
1007       break;
1008     case kAVXFloat64Sub:
1009       ASSEMBLE_AVX_BINOP(vsubsd);
1010       break;
1011     case kAVXFloat64Mul:
1012       ASSEMBLE_AVX_BINOP(vmulsd);
1013       break;
1014     case kAVXFloat64Div:
1015       ASSEMBLE_AVX_BINOP(vdivsd);
1016       // Don't delete this mov. It may improve performance on some CPUs,
1017       // when there is a (v)mulsd depending on the result.
1018       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1019       break;
1020     case kAVXFloat64Max:
1021       ASSEMBLE_AVX_BINOP(vmaxsd);
1022       break;
1023     case kAVXFloat64Min:
1024       ASSEMBLE_AVX_BINOP(vminsd);
1025       break;
1026     case kAVXFloat32Abs: {
1027       // TODO(bmeurer): Use RIP relative 128-bit constants.
1028       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1029       __ psrlq(kScratchDoubleReg, 33);
1030       CpuFeatureScope avx_scope(masm(), AVX);
1031       if (instr->InputAt(0)->IsDoubleRegister()) {
1032         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1033                   i.InputDoubleRegister(0));
1034       } else {
1035         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1036                   i.InputOperand(0));
1037       }
1038       break;
1039     }
1040     case kAVXFloat32Neg: {
1041       // TODO(bmeurer): Use RIP relative 128-bit constants.
1042       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1043       __ psllq(kScratchDoubleReg, 31);
1044       CpuFeatureScope avx_scope(masm(), AVX);
1045       if (instr->InputAt(0)->IsDoubleRegister()) {
1046         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1047                   i.InputDoubleRegister(0));
1048       } else {
1049         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1050                   i.InputOperand(0));
1051       }
1052       break;
1053     }
1054     case kAVXFloat64Abs: {
1055       // TODO(bmeurer): Use RIP relative 128-bit constants.
1056       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1057       __ psrlq(kScratchDoubleReg, 1);
1058       CpuFeatureScope avx_scope(masm(), AVX);
1059       if (instr->InputAt(0)->IsDoubleRegister()) {
1060         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1061                   i.InputDoubleRegister(0));
1062       } else {
1063         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1064                   i.InputOperand(0));
1065       }
1066       break;
1067     }
1068     case kAVXFloat64Neg: {
1069       // TODO(bmeurer): Use RIP relative 128-bit constants.
1070       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1071       __ psllq(kScratchDoubleReg, 63);
1072       CpuFeatureScope avx_scope(masm(), AVX);
1073       if (instr->InputAt(0)->IsDoubleRegister()) {
1074         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1075                   i.InputDoubleRegister(0));
1076       } else {
1077         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1078                   i.InputOperand(0));
1079       }
1080       break;
1081     }
1082     case kX64Movsxbl:
1083       ASSEMBLE_MOVX(movsxbl);
1084       __ AssertZeroExtended(i.OutputRegister());
1085       break;
1086     case kX64Movzxbl:
1087       ASSEMBLE_MOVX(movzxbl);
1088       __ AssertZeroExtended(i.OutputRegister());
1089       break;
1090     case kX64Movb: {
1091       size_t index = 0;
1092       Operand operand = i.MemoryOperand(&index);
1093       if (HasImmediateInput(instr, index)) {
1094         __ movb(operand, Immediate(i.InputInt8(index)));
1095       } else {
1096         __ movb(operand, i.InputRegister(index));
1097       }
1098       break;
1099     }
1100     case kX64Movsxwl:
1101       ASSEMBLE_MOVX(movsxwl);
1102       __ AssertZeroExtended(i.OutputRegister());
1103       break;
1104     case kX64Movzxwl:
1105       ASSEMBLE_MOVX(movzxwl);
1106       __ AssertZeroExtended(i.OutputRegister());
1107       break;
1108     case kX64Movw: {
1109       size_t index = 0;
1110       Operand operand = i.MemoryOperand(&index);
1111       if (HasImmediateInput(instr, index)) {
1112         __ movw(operand, Immediate(i.InputInt16(index)));
1113       } else {
1114         __ movw(operand, i.InputRegister(index));
1115       }
1116       break;
1117     }
1118     case kX64Movl:
1119       if (instr->HasOutput()) {
1120         if (instr->addressing_mode() == kMode_None) {
1121           if (instr->InputAt(0)->IsRegister()) {
1122             __ movl(i.OutputRegister(), i.InputRegister(0));
1123           } else {
1124             __ movl(i.OutputRegister(), i.InputOperand(0));
1125           }
1126         } else {
1127           __ movl(i.OutputRegister(), i.MemoryOperand());
1128         }
1129         __ AssertZeroExtended(i.OutputRegister());
1130       } else {
1131         size_t index = 0;
1132         Operand operand = i.MemoryOperand(&index);
1133         if (HasImmediateInput(instr, index)) {
1134           __ movl(operand, i.InputImmediate(index));
1135         } else {
1136           __ movl(operand, i.InputRegister(index));
1137         }
1138       }
1139       break;
1140     case kX64Movsxlq:
1141       ASSEMBLE_MOVX(movsxlq);
1142       break;
1143     case kX64Movq:
1144       if (instr->HasOutput()) {
1145         __ movq(i.OutputRegister(), i.MemoryOperand());
1146       } else {
1147         size_t index = 0;
1148         Operand operand = i.MemoryOperand(&index);
1149         if (HasImmediateInput(instr, index)) {
1150           __ movq(operand, i.InputImmediate(index));
1151         } else {
1152           __ movq(operand, i.InputRegister(index));
1153         }
1154       }
1155       break;
1156     case kX64Movss:
1157       if (instr->HasOutput()) {
1158         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1159       } else {
1160         size_t index = 0;
1161         Operand operand = i.MemoryOperand(&index);
1162         __ movss(operand, i.InputDoubleRegister(index));
1163       }
1164       break;
1165     case kX64Movsd:
1166       if (instr->HasOutput()) {
1167         __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1168       } else {
1169         size_t index = 0;
1170         Operand operand = i.MemoryOperand(&index);
1171         __ movsd(operand, i.InputDoubleRegister(index));
1172       }
1173       break;
1174     case kX64Lea32: {
1175       AddressingMode mode = AddressingModeField::decode(instr->opcode());
1176       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
1177       // and addressing mode just happens to work out. The "addl"/"subl" forms
1178       // in these cases are faster based on measurements.
1179       if (i.InputRegister(0).is(i.OutputRegister())) {
1180         if (mode == kMode_MRI) {
1181           int32_t constant_summand = i.InputInt32(1);
1182           if (constant_summand > 0) {
1183             __ addl(i.OutputRegister(), Immediate(constant_summand));
1184           } else if (constant_summand < 0) {
1185             __ subl(i.OutputRegister(), Immediate(-constant_summand));
1186           }
1187         } else if (mode == kMode_MR1) {
1188           if (i.InputRegister(1).is(i.OutputRegister())) {
1189             __ shll(i.OutputRegister(), Immediate(1));
1190           } else {
1191             __ leal(i.OutputRegister(), i.MemoryOperand());
1192           }
1193         } else if (mode == kMode_M2) {
1194           __ shll(i.OutputRegister(), Immediate(1));
1195         } else if (mode == kMode_M4) {
1196           __ shll(i.OutputRegister(), Immediate(2));
1197         } else if (mode == kMode_M8) {
1198           __ shll(i.OutputRegister(), Immediate(3));
1199         } else {
1200           __ leal(i.OutputRegister(), i.MemoryOperand());
1201         }
1202       } else {
1203         __ leal(i.OutputRegister(), i.MemoryOperand());
1204       }
1205       __ AssertZeroExtended(i.OutputRegister());
1206       break;
1207     }
1208     case kX64Lea:
1209       __ leaq(i.OutputRegister(), i.MemoryOperand());
1210       break;
1211     case kX64Dec32:
1212       __ decl(i.OutputRegister());
1213       break;
1214     case kX64Inc32:
1215       __ incl(i.OutputRegister());
1216       break;
1217     case kX64Push:
1218       if (HasImmediateInput(instr, 0)) {
1219         __ pushq(i.InputImmediate(0));
1220       } else {
1221         if (instr->InputAt(0)->IsRegister()) {
1222           __ pushq(i.InputRegister(0));
1223         } else if (instr->InputAt(0)->IsDoubleRegister()) {
1224           // TODO(titzer): use another machine instruction?
1225           __ subq(rsp, Immediate(kDoubleSize));
1226           __ movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1227         } else {
1228           __ pushq(i.InputOperand(0));
1229         }
1230       }
1231       break;
1232     case kX64Poke: {
1233       int const slot = MiscField::decode(instr->opcode());
1234       if (HasImmediateInput(instr, 0)) {
1235         __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
1236       } else {
1237         __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
1238       }
1239       break;
1240     }
1241     case kX64StoreWriteBarrier: {
1242       Register object = i.InputRegister(0);
1243       Register value = i.InputRegister(2);
1244       SaveFPRegsMode mode =
1245           frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
1246       if (HasImmediateInput(instr, 1)) {
1247         int index = i.InputInt32(1);
1248         Register scratch = i.TempRegister(1);
1249         __ movq(Operand(object, index), value);
1250         __ RecordWriteContextSlot(object, index, value, scratch, mode);
1251       } else {
1252         Register index = i.InputRegister(1);
1253         __ movq(Operand(object, index, times_1, 0), value);
1254         __ leaq(index, Operand(object, index, times_1, 0));
1255         __ RecordWrite(object, index, value, mode);
1256       }
1257       break;
1258     }
1259     case kCheckedLoadInt8:
1260       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
1261       break;
1262     case kCheckedLoadUint8:
1263       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
1264       break;
1265     case kCheckedLoadInt16:
1266       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
1267       break;
1268     case kCheckedLoadUint16:
1269       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
1270       break;
1271     case kCheckedLoadWord32:
1272       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
1273       break;
1274     case kCheckedLoadWord64:
1275       ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
1276       break;
1277     case kCheckedLoadFloat32:
1278       ASSEMBLE_CHECKED_LOAD_FLOAT(movss);
1279       break;
1280     case kCheckedLoadFloat64:
1281       ASSEMBLE_CHECKED_LOAD_FLOAT(movsd);
1282       break;
1283     case kCheckedStoreWord8:
1284       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
1285       break;
1286     case kCheckedStoreWord16:
1287       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
1288       break;
1289     case kCheckedStoreWord32:
1290       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
1291       break;
1292     case kCheckedStoreWord64:
1293       ASSEMBLE_CHECKED_STORE_INTEGER(movq);
1294       break;
1295     case kCheckedStoreFloat32:
1296       ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1297       break;
1298     case kCheckedStoreFloat64:
1299       ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1300       break;
1301     case kX64StackCheck:
1302       __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
1303       break;
1304   }
1305 }  // NOLINT(readability/fn_size)
1306
1307
1308 // Assembles branches after this instruction.
1309 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1310   X64OperandConverter i(this, instr);
1311   Label::Distance flabel_distance =
1312       branch->fallthru ? Label::kNear : Label::kFar;
1313   Label* tlabel = branch->true_label;
1314   Label* flabel = branch->false_label;
1315   switch (branch->condition) {
1316     case kUnorderedEqual:
1317       __ j(parity_even, flabel, flabel_distance);
1318     // Fall through.
1319     case kEqual:
1320       __ j(equal, tlabel);
1321       break;
1322     case kUnorderedNotEqual:
1323       __ j(parity_even, tlabel);
1324     // Fall through.
1325     case kNotEqual:
1326       __ j(not_equal, tlabel);
1327       break;
1328     case kSignedLessThan:
1329       __ j(less, tlabel);
1330       break;
1331     case kSignedGreaterThanOrEqual:
1332       __ j(greater_equal, tlabel);
1333       break;
1334     case kSignedLessThanOrEqual:
1335       __ j(less_equal, tlabel);
1336       break;
1337     case kSignedGreaterThan:
1338       __ j(greater, tlabel);
1339       break;
1340     case kUnsignedLessThan:
1341       __ j(below, tlabel);
1342       break;
1343     case kUnsignedGreaterThanOrEqual:
1344       __ j(above_equal, tlabel);
1345       break;
1346     case kUnsignedLessThanOrEqual:
1347       __ j(below_equal, tlabel);
1348       break;
1349     case kUnsignedGreaterThan:
1350       __ j(above, tlabel);
1351       break;
1352     case kOverflow:
1353       __ j(overflow, tlabel);
1354       break;
1355     case kNotOverflow:
1356       __ j(no_overflow, tlabel);
1357       break;
1358   }
1359   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
1360 }
1361
1362
1363 void CodeGenerator::AssembleArchJump(RpoNumber target) {
1364   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
1365 }
1366
1367
1368 // Assembles boolean materializations after this instruction.
1369 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1370                                         FlagsCondition condition) {
1371   X64OperandConverter i(this, instr);
1372   Label done;
1373
1374   // Materialize a full 64-bit 1 or 0 value. The result register is always the
1375   // last output of the instruction.
1376   Label check;
1377   DCHECK_NE(0u, instr->OutputCount());
1378   Register reg = i.OutputRegister(instr->OutputCount() - 1);
1379   Condition cc = no_condition;
1380   switch (condition) {
1381     case kUnorderedEqual:
1382       __ j(parity_odd, &check, Label::kNear);
1383       __ movl(reg, Immediate(0));
1384       __ jmp(&done, Label::kNear);
1385     // Fall through.
1386     case kEqual:
1387       cc = equal;
1388       break;
1389     case kUnorderedNotEqual:
1390       __ j(parity_odd, &check, Label::kNear);
1391       __ movl(reg, Immediate(1));
1392       __ jmp(&done, Label::kNear);
1393     // Fall through.
1394     case kNotEqual:
1395       cc = not_equal;
1396       break;
1397     case kSignedLessThan:
1398       cc = less;
1399       break;
1400     case kSignedGreaterThanOrEqual:
1401       cc = greater_equal;
1402       break;
1403     case kSignedLessThanOrEqual:
1404       cc = less_equal;
1405       break;
1406     case kSignedGreaterThan:
1407       cc = greater;
1408       break;
1409     case kUnsignedLessThan:
1410       cc = below;
1411       break;
1412     case kUnsignedGreaterThanOrEqual:
1413       cc = above_equal;
1414       break;
1415     case kUnsignedLessThanOrEqual:
1416       cc = below_equal;
1417       break;
1418     case kUnsignedGreaterThan:
1419       cc = above;
1420       break;
1421     case kOverflow:
1422       cc = overflow;
1423       break;
1424     case kNotOverflow:
1425       cc = no_overflow;
1426       break;
1427   }
1428   __ bind(&check);
1429   __ setcc(cc, reg);
1430   __ movzxbl(reg, reg);
1431   __ bind(&done);
1432 }
1433
1434
1435 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1436   X64OperandConverter i(this, instr);
1437   Register input = i.InputRegister(0);
1438   for (size_t index = 2; index < instr->InputCount(); index += 2) {
1439     __ cmpl(input, Immediate(i.InputInt32(index + 0)));
1440     __ j(equal, GetLabel(i.InputRpo(index + 1)));
1441   }
1442   AssembleArchJump(i.InputRpo(1));
1443 }
1444
1445
1446 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1447   X64OperandConverter i(this, instr);
1448   Register input = i.InputRegister(0);
1449   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
1450   Label** cases = zone()->NewArray<Label*>(case_count);
1451   for (int32_t index = 0; index < case_count; ++index) {
1452     cases[index] = GetLabel(i.InputRpo(index + 2));
1453   }
1454   Label* const table = AddJumpTable(cases, case_count);
1455   __ cmpl(input, Immediate(case_count));
1456   __ j(above_equal, GetLabel(i.InputRpo(1)));
1457   __ leaq(kScratchRegister, Operand(table));
1458   __ jmp(Operand(kScratchRegister, input, times_8, 0));
1459 }
1460
1461
1462 void CodeGenerator::AssembleDeoptimizerCall(
1463     int deoptimization_id, Deoptimizer::BailoutType bailout_type) {
1464   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1465       isolate(), deoptimization_id, bailout_type);
1466   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1467 }
1468
1469
1470 namespace {
1471
1472 static const int kQuadWordSize = 16;
1473
1474 }  // namespace
1475
1476
1477 void CodeGenerator::AssemblePrologue() {
1478   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1479   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1480     __ pushq(rbp);
1481     __ movq(rbp, rsp);
1482   } else if (descriptor->IsJSFunctionCall()) {
1483     CompilationInfo* info = this->info();
1484     __ Prologue(info->IsCodePreAgingActive());
1485   } else if (needs_frame_) {
1486     __ StubPrologue();
1487   } else {
1488     frame()->SetElidedFrameSizeInSlots(kPCOnStackSize / kPointerSize);
1489   }
1490
1491   int stack_shrink_slots = frame()->GetSpillSlotCount();
1492   if (info()->is_osr()) {
1493     // TurboFan OSR-compiled functions cannot be entered directly.
1494     __ Abort(kShouldNotDirectlyEnterOsrFunction);
1495
1496     // Unoptimized code jumps directly to this entrypoint while the unoptimized
1497     // frame is still on the stack. Optimized code uses OSR values directly from
1498     // the unoptimized frame. Thus, all that needs to be done is to allocate the
1499     // remaining stack slots.
1500     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1501     osr_pc_offset_ = __ pc_offset();
1502     // TODO(titzer): cannot address target function == local #-1
1503     __ movq(rdi, Operand(rbp, JavaScriptFrameConstants::kFunctionOffset));
1504     stack_shrink_slots -=
1505         static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots());
1506   }
1507
1508   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
1509   if (saves_fp != 0) {
1510     stack_shrink_slots += frame()->AlignSavedCalleeRegisterSlots();
1511   }
1512   if (stack_shrink_slots > 0) {
1513     __ subq(rsp, Immediate(stack_shrink_slots * kPointerSize));
1514   }
1515
1516   if (saves_fp != 0) {  // Save callee-saved XMM registers.
1517     const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
1518     const int stack_size = saves_fp_count * kQuadWordSize;
1519     // Adjust the stack pointer.
1520     __ subp(rsp, Immediate(stack_size));
1521     // Store the registers on the stack.
1522     int slot_idx = 0;
1523     for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
1524       if (!((1 << i) & saves_fp)) continue;
1525       __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
1526                 XMMRegister::from_code(i));
1527       slot_idx++;
1528     }
1529     frame()->AllocateSavedCalleeRegisterSlots(saves_fp_count *
1530                                               (kQuadWordSize / kPointerSize));
1531   }
1532
1533   const RegList saves = descriptor->CalleeSavedRegisters();
1534   if (saves != 0) {  // Save callee-saved registers.
1535     for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
1536       if (!((1 << i) & saves)) continue;
1537       __ pushq(Register::from_code(i));
1538       frame()->AllocateSavedCalleeRegisterSlots(1);
1539     }
1540   }
1541 }
1542
1543
1544 void CodeGenerator::AssembleReturn() {
1545   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1546
1547   // Restore registers.
1548   const RegList saves = descriptor->CalleeSavedRegisters();
1549   if (saves != 0) {
1550     for (int i = 0; i < Register::kNumRegisters; i++) {
1551       if (!((1 << i) & saves)) continue;
1552       __ popq(Register::from_code(i));
1553     }
1554   }
1555   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
1556   if (saves_fp != 0) {
1557     const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
1558     const int stack_size = saves_fp_count * kQuadWordSize;
1559     // Load the registers from the stack.
1560     int slot_idx = 0;
1561     for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
1562       if (!((1 << i) & saves_fp)) continue;
1563       __ movdqu(XMMRegister::from_code(i),
1564                 Operand(rsp, kQuadWordSize * slot_idx));
1565       slot_idx++;
1566     }
1567     // Adjust the stack pointer.
1568     __ addp(rsp, Immediate(stack_size));
1569   }
1570
1571   if (descriptor->kind() == CallDescriptor::kCallAddress) {
1572     __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1573     __ popq(rbp);       // Pop caller's frame pointer.
1574   } else if (descriptor->IsJSFunctionCall() || needs_frame_) {
1575     // Canonicalize JSFunction return sites for now.
1576     if (return_label_.is_bound()) {
1577       __ jmp(&return_label_);
1578       return;
1579     } else {
1580       __ bind(&return_label_);
1581       __ movq(rsp, rbp);  // Move stack pointer back to frame pointer.
1582       __ popq(rbp);       // Pop caller's frame pointer.
1583     }
1584   }
1585   size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
1586   // Might need rcx for scratch if pop_size is too big.
1587   DCHECK_EQ(0, descriptor->CalleeSavedRegisters() & rcx.bit());
1588   __ Ret(static_cast<int>(pop_size), rcx);
1589 }
1590
1591
1592 void CodeGenerator::AssembleMove(InstructionOperand* source,
1593                                  InstructionOperand* destination) {
1594   X64OperandConverter g(this, NULL);
1595   // Dispatch on the source and destination operand kinds.  Not all
1596   // combinations are possible.
1597   if (source->IsRegister()) {
1598     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1599     Register src = g.ToRegister(source);
1600     if (destination->IsRegister()) {
1601       __ movq(g.ToRegister(destination), src);
1602     } else {
1603       __ movq(g.ToOperand(destination), src);
1604     }
1605   } else if (source->IsStackSlot()) {
1606     DCHECK(destination->IsRegister() || destination->IsStackSlot());
1607     Operand src = g.ToOperand(source);
1608     if (destination->IsRegister()) {
1609       Register dst = g.ToRegister(destination);
1610       __ movq(dst, src);
1611     } else {
1612       // Spill on demand to use a temporary register for memory-to-memory
1613       // moves.
1614       Register tmp = kScratchRegister;
1615       Operand dst = g.ToOperand(destination);
1616       __ movq(tmp, src);
1617       __ movq(dst, tmp);
1618     }
1619   } else if (source->IsConstant()) {
1620     ConstantOperand* constant_source = ConstantOperand::cast(source);
1621     Constant src = g.ToConstant(constant_source);
1622     if (destination->IsRegister() || destination->IsStackSlot()) {
1623       Register dst = destination->IsRegister() ? g.ToRegister(destination)
1624                                                : kScratchRegister;
1625       switch (src.type()) {
1626         case Constant::kInt32:
1627           // TODO(dcarney): don't need scratch in this case.
1628           __ Set(dst, src.ToInt32());
1629           break;
1630         case Constant::kInt64:
1631           __ Set(dst, src.ToInt64());
1632           break;
1633         case Constant::kFloat32:
1634           __ Move(dst,
1635                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
1636           break;
1637         case Constant::kFloat64:
1638           __ Move(dst,
1639                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
1640           break;
1641         case Constant::kExternalReference:
1642           __ Move(dst, src.ToExternalReference());
1643           break;
1644         case Constant::kHeapObject: {
1645           Handle<HeapObject> src_object = src.ToHeapObject();
1646           Heap::RootListIndex index;
1647           int offset;
1648           if (IsMaterializableFromFrame(src_object, &offset)) {
1649             __ movp(dst, Operand(rbp, offset));
1650           } else if (IsMaterializableFromRoot(src_object, &index)) {
1651             __ LoadRoot(dst, index);
1652           } else {
1653             __ Move(dst, src_object);
1654           }
1655           break;
1656         }
1657         case Constant::kRpoNumber:
1658           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
1659           break;
1660       }
1661       if (destination->IsStackSlot()) {
1662         __ movq(g.ToOperand(destination), kScratchRegister);
1663       }
1664     } else if (src.type() == Constant::kFloat32) {
1665       // TODO(turbofan): Can we do better here?
1666       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
1667       if (destination->IsDoubleRegister()) {
1668         __ Move(g.ToDoubleRegister(destination), src_const);
1669       } else {
1670         DCHECK(destination->IsDoubleStackSlot());
1671         Operand dst = g.ToOperand(destination);
1672         __ movl(dst, Immediate(src_const));
1673       }
1674     } else {
1675       DCHECK_EQ(Constant::kFloat64, src.type());
1676       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
1677       if (destination->IsDoubleRegister()) {
1678         __ Move(g.ToDoubleRegister(destination), src_const);
1679       } else {
1680         DCHECK(destination->IsDoubleStackSlot());
1681         __ movq(kScratchRegister, src_const);
1682         __ movq(g.ToOperand(destination), kScratchRegister);
1683       }
1684     }
1685   } else if (source->IsDoubleRegister()) {
1686     XMMRegister src = g.ToDoubleRegister(source);
1687     if (destination->IsDoubleRegister()) {
1688       XMMRegister dst = g.ToDoubleRegister(destination);
1689       __ movaps(dst, src);
1690     } else {
1691       DCHECK(destination->IsDoubleStackSlot());
1692       Operand dst = g.ToOperand(destination);
1693       __ movsd(dst, src);
1694     }
1695   } else if (source->IsDoubleStackSlot()) {
1696     DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
1697     Operand src = g.ToOperand(source);
1698     if (destination->IsDoubleRegister()) {
1699       XMMRegister dst = g.ToDoubleRegister(destination);
1700       __ movsd(dst, src);
1701     } else {
1702       // We rely on having xmm0 available as a fixed scratch register.
1703       Operand dst = g.ToOperand(destination);
1704       __ movsd(xmm0, src);
1705       __ movsd(dst, xmm0);
1706     }
1707   } else {
1708     UNREACHABLE();
1709   }
1710 }
1711
1712
1713 void CodeGenerator::AssembleSwap(InstructionOperand* source,
1714                                  InstructionOperand* destination) {
1715   X64OperandConverter g(this, NULL);
1716   // Dispatch on the source and destination operand kinds.  Not all
1717   // combinations are possible.
1718   if (source->IsRegister() && destination->IsRegister()) {
1719     // Register-register.
1720     __ xchgq(g.ToRegister(source), g.ToRegister(destination));
1721   } else if (source->IsRegister() && destination->IsStackSlot()) {
1722     Register src = g.ToRegister(source);
1723     Operand dst = g.ToOperand(destination);
1724     __ xchgq(src, dst);
1725   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
1726              (source->IsDoubleStackSlot() &&
1727               destination->IsDoubleStackSlot())) {
1728     // Memory-memory.
1729     Register tmp = kScratchRegister;
1730     Operand src = g.ToOperand(source);
1731     Operand dst = g.ToOperand(destination);
1732     __ movq(tmp, dst);
1733     __ xchgq(tmp, src);
1734     __ movq(dst, tmp);
1735   } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) {
1736     // XMM register-register swap. We rely on having xmm0
1737     // available as a fixed scratch register.
1738     XMMRegister src = g.ToDoubleRegister(source);
1739     XMMRegister dst = g.ToDoubleRegister(destination);
1740     __ movaps(xmm0, src);
1741     __ movaps(src, dst);
1742     __ movaps(dst, xmm0);
1743   } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) {
1744     // XMM register-memory swap.  We rely on having xmm0
1745     // available as a fixed scratch register.
1746     XMMRegister src = g.ToDoubleRegister(source);
1747     Operand dst = g.ToOperand(destination);
1748     __ movsd(xmm0, src);
1749     __ movsd(src, dst);
1750     __ movsd(dst, xmm0);
1751   } else {
1752     // No other combinations are possible.
1753     UNREACHABLE();
1754   }
1755 }
1756
1757
1758 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
1759   for (size_t index = 0; index < target_count; ++index) {
1760     __ dq(targets[index]);
1761   }
1762 }
1763
1764
1765 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); }
1766
1767
1768 void CodeGenerator::EnsureSpaceForLazyDeopt() {
1769   if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
1770     return;
1771   }
1772
1773   int space_needed = Deoptimizer::patch_size();
1774   // Ensure that we have enough space after the previous lazy-bailout
1775   // instruction for patching the code here.
1776   int current_pc = masm()->pc_offset();
1777   if (current_pc < last_lazy_deopt_pc_ + space_needed) {
1778     int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
1779     __ Nop(padding_size);
1780   }
1781 }
1782
1783 #undef __
1784
1785 }  // namespace internal
1786 }  // namespace compiler
1787 }  // namespace v8