}
OS::SNPrintF(Vector<char>(name_, len),
- "GenericBinaryOpStub_%s_%s%s_%s%s_%s%s",
+ "GenericBinaryOpStub_%s_%s%s_%s%s_%s%s_%s",
op_name,
overwrite_name,
(flags_ & NO_SMI_CODE_IN_STUB) ? "_NoSmiInStub" : "",
args_in_registers_ ? "RegArgs" : "StackArgs",
args_reversed_ ? "_R" : "",
use_sse3_ ? "SSE3" : "SSE2",
- operands_type_.ToString());
+ static_operands_type_.ToString(),
+ BinaryOpIC::GetName(runtime_operands_type_));
return name_;
}
void GenericBinaryOpStub::GenerateSmiCode(MacroAssembler* masm, Label* slow) {
- // 1. Move arguments into edx, eax except for DIV and MOD, which need the
- // dividend in eax and edx free for the division. Use eax, ebx for those.
+ // 1. Move arguments into rdx, rax except for DIV and MOD, which need the
+ // dividend in rax and rdx free for the division. Use rax, rbx for those.
Comment load_comment(masm, "-- Load arguments");
Register left = rdx;
Register right = rax;
break;
}
- // 4. Emit return of result in eax.
+ // 4. Emit return of result in rax.
GenerateReturn(masm);
// 5. For some operations emit inline code to perform floating point
void GenericBinaryOpStub::Generate(MacroAssembler* masm) {
Label call_runtime;
- if (HasSmiCodeInStub()) {
+
+ if (ShouldGenerateSmiCode()) {
GenerateSmiCode(masm, &call_runtime);
} else if (op_ != Token::MOD) {
- GenerateLoadArguments(masm);
+ if (!HasArgsInRegisters()) {
+ GenerateLoadArguments(masm);
+ }
}
// Floating point case.
- switch (op_) {
- case Token::ADD:
- case Token::SUB:
- case Token::MUL:
- case Token::DIV: {
- // rax: y
- // rdx: x
- if (operands_type_.IsNumber()) {
+ if (ShouldGenerateFPCode()) {
+ switch (op_) {
+ case Token::ADD:
+ case Token::SUB:
+ case Token::MUL:
+ case Token::DIV: {
+ if (runtime_operands_type_ == BinaryOpIC::DEFAULT &&
+ HasSmiCodeInStub()) {
+ // Execution reaches this point when the first non-smi argument occurs
+ // (and only if smi code is generated). This is the right moment to
+ // patch to HEAP_NUMBERS state. The transition is attempted only for
+ // the four basic operations. The stub stays in the DEFAULT state
+ // forever for all other operations (also if smi code is skipped).
+ GenerateTypeTransition(masm);
+ }
+
+ Label not_floats;
+ // rax: y
+ // rdx: x
+ if (static_operands_type_.IsNumber()) {
if (FLAG_debug_code) {
// Assert at runtime that inputs are only numbers.
__ AbortIfNotNumber(rdx, "GenericBinaryOpStub operand not a number.");
} else {
FloatingPointHelper::CheckNumberOperands(masm, &call_runtime);
}
- // Fast-case: Both operands are numbers.
- // xmm4 and xmm5 are volatile XMM registers.
- FloatingPointHelper::LoadFloatOperands(masm, xmm4, xmm5);
-
- switch (op_) {
- case Token::ADD: __ addsd(xmm4, xmm5); break;
- case Token::SUB: __ subsd(xmm4, xmm5); break;
- case Token::MUL: __ mulsd(xmm4, xmm5); break;
- case Token::DIV: __ divsd(xmm4, xmm5); break;
- default: UNREACHABLE();
- }
- // Allocate a heap number, if needed.
- Label skip_allocation;
- OverwriteMode mode = mode_;
- if (HasArgsReversed()) {
- if (mode == OVERWRITE_RIGHT) {
- mode = OVERWRITE_LEFT;
- } else if (mode == OVERWRITE_LEFT) {
- mode = OVERWRITE_RIGHT;
+ // Fast-case: Both operands are numbers.
+ // xmm4 and xmm5 are volatile XMM registers.
+ FloatingPointHelper::LoadFloatOperands(masm, xmm4, xmm5);
+
+ switch (op_) {
+ case Token::ADD: __ addsd(xmm4, xmm5); break;
+ case Token::SUB: __ subsd(xmm4, xmm5); break;
+ case Token::MUL: __ mulsd(xmm4, xmm5); break;
+ case Token::DIV: __ divsd(xmm4, xmm5); break;
+ default: UNREACHABLE();
}
- }
- switch (mode) {
- case OVERWRITE_LEFT:
- __ JumpIfNotSmi(rdx, &skip_allocation);
- __ AllocateHeapNumber(rbx, rcx, &call_runtime);
- __ movq(rdx, rbx);
- __ bind(&skip_allocation);
- __ movq(rax, rdx);
- break;
- case OVERWRITE_RIGHT:
- // If the argument in rax is already an object, we skip the
- // allocation of a heap number.
- __ JumpIfNotSmi(rax, &skip_allocation);
- // Fall through!
- case NO_OVERWRITE:
- // Allocate a heap number for the result. Keep rax and rdx intact
- // for the possible runtime call.
- __ AllocateHeapNumber(rbx, rcx, &call_runtime);
- __ movq(rax, rbx);
- __ bind(&skip_allocation);
- break;
- default: UNREACHABLE();
- }
- __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm4);
- GenerateReturn(masm);
- }
- case Token::MOD: {
- // For MOD we go directly to runtime in the non-smi case.
- break;
- }
- case Token::BIT_OR:
- case Token::BIT_AND:
- case Token::BIT_XOR:
- case Token::SAR:
- case Token::SHL:
- case Token::SHR: {
- Label skip_allocation, non_smi_result;
- FloatingPointHelper::LoadAsIntegers(masm, use_sse3_, &call_runtime);
- switch (op_) {
- case Token::BIT_OR: __ orl(rax, rcx); break;
- case Token::BIT_AND: __ andl(rax, rcx); break;
- case Token::BIT_XOR: __ xorl(rax, rcx); break;
- case Token::SAR: __ sarl_cl(rax); break;
- case Token::SHL: __ shll_cl(rax); break;
- case Token::SHR: __ shrl_cl(rax); break;
- default: UNREACHABLE();
- }
- if (op_ == Token::SHR) {
- // Check if result is non-negative. This can only happen for a shift
- // by zero, which also doesn't update the sign flag.
- __ testl(rax, rax);
- __ j(negative, &non_smi_result);
- }
- __ JumpIfNotValidSmiValue(rax, &non_smi_result);
- // Tag smi result, if possible, and return.
- __ Integer32ToSmi(rax, rax);
- GenerateReturn(masm);
-
- // All ops except SHR return a signed int32 that we load in a HeapNumber.
- if (op_ != Token::SHR && non_smi_result.is_linked()) {
- __ bind(&non_smi_result);
- // Allocate a heap number if needed.
- __ movsxlq(rbx, rax); // rbx: sign extended 32-bit result
- switch (mode_) {
+ // Allocate a heap number, if needed.
+ Label skip_allocation;
+ OverwriteMode mode = mode_;
+ if (HasArgsReversed()) {
+ if (mode == OVERWRITE_RIGHT) {
+ mode = OVERWRITE_LEFT;
+ } else if (mode == OVERWRITE_LEFT) {
+ mode = OVERWRITE_RIGHT;
+ }
+ }
+ switch (mode) {
case OVERWRITE_LEFT:
+ __ JumpIfNotSmi(rdx, &skip_allocation);
+ __ AllocateHeapNumber(rbx, rcx, &call_runtime);
+ __ movq(rdx, rbx);
+ __ bind(&skip_allocation);
+ __ movq(rax, rdx);
+ break;
case OVERWRITE_RIGHT:
- // If the operand was an object, we skip the
+ // If the argument in rax is already an object, we skip the
// allocation of a heap number.
- __ movq(rax, Operand(rsp, mode_ == OVERWRITE_RIGHT ?
- 1 * kPointerSize : 2 * kPointerSize));
__ JumpIfNotSmi(rax, &skip_allocation);
// Fall through!
case NO_OVERWRITE:
- __ AllocateHeapNumber(rax, rcx, &call_runtime);
+ // Allocate a heap number for the result. Keep rax and rdx intact
+ // for the possible runtime call.
+ __ AllocateHeapNumber(rbx, rcx, &call_runtime);
+ __ movq(rax, rbx);
__ bind(&skip_allocation);
break;
default: UNREACHABLE();
}
- // Store the result in the HeapNumber and return.
- __ movq(Operand(rsp, 1 * kPointerSize), rbx);
- __ fild_s(Operand(rsp, 1 * kPointerSize));
- __ fstp_d(FieldOperand(rax, HeapNumber::kValueOffset));
+ __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm4);
GenerateReturn(masm);
+ __ bind(¬_floats);
+ if (runtime_operands_type_ == BinaryOpIC::DEFAULT &&
+ !HasSmiCodeInStub()) {
+ // Execution reaches this point when the first non-number argument
+ // occurs (and only if smi code is skipped from the stub, otherwise
+ // the patching has already been done earlier in this case branch).
+ // A perfect moment to try patching to STRINGS for ADD operation.
+ if (op_ == Token::ADD) {
+ GenerateTypeTransition(masm);
+ }
+ }
+ break;
}
+ case Token::MOD: {
+ // For MOD we go directly to runtime in the non-smi case.
+ break;
+ }
+ case Token::BIT_OR:
+ case Token::BIT_AND:
+ case Token::BIT_XOR:
+ case Token::SAR:
+ case Token::SHL:
+ case Token::SHR: {
+ Label skip_allocation, non_smi_result;
+ FloatingPointHelper::LoadAsIntegers(masm, use_sse3_, &call_runtime);
+ switch (op_) {
+ case Token::BIT_OR: __ orl(rax, rcx); break;
+ case Token::BIT_AND: __ andl(rax, rcx); break;
+ case Token::BIT_XOR: __ xorl(rax, rcx); break;
+ case Token::SAR: __ sarl_cl(rax); break;
+ case Token::SHL: __ shll_cl(rax); break;
+ case Token::SHR: __ shrl_cl(rax); break;
+ default: UNREACHABLE();
+ }
+ if (op_ == Token::SHR) {
+ // Check if result is non-negative. This can only happen for a shift
+ // by zero, which also doesn't update the sign flag.
+ __ testl(rax, rax);
+ __ j(negative, &non_smi_result);
+ }
+ __ JumpIfNotValidSmiValue(rax, &non_smi_result);
+ // Tag smi result, if possible, and return.
+ __ Integer32ToSmi(rax, rax);
+ GenerateReturn(masm);
- // SHR should return uint32 - go to runtime for non-smi/negative result.
- if (op_ == Token::SHR) {
- __ bind(&non_smi_result);
+ // All ops except SHR return a signed int32 that we load in
+ // a HeapNumber.
+ if (op_ != Token::SHR && non_smi_result.is_linked()) {
+ __ bind(&non_smi_result);
+ // Allocate a heap number if needed.
+ __ movsxlq(rbx, rax); // rbx: sign extended 32-bit result
+ switch (mode_) {
+ case OVERWRITE_LEFT:
+ case OVERWRITE_RIGHT:
+ // If the operand was an object, we skip the
+ // allocation of a heap number.
+ __ movq(rax, Operand(rsp, mode_ == OVERWRITE_RIGHT ?
+ 1 * kPointerSize : 2 * kPointerSize));
+ __ JumpIfNotSmi(rax, &skip_allocation);
+ // Fall through!
+ case NO_OVERWRITE:
+ __ AllocateHeapNumber(rax, rcx, &call_runtime);
+ __ bind(&skip_allocation);
+ break;
+ default: UNREACHABLE();
+ }
+ // Store the result in the HeapNumber and return.
+ __ movq(Operand(rsp, 1 * kPointerSize), rbx);
+ __ fild_s(Operand(rsp, 1 * kPointerSize));
+ __ fstp_d(FieldOperand(rax, HeapNumber::kValueOffset));
+ GenerateReturn(masm);
+ }
+
+ // SHR should return uint32 - go to runtime for non-smi/negative result.
+ if (op_ == Token::SHR) {
+ __ bind(&non_smi_result);
+ }
+ break;
}
- break;
+ default: UNREACHABLE(); break;
}
- default: UNREACHABLE(); break;
}
// If all else fails, use the runtime system to get the correct
__ bind(&call_runtime);
if (HasArgsInRegisters()) {
- __ pop(rcx);
- if (HasArgsReversed()) {
- __ push(rax);
- __ push(rdx);
- } else {
- __ push(rdx);
- __ push(rax);
- }
- __ push(rcx);
+ GenerateRegisterArgsPush(masm);
}
switch (op_) {
// Test for string arguments before calling runtime.
Label not_strings, both_strings, not_string1, string1, string1_smi2;
+
+ // If this stub has already generated FP-specific code then the arguments
+ // are already in rdx, rax
+ if (!ShouldGenerateFPCode() && !HasArgsInRegisters()) {
+ GenerateLoadArguments(masm);
+ }
+
Condition is_smi;
- Result answer;
is_smi = masm->CheckSmi(lhs);
__ j(is_smi, ¬_string1);
__ CmpObjectType(lhs, FIRST_NONSTRING_TYPE, r8);
default:
UNREACHABLE();
}
+
+ // TODO(kaznacheev) Remove this (along with clearing) if it does not harm
+ // performance.
+ // Generate an unreachable reference to the DEFAULT stub so that it can be
+ // found at the end of this stub when clearing ICs at GC.
+ if (runtime_operands_type_ != BinaryOpIC::DEFAULT) {
+ GenericBinaryOpStub uninit(MinorKey(), BinaryOpIC::DEFAULT);
+ __ TailCallStub(&uninit);
+ }
}
void GenericBinaryOpStub::GenerateLoadArguments(MacroAssembler* masm) {
- // If arguments are not passed in registers read them from the stack.
- if (!HasArgsInRegisters()) {
- __ movq(rax, Operand(rsp, 1 * kPointerSize));
- __ movq(rdx, Operand(rsp, 2 * kPointerSize));
- }
+ ASSERT(!HasArgsInRegisters());
+ __ movq(rax, Operand(rsp, 1 * kPointerSize));
+ __ movq(rdx, Operand(rsp, 2 * kPointerSize));
}
}
+void GenericBinaryOpStub::GenerateRegisterArgsPush(MacroAssembler* masm) {
+ ASSERT(HasArgsInRegisters());
+ __ pop(rcx);
+ if (HasArgsReversed()) {
+ __ push(rax);
+ __ push(rdx);
+ } else {
+ __ push(rdx);
+ __ push(rax);
+ }
+ __ push(rcx);
+}
+
+
+void GenericBinaryOpStub::GenerateTypeTransition(MacroAssembler* masm) {
+ Label get_result;
+
+ // Keep a copy of operands on the stack and make sure they are also in
+ // rdx, rax.
+ if (HasArgsInRegisters()) {
+ GenerateRegisterArgsPush(masm);
+ } else {
+ GenerateLoadArguments(masm);
+ }
+
+ // Internal frame is necessary to handle exceptions properly.
+ __ EnterInternalFrame();
+
+ // Push arguments on stack if the stub expects them there.
+ if (!HasArgsInRegisters()) {
+ __ push(rdx);
+ __ push(rax);
+ }
+ // Call the stub proper to get the result in rax.
+ __ call(&get_result);
+ __ LeaveInternalFrame();
+
+ // Left and right arguments are already on stack.
+ __ pop(rcx);
+ // Push the operation result. The tail call to BinaryOp_Patch will
+ // return it to the original caller..
+ __ push(rax);
+
+ // Push this stub's key.
+ __ movq(rax, Immediate(MinorKey()));
+ __ Integer32ToSmi(rax, rax);
+ __ push(rax);
+
+ // Although the operation and the type info are encoded into the key,
+ // the encoding is opaque, so push them too.
+ __ movq(rax, Immediate(op_));
+ __ Integer32ToSmi(rax, rax);
+ __ push(rax);
+
+ __ movq(rax, Immediate(runtime_operands_type_));
+ __ Integer32ToSmi(rax, rax);
+ __ push(rax);
+
+ __ push(rcx);
+
+ // Perform patching to an appropriate fast case and return the result.
+ __ TailCallExternalReference(
+ ExternalReference(IC_Utility(IC::kBinaryOp_Patch)),
+ 6,
+ 1);
+
+ // The entry point for the result calculation is assumed to be immediately
+ // after this sequence.
+ __ bind(&get_result);
+}
+
+
Handle<Code> GetBinaryOpStub(int key, BinaryOpIC::TypeInfo type_info) {
- return Handle<Code>::null();
+ GenericBinaryOpStub stub(key, type_info);
+ return stub.GetCode();
}
#ifndef V8_X64_CODEGEN_X64_H_
#define V8_X64_CODEGEN_X64_H_
+#include "ic-inl.h"
+
namespace v8 {
namespace internal {
flags_(flags),
args_in_registers_(false),
args_reversed_(false),
- name_(NULL),
- operands_type_(operands_type) {
+ static_operands_type_(operands_type),
+ runtime_operands_type_(BinaryOpIC::DEFAULT),
+ name_(NULL) {
use_sse3_ = CpuFeatures::IsSupported(SSE3);
ASSERT(OpBits::is_valid(Token::NUM_TOKENS));
}
+ GenericBinaryOpStub(int key, BinaryOpIC::TypeInfo type_info)
+ : op_(OpBits::decode(key)),
+ mode_(ModeBits::decode(key)),
+ flags_(FlagBits::decode(key)),
+ args_in_registers_(ArgsInRegistersBits::decode(key)),
+ args_reversed_(ArgsReversedBits::decode(key)),
+ use_sse3_(SSE3Bits::decode(key)),
+ static_operands_type_(NumberInfo::ExpandedRepresentation(
+ StaticTypeInfoBits::decode(key))),
+ runtime_operands_type_(type_info),
+ name_(NULL) {
+ }
+
// Generate code to call the stub with the supplied arguments. This will add
// code at the call site to prepare arguments either in registers or on the
// stack together with the actual call.
bool args_in_registers_; // Arguments passed in registers not on the stack.
bool args_reversed_; // Left and right argument are swapped.
bool use_sse3_;
+
+ // Number type information of operands, determined by code generator.
+ NumberInfo static_operands_type_;
+
+ // Operand type information determined at runtime.
+ BinaryOpIC::TypeInfo runtime_operands_type_;
+
char* name_;
- NumberInfo operands_type_;
const char* GetName();
static_cast<int>(flags_),
static_cast<int>(args_in_registers_),
static_cast<int>(args_reversed_),
- operands_type_.ToString());
+ static_operands_type_.ToString());
}
#endif
- // Minor key encoding in 16 bits NNNFRASOOOOOOOMM.
+ // Minor key encoding in 18 bits TTNNNFRASOOOOOOOMM.
class ModeBits: public BitField<OverwriteMode, 0, 2> {};
class OpBits: public BitField<Token::Value, 2, 7> {};
class SSE3Bits: public BitField<bool, 9, 1> {};
class ArgsInRegistersBits: public BitField<bool, 10, 1> {};
class ArgsReversedBits: public BitField<bool, 11, 1> {};
class FlagBits: public BitField<GenericBinaryFlags, 12, 1> {};
- class NumberInfoBits: public BitField<int, 13, 3> {};
+ class StaticTypeInfoBits: public BitField<int, 13, 3> {};
+ class RuntimeTypeInfoBits: public BitField<BinaryOpIC::TypeInfo, 16, 2> {};
Major MajorKey() { return GenericBinaryOp; }
int MinorKey() {
- // Encode the parameters in a unique 16 bit value.
+ // Encode the parameters in a unique 18 bit value.
return OpBits::encode(op_)
| ModeBits::encode(mode_)
| FlagBits::encode(flags_)
| SSE3Bits::encode(use_sse3_)
| ArgsInRegistersBits::encode(args_in_registers_)
| ArgsReversedBits::encode(args_reversed_)
- | NumberInfoBits::encode(operands_type_.ThreeBitRepresentation());
+ | StaticTypeInfoBits::encode(
+ static_operands_type_.ThreeBitRepresentation())
+ | RuntimeTypeInfoBits::encode(runtime_operands_type_);
}
void Generate(MacroAssembler* masm);
void GenerateSmiCode(MacroAssembler* masm, Label* slow);
void GenerateLoadArguments(MacroAssembler* masm);
void GenerateReturn(MacroAssembler* masm);
+ void GenerateRegisterArgsPush(MacroAssembler* masm);
+ void GenerateTypeTransition(MacroAssembler* masm);
bool ArgsInRegistersSupported() {
return (op_ == Token::ADD) || (op_ == Token::SUB)
bool HasSmiCodeInStub() { return (flags_ & NO_SMI_CODE_IN_STUB) == 0; }
bool HasArgsInRegisters() { return args_in_registers_; }
bool HasArgsReversed() { return args_reversed_; }
+
+ bool ShouldGenerateSmiCode() {
+ return HasSmiCodeInStub() &&
+ runtime_operands_type_ != BinaryOpIC::HEAP_NUMBERS &&
+ runtime_operands_type_ != BinaryOpIC::STRINGS;
+ }
+
+ bool ShouldGenerateFPCode() {
+ return runtime_operands_type_ != BinaryOpIC::STRINGS;
+ }
+
+ virtual int GetCodeKind() { return Code::BINARY_OP_IC; }
+
+ virtual InlineCacheState GetICState() {
+ return BinaryOpIC::ToState(runtime_operands_type_);
+ }
};
projects / platform / upstream / v8.git / commitdiff