From a91499fec76b6aaacb8f1b33a76bca7df3969cf8 Mon Sep 17 00:00:00 2001 From: "bmeurer@chromium.org" Date: Thu, 2 Jan 2014 09:17:52 +0000 Subject: [PATCH] Allocation site support for monomorphic StringAdds in BinaryOps. R=mvstanton@chromium.org Review URL: https://codereview.chromium.org/106453003 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@18431 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/arm/code-stubs-arm.cc | 68 +++- src/ast.h | 5 + src/code-stubs-hydrogen.cc | 40 ++- src/code-stubs.cc | 22 ++ src/code-stubs.h | 103 +++++- src/hydrogen-instructions.cc | 22 +- src/hydrogen-instructions.h | 50 +-- src/hydrogen.cc | 545 ++++++++++++++++++------------- src/hydrogen.h | 62 +++- src/ia32/code-stubs-ia32.cc | 42 +++ src/ia32/lithium-codegen-ia32.cc | 2 +- src/ic.cc | 70 +++- src/ic.h | 23 +- src/objects-inl.h | 4 +- src/objects.cc | 6 + src/objects.h | 8 +- src/type-info.cc | 9 + src/type-info.h | 1 + src/typing.cc | 5 +- src/x64/code-stubs-x64.cc | 65 +++- 20 files changed, 846 insertions(+), 306 deletions(-) diff --git a/src/arm/code-stubs-arm.cc b/src/arm/code-stubs-arm.cc index f658291f8..f70271bd5 100644 --- a/src/arm/code-stubs-arm.cc +++ b/src/arm/code-stubs-arm.cc @@ -193,18 +193,6 @@ void CompareNilICStub::InitializeInterfaceDescriptor( } -void BinaryOpICStub::InitializeInterfaceDescriptor( - Isolate* isolate, - CodeStubInterfaceDescriptor* descriptor) { - static Register registers[] = { r1, r0 }; - descriptor->register_param_count_ = 2; - descriptor->register_params_ = registers; - descriptor->deoptimization_handler_ = FUNCTION_ADDR(BinaryOpIC_Miss); - descriptor->SetMissHandler( - ExternalReference(IC_Utility(IC::kBinaryOpIC_Miss), isolate)); -} - - static void InitializeArrayConstructorDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor, @@ -339,6 +327,29 @@ void ElementsTransitionAndStoreStub::InitializeInterfaceDescriptor( } +void BinaryOpICStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { r1, r0 }; + descriptor->register_param_count_ = 2; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = FUNCTION_ADDR(BinaryOpIC_Miss); + descriptor->SetMissHandler( + ExternalReference(IC_Utility(IC::kBinaryOpIC_Miss), isolate)); +} + + +void BinaryOpWithAllocationSiteStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { r2, r1, r0 }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(BinaryOpIC_MissWithAllocationSite); +} + + void NewStringAddStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { @@ -1442,6 +1453,7 @@ void CodeStub::GenerateStubsAheadOfTime(Isolate* isolate) { ArrayConstructorStubBase::GenerateStubsAheadOfTime(isolate); CreateAllocationSiteStub::GenerateAheadOfTime(isolate); BinaryOpICStub::GenerateAheadOfTime(isolate); + BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime(isolate); } @@ -4162,6 +4174,38 @@ void StringCompareStub::Generate(MacroAssembler* masm) { } +void BinaryOpICWithAllocationSiteStub::Generate(MacroAssembler* masm) { + // ----------- S t a t e ------------- + // -- r1 : left + // -- r0 : right + // -- lr : return address + // ----------------------------------- + Isolate* isolate = masm->isolate(); + + // Load r2 with the allocation site. We stick an undefined dummy value here + // and replace it with the real allocation site later when we instantiate this + // stub in BinaryOpICWithAllocationSiteStub::GetCodeCopyFromTemplate(). + __ Move(r2, handle(isolate->heap()->undefined_value())); + + // Make sure that we actually patched the allocation site. + if (FLAG_debug_code) { + __ tst(r2, Operand(kSmiTagMask)); + __ Assert(ne, kExpectedAllocationSite); + __ push(r2); + __ ldr(r2, FieldMemOperand(r2, HeapObject::kMapOffset)); + __ LoadRoot(ip, Heap::kAllocationSiteMapRootIndex); + __ cmp(r2, ip); + __ pop(r2); + __ Assert(eq, kExpectedAllocationSite); + } + + // Tail call into the stub that handles binary operations with allocation + // sites. + BinaryOpWithAllocationSiteStub stub(state_); + __ TailCallStub(&stub); +} + + void StringAddStub::Generate(MacroAssembler* masm) { Label call_runtime, call_builtin; Builtins::JavaScript builtin_id = Builtins::ADD; diff --git a/src/ast.h b/src/ast.h index f8de9be0e..3fabfddec 100644 --- a/src/ast.h +++ b/src/ast.h @@ -1936,6 +1936,10 @@ class BinaryOperation V8_FINAL : public Expression { Token::Value op() const { return op_; } Expression* left() const { return left_; } Expression* right() const { return right_; } + Handle allocation_site() const { return allocation_site_; } + void set_allocation_site(Handle allocation_site) { + allocation_site_ = allocation_site; + } BailoutId RightId() const { return right_id_; } @@ -1964,6 +1968,7 @@ class BinaryOperation V8_FINAL : public Expression { Token::Value op_; Expression* left_; Expression* right_; + Handle allocation_site_; // TODO(rossberg): the fixed arg should probably be represented as a Constant // type for the RHS. diff --git a/src/code-stubs-hydrogen.cc b/src/code-stubs-hydrogen.cc index f66a9e3a2..374f55f06 100644 --- a/src/code-stubs-hydrogen.cc +++ b/src/code-stubs-hydrogen.cc @@ -911,6 +911,7 @@ HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { (state.HasSideEffects() || !result_type->Is(Type::None()))); HValue* result = NULL; + HAllocationMode allocation_mode(NOT_TENURED); if (state.op() == Token::ADD && (left_type->Maybe(Type::String()) || right_type->Maybe(Type::String())) && !left_type->Is(Type::String()) && !right_type->Is(Type::String())) { @@ -924,14 +925,15 @@ HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { Push(BuildBinaryOperation( state.op(), left, right, handle(Type::String(), isolate()), right_type, - result_type, state.fixed_right_arg())); + result_type, state.fixed_right_arg(), + allocation_mode)); } if_leftisstring.Else(); { Push(BuildBinaryOperation( state.op(), left, right, left_type, right_type, result_type, - state.fixed_right_arg())); + state.fixed_right_arg(), allocation_mode)); } if_leftisstring.End(); result = Pop(); @@ -943,14 +945,15 @@ HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { Push(BuildBinaryOperation( state.op(), left, right, left_type, handle(Type::String(), isolate()), - result_type, state.fixed_right_arg())); + result_type, state.fixed_right_arg(), + allocation_mode)); } if_rightisstring.Else(); { Push(BuildBinaryOperation( state.op(), left, right, left_type, right_type, result_type, - state.fixed_right_arg())); + state.fixed_right_arg(), allocation_mode)); } if_rightisstring.End(); result = Pop(); @@ -959,7 +962,7 @@ HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { result = BuildBinaryOperation( state.op(), left, right, left_type, right_type, result_type, - state.fixed_right_arg()); + state.fixed_right_arg(), allocation_mode); } // If we encounter a generic argument, the number conversion is @@ -1006,6 +1009,31 @@ Handle BinaryOpICStub::GenerateCode(Isolate* isolate) { } +template <> +HValue* CodeStubGraphBuilder::BuildCodeStub() { + BinaryOpIC::State state = casted_stub()->state(); + + HValue* allocation_site = GetParameter( + BinaryOpWithAllocationSiteStub::kAllocationSite); + HValue* left = GetParameter(BinaryOpWithAllocationSiteStub::kLeft); + HValue* right = GetParameter(BinaryOpWithAllocationSiteStub::kRight); + + Handle left_type = state.GetLeftType(isolate()); + Handle right_type = state.GetRightType(isolate()); + Handle result_type = state.GetResultType(isolate()); + HAllocationMode allocation_mode(allocation_site); + + return BuildBinaryOperation(state.op(), left, right, + left_type, right_type, result_type, + state.fixed_right_arg(), allocation_mode); +} + + +Handle BinaryOpWithAllocationSiteStub::GenerateCode(Isolate* isolate) { + return DoGenerateCode(isolate, this); +} + + template <> HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { NewStringAddStub* stub = casted_stub(); @@ -1023,7 +1051,7 @@ HValue* CodeStubGraphBuilder::BuildCodeInitializedStub() { right = BuildCheckString(right); } - return BuildStringAdd(left, right, pretenure_flag); + return BuildStringAdd(left, right, HAllocationMode(pretenure_flag)); } diff --git a/src/code-stubs.cc b/src/code-stubs.cc index 900385240..e366e88b4 100644 --- a/src/code-stubs.cc +++ b/src/code-stubs.cc @@ -247,6 +247,28 @@ void BinaryOpICStub::GenerateAheadOfTime(Isolate* isolate, } +// static +void BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime(Isolate* isolate) { + // Generate special versions of the stub. + BinaryOpIC::State::GenerateAheadOfTime(isolate, &GenerateAheadOfTime); +} + + +void BinaryOpICWithAllocationSiteStub::PrintState(StringStream* stream) { + state_.Print(stream); +} + + +// static +void BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime( + Isolate* isolate, const BinaryOpIC::State& state) { + if (state.CouldCreateAllocationMementos()) { + BinaryOpICWithAllocationSiteStub stub(state); + stub.GetCode(isolate); + } +} + + void NewStringAddStub::PrintBaseName(StringStream* stream) { stream->Add("NewStringAddStub"); if ((flags() & STRING_ADD_CHECK_BOTH) == STRING_ADD_CHECK_BOTH) { diff --git a/src/code-stubs.h b/src/code-stubs.h index 4e4968065..106ce329d 100644 --- a/src/code-stubs.h +++ b/src/code-stubs.h @@ -42,6 +42,8 @@ namespace internal { V(CallFunction) \ V(CallConstruct) \ V(BinaryOpIC) \ + V(BinaryOpICWithAllocationSite) \ + V(BinaryOpWithAllocationSite) \ V(StringAdd) \ V(NewStringAdd) \ V(SubString) \ @@ -1062,7 +1064,7 @@ class KeyedArrayCallStub: public HICStub { }; -class BinaryOpICStub V8_FINAL : public HydrogenCodeStub { +class BinaryOpICStub : public HydrogenCodeStub { public: BinaryOpICStub(Token::Value op, OverwriteMode mode) : HydrogenCodeStub(UNINITIALIZED), state_(op, mode) {} @@ -1076,19 +1078,19 @@ class BinaryOpICStub V8_FINAL : public HydrogenCodeStub { static void InstallDescriptors(Isolate* isolate); - virtual Code::Kind GetCodeKind() const V8_OVERRIDE { + virtual Code::Kind GetCodeKind() const V8_FINAL V8_OVERRIDE { return Code::BINARY_OP_IC; } - virtual InlineCacheState GetICState() V8_OVERRIDE { + virtual InlineCacheState GetICState() V8_FINAL V8_OVERRIDE { return state_.GetICState(); } - virtual ExtraICState GetExtraICState() V8_OVERRIDE { + virtual ExtraICState GetExtraICState() V8_FINAL V8_OVERRIDE { return state_.GetExtraICState(); } - virtual void VerifyPlatformFeatures(Isolate* isolate) V8_OVERRIDE { + virtual void VerifyPlatformFeatures(Isolate* isolate) V8_FINAL V8_OVERRIDE { ASSERT(CpuFeatures::VerifyCrossCompiling(SSE2)); } @@ -1096,7 +1098,12 @@ class BinaryOpICStub V8_FINAL : public HydrogenCodeStub { const BinaryOpIC::State& state() const { return state_; } - virtual void PrintState(StringStream* stream) V8_OVERRIDE; + virtual void PrintState(StringStream* stream) V8_FINAL V8_OVERRIDE; + + virtual Major MajorKey() V8_OVERRIDE { return BinaryOpIC; } + virtual int NotMissMinorKey() V8_FINAL V8_OVERRIDE { + return GetExtraICState(); + } // Parameters accessed via CodeStubGraphBuilder::GetParameter() static const int kLeft = 0; @@ -1106,15 +1113,89 @@ class BinaryOpICStub V8_FINAL : public HydrogenCodeStub { static void GenerateAheadOfTime(Isolate* isolate, const BinaryOpIC::State& state); - virtual Major MajorKey() V8_OVERRIDE { return BinaryOpIC; } - virtual int NotMissMinorKey() V8_OVERRIDE { return GetExtraICState(); } - BinaryOpIC::State state_; DISALLOW_COPY_AND_ASSIGN(BinaryOpICStub); }; +// TODO(bmeurer): Merge this into the BinaryOpICStub once we have proper tail +// call support for stubs in Hydrogen. +class BinaryOpICWithAllocationSiteStub V8_FINAL : public PlatformCodeStub { + public: + explicit BinaryOpICWithAllocationSiteStub(const BinaryOpIC::State& state) + : state_(state) {} + + static void GenerateAheadOfTime(Isolate* isolate); + + Handle GetCodeCopyFromTemplate(Isolate* isolate, + Handle allocation_site) { + Handle code = CodeStub::GetCodeCopyFromTemplate(isolate); + // Replace the placeholder oddball with the actual allocation site. + code->ReplaceNthObject(1, isolate->heap()->oddball_map(), *allocation_site); + return code; + } + + virtual Code::Kind GetCodeKind() const V8_OVERRIDE { + return Code::BINARY_OP_IC; + } + + virtual InlineCacheState GetICState() V8_OVERRIDE { + return state_.GetICState(); + } + + virtual ExtraICState GetExtraICState() V8_OVERRIDE { + return state_.GetExtraICState(); + } + + virtual void VerifyPlatformFeatures(Isolate* isolate) V8_OVERRIDE { + ASSERT(CpuFeatures::VerifyCrossCompiling(SSE2)); + } + + virtual void Generate(MacroAssembler* masm) V8_OVERRIDE; + + virtual void PrintState(StringStream* stream) V8_OVERRIDE; + + virtual Major MajorKey() V8_OVERRIDE { return BinaryOpICWithAllocationSite; } + virtual int MinorKey() V8_OVERRIDE { return GetExtraICState(); } + + private: + static void GenerateAheadOfTime(Isolate* isolate, + const BinaryOpIC::State& state); + + BinaryOpIC::State state_; + + DISALLOW_COPY_AND_ASSIGN(BinaryOpICWithAllocationSiteStub); +}; + + +class BinaryOpWithAllocationSiteStub V8_FINAL : public BinaryOpICStub { + public: + explicit BinaryOpWithAllocationSiteStub(const BinaryOpIC::State& state) + : BinaryOpICStub(state) {} + + virtual void InitializeInterfaceDescriptor( + Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) V8_OVERRIDE; + + static void InstallDescriptors(Isolate* isolate); + + virtual Code::Kind GetCodeKind() const V8_FINAL V8_OVERRIDE { + return Code::STUB; + } + + virtual Handle GenerateCode(Isolate* isolate) V8_OVERRIDE; + + virtual Major MajorKey() V8_OVERRIDE { + return BinaryOpWithAllocationSite; + } + + // Parameters accessed via CodeStubGraphBuilder::GetParameter() + static const int kAllocationSite = 0; + static const int kLeft = 1; + static const int kRight = 2; +}; + + // TODO(bmeurer): Rename to StringAddStub once we dropped the old StringAddStub. class NewStringAddStub V8_FINAL : public HydrogenCodeStub { public: @@ -1130,6 +1211,10 @@ class NewStringAddStub V8_FINAL : public HydrogenCodeStub { return PretenureFlagBits::decode(bit_field_); } + virtual void VerifyPlatformFeatures(Isolate* isolate) V8_OVERRIDE { + ASSERT(CpuFeatures::VerifyCrossCompiling(SSE2)); + } + virtual Handle GenerateCode(Isolate* isolate) V8_OVERRIDE; virtual void InitializeInterfaceDescriptor( diff --git a/src/hydrogen-instructions.cc b/src/hydrogen-instructions.cc index a3bc56ca4..499166876 100644 --- a/src/hydrogen-instructions.cc +++ b/src/hydrogen-instructions.cc @@ -3815,7 +3815,9 @@ HInstruction* HStringAdd::New(Zone* zone, HValue* context, HValue* left, HValue* right, - StringAddFlags flags) { + PretenureFlag pretenure_flag, + StringAddFlags flags, + Handle allocation_site) { if (FLAG_fold_constants && left->IsConstant() && right->IsConstant()) { HConstant* c_right = HConstant::cast(right); HConstant* c_left = HConstant::cast(left); @@ -3825,7 +3827,23 @@ HInstruction* HStringAdd::New(Zone* zone, return HConstant::New(zone, context, concat); } } - return new(zone) HStringAdd(context, left, right, flags); + return new(zone) HStringAdd( + context, left, right, pretenure_flag, flags, allocation_site); +} + + +void HStringAdd::PrintDataTo(StringStream* stream) { + if ((flags() & STRING_ADD_CHECK_BOTH) == STRING_ADD_CHECK_BOTH) { + stream->Add("_CheckBoth"); + } else if ((flags() & STRING_ADD_CHECK_BOTH) == STRING_ADD_CHECK_LEFT) { + stream->Add("_CheckLeft"); + } else if ((flags() & STRING_ADD_CHECK_BOTH) == STRING_ADD_CHECK_RIGHT) { + stream->Add("_CheckRight"); + } + stream->Add(" ("); + if (pretenure_flag() == NOT_TENURED) stream->Add("N"); + else if (pretenure_flag() == TENURED) stream->Add("D"); + stream->Add(")"); } diff --git a/src/hydrogen-instructions.h b/src/hydrogen-instructions.h index 086f9a1a4..a32a41860 100644 --- a/src/hydrogen-instructions.h +++ b/src/hydrogen-instructions.h @@ -6992,45 +6992,55 @@ class HStringAdd V8_FINAL : public HBinaryOperation { HValue* context, HValue* left, HValue* right, - StringAddFlags flags = STRING_ADD_CHECK_NONE); + PretenureFlag pretenure_flag = NOT_TENURED, + StringAddFlags flags = STRING_ADD_CHECK_BOTH, + Handle allocation_site = + Handle::null()); StringAddFlags flags() const { return flags_; } + PretenureFlag pretenure_flag() const { return pretenure_flag_; } virtual Representation RequiredInputRepresentation(int index) V8_OVERRIDE { return Representation::Tagged(); } + virtual void PrintDataTo(StringStream* stream) V8_OVERRIDE; + DECLARE_CONCRETE_INSTRUCTION(StringAdd) protected: - virtual bool DataEquals(HValue* other) V8_OVERRIDE { return true; } + virtual bool DataEquals(HValue* other) V8_OVERRIDE { + return flags_ == HStringAdd::cast(other)->flags_ && + pretenure_flag_ == HStringAdd::cast(other)->pretenure_flag_; + } private: - HStringAdd(HValue* context, HValue* left, HValue* right, StringAddFlags flags) - : HBinaryOperation(context, left, right, HType::String()), flags_(flags) { + HStringAdd(HValue* context, + HValue* left, + HValue* right, + PretenureFlag pretenure_flag, + StringAddFlags flags, + Handle allocation_site) + : HBinaryOperation(context, left, right, HType::String()), + flags_(flags), pretenure_flag_(pretenure_flag) { set_representation(Representation::Tagged()); - if (MightHaveSideEffects()) { - SetAllSideEffects(); - } else { - SetFlag(kUseGVN); - SetGVNFlag(kDependsOnMaps); - SetGVNFlag(kChangesNewSpacePromotion); + SetFlag(kUseGVN); + SetGVNFlag(kDependsOnMaps); + SetGVNFlag(kChangesNewSpacePromotion); + if (FLAG_trace_pretenuring) { + PrintF("HStringAdd with AllocationSite %p %s\n", + allocation_site.is_null() + ? static_cast(NULL) + : static_cast(*allocation_site), + pretenure_flag == TENURED ? "tenured" : "not tenured"); } } - bool MightHaveSideEffects() const { - return flags_ != STRING_ADD_CHECK_NONE && - (left()->ToStringCanBeObserved() || right()->ToStringCanBeObserved()); - } - // No side-effects except possible allocation: - // NOTE: this instruction does not call ToString() on its inputs, when flags_ - // is set to STRING_ADD_CHECK_NONE. - virtual bool IsDeletable() const V8_OVERRIDE { - return !MightHaveSideEffects(); - } + virtual bool IsDeletable() const V8_OVERRIDE { return true; } const StringAddFlags flags_; + const PretenureFlag pretenure_flag_; }; diff --git a/src/hydrogen.cc b/src/hydrogen.cc index 3fcce8046..7bdc02f79 100644 --- a/src/hydrogen.cc +++ b/src/hydrogen.cc @@ -1693,22 +1693,131 @@ HValue* HGraphBuilder::BuildNumberToString(HValue* object, } -HValue* HGraphBuilder::BuildSeqStringSizeFor(HValue* length, - String::Encoding encoding) { - STATIC_ASSERT((SeqString::kHeaderSize & kObjectAlignmentMask) == 0); - HValue* size = length; - if (encoding == String::TWO_BYTE_ENCODING) { - size = AddUncasted(length, graph()->GetConstant1()); +HAllocate* HGraphBuilder::BuildAllocate( + HValue* object_size, + HType type, + InstanceType instance_type, + HAllocationMode allocation_mode) { + // Compute the effective allocation size. + HValue* size = object_size; + if (allocation_mode.CreateAllocationMementos()) { + size = AddUncasted(size, Add(AllocationMemento::kSize)); size->ClearFlag(HValue::kCanOverflow); - size->SetFlag(HValue::kUint32); } - size = AddUncasted(size, Add(static_cast( - SeqString::kHeaderSize + kObjectAlignmentMask))); - size->ClearFlag(HValue::kCanOverflow); - size = AddUncasted( - Token::BIT_AND, size, Add(static_cast( - ~kObjectAlignmentMask))); - return size; + + // Perform the actual allocation. + HAllocate* object = Add( + size, type, allocation_mode.GetPretenureMode(), + instance_type, allocation_mode.feedback_site()); + + // Setup the allocation memento. + if (allocation_mode.CreateAllocationMementos()) { + BuildCreateAllocationMemento( + object, object_size, allocation_mode.current_site()); + } + + return object; +} + + +HValue* HGraphBuilder::BuildAddStringLengths(HValue* left_length, + HValue* right_length) { + // Compute the combined string length. If the result is larger than the max + // supported string length, we bailout to the runtime. This is done implicitly + // when converting the result back to a smi in case the max string length + // equals the max smi value. Otherwise, for platforms with 32-bit smis, we do + HValue* length = AddUncasted(left_length, right_length); + STATIC_ASSERT(String::kMaxLength <= Smi::kMaxValue); + if (String::kMaxLength != Smi::kMaxValue) { + IfBuilder if_nooverflow(this); + if_nooverflow.If( + length, Add(String::kMaxLength), Token::LTE); + if_nooverflow.Then(); + if_nooverflow.ElseDeopt("String length exceeds limit"); + } + return length; +} + + +HValue* HGraphBuilder::BuildCreateConsString( + HValue* length, + HValue* left, + HValue* right, + HAllocationMode allocation_mode) { + // Determine the string instance types. + HInstruction* left_instance_type = AddLoadStringInstanceType(left); + HInstruction* right_instance_type = AddLoadStringInstanceType(right); + + // Allocate the cons string object. HAllocate does not care whether we + // pass CONS_STRING_TYPE or CONS_ASCII_STRING_TYPE here, so we just use + // CONS_STRING_TYPE here. Below we decide whether the cons string is + // one-byte or two-byte and set the appropriate map. + ASSERT(HAllocate::CompatibleInstanceTypes(CONS_STRING_TYPE, + CONS_ASCII_STRING_TYPE)); + HAllocate* result = BuildAllocate(Add(ConsString::kSize), + HType::String(), CONS_STRING_TYPE, + allocation_mode); + + // Compute intersection and difference of instance types. + HValue* anded_instance_types = AddUncasted( + Token::BIT_AND, left_instance_type, right_instance_type); + HValue* xored_instance_types = AddUncasted( + Token::BIT_XOR, left_instance_type, right_instance_type); + + // We create a one-byte cons string if + // 1. both strings are one-byte, or + // 2. at least one of the strings is two-byte, but happens to contain only + // one-byte characters. + // To do this, we check + // 1. if both strings are one-byte, or if the one-byte data hint is set in + // both strings, or + // 2. if one of the strings has the one-byte data hint set and the other + // string is one-byte. + IfBuilder if_onebyte(this); + STATIC_ASSERT(kOneByteStringTag != 0); + STATIC_ASSERT(kOneByteDataHintMask != 0); + if_onebyte.If( + AddUncasted( + Token::BIT_AND, anded_instance_types, + Add(static_cast( + kStringEncodingMask | kOneByteDataHintMask))), + graph()->GetConstant0(), Token::NE); + if_onebyte.Or(); + STATIC_ASSERT(kOneByteStringTag != 0 && + kOneByteDataHintTag != 0 && + kOneByteDataHintTag != kOneByteStringTag); + if_onebyte.If( + AddUncasted( + Token::BIT_AND, xored_instance_types, + Add(static_cast( + kOneByteStringTag | kOneByteDataHintTag))), + Add(static_cast( + kOneByteStringTag | kOneByteDataHintTag)), Token::EQ); + if_onebyte.Then(); + { + // We can safely skip the write barrier for storing the map here. + Handle map = isolate()->factory()->cons_ascii_string_map(); + AddStoreMapConstantNoWriteBarrier(result, map); + } + if_onebyte.Else(); + { + // We can safely skip the write barrier for storing the map here. + Handle map = isolate()->factory()->cons_string_map(); + AddStoreMapConstantNoWriteBarrier(result, map); + } + if_onebyte.End(); + + // Initialize the cons string fields. + Add(result, HObjectAccess::ForStringHashField(), + Add(String::kEmptyHashField)); + Add(result, HObjectAccess::ForStringLength(), length); + Add(result, HObjectAccess::ForConsStringFirst(), left); + Add(result, HObjectAccess::ForConsStringSecond(), right); + + // Count the native string addition. + AddIncrementCounter(isolate()->counters()->string_add_native()); + + return result; } @@ -1734,40 +1843,33 @@ void HGraphBuilder::BuildCopySeqStringChars(HValue* src, } -HValue* HGraphBuilder::BuildUncheckedStringAdd(HValue* left, - HValue* right, - PretenureFlag pretenure_flag) { +HValue* HGraphBuilder::BuildUncheckedStringAdd( + HValue* left, + HValue* right, + HAllocationMode allocation_mode) { // Determine the string lengths. - HValue* left_length = Add( - left, HObjectAccess::ForStringLength()); - HValue* right_length = Add( - right, HObjectAccess::ForStringLength()); + HValue* left_length = AddLoadStringLength(left); + HValue* right_length = AddLoadStringLength(right); - // Compute the combined string length. If the result is larger than the max - // supported string length, we bailout to the runtime. This is done implicitly - // when converting the result back to a smi in case the max string length - // equals the max smi valie. Otherwise, for platforms with 32-bit smis, we do - HValue* length = AddUncasted(left_length, right_length); - STATIC_ASSERT(String::kMaxLength <= Smi::kMaxValue); - if (String::kMaxLength != Smi::kMaxValue) { - IfBuilder if_nooverflow(this); - if_nooverflow.If( - length, Add(String::kMaxLength), Token::LTE); - if_nooverflow.Then(); - if_nooverflow.ElseDeopt("String length exceeds limit"); - } - - // Determine the string instance types. - HLoadNamedField* left_instance_type = Add( - Add(left, HObjectAccess::ForMap()), - HObjectAccess::ForMapInstanceType()); - HLoadNamedField* right_instance_type = Add( - Add(right, HObjectAccess::ForMap()), - HObjectAccess::ForMapInstanceType()); + // Compute the combined string length. + HValue* length = BuildAddStringLengths(left_length, right_length); - // Compute difference of instance types. - HValue* xored_instance_types = AddUncasted( - Token::BIT_XOR, left_instance_type, right_instance_type); + // Do some manual constant folding here. + if (left_length->IsConstant()) { + HConstant* c_left_length = HConstant::cast(left_length); + ASSERT_NE(0, c_left_length->Integer32Value()); + if (c_left_length->Integer32Value() + 1 >= ConsString::kMinLength) { + // The right string contains at least one character. + return BuildCreateConsString(length, left, right, allocation_mode); + } + } else if (right_length->IsConstant()) { + HConstant* c_right_length = HConstant::cast(right_length); + ASSERT_NE(0, c_right_length->Integer32Value()); + if (c_right_length->Integer32Value() + 1 >= ConsString::kMinLength) { + // The left string contains at least one character. + return BuildCreateConsString(length, left, right, allocation_mode); + } + } // Check if we should create a cons string. IfBuilder if_createcons(this); @@ -1775,82 +1877,20 @@ HValue* HGraphBuilder::BuildUncheckedStringAdd(HValue* left, length, Add(ConsString::kMinLength), Token::GTE); if_createcons.Then(); { - // Allocate the cons string object. HAllocate does not care whether we - // pass CONS_STRING_TYPE or CONS_ASCII_STRING_TYPE here, so we just use - // CONS_STRING_TYPE here. Below we decide whether the cons string is - // one-byte or two-byte and set the appropriate map. - ASSERT(HAllocate::CompatibleInstanceTypes(CONS_STRING_TYPE, - CONS_ASCII_STRING_TYPE)); - HAllocate* string = Add(Add(ConsString::kSize), - HType::String(), pretenure_flag, - CONS_STRING_TYPE); - - // Compute the intersection of instance types. - HValue* anded_instance_types = AddUncasted( - Token::BIT_AND, left_instance_type, right_instance_type); - - // We create a one-byte cons string if - // 1. both strings are one-byte, or - // 2. at least one of the strings is two-byte, but happens to contain only - // one-byte characters. - // To do this, we check - // 1. if both strings are one-byte, or if the one-byte data hint is set in - // both strings, or - // 2. if one of the strings has the one-byte data hint set and the other - // string is one-byte. - IfBuilder if_onebyte(this); - STATIC_ASSERT(kOneByteStringTag != 0); - STATIC_ASSERT(kOneByteDataHintMask != 0); - if_onebyte.If( - AddUncasted( - Token::BIT_AND, anded_instance_types, - Add(static_cast( - kStringEncodingMask | kOneByteDataHintMask))), - graph()->GetConstant0(), Token::NE); - if_onebyte.Or(); - STATIC_ASSERT(kOneByteStringTag != 0 && - kOneByteDataHintTag != 0 && - kOneByteDataHintTag != kOneByteStringTag); - if_onebyte.If( - AddUncasted( - Token::BIT_AND, xored_instance_types, - Add(static_cast( - kOneByteStringTag | kOneByteDataHintTag))), - Add(static_cast( - kOneByteStringTag | kOneByteDataHintTag)), Token::EQ); - if_onebyte.Then(); - { - // We can safely skip the write barrier for storing the map here. - Handle map = isolate()->factory()->cons_ascii_string_map(); - AddStoreMapConstantNoWriteBarrier(string, map); - } - if_onebyte.Else(); - { - // We can safely skip the write barrier for storing the map here. - Handle map = isolate()->factory()->cons_string_map(); - AddStoreMapConstantNoWriteBarrier(string, map); - } - if_onebyte.End(); - - // Initialize the cons string fields. - Add(string, HObjectAccess::ForStringHashField(), - Add(String::kEmptyHashField)); - Add(string, HObjectAccess::ForStringLength(), length); - Add(string, HObjectAccess::ForConsStringFirst(), left); - Add(string, HObjectAccess::ForConsStringSecond(), - right); - - // Count the native string addition. - AddIncrementCounter(isolate()->counters()->string_add_native()); - - // Cons string is result. - Push(string); + // Create a cons string. + Push(BuildCreateConsString(length, left, right, allocation_mode)); } if_createcons.Else(); { - // Compute union of instance types. + // Determine the string instance types. + HValue* left_instance_type = AddLoadStringInstanceType(left); + HValue* right_instance_type = AddLoadStringInstanceType(right); + + // Compute union and difference of instance types. HValue* ored_instance_types = AddUncasted( Token::BIT_OR, left_instance_type, right_instance_type); + HValue* xored_instance_types = AddUncasted( + Token::BIT_XOR, left_instance_type, right_instance_type); // Check if both strings have the same encoding and both are // sequential. @@ -1869,7 +1909,12 @@ HValue* HGraphBuilder::BuildUncheckedStringAdd(HValue* left, graph()->GetConstant0(), Token::EQ); if_sameencodingandsequential.Then(); { - // Check if the result is a one-byte string. + HConstant* string_map = + Add(isolate()->factory()->string_map()); + HConstant* ascii_string_map = + Add(isolate()->factory()->ascii_string_map()); + + // Determine map and size depending on whether result is one-byte string. IfBuilder if_onebyte(this); STATIC_ASSERT(kOneByteStringTag != 0); if_onebyte.If( @@ -1879,90 +1924,84 @@ HValue* HGraphBuilder::BuildUncheckedStringAdd(HValue* left, graph()->GetConstant0(), Token::NE); if_onebyte.Then(); { - // Calculate the number of bytes needed for the characters in the - // string while observing object alignment. - HValue* size = BuildSeqStringSizeFor( - length, String::ONE_BYTE_ENCODING); - - // Allocate the ASCII string object. - Handle map = isolate()->factory()->ascii_string_map(); - HAllocate* string = Add(size, HType::String(), - pretenure_flag, ASCII_STRING_TYPE); - string->set_known_initial_map(map); - - // We can safely skip the write barrier for storing map here. - AddStoreMapConstantNoWriteBarrier(string, map); - - // Length must be stored into the string before we copy characters to - // make debug verification code happy. - Add(string, HObjectAccess::ForStringLength(), - length); - - // Copy bytes from the left string. + // Allocate sequential one-byte string object. + Push(length); + Push(ascii_string_map); + } + if_onebyte.Else(); + { + // Allocate sequential two-byte string object. + HValue* size = AddUncasted(length, graph()->GetConstant1()); + size->ClearFlag(HValue::kCanOverflow); + size->SetFlag(HValue::kUint32); + Push(size); + Push(string_map); + } + if_onebyte.End(); + HValue* map = Pop(); + + // Calculate the number of bytes needed for the characters in the + // string while observing object alignment. + STATIC_ASSERT((SeqString::kHeaderSize & kObjectAlignmentMask) == 0); + HValue* size = Pop(); + size = AddUncasted(size, Add(static_cast( + SeqString::kHeaderSize + kObjectAlignmentMask))); + size->ClearFlag(HValue::kCanOverflow); + size = AddUncasted( + Token::BIT_AND, size, Add(static_cast( + ~kObjectAlignmentMask))); + + // Allocate the string object. HAllocate does not care whether we pass + // STRING_TYPE or ASCII_STRING_TYPE here, so we just use STRING_TYPE here. + HAllocate* result = BuildAllocate( + size, HType::String(), STRING_TYPE, allocation_mode); + + // We can safely skip the write barrier for storing map here. + AddStoreMapNoWriteBarrier(result, map); + + // Initialize the string fields. + Add(result, HObjectAccess::ForStringHashField(), + Add(String::kEmptyHashField)); + Add(result, HObjectAccess::ForStringLength(), length); + + // Copy characters to the result string. + IfBuilder if_twobyte(this); + if_twobyte.If(map, string_map); + if_twobyte.Then(); + { + // Copy characters from the left string. BuildCopySeqStringChars( - left, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, - string, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, + left, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, + result, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, left_length); - // Copy bytes from the right string. + // Copy characters from the right string. BuildCopySeqStringChars( - right, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, - string, left_length, String::ONE_BYTE_ENCODING, + right, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, + result, left_length, String::TWO_BYTE_ENCODING, right_length); - - // Count the native string addition. - AddIncrementCounter(isolate()->counters()->string_add_native()); - - // Return the string. - Push(string); } - if_onebyte.Else(); + if_twobyte.Else(); { - // Calculate the number of bytes needed for the characters in the - // string while observing object alignment. - HValue* size = BuildSeqStringSizeFor( - length, String::TWO_BYTE_ENCODING); - - // Allocate the two-byte string object. - Handle map = isolate()->factory()->string_map(); - HAllocate* string = Add(size, HType::String(), - pretenure_flag, STRING_TYPE); - string->set_known_initial_map(map); - - // We can safely skip the write barrier for storing map here. - AddStoreMapConstantNoWriteBarrier(string, map); - - // Length must be stored into the string before we copy characters to - // make debug verification code happy. - Add(string, HObjectAccess::ForStringLength(), - length); - - // Copy bytes from the left string. + // Copy characters from the left string. BuildCopySeqStringChars( - left, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, - string, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, + left, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, + result, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, left_length); - // Copy bytes from the right string. + // Copy characters from the right string. BuildCopySeqStringChars( - right, graph()->GetConstant0(), String::TWO_BYTE_ENCODING, - string, left_length, String::TWO_BYTE_ENCODING, + right, graph()->GetConstant0(), String::ONE_BYTE_ENCODING, + result, left_length, String::ONE_BYTE_ENCODING, right_length); - - // Return the string. - Push(string); } - if_onebyte.End(); - - // Initialize the (common) string fields. - HValue* string = Pop(); - Add(string, HObjectAccess::ForStringHashField(), - Add(String::kEmptyHashField)); + if_twobyte.End(); // Count the native string addition. AddIncrementCounter(isolate()->counters()->string_add_native()); - Push(string); + // Return the sequential string. + Push(result); } if_sameencodingandsequential.Else(); { @@ -1981,20 +2020,21 @@ HValue* HGraphBuilder::BuildUncheckedStringAdd(HValue* left, } -HValue* HGraphBuilder::BuildStringAdd(HValue* left, - HValue* right, - PretenureFlag pretenure_flag) { - // Determine the string lengths. - HValue* left_length = Add( - left, HObjectAccess::ForStringLength()); - HValue* right_length = Add( - right, HObjectAccess::ForStringLength()); +HValue* HGraphBuilder::BuildStringAdd( + HValue* left, + HValue* right, + HAllocationMode allocation_mode) { + NoObservableSideEffectsScope no_effects(this); + + // Determine string lengths. + HValue* left_length = AddLoadStringLength(left); + HValue* right_length = AddLoadStringLength(right); // Check if left string is empty. - IfBuilder if_leftisempty(this); - if_leftisempty.If( + IfBuilder if_leftempty(this); + if_leftempty.If( left_length, graph()->GetConstant0(), Token::EQ); - if_leftisempty.Then(); + if_leftempty.Then(); { // Count the native string addition. AddIncrementCounter(isolate()->counters()->string_add_native()); @@ -2002,13 +2042,13 @@ HValue* HGraphBuilder::BuildStringAdd(HValue* left, // Just return the right string. Push(right); } - if_leftisempty.Else(); + if_leftempty.Else(); { // Check if right string is empty. - IfBuilder if_rightisempty(this); - if_rightisempty.If( + IfBuilder if_rightempty(this); + if_rightempty.If( right_length, graph()->GetConstant0(), Token::EQ); - if_rightisempty.Then(); + if_rightempty.Then(); { // Count the native string addition. AddIncrementCounter(isolate()->counters()->string_add_native()); @@ -2016,14 +2056,14 @@ HValue* HGraphBuilder::BuildStringAdd(HValue* left, // Just return the left string. Push(left); } - if_rightisempty.Else(); + if_rightempty.Else(); { - // Concatenate the two non-empty strings. - Push(BuildUncheckedStringAdd(left, right, pretenure_flag)); + // Add the two non-empty strings. + Push(BuildUncheckedStringAdd(left, right, allocation_mode)); } - if_rightisempty.End(); + if_rightempty.End(); } - if_leftisempty.End(); + if_leftempty.End(); return Pop(); } @@ -6100,15 +6140,27 @@ HInstruction* HGraphBuilder::AddLoadNamedField(HValue* object, } -HInstruction* HGraphBuilder::BuildLoadStringLength(HValue* object, - HValue* checked_string) { - if (FLAG_fold_constants && object->IsConstant()) { - HConstant* constant = HConstant::cast(object); - if (constant->HasStringValue()) { - return New(constant->StringValue()->length()); +HInstruction* HGraphBuilder::AddLoadStringInstanceType(HValue* string) { + if (string->IsConstant()) { + HConstant* c_string = HConstant::cast(string); + if (c_string->HasStringValue()) { + return Add(c_string->StringValue()->map()->instance_type()); } } - return BuildLoadNamedField(checked_string, HObjectAccess::ForStringLength()); + return AddLoadNamedField( + AddLoadNamedField(string, HObjectAccess::ForMap()), + HObjectAccess::ForMapInstanceType()); +} + + +HInstruction* HGraphBuilder::AddLoadStringLength(HValue* string) { + if (string->IsConstant()) { + HConstant* c_string = HConstant::cast(string); + if (c_string->HasStringValue()) { + return Add(c_string->StringValue()->length()); + } + } + return AddLoadNamedField(string, HObjectAccess::ForStringLength()); } @@ -8556,13 +8608,9 @@ HInstruction* HOptimizedGraphBuilder::BuildStringCharCodeAt( return New(s->Get(i)); } } - BuildCheckHeapObject(string); - HValue* checkstring = - Add(string, HCheckInstanceType::IS_STRING); - HInstruction* length = BuildLoadStringLength(string, checkstring); - AddInstruction(length); - HInstruction* checked_index = Add(index, length); - return New(string, checked_index); + string = BuildCheckString(string); + index = Add(index, AddLoadStringLength(string)); + return New(string, index); } @@ -8693,10 +8741,18 @@ HValue* HOptimizedGraphBuilder::BuildBinaryOperation( Handle right_type = expr->right()->bounds().lower; Handle result_type = expr->bounds().lower; Maybe fixed_right_arg = expr->fixed_right_arg(); + Handle allocation_site = expr->allocation_site(); + + HAllocationMode allocation_mode = + FLAG_allocation_site_pretenuring + ? (allocation_site.is_null() + ? HAllocationMode(NOT_TENURED) + : HAllocationMode(allocation_site)) + : HAllocationMode(isolate()->heap()->GetPretenureMode()); HValue* result = HGraphBuilder::BuildBinaryOperation( - expr->op(), left, right, left_type, right_type, - result_type, fixed_right_arg); + expr->op(), left, right, left_type, right_type, result_type, + fixed_right_arg, allocation_mode); // Add a simulate after instructions with observable side effects, and // after phis, which are the result of BuildBinaryOperation when we // inlined some complex subgraph. @@ -8716,7 +8772,8 @@ HValue* HGraphBuilder::BuildBinaryOperation( Handle left_type, Handle right_type, Handle result_type, - Maybe fixed_right_arg) { + Maybe fixed_right_arg, + HAllocationMode allocation_mode) { Representation left_rep = Representation::FromType(left_type); Representation right_rep = Representation::FromType(right_type); @@ -8782,7 +8839,48 @@ HValue* HGraphBuilder::BuildBinaryOperation( return AddUncasted(function, 2); } - return AddUncasted(left, right, STRING_ADD_CHECK_NONE); + // Inline the string addition into the stub when creating allocation + // mementos to gather allocation site feedback. + if (graph()->info()->IsStub() && + allocation_mode.CreateAllocationMementos()) { + return BuildStringAdd(left, right, allocation_mode); + } + + // Register the dependent code with the allocation site. + if (!allocation_mode.feedback_site().is_null()) { + ASSERT(!graph()->info()->IsStub()); + allocation_mode.feedback_site()->AddDependentCompilationInfo( + AllocationSite::TENURING, top_info()); + } + + // Inline string addition if we know that we'll create a cons string. + if (left->IsConstant()) { + HConstant* c_left = HConstant::cast(left); + if (c_left->HasStringValue()) { + int c_left_length = c_left->StringValue()->length(); + if (c_left_length == 0) { + return right; + } else if (c_left_length + 1 >= ConsString::kMinLength) { + return BuildStringAdd(left, right, allocation_mode); + } + } + } + if (right->IsConstant()) { + HConstant* c_right = HConstant::cast(right); + if (c_right->HasStringValue()) { + int c_right_length = c_right->StringValue()->length(); + if (c_right_length == 0) { + return left; + } else if (c_right_length + 1 >= ConsString::kMinLength) { + return BuildStringAdd(left, right, allocation_mode); + } + } + } + + // Fallback to using the string add stub. + return AddUncasted( + left, right, allocation_mode.GetPretenureMode(), + STRING_ADD_CHECK_NONE, allocation_mode.feedback_site()); } if (graph()->info()->IsStub()) { @@ -10011,8 +10109,7 @@ void HOptimizedGraphBuilder::GenerateStringAdd(CallRuntime* call) { CHECK_ALIVE(VisitForValue(call->arguments()->at(1))); HValue* right = Pop(); HValue* left = Pop(); - HInstruction* result = - NewUncasted(left, right, STRING_ADD_CHECK_BOTH); + HInstruction* result = NewUncasted(left, right); return ast_context()->ReturnInstruction(result, call->id()); } diff --git a/src/hydrogen.h b/src/hydrogen.h index ea4da9abf..ca3fb4349 100644 --- a/src/hydrogen.h +++ b/src/hydrogen.h @@ -995,6 +995,35 @@ class HIfContinuation V8_FINAL { }; +class HAllocationMode V8_FINAL BASE_EMBEDDED { + public: + explicit HAllocationMode(Handle feedback_site) + : current_site_(NULL), feedback_site_(feedback_site), + pretenure_flag_(NOT_TENURED) {} + explicit HAllocationMode(HValue* current_site) + : current_site_(current_site), pretenure_flag_(NOT_TENURED) {} + explicit HAllocationMode(PretenureFlag pretenure_flag) + : current_site_(NULL), pretenure_flag_(pretenure_flag) {} + + HValue* current_site() const { return current_site_; } + Handle feedback_site() const { return feedback_site_; } + + bool CreateAllocationMementos() const V8_WARN_UNUSED_RESULT { + return current_site() != NULL; + } + + PretenureFlag GetPretenureMode() const V8_WARN_UNUSED_RESULT { + if (!feedback_site().is_null()) return feedback_site()->GetPretenureMode(); + return pretenure_flag_; + } + + private: + HValue* current_site_; + Handle feedback_site_; + PretenureFlag pretenure_flag_; +}; + + class HGraphBuilder { public: explicit HGraphBuilder(CompilationInfo* info) @@ -1291,9 +1320,18 @@ class HGraphBuilder { HValue* BuildUncheckedDictionaryElementLoad(HValue* receiver, HValue* key); - // Computes the size for a sequential string of the given length and encoding. - HValue* BuildSeqStringSizeFor(HValue* length, - String::Encoding encoding); + // Allocates a new object according with the given allocation properties. + HAllocate* BuildAllocate(HValue* object_size, + HType type, + InstanceType instance_type, + HAllocationMode allocation_mode); + // Computes the sum of two string lengths, taking care of overflow handling. + HValue* BuildAddStringLengths(HValue* left_length, HValue* right_length); + // Creates a cons string using the two input strings. + HValue* BuildCreateConsString(HValue* length, + HValue* left, + HValue* right, + HAllocationMode allocation_mode); // Copies characters from one sequential string to another. void BuildCopySeqStringChars(HValue* src, HValue* src_offset, @@ -1305,11 +1343,11 @@ class HGraphBuilder { // Both operands are non-empty strings. HValue* BuildUncheckedStringAdd(HValue* left, HValue* right, - PretenureFlag pretenure_flag); - // Both operands are strings. + HAllocationMode allocation_mode); + // Add two strings using allocation mode, validating type feedback. HValue* BuildStringAdd(HValue* left, HValue* right, - PretenureFlag pretenure_flag); + HAllocationMode allocation_mode); HInstruction* BuildUncheckedMonomorphicElementAccess( HValue* checked_object, @@ -1332,7 +1370,14 @@ class HGraphBuilder { HLoadNamedField* BuildLoadNamedField(HValue* object, HObjectAccess access); HInstruction* AddLoadNamedField(HValue* object, HObjectAccess access); - HInstruction* BuildLoadStringLength(HValue* object, HValue* checked_value); + HInstruction* AddLoadStringInstanceType(HValue* string); + HInstruction* AddLoadStringLength(HValue* string); + HStoreNamedField* AddStoreMapNoWriteBarrier(HValue* object, HValue* map) { + HStoreNamedField* store_map = Add( + object, HObjectAccess::ForMap(), map); + store_map->SkipWriteBarrier(); + return store_map; + } HStoreNamedField* AddStoreMapConstant(HValue* object, Handle map); HStoreNamedField* AddStoreMapConstantNoWriteBarrier(HValue* object, Handle map) { @@ -1353,7 +1398,8 @@ class HGraphBuilder { Handle left_type, Handle right_type, Handle result_type, - Maybe fixed_right_arg); + Maybe fixed_right_arg, + HAllocationMode allocation_mode); HLoadNamedField* AddLoadFixedArrayLength(HValue *object); diff --git a/src/ia32/code-stubs-ia32.cc b/src/ia32/code-stubs-ia32.cc index 208bd91a0..305a6d854 100644 --- a/src/ia32/code-stubs-ia32.cc +++ b/src/ia32/code-stubs-ia32.cc @@ -342,6 +342,17 @@ void BinaryOpICStub::InitializeInterfaceDescriptor( } +void BinaryOpWithAllocationSiteStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { ecx, edx, eax }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(BinaryOpIC_MissWithAllocationSite); +} + + void NewStringAddStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { @@ -2652,8 +2663,10 @@ void CodeStub::GenerateStubsAheadOfTime(Isolate* isolate) { if (Serializer::enabled()) { PlatformFeatureScope sse2(SSE2); BinaryOpICStub::GenerateAheadOfTime(isolate); + BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime(isolate); } else { BinaryOpICStub::GenerateAheadOfTime(isolate); + BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime(isolate); } } @@ -4385,6 +4398,35 @@ void StringCompareStub::Generate(MacroAssembler* masm) { } +void BinaryOpICWithAllocationSiteStub::Generate(MacroAssembler* masm) { + // ----------- S t a t e ------------- + // -- edx : left + // -- eax : right + // -- esp[0] : return address + // ----------------------------------- + Isolate* isolate = masm->isolate(); + + // Load ecx with the allocation site. We stick an undefined dummy value here + // and replace it with the real allocation site later when we instantiate this + // stub in BinaryOpICWithAllocationSiteStub::GetCodeCopyFromTemplate(). + __ mov(ecx, handle(isolate->heap()->undefined_value())); + + // Make sure that we actually patched the allocation site. + if (FLAG_debug_code) { + __ test(ecx, Immediate(kSmiTagMask)); + __ Assert(not_equal, kExpectedAllocationSite); + __ cmp(FieldOperand(ecx, HeapObject::kMapOffset), + isolate->factory()->allocation_site_map()); + __ Assert(equal, kExpectedAllocationSite); + } + + // Tail call into the stub that handles binary operations with allocation + // sites. + BinaryOpWithAllocationSiteStub stub(state_); + __ TailCallStub(&stub); +} + + void ICCompareStub::GenerateSmis(MacroAssembler* masm) { ASSERT(state_ == CompareIC::SMI); Label miss; diff --git a/src/ia32/lithium-codegen-ia32.cc b/src/ia32/lithium-codegen-ia32.cc index 86118ebf6..1a20fc3b6 100644 --- a/src/ia32/lithium-codegen-ia32.cc +++ b/src/ia32/lithium-codegen-ia32.cc @@ -4912,7 +4912,7 @@ void LCodeGen::DoStringAdd(LStringAdd* instr) { ASSERT(ToRegister(instr->left()).is(edx)); ASSERT(ToRegister(instr->right()).is(eax)); NewStringAddStub stub(instr->hydrogen()->flags(), - isolate()->heap()->GetPretenureMode()); + instr->hydrogen()->pretenure_flag()); CallCode(stub.GetCode(isolate()), RelocInfo::CODE_TARGET, instr); } else { EmitPushTaggedOperand(instr->left()); diff --git a/src/ic.cc b/src/ic.cc index af9b19eac..b548aa64c 100644 --- a/src/ic.cc +++ b/src/ic.cc @@ -2398,6 +2398,11 @@ void BinaryOpIC::State::GenerateAheadOfTime( GENERATE(Token::ADD, SMI, NUMBER, NUMBER, OVERWRITE_RIGHT); GENERATE(Token::ADD, SMI, SMI, INT32, OVERWRITE_LEFT); GENERATE(Token::ADD, SMI, SMI, SMI, OVERWRITE_RIGHT); + GENERATE(Token::ADD, STRING, SMI, STRING, NO_OVERWRITE); + GENERATE(Token::ADD, SMI, STRING, STRING, NO_OVERWRITE); + GENERATE(Token::ADD, STRING, NUMBER, STRING, NO_OVERWRITE); + GENERATE(Token::ADD, NUMBER, STRING, STRING, NO_OVERWRITE); + GENERATE(Token::ADD, STRING, STRING, STRING, NO_OVERWRITE); GENERATE(Token::BIT_AND, INT32, INT32, INT32, NO_OVERWRITE); GENERATE(Token::BIT_AND, INT32, INT32, INT32, OVERWRITE_LEFT); GENERATE(Token::BIT_AND, INT32, INT32, INT32, OVERWRITE_RIGHT); @@ -2597,6 +2602,7 @@ void BinaryOpIC::State::Print(StringStream* stream) const { stream->Add("(%s", Token::Name(op_)); if (mode_ == OVERWRITE_LEFT) stream->Add("_ReuseLeft"); else if (mode_ == OVERWRITE_RIGHT) stream->Add("_ReuseRight"); + if (CouldCreateAllocationMementos()) stream->Add("_CreateAllocationMementos"); stream->Add(":%s*", KindToString(left_kind_)); if (fixed_right_arg_.has_value) { stream->Add("%d", fixed_right_arg_.value); @@ -2636,6 +2642,18 @@ void BinaryOpIC::State::Update(Handle left, } } + // We don't want to distinguish INT32 and NUMBER for string add (because + // NumberToString can't make use of this anyway). + if (left_kind_ == STRING && right_kind_ == INT32) { + ASSERT_EQ(STRING, result_kind_); + ASSERT_EQ(Token::ADD, op_); + right_kind_ = NUMBER; + } else if (right_kind_ == STRING && left_kind_ == INT32) { + ASSERT_EQ(STRING, result_kind_); + ASSERT_EQ(Token::ADD, op_); + left_kind_ = NUMBER; + } + // Reset overwrite mode unless we can actually make use of it, or may be able // to make use of it at some point in the future. if ((mode_ == OVERWRITE_LEFT && left_kind_ > NUMBER) || @@ -2721,7 +2739,9 @@ Handle BinaryOpIC::State::KindToType(Kind kind, Isolate* isolate) { } -MaybeObject* BinaryOpIC::Transition(Handle left, Handle right) { +MaybeObject* BinaryOpIC::Transition(Handle allocation_site, + Handle left, + Handle right) { State state(target()->extended_extra_ic_state()); // Compute the actual result using the builtin for the binary operation. @@ -2737,9 +2757,29 @@ MaybeObject* BinaryOpIC::Transition(Handle left, Handle right) { State old_state = state; state.Update(left, right, result); - // Install the new stub. - BinaryOpICStub stub(state); - set_target(*stub.GetCode(isolate())); + // Check if we have a string operation here. + Handle target; + if (!allocation_site.is_null() || state.ShouldCreateAllocationMementos()) { + // Setup the allocation site on-demand. + if (allocation_site.is_null()) { + allocation_site = isolate()->factory()->NewAllocationSite(); + } + + // Install the stub with an allocation site. + BinaryOpICWithAllocationSiteStub stub(state); + target = stub.GetCodeCopyFromTemplate(isolate(), allocation_site); + + // Sanity check the trampoline stub. + ASSERT_EQ(*allocation_site, target->FindFirstAllocationSite()); + } else { + // Install the generic stub. + BinaryOpICStub stub(state); + target = stub.GetCode(isolate()); + + // Sanity check the generic stub. + ASSERT_EQ(NULL, target->FindFirstAllocationSite()); + } + set_target(*target); if (FLAG_trace_ic) { char buffer[150]; @@ -2750,9 +2790,12 @@ MaybeObject* BinaryOpIC::Transition(Handle left, Handle right) { old_state.Print(&stream); stream.Add(" => "); state.Print(&stream); - stream.Add(" @ %p <- ", static_cast(*target())); + stream.Add(" @ %p <- ", static_cast(*target)); stream.OutputToStdOut(); JavaScriptFrame::PrintTop(isolate(), stdout, false, true); + if (!allocation_site.is_null()) { + PrintF(" using allocation site %p", static_cast(*allocation_site)); + } PrintF("]\n"); } @@ -2769,10 +2812,25 @@ MaybeObject* BinaryOpIC::Transition(Handle left, Handle right) { RUNTIME_FUNCTION(MaybeObject*, BinaryOpIC_Miss) { HandleScope scope(isolate); + ASSERT_EQ(2, args.length()); Handle left = args.at(BinaryOpICStub::kLeft); Handle right = args.at(BinaryOpICStub::kRight); BinaryOpIC ic(isolate); - return ic.Transition(left, right); + return ic.Transition(Handle::null(), left, right); +} + + +RUNTIME_FUNCTION(MaybeObject*, BinaryOpIC_MissWithAllocationSite) { + HandleScope scope(isolate); + ASSERT_EQ(3, args.length()); + Handle allocation_site = args.at( + BinaryOpWithAllocationSiteStub::kAllocationSite); + Handle left = args.at( + BinaryOpWithAllocationSiteStub::kLeft); + Handle right = args.at( + BinaryOpWithAllocationSiteStub::kRight); + BinaryOpIC ic(isolate); + return ic.Transition(allocation_site, left, right); } diff --git a/src/ic.h b/src/ic.h index d1d2b6139..308ab9530 100644 --- a/src/ic.h +++ b/src/ic.h @@ -861,10 +861,27 @@ class BinaryOpIC: public IC { right_kind_ > SMI && right_kind_ <= NUMBER)); } + // Returns true if the IC _could_ create allocation mementos. + bool CouldCreateAllocationMementos() const { + if (left_kind_ == STRING || right_kind_ == STRING) { + ASSERT_EQ(Token::ADD, op_); + return true; + } + return false; + } + + // Returns true if the IC _should_ create allocation mementos. + bool ShouldCreateAllocationMementos() const { + return FLAG_allocation_site_pretenuring && + CouldCreateAllocationMementos(); + } + bool HasSideEffects() const { return Max(left_kind_, right_kind_) == GENERIC; } + // Returns true if the IC should enable the inline smi code (i.e. if either + // parameter may be a smi). bool UseInlinedSmiCode() const { return KindMaybeSmi(left_kind_) || KindMaybeSmi(right_kind_); } @@ -926,8 +943,9 @@ class BinaryOpIC: public IC { static Builtins::JavaScript TokenToJSBuiltin(Token::Value op); - MUST_USE_RESULT MaybeObject* Transition(Handle left, - Handle right); + MaybeObject* Transition(Handle allocation_site, + Handle left, + Handle right) V8_WARN_UNUSED_RESULT; }; @@ -1036,6 +1054,7 @@ DECLARE_RUNTIME_FUNCTION(MaybeObject*, StoreIC_MissFromStubFailure); DECLARE_RUNTIME_FUNCTION(MaybeObject*, KeyedCallIC_MissFromStubFailure); DECLARE_RUNTIME_FUNCTION(MaybeObject*, ElementsTransitionAndStoreIC_Miss); DECLARE_RUNTIME_FUNCTION(MaybeObject*, BinaryOpIC_Miss); +DECLARE_RUNTIME_FUNCTION(MaybeObject*, BinaryOpIC_MissWithAllocationSite); DECLARE_RUNTIME_FUNCTION(MaybeObject*, CompareNilIC_Miss); DECLARE_RUNTIME_FUNCTION(MaybeObject*, ToBooleanIC_Miss); diff --git a/src/objects-inl.h b/src/objects-inl.h index 5975d80fa..dbf21d2c8 100644 --- a/src/objects-inl.h +++ b/src/objects-inl.h @@ -1349,7 +1349,9 @@ AllocationSiteMode AllocationSite::GetMode(ElementsKind from, inline bool AllocationSite::CanTrack(InstanceType type) { if (FLAG_allocation_site_pretenuring) { - return type == JS_ARRAY_TYPE || type == JS_OBJECT_TYPE; + return type == JS_ARRAY_TYPE || + type == JS_OBJECT_TYPE || + type < FIRST_NONSTRING_TYPE; } return type == JS_ARRAY_TYPE; } diff --git a/src/objects.cc b/src/objects.cc index b295bae9f..cf21e802e 100644 --- a/src/objects.cc +++ b/src/objects.cc @@ -10477,6 +10477,12 @@ Object* Code::FindNthObject(int n, Map* match_map) { } +AllocationSite* Code::FindFirstAllocationSite() { + Object* result = FindNthObject(1, GetHeap()->allocation_site_map()); + return (result != NULL) ? AllocationSite::cast(result) : NULL; +} + + Map* Code::FindFirstMap() { Object* result = FindNthObject(1, GetHeap()->meta_map()); return (result != NULL) ? Map::cast(result) : NULL; diff --git a/src/objects.h b/src/objects.h index 17b92d0cc..91a2aea2b 100644 --- a/src/objects.h +++ b/src/objects.h @@ -1104,9 +1104,10 @@ class MaybeObject BASE_EMBEDDED { "DoPushArgument not implemented for double type") \ V(kEmitLoadRegisterUnsupportedDoubleImmediate, \ "EmitLoadRegister: Unsupported double immediate") \ - V(kEval, "Eval") \ + V(kEval, "eval") \ V(kExpected0AsASmiSentinel, "Expected 0 as a Smi sentinel") \ - V(kExpectedAlignmentMarker, "Expected alignment marker") \ + V(kExpectedAlignmentMarker, "expected alignment marker") \ + V(kExpectedAllocationSite, "expected allocation site") \ V(kExpectedAllocationSiteInCell, \ "Expected AllocationSite in property cell") \ V(kExpectedPropertyCellInRegisterA2, \ @@ -5247,6 +5248,9 @@ class Code: public HeapObject { Object* FindNthObject(int n, Map* match_map); void ReplaceNthObject(int n, Map* match_map, Object* replace_with); + // Find the first allocation site in an IC stub. + AllocationSite* FindFirstAllocationSite(); + // Find the first map in an IC stub. Map* FindFirstMap(); void FindAllMaps(MapHandleList* maps); diff --git a/src/type-info.cc b/src/type-info.cc index 3cdcf8e43..7f1d3b5ea 100644 --- a/src/type-info.cc +++ b/src/type-info.cc @@ -283,6 +283,7 @@ void TypeFeedbackOracle::BinaryType(TypeFeedbackId id, Handle* right, Handle* result, Maybe* fixed_right_arg, + Handle* allocation_site, Token::Value op) { Handle object = GetInfo(id); if (!object->IsCode()) { @@ -292,6 +293,7 @@ void TypeFeedbackOracle::BinaryType(TypeFeedbackId id, op > BinaryOpIC::State::LAST_TOKEN); *left = *right = *result = handle(Type::None(), isolate_); *fixed_right_arg = Maybe(); + *allocation_site = Handle::null(); return; } Handle code = Handle::cast(object); @@ -303,6 +305,13 @@ void TypeFeedbackOracle::BinaryType(TypeFeedbackId id, *right = state.GetRightType(isolate()); *result = state.GetResultType(isolate()); *fixed_right_arg = state.fixed_right_arg(); + + AllocationSite* first_allocation_site = code->FindFirstAllocationSite(); + if (first_allocation_site != NULL) { + *allocation_site = handle(first_allocation_site); + } else { + *allocation_site = Handle::null(); + } } diff --git a/src/type-info.h b/src/type-info.h index 635be1a80..6e6c67d16 100644 --- a/src/type-info.h +++ b/src/type-info.h @@ -112,6 +112,7 @@ class TypeFeedbackOracle: public ZoneObject { Handle* right, Handle* result, Maybe* fixed_right_arg, + Handle* allocation_site, Token::Value operation); void CompareType(TypeFeedbackId id, diff --git a/src/typing.cc b/src/typing.cc index de9f4041e..c692f485e 100644 --- a/src/typing.cc +++ b/src/typing.cc @@ -629,11 +629,14 @@ void AstTyper::VisitBinaryOperation(BinaryOperation* expr) { // Collect type feedback. Handle type, left_type, right_type; Maybe fixed_right_arg; + Handle allocation_site; oracle()->BinaryType(expr->BinaryOperationFeedbackId(), - &left_type, &right_type, &type, &fixed_right_arg, expr->op()); + &left_type, &right_type, &type, &fixed_right_arg, + &allocation_site, expr->op()); NarrowLowerType(expr, type); NarrowLowerType(expr->left(), left_type); NarrowLowerType(expr->right(), right_type); + expr->set_allocation_site(allocation_site); expr->set_fixed_right_arg(fixed_right_arg); if (expr->op() == Token::OR || expr->op() == Token::AND) { expr->left()->RecordToBooleanTypeFeedback(oracle()); diff --git a/src/x64/code-stubs-x64.cc b/src/x64/code-stubs-x64.cc index 5b5212c7e..920843dfe 100644 --- a/src/x64/code-stubs-x64.cc +++ b/src/x64/code-stubs-x64.cc @@ -180,18 +180,6 @@ void TransitionElementsKindStub::InitializeInterfaceDescriptor( } -void BinaryOpICStub::InitializeInterfaceDescriptor( - Isolate* isolate, - CodeStubInterfaceDescriptor* descriptor) { - static Register registers[] = { rdx, rax }; - descriptor->register_param_count_ = 2; - descriptor->register_params_ = registers; - descriptor->deoptimization_handler_ = FUNCTION_ADDR(BinaryOpIC_Miss); - descriptor->SetMissHandler( - ExternalReference(IC_Utility(IC::kBinaryOpIC_Miss), isolate)); -} - - static void InitializeArrayConstructorDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor, @@ -339,6 +327,29 @@ void ElementsTransitionAndStoreStub::InitializeInterfaceDescriptor( } +void BinaryOpICStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { rdx, rax }; + descriptor->register_param_count_ = 2; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = FUNCTION_ADDR(BinaryOpIC_Miss); + descriptor->SetMissHandler( + ExternalReference(IC_Utility(IC::kBinaryOpIC_Miss), isolate)); +} + + +void BinaryOpWithAllocationSiteStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { rcx, rdx, rax }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(BinaryOpIC_MissWithAllocationSite); +} + + void NewStringAddStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { @@ -2483,6 +2494,7 @@ void CodeStub::GenerateStubsAheadOfTime(Isolate* isolate) { ArrayConstructorStubBase::GenerateStubsAheadOfTime(isolate); CreateAllocationSiteStub::GenerateAheadOfTime(isolate); BinaryOpICStub::GenerateAheadOfTime(isolate); + BinaryOpICWithAllocationSiteStub::GenerateAheadOfTime(isolate); } @@ -4229,6 +4241,35 @@ void StringCompareStub::Generate(MacroAssembler* masm) { } +void BinaryOpICWithAllocationSiteStub::Generate(MacroAssembler* masm) { + // ----------- S t a t e ------------- + // -- rdx : left + // -- rax : right + // -- rsp[0] : return address + // ----------------------------------- + Isolate* isolate = masm->isolate(); + + // Load rcx with the allocation site. We stick an undefined dummy value here + // and replace it with the real allocation site later when we instantiate this + // stub in BinaryOpICWithAllocationSiteStub::GetCodeCopyFromTemplate(). + __ Move(rcx, handle(isolate->heap()->undefined_value())); + + // Make sure that we actually patched the allocation site. + if (FLAG_debug_code) { + __ testb(rcx, Immediate(kSmiTagMask)); + __ Assert(zero, kExpectedAllocationSite); + __ Cmp(FieldOperand(rcx, HeapObject::kMapOffset), + isolate->factory()->allocation_site_map()); + __ Assert(equal, kExpectedAllocationSite); + } + + // Tail call into the stub that handles binary operations with allocation + // sites. + BinaryOpWithAllocationSiteStub stub(state_); + __ TailCallStub(&stub); +} + + void ICCompareStub::GenerateSmis(MacroAssembler* masm) { ASSERT(state_ == CompareIC::SMI); Label miss; -- 2.34.1