From ffd0c712e829280b152143cf6d6514e35cf61412 Mon Sep 17 00:00:00 2001 From: "danno@chromium.org" Date: Wed, 20 Mar 2013 10:37:13 +0000 Subject: [PATCH] Implement many KeyedStoreStubs using Crankshaft - Addition of a compiled hydrogen stub for KeyedStores. - Inlining of "grow" stubs into OPTIMIZED_FUNCTIONs - Addition of new "ignore OOB" ic stub that silently swallows out-of-bounds stores to external typed arrays. - Addition of new "copy-on-write" ic stub that inlines allocation and copying operations for cow array - New stub are generated with Crankshaft, so they are automatically inlined into OPTIMIZED_FUNCTIONs Review URL: https://codereview.chromium.org/12221064 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@14001 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/arm/code-stubs-arm.cc | 11 ++ src/arm/lithium-codegen-arm.cc | 14 +- src/arm/lithium-codegen-arm.h | 4 + src/ast.cc | 6 +- src/ast.h | 23 ++- src/code-stubs-hydrogen.cc | 76 ++++---- src/code-stubs.h | 41 ++++ src/flag-definitions.h | 2 + src/heap.h | 1 + src/hydrogen.cc | 426 +++++++++++++++++++++++++++++++++++------ src/hydrogen.h | 65 +++++-- src/ia32/code-stubs-ia32.cc | 11 ++ src/ic.cc | 55 ++++-- src/ic.h | 6 +- src/runtime.cc | 3 +- src/stub-cache.cc | 42 +++- src/type-info.cc | 12 +- src/x64/code-stubs-x64.cc | 11 ++ test/mjsunit/elements-kind.js | 7 +- 19 files changed, 660 insertions(+), 156 deletions(-) diff --git a/src/arm/code-stubs-arm.cc b/src/arm/code-stubs-arm.cc index b1ffaea..9fab908 100644 --- a/src/arm/code-stubs-arm.cc +++ b/src/arm/code-stubs-arm.cc @@ -61,6 +61,17 @@ void KeyedLoadFastElementStub::InitializeInterfaceDescriptor( } +void KeyedStoreFastElementStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { r2, r1, r0 }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(KeyedStoreIC_MissFromStubFailure); +} + + void TransitionElementsKindStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { diff --git a/src/arm/lithium-codegen-arm.cc b/src/arm/lithium-codegen-arm.cc index 3ad86cf..8040791 100644 --- a/src/arm/lithium-codegen-arm.cc +++ b/src/arm/lithium-codegen-arm.cc @@ -239,7 +239,7 @@ bool LCodeGen::GeneratePrologue() { __ str(r0, target); // Update the write barrier. This clobbers r3 and r0. __ RecordWriteContextSlot( - cp, target.offset(), r0, r3, kLRHasBeenSaved, kSaveFPRegs); + cp, target.offset(), r0, r3, GetLinkRegisterState(), kSaveFPRegs); } } Comment(";;; End allocate local context"); @@ -3075,7 +3075,7 @@ void LCodeGen::DoStoreContextSlot(LStoreContextSlot* instr) { target.offset(), value, scratch, - kLRHasBeenSaved, + GetLinkRegisterState(), kSaveFPRegs, EMIT_REMEMBERED_SET, check_needed); @@ -4367,7 +4367,7 @@ void LCodeGen::DoStoreNamedField(LStoreNamedField* instr) { HeapObject::kMapOffset, scratch, temp, - kLRHasBeenSaved, + GetLinkRegisterState(), kSaveFPRegs, OMIT_REMEMBERED_SET, OMIT_SMI_CHECK); @@ -4386,7 +4386,7 @@ void LCodeGen::DoStoreNamedField(LStoreNamedField* instr) { offset, value, scratch, - kLRHasBeenSaved, + GetLinkRegisterState(), kSaveFPRegs, EMIT_REMEMBERED_SET, check_needed); @@ -4401,7 +4401,7 @@ void LCodeGen::DoStoreNamedField(LStoreNamedField* instr) { offset, value, object, - kLRHasBeenSaved, + GetLinkRegisterState(), kSaveFPRegs, EMIT_REMEMBERED_SET, check_needed); @@ -4602,7 +4602,7 @@ void LCodeGen::DoStoreKeyedFixedArray(LStoreKeyed* instr) { __ RecordWrite(elements, key, value, - kLRHasBeenSaved, + GetLinkRegisterState(), kSaveFPRegs, EMIT_REMEMBERED_SET, check_needed); @@ -4654,7 +4654,7 @@ void LCodeGen::DoTransitionElementsKind(LTransitionElementsKind* instr) { __ str(new_map_reg, FieldMemOperand(object_reg, HeapObject::kMapOffset)); // Write barrier. __ RecordWriteField(object_reg, HeapObject::kMapOffset, new_map_reg, - scratch, kLRHasBeenSaved, kDontSaveFPRegs); + scratch, GetLinkRegisterState(), kDontSaveFPRegs); } else if (FLAG_compiled_transitions) { PushSafepointRegistersScope scope(this, Safepoint::kWithRegisters); __ Move(r0, object_reg); diff --git a/src/arm/lithium-codegen-arm.h b/src/arm/lithium-codegen-arm.h index d1f712a..686241d 100644 --- a/src/arm/lithium-codegen-arm.h +++ b/src/arm/lithium-codegen-arm.h @@ -87,6 +87,10 @@ class LCodeGen BASE_EMBEDDED { return !NeedsEagerFrame() && info()->is_deferred_calling(); } + LinkRegisterStatus GetLinkRegisterState() const { + return frame_is_built_ ? kLRHasBeenSaved : kLRHasNotBeenSaved; + } + // Support for converting LOperands to assembler types. // LOperand must be a register. Register ToRegister(LOperand* op) const; diff --git a/src/ast.cc b/src/ast.cc index 712bfd1..a6984dd 100644 --- a/src/ast.cc +++ b/src/ast.cc @@ -128,7 +128,8 @@ Assignment::Assignment(Isolate* isolate, pos_(pos), binary_operation_(NULL), assignment_id_(GetNextId(isolate)), - is_monomorphic_(false) { } + is_monomorphic_(false), + store_mode_(STANDARD_STORE) { } Token::Value Assignment::binary_op() const { @@ -455,9 +456,11 @@ void Assignment::RecordTypeFeedback(TypeFeedbackOracle* oracle, } else if (is_monomorphic_) { // Record receiver type for monomorphic keyed stores. receiver_types_.Add(oracle->StoreMonomorphicReceiverType(id), zone); + store_mode_ = oracle->GetStoreMode(id); } else if (oracle->StoreIsPolymorphic(id)) { receiver_types_.Reserve(kMaxKeyedPolymorphism, zone); oracle->CollectKeyedReceiverTypes(id, &receiver_types_); + store_mode_ = oracle->GetStoreMode(id); } } @@ -475,6 +478,7 @@ void CountOperation::RecordTypeFeedback(TypeFeedbackOracle* oracle, receiver_types_.Reserve(kMaxKeyedPolymorphism, zone); oracle->CollectKeyedReceiverTypes(id, &receiver_types_); } + store_mode_ = oracle->GetStoreMode(id); } diff --git a/src/ast.h b/src/ast.h index 5debc74..88cd696 100644 --- a/src/ast.h +++ b/src/ast.h @@ -349,6 +349,10 @@ class Expression: public AstNode { ASSERT(types != NULL && types->length() == 1); return types->at(0); } + virtual KeyedAccessStoreMode GetStoreMode() { + UNREACHABLE(); + return STANDARD_STORE; + } BailoutId id() const { return id_; } TypeFeedbackId test_id() const { return test_id_; } @@ -1481,6 +1485,9 @@ class Property: public Expression { void RecordTypeFeedback(TypeFeedbackOracle* oracle, Zone* zone); virtual bool IsMonomorphic() { return is_monomorphic_; } virtual SmallMapList* GetReceiverTypes() { return &receiver_types_; } + virtual KeyedAccessStoreMode GetStoreMode() { + return STANDARD_STORE; + } bool IsArrayLength() { return is_array_length_; } bool IsUninitialized() { return is_uninitialized_; } TypeFeedbackId PropertyFeedbackId() { return reuse(id()); } @@ -1773,6 +1780,9 @@ class CountOperation: public Expression { void RecordTypeFeedback(TypeFeedbackOracle* oracle, Zone* znoe); virtual bool IsMonomorphic() { return is_monomorphic_; } virtual SmallMapList* GetReceiverTypes() { return &receiver_types_; } + virtual KeyedAccessStoreMode GetStoreMode() { + return store_mode_; + } BailoutId AssignmentId() const { return assignment_id_; } @@ -1788,6 +1798,8 @@ class CountOperation: public Expression { : Expression(isolate), op_(op), is_prefix_(is_prefix), + is_monomorphic_(false), + store_mode_(STANDARD_STORE), expression_(expr), pos_(pos), assignment_id_(GetNextId(isolate)), @@ -1795,8 +1807,9 @@ class CountOperation: public Expression { private: Token::Value op_; - bool is_prefix_; - bool is_monomorphic_; + bool is_prefix_ : 1; + bool is_monomorphic_ : 1; + KeyedAccessStoreMode store_mode_: 4; Expression* expression_; int pos_; const BailoutId assignment_id_; @@ -1909,6 +1922,9 @@ class Assignment: public Expression { void RecordTypeFeedback(TypeFeedbackOracle* oracle, Zone* zone); virtual bool IsMonomorphic() { return is_monomorphic_; } virtual SmallMapList* GetReceiverTypes() { return &receiver_types_; } + virtual KeyedAccessStoreMode GetStoreMode() { + return store_mode_; + } protected: Assignment(Isolate* isolate, @@ -1934,7 +1950,8 @@ class Assignment: public Expression { BinaryOperation* binary_operation_; const BailoutId assignment_id_; - bool is_monomorphic_; + bool is_monomorphic_ : 1; + KeyedAccessStoreMode store_mode_ : 4; SmallMapList receiver_types_; }; diff --git a/src/code-stubs-hydrogen.cc b/src/code-stubs-hydrogen.cc index ae198bc..6e61c78 100644 --- a/src/code-stubs-hydrogen.cc +++ b/src/code-stubs-hydrogen.cc @@ -111,6 +111,8 @@ bool CodeStubGraphBuilderBase::BuildGraph() { next_block->SetJoinId(BailoutId::StubEntry()); set_current_block(next_block); + start_environment->set_ast_id(BailoutId::StubEntry()); + HConstant* undefined_constant = new(zone) HConstant( isolate()->factory()->undefined_value(), Representation::Tagged()); AddInstruction(undefined_constant); @@ -227,8 +229,7 @@ HValue* CodeStubGraphBuilder::BuildCodeStub() { HInstruction* load = BuildUncheckedMonomorphicElementAccess( GetParameter(0), GetParameter(1), NULL, NULL, casted_stub()->is_js_array(), casted_stub()->elements_kind(), - false, Representation::Tagged()); - AddInstruction(load); + false, STANDARD_STORE, Representation::Tagged()); return load; } @@ -241,6 +242,25 @@ Handle KeyedLoadFastElementStub::GenerateCode() { template <> +HValue* CodeStubGraphBuilder::BuildCodeStub() { + BuildUncheckedMonomorphicElementAccess( + GetParameter(0), GetParameter(1), GetParameter(2), NULL, + casted_stub()->is_js_array(), casted_stub()->elements_kind(), + true, casted_stub()->store_mode(), Representation::Tagged()); + AddSimulate(BailoutId::StubEntry(), REMOVABLE_SIMULATE); + + return GetParameter(2); +} + + +Handle KeyedStoreFastElementStub::GenerateCode() { + CodeStubGraphBuilder builder(this); + LChunk* chunk = OptimizeGraph(builder.CreateGraph()); + return chunk->Codegen(Code::COMPILED_STUB); +} + + +template <> HValue* CodeStubGraphBuilder::BuildCodeStub() { Zone* zone = this->zone(); @@ -256,27 +276,16 @@ HValue* CodeStubGraphBuilder::BuildCodeStub() { js_array, HType::Smi())); - Heap* heap = isolate()->heap(); - const int kMinFreeNewSpaceAfterGC = - ((heap->InitialSemiSpaceSize() - sizeof(FixedArrayBase)) / 2) / - kDoubleSize; - - HConstant* max_alloc_size = - new(zone) HConstant(kMinFreeNewSpaceAfterGC, Representation::Integer32()); - AddInstruction(max_alloc_size); - // Since we're forcing Integer32 representation for this HBoundsCheck, - // there's no need to Smi-check the index. - AddInstruction( - new(zone) HBoundsCheck(array_length, max_alloc_size, - DONT_ALLOW_SMI_KEY, Representation::Integer32())); + ElementsKind to_kind = casted_stub()->to_kind(); + BuildNewSpaceArrayCheck(array_length, to_kind); IfBuilder if_builder(this, BailoutId::StubEntry()); - if_builder.BeginTrue(array_length, graph()->GetConstant0(), Token::EQ); + if_builder.BeginIf(array_length, graph()->GetConstant0(), Token::EQ); // Nothing to do, just change the map. - if_builder.BeginFalse(); + if_builder.BeginElse(); HInstruction* elements = AddInstruction(new(zone) HLoadElements(js_array, js_array)); @@ -284,37 +293,16 @@ HValue* CodeStubGraphBuilder::BuildCodeStub() { HInstruction* elements_length = AddInstruction(new(zone) HFixedArrayBaseLength(elements)); - ElementsKind to_kind = casted_stub()->to_kind(); HValue* new_elements = - BuildAllocateElements(context(), to_kind, elements_length); - - // Fast elements kinds need to be initialized in case statements below cause a - // garbage collection. - Factory* factory = isolate()->factory(); - - ASSERT(!IsFastSmiElementsKind(to_kind)); - double nan_double = FixedDoubleArray::hole_nan_as_double(); - HValue* hole = IsFastObjectElementsKind(to_kind) - ? AddInstruction(new(zone) HConstant(factory->the_hole_value(), - Representation::Tagged())) - : AddInstruction(new(zone) HConstant(nan_double, - Representation::Double())); - - LoopBuilder builder(this, context(), LoopBuilder::kPostIncrement, - BailoutId::StubEntry()); - - HValue* zero = graph()->GetConstant0(); - HValue* start = IsFastElementsKind(to_kind) ? zero : array_length; - HValue* key = builder.BeginBody(start, elements_length, Token::LT); - - AddInstruction(new(zone) HStoreKeyed(new_elements, key, hole, to_kind)); - AddSimulate(BailoutId::StubEntry(), REMOVABLE_SIMULATE); - - builder.EndBody(); + BuildAllocateElements(context(), to_kind, elements_length, + BailoutId::StubEntry()); BuildCopyElements(context(), elements, casted_stub()->from_kind(), new_elements, - to_kind, array_length); + to_kind, array_length, elements_length, + BailoutId::StubEntry()); + + Factory* factory = isolate()->factory(); AddInstruction(new(zone) HStoreNamedField(js_array, factory->elements_field_string(), diff --git a/src/code-stubs.h b/src/code-stubs.h index e91b241..6e95780 100644 --- a/src/code-stubs.h +++ b/src/code-stubs.h @@ -1312,6 +1312,47 @@ class KeyedLoadFastElementStub : public HydrogenCodeStub { }; +class KeyedStoreFastElementStub : public HydrogenCodeStub { + public: + KeyedStoreFastElementStub(bool is_js_array, + ElementsKind elements_kind, + KeyedAccessStoreMode mode) { + bit_field_ = ElementsKindBits::encode(elements_kind) | + IsJSArrayBits::encode(is_js_array) | + StoreModeBits::encode(mode); + } + + Major MajorKey() { return KeyedStoreElement; } + int MinorKey() { return bit_field_; } + + bool is_js_array() const { + return IsJSArrayBits::decode(bit_field_); + } + + ElementsKind elements_kind() const { + return ElementsKindBits::decode(bit_field_); + } + + KeyedAccessStoreMode store_mode() const { + return StoreModeBits::decode(bit_field_); + } + + virtual Handle GenerateCode(); + + virtual void InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor); + + private: + class ElementsKindBits: public BitField {}; + class StoreModeBits: public BitField {}; + class IsJSArrayBits: public BitField {}; + uint32_t bit_field_; + + DISALLOW_COPY_AND_ASSIGN(KeyedStoreFastElementStub); +}; + + class TransitionElementsKindStub : public HydrogenCodeStub { public: TransitionElementsKindStub(ElementsKind from_kind, diff --git a/src/flag-definitions.h b/src/flag-definitions.h index c4b560b..dbb3f7f 100644 --- a/src/flag-definitions.h +++ b/src/flag-definitions.h @@ -163,6 +163,8 @@ DEFINE_bool(packed_arrays, true, "optimizes arrays that have no holes") DEFINE_bool(smi_only_arrays, true, "tracks arrays with only smi values") DEFINE_bool(compiled_transitions, false, "use optimizing compiler to " "generate array elements transition stubs") +DEFINE_bool(compiled_keyed_stores, false, "use optimizing compiler to " + "generate keyed store stubs") DEFINE_bool(clever_optimizations, true, "Optimize object size, Array shift, DOM strings and string +") diff --git a/src/heap.h b/src/heap.h index d26c384..90f2e60 100644 --- a/src/heap.h +++ b/src/heap.h @@ -520,6 +520,7 @@ class Heap { int InitialSemiSpaceSize() { return initial_semispace_size_; } intptr_t MaxOldGenerationSize() { return max_old_generation_size_; } intptr_t MaxExecutableSize() { return max_executable_size_; } + int MaxNewSpaceAllocationSize() { return InitialSemiSpaceSize() * 3/4; } // Returns the capacity of the heap in bytes w/o growing. Heap grows when // more spaces are needed until it reaches the limit. diff --git a/src/hydrogen.cc b/src/hydrogen.cc index 14feb5f..6fac302 100644 --- a/src/hydrogen.cc +++ b/src/hydrogen.cc @@ -114,6 +114,8 @@ void HBasicBlock::AddInstruction(HInstruction* instr) { ASSERT(!instr->IsLinked()); ASSERT(!IsFinished()); if (first_ == NULL) { + ASSERT(last_environment() != NULL); + ASSERT(!last_environment()->ast_id().IsNone()); HBlockEntry* entry = new(zone()) HBlockEntry(); entry->InitializeAsFirst(this); first_ = last_ = entry; @@ -231,6 +233,7 @@ void HBasicBlock::SetJoinId(BailoutId ast_id) { predecessor->last_environment()->closure()->shared() ->VerifyBailoutId(ast_id))); simulate->set_ast_id(ast_id); + predecessor->last_environment()->set_ast_id(ast_id); } } @@ -642,37 +645,51 @@ HGraphBuilder::CheckBuilder::CheckBuilder(HGraphBuilder* builder, BailoutId id) finished_(false), id_(id) { HEnvironment* env = builder->environment(); - failure_block_ = builder->CreateBasicBlock(env->Copy()); - merge_block_ = builder->CreateBasicBlock(env->Copy()); + failure_block_ = builder->CreateBasicBlock(env->CopyWithoutHistory()); + merge_block_ = builder->CreateBasicBlock(env->CopyWithoutHistory()); } -void HGraphBuilder::CheckBuilder::CheckNotUndefined(HValue* value) { +HValue* HGraphBuilder::CheckBuilder::CheckNotUndefined(HValue* value) { HEnvironment* env = builder_->environment(); HIsNilAndBranch* compare = new(zone()) HIsNilAndBranch(value, kStrictEquality, kUndefinedValue); - HBasicBlock* success_block = builder_->CreateBasicBlock(env->Copy()); - HBasicBlock* failure_block = builder_->CreateBasicBlock(env->Copy()); + HBasicBlock* success_block = + builder_->CreateBasicBlock(env->CopyWithoutHistory()); + HBasicBlock* failure_block = + builder_->CreateBasicBlock(env->CopyWithoutHistory()); compare->SetSuccessorAt(0, failure_block); compare->SetSuccessorAt(1, success_block); failure_block->Goto(failure_block_); builder_->current_block()->Finish(compare); builder_->set_current_block(success_block); + return compare; } -void HGraphBuilder::CheckBuilder::CheckIntegerEq(HValue* left, HValue* right) { +HValue* HGraphBuilder::CheckBuilder::CheckIntegerCompare(HValue* left, + HValue* right, + Token::Value op) { HEnvironment* env = builder_->environment(); HCompareIDAndBranch* compare = - new(zone()) HCompareIDAndBranch(left, right, Token::EQ); + new(zone()) HCompareIDAndBranch(left, right, op); compare->AssumeRepresentation(Representation::Integer32()); - HBasicBlock* success_block = builder_->CreateBasicBlock(env->Copy()); - HBasicBlock* failure_block = builder_->CreateBasicBlock(env->Copy()); + HBasicBlock* success_block = + builder_->CreateBasicBlock(env->CopyWithoutHistory()); + HBasicBlock* failure_block = + builder_->CreateBasicBlock(env->CopyWithoutHistory()); compare->SetSuccessorAt(0, success_block); compare->SetSuccessorAt(1, failure_block); failure_block->Goto(failure_block_); builder_->current_block()->Finish(compare); builder_->set_current_block(success_block); + return compare; +} + + +HValue* HGraphBuilder::CheckBuilder::CheckIntegerEq(HValue* left, + HValue* right) { + return CheckIntegerCompare(left, right, Token::EQ); } @@ -695,6 +712,7 @@ HConstant* HGraph::GetInvalidContext() { HGraphBuilder::IfBuilder::IfBuilder(HGraphBuilder* builder, BailoutId id) : builder_(builder), finished_(false), + did_else_(false), id_(id) { HEnvironment* env = builder->environment(); first_true_block_ = builder->CreateBasicBlock(env->Copy()); @@ -703,7 +721,7 @@ HGraphBuilder::IfBuilder::IfBuilder(HGraphBuilder* builder, BailoutId id) } -HInstruction* HGraphBuilder::IfBuilder::BeginTrue( +HInstruction* HGraphBuilder::IfBuilder::BeginIf( HValue* left, HValue* right, Token::Value token, @@ -721,20 +739,45 @@ HInstruction* HGraphBuilder::IfBuilder::BeginTrue( } -void HGraphBuilder::IfBuilder::BeginFalse() { +HInstruction* HGraphBuilder::IfBuilder::BeginIfObjectsEqual( + HValue* left, + HValue* right) { + HCompareObjectEqAndBranch* compare = + new(zone()) HCompareObjectEqAndBranch(left, right); + compare->SetSuccessorAt(0, first_true_block_); + compare->SetSuccessorAt(1, first_false_block_); + builder_->current_block()->Finish(compare); + builder_->set_current_block(first_true_block_); + return compare; +} + + +HInstruction* HGraphBuilder::IfBuilder::BeginIfMapEquals(HValue* value, + Handle map) { + HCompareMap* compare = new(zone()) + HCompareMap(value, map, first_true_block_, first_false_block_); + builder_->current_block()->Finish(compare); + builder_->set_current_block(first_true_block_); + return compare; +} + + +void HGraphBuilder::IfBuilder::BeginElse() { last_true_block_ = builder_->current_block(); ASSERT(!last_true_block_->IsFinished()); builder_->set_current_block(first_false_block_); + did_else_ = true; } void HGraphBuilder::IfBuilder::End() { ASSERT(!finished_); + if (!did_else_) BeginElse(); ASSERT(!last_true_block_->IsFinished()); HBasicBlock* last_false_block = builder_->current_block(); ASSERT(!last_false_block->IsFinished()); HEnvironment* merge_env = - last_true_block_->last_environment()->Copy(); + last_true_block_->last_environment()->CopyWithoutHistory(); merge_block_ = builder_->CreateBasicBlock(merge_env); last_true_block_->Goto(merge_block_); last_false_block->Goto(merge_block_); @@ -856,6 +899,7 @@ void HGraphBuilder::AddSimulate(BailoutId id, RemovableSimulate removable) { ASSERT(current_block() != NULL); current_block()->AddSimulate(id, removable); + environment()->set_ast_id(id); } @@ -965,7 +1009,8 @@ HInstruction* HGraphBuilder::BuildFastElementAccess( HValue* val, HValue* load_dependency, ElementsKind elements_kind, - bool is_store) { + bool is_store, + KeyedAccessStoreMode store_mode) { Zone* zone = this->zone(); if (is_store) { ASSERT(val != NULL); @@ -993,6 +1038,100 @@ HInstruction* HGraphBuilder::BuildFastElementAccess( } +HValue* HGraphBuilder::BuildCheckForCapacityGrow(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length, + HValue* key, + bool is_js_array) { + BailoutId ast_id = environment()->ast_id(); + Zone* zone = this->zone(); + IfBuilder length_checker(this, ast_id); + + length_checker.BeginIf(length, key, Token::EQ); + + HValue* current_capacity = + AddInstruction(new(zone) HFixedArrayBaseLength(elements)); + + IfBuilder capacity_checker(this, ast_id); + + capacity_checker.BeginIf(length, current_capacity, Token::EQ); + + HValue* context = environment()->LookupContext(); + + HValue* new_capacity = + BuildNewElementsCapacity(context, current_capacity); + + HValue* new_elements = BuildGrowElementsCapacity(object, elements, + kind, length, + new_capacity, ast_id); + + environment()->Push(new_elements); + capacity_checker.BeginElse(); + + environment()->Push(elements); + capacity_checker.End(); + + if (is_js_array) { + HValue* new_length = AddInstruction( + HAdd::New(zone, context, length, graph_->GetConstant1())); + new_length->ChangeRepresentation(Representation::Integer32()); + new_length->ClearFlag(HValue::kCanOverflow); + AddSimulate(ast_id, REMOVABLE_SIMULATE); + + Factory* factory = isolate()->factory(); + HInstruction* length_store = AddInstruction(new(zone) HStoreNamedField( + object, + factory->length_field_string(), + new_length, true, + JSArray::kLengthOffset)); + length_store->SetGVNFlag(kChangesArrayLengths); + AddSimulate(ast_id, REMOVABLE_SIMULATE); + } + + length_checker.BeginElse(); + + AddBoundsCheck(key, length, ALLOW_SMI_KEY); + environment()->Push(elements); + + length_checker.End(); + + return environment()->Pop(); +} + + +HValue* HGraphBuilder::BuildCopyElementsOnWrite(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length) { + BailoutId ast_id = environment()->ast_id(); + Zone* zone = this->zone(); + Heap* heap = isolate()->heap(); + + IfBuilder cow_checker(this, ast_id); + + cow_checker.BeginIfMapEquals(elements, + Handle(heap->fixed_cow_array_map())); + + HValue* capacity = + AddInstruction(new(zone) HFixedArrayBaseLength(elements)); + + HValue* new_elements = BuildGrowElementsCapacity(object, elements, + kind, length, + capacity, ast_id); + + environment()->Push(new_elements); + + cow_checker.BeginElse(); + + environment()->Push(elements); + + cow_checker.End(); + + return environment()->Pop(); +} + + HInstruction* HGraphBuilder::BuildUncheckedMonomorphicElementAccess( HValue* object, HValue* key, @@ -1001,7 +1140,9 @@ HInstruction* HGraphBuilder::BuildUncheckedMonomorphicElementAccess( bool is_js_array, ElementsKind elements_kind, bool is_store, + KeyedAccessStoreMode store_mode, Representation checked_index_representation) { + ASSERT(!IsExternalArrayElementsKind(elements_kind) || !is_js_array); Zone* zone = this->zone(); // No GVNFlag is necessary for ElementsKind if there is an explicit dependency // on a HElementsTransition instruction. The flag can also be removed if the @@ -1017,46 +1158,93 @@ HInstruction* HGraphBuilder::BuildUncheckedMonomorphicElementAccess( } bool fast_smi_only_elements = IsFastSmiElementsKind(elements_kind); bool fast_elements = IsFastObjectElementsKind(elements_kind); - HInstruction* elements = + HValue* elements = AddInstruction(new(zone) HLoadElements(object, mapcheck)); - if (is_store && (fast_elements || fast_smi_only_elements)) { + if (is_store && (fast_elements || fast_smi_only_elements) && + store_mode != STORE_NO_TRANSITION_HANDLE_COW) { HCheckMaps* check_cow_map = new(zone) HCheckMaps( elements, isolate()->factory()->fixed_array_map(), zone); check_cow_map->ClearGVNFlag(kDependsOnElementsKind); AddInstruction(check_cow_map); } HInstruction* length = NULL; - HInstruction* checked_key = NULL; - if (IsExternalArrayElementsKind(elements_kind)) { + if (is_js_array) { + length = AddInstruction(new(zone) HJSArrayLength(object, mapcheck, + HType::Smi())); + } else { length = AddInstruction(new(zone) HFixedArrayBaseLength(elements)); - checked_key = AddBoundsCheck( - key, length, ALLOW_SMI_KEY, checked_index_representation); - HLoadExternalArrayPointer* external_elements = - new(zone) HLoadExternalArrayPointer(elements); - AddInstruction(external_elements); - return BuildExternalArrayElementAccess( - external_elements, checked_key, val, mapcheck, - elements_kind, is_store); + } + HValue* checked_key = NULL; + if (IsExternalArrayElementsKind(elements_kind)) { + if (store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS) { + HLoadExternalArrayPointer* external_elements = + new(zone) HLoadExternalArrayPointer(elements); + AddInstruction(external_elements); + BailoutId previous_id = environment()->ast_id(); + ASSERT(!previous_id.IsNone()); + IfBuilder length_checker(this, previous_id); + length_checker.BeginIf(key, length, Token::LT); + CheckBuilder negative_checker(this, previous_id); + HValue* bounds_check = negative_checker.CheckIntegerCompare( + key, graph()->GetConstant0(), Token::GTE); + negative_checker.End(); + HInstruction* result = BuildExternalArrayElementAccess( + external_elements, key, val, bounds_check, + elements_kind, is_store); + AddInstruction(result); + length_checker.End(); + return result; + } else { + ASSERT(store_mode == STANDARD_STORE); + checked_key = AddBoundsCheck( + key, length, ALLOW_SMI_KEY, checked_index_representation); + HLoadExternalArrayPointer* external_elements = + new(zone) HLoadExternalArrayPointer(elements); + AddInstruction(external_elements); + return AddInstruction(BuildExternalArrayElementAccess( + external_elements, checked_key, val, mapcheck, + elements_kind, is_store)); + } } ASSERT(fast_smi_only_elements || fast_elements || IsFastDoubleElementsKind(elements_kind)); - if (is_js_array) { - length = AddInstruction(new(zone) HJSArrayLength(object, mapcheck, - HType::Smi())); + + if (is_store && IsFastSmiElementsKind(elements_kind) && + !val->type().IsSmi()) { + AddInstruction(new(zone) HCheckSmi(val)); + } + + if (IsGrowStoreMode(store_mode)) { + elements = BuildCheckForCapacityGrow(object, elements, elements_kind, + length, key, is_js_array); + checked_key = key; } else { - length = AddInstruction(new(zone) HFixedArrayBaseLength(elements)); + checked_key = AddBoundsCheck( + key, length, ALLOW_SMI_KEY, checked_index_representation); + + if (is_store && (fast_elements || fast_smi_only_elements)) { + if (store_mode == STORE_NO_TRANSITION_HANDLE_COW) { + elements = BuildCopyElementsOnWrite(object, elements, elements_kind, + length); + } else { + HCheckMaps* check_cow_map = new(zone) HCheckMaps( + elements, isolate()->factory()->fixed_array_map(), zone); + check_cow_map->ClearGVNFlag(kDependsOnElementsKind); + AddInstruction(check_cow_map); + } + } } - checked_key = AddBoundsCheck( - key, length, ALLOW_SMI_KEY, checked_index_representation); - return BuildFastElementAccess(elements, checked_key, val, mapcheck, - elements_kind, is_store); + return AddInstruction( + BuildFastElementAccess(elements, checked_key, val, mapcheck, + elements_kind, is_store, store_mode)); } -HValue* HGraphBuilder::BuildAllocateElements(HContext* context, +HValue* HGraphBuilder::BuildAllocateElements(HValue* context, ElementsKind kind, - HValue* capacity) { + HValue* capacity, + BailoutId ast_id) { Zone* zone = this->zone(); int elements_size = IsFastDoubleElementsKind(kind) @@ -1096,14 +1284,14 @@ HValue* HGraphBuilder::BuildAllocateElements(HContext* context, Handle map = IsFastDoubleElementsKind(kind) ? factory->fixed_double_array_map() : factory->fixed_array_map(); - BuildStoreMap(elements, map, BailoutId::StubEntry()); + BuildStoreMap(elements, map, ast_id); Handle fixed_array_length_field_name = factory->length_field_string(); HInstruction* store_length = new(zone) HStoreNamedField(elements, fixed_array_length_field_name, capacity, true, FixedArray::kLengthOffset); AddInstruction(store_length); - AddSimulate(BailoutId::StubEntry(), FIXED_SIMULATE); + AddSimulate(ast_id, REMOVABLE_SIMULATE); return elements; } @@ -1120,7 +1308,7 @@ HInstruction* HGraphBuilder::BuildStoreMap(HValue* object, true, JSObject::kMapOffset); store_map->SetGVNFlag(kChangesMaps); AddInstruction(store_map); - AddSimulate(id, FIXED_SIMULATE); + AddSimulate(id, REMOVABLE_SIMULATE); return store_map; } @@ -1135,17 +1323,132 @@ HInstruction* HGraphBuilder::BuildStoreMap(HValue* object, } -void HGraphBuilder::BuildCopyElements(HContext* context, +HValue* HGraphBuilder::BuildNewElementsCapacity(HValue* context, + HValue* old_capacity) { + Zone* zone = this->zone(); + HValue* half_old_capacity = + AddInstruction(HShr::New(zone, context, old_capacity, + graph_->GetConstant1())); + half_old_capacity->ChangeRepresentation(Representation::Integer32()); + half_old_capacity->ClearFlag(HValue::kCanOverflow); + + HValue* new_capacity = AddInstruction( + HAdd::New(zone, context, half_old_capacity, old_capacity)); + new_capacity->ChangeRepresentation(Representation::Integer32()); + new_capacity->ClearFlag(HValue::kCanOverflow); + + HValue* min_growth = + AddInstruction(new(zone) HConstant(16, Representation::Integer32())); + + new_capacity = AddInstruction( + HAdd::New(zone, context, new_capacity, min_growth)); + new_capacity->ChangeRepresentation(Representation::Integer32()); + new_capacity->ClearFlag(HValue::kCanOverflow); + + return new_capacity; +} + + +void HGraphBuilder::BuildNewSpaceArrayCheck(HValue* length, ElementsKind kind) { + Zone* zone = this->zone(); + Heap* heap = isolate()->heap(); + int element_size = IsFastDoubleElementsKind(kind) ? kDoubleSize + : kPointerSize; + int max_size = heap->MaxNewSpaceAllocationSize() / element_size; + max_size -= JSArray::kSize / element_size; + HConstant* max_size_constant = + new(zone) HConstant(max_size, Representation::Integer32()); + AddInstruction(max_size_constant); + // Since we're forcing Integer32 representation for this HBoundsCheck, + // there's no need to Smi-check the index. + AddInstruction(new(zone) + HBoundsCheck(length, max_size_constant, + DONT_ALLOW_SMI_KEY, Representation::Integer32())); +} + + +HValue* HGraphBuilder::BuildGrowElementsCapacity(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length, + HValue* new_capacity, + BailoutId ast_id) { + Zone* zone = this->zone(); + HValue* context = environment()->LookupContext(); + + BuildNewSpaceArrayCheck(new_capacity, kind); + + HValue* new_elements = + BuildAllocateElements(context, kind, new_capacity, ast_id); + + BuildCopyElements(context, elements, kind, + new_elements, kind, + length, new_capacity, ast_id); + + Factory* factory = isolate()->factory(); + HInstruction* elements_store = AddInstruction(new(zone) HStoreNamedField( + object, + factory->elements_field_string(), + new_elements, true, + JSArray::kElementsOffset)); + elements_store->SetGVNFlag(kChangesElementsPointer); + + return new_elements; +} + + +void HGraphBuilder::BuildFillElementsWithHole(HValue* context, + HValue* elements, + ElementsKind elements_kind, + HValue* from, + HValue* to, + BailoutId ast_id) { + // Fast elements kinds need to be initialized in case statements below cause + // a garbage collection. + Factory* factory = isolate()->factory(); + + double nan_double = FixedDoubleArray::hole_nan_as_double(); + Zone* zone = this->zone(); + HValue* hole = IsFastSmiOrObjectElementsKind(elements_kind) + ? AddInstruction(new(zone) HConstant(factory->the_hole_value(), + Representation::Tagged())) + : AddInstruction(new(zone) HConstant(nan_double, + Representation::Double())); + + LoopBuilder builder(this, context, LoopBuilder::kPostIncrement, ast_id); + + HValue* key = builder.BeginBody(from, to, Token::LT); + + AddInstruction(new(zone) HStoreKeyed(elements, key, hole, elements_kind)); + AddSimulate(ast_id, REMOVABLE_SIMULATE); + + builder.EndBody(); +} + + +void HGraphBuilder::BuildCopyElements(HValue* context, HValue* from_elements, ElementsKind from_elements_kind, HValue* to_elements, ElementsKind to_elements_kind, - HValue* length) { - LoopBuilder builder(this, context, LoopBuilder::kPostIncrement, - BailoutId::StubEntry()); + HValue* length, + HValue* capacity, + BailoutId ast_id) { + bool pre_fill_with_holes = + IsFastDoubleElementsKind(from_elements_kind) && + IsFastObjectElementsKind(to_elements_kind); + + if (pre_fill_with_holes) { + // If the copy might trigger a GC, make sure that the FixedArray is + // pre-initialized with holes to make sure that it's always in a consistent + // state. + BuildFillElementsWithHole(context, to_elements, to_elements_kind, + graph()->GetConstant0(), capacity, ast_id); + } - HValue* key = builder.BeginBody(graph()->GetConstant0(), - length, Token::LT); + LoopBuilder builder(this, context, LoopBuilder::kPostIncrement, ast_id); + + HValue* key = builder.BeginBody(graph()->GetConstant0(), length, Token::LT); HValue* element = AddInstruction(new(zone()) HLoadKeyed(from_elements, key, NULL, @@ -1154,9 +1457,15 @@ void HGraphBuilder::BuildCopyElements(HContext* context, AddInstruction(new(zone()) HStoreKeyed(to_elements, key, element, to_elements_kind)); - AddSimulate(BailoutId::StubEntry(), REMOVABLE_SIMULATE); + AddSimulate(ast_id, REMOVABLE_SIMULATE); builder.EndBody(); + + if (!pre_fill_with_holes && length != capacity) { + // Fill unused capacity with the hole. + BuildFillElementsWithHole(context, to_elements, to_elements_kind, + key, capacity, ast_id); + } } @@ -6806,7 +7115,8 @@ HInstruction* HOptimizedGraphBuilder::BuildMonomorphicElementAccess( HValue* val, HValue* dependency, Handle map, - bool is_store) { + bool is_store, + KeyedAccessStoreMode store_mode) { HCheckMaps* mapcheck = new(zone()) HCheckMaps(object, map, zone(), dependency); AddInstruction(mapcheck); @@ -6816,7 +7126,7 @@ HInstruction* HOptimizedGraphBuilder::BuildMonomorphicElementAccess( return BuildUncheckedMonomorphicElementAccess( object, key, val, mapcheck, map->instance_type() == JS_ARRAY_TYPE, - map->elements_kind(), is_store); + map->elements_kind(), is_store, store_mode); } @@ -6871,7 +7181,7 @@ HInstruction* HOptimizedGraphBuilder::TryBuildConsolidatedElementLoad( object, key, val, check_maps, most_general_consolidated_map->instance_type() == JS_ARRAY_TYPE, most_general_consolidated_map->elements_kind(), - false); + false, STANDARD_STORE); return instr; } @@ -6884,6 +7194,7 @@ HValue* HOptimizedGraphBuilder::HandlePolymorphicElementAccess( BailoutId ast_id, int position, bool is_store, + KeyedAccessStoreMode store_mode, bool* has_side_effects) { *has_side_effects = false; AddInstruction(new(zone()) HCheckNonSmi(object)); @@ -6894,7 +7205,6 @@ HValue* HOptimizedGraphBuilder::HandlePolymorphicElementAccess( HInstruction* consolidated_load = TryBuildConsolidatedElementLoad(object, key, val, maps); if (consolidated_load != NULL) { - AddInstruction(consolidated_load); *has_side_effects |= consolidated_load->HasObservableSideEffects(); if (position != RelocInfo::kNoPosition) { consolidated_load->set_position(position); @@ -6961,8 +7271,9 @@ HValue* HOptimizedGraphBuilder::HandlePolymorphicElementAccess( instr = AddInstruction(is_store ? BuildStoreKeyedGeneric(object, key, val) : BuildLoadKeyedGeneric(object, key)); } else { - instr = AddInstruction(BuildMonomorphicElementAccess( - object, key, val, transition, untransitionable_map, is_store)); + instr = BuildMonomorphicElementAccess( + object, key, val, transition, untransitionable_map, is_store, + store_mode); } *has_side_effects |= instr->HasObservableSideEffects(); if (position != RelocInfo::kNoPosition) instr->set_position(position); @@ -7043,7 +7354,7 @@ HValue* HOptimizedGraphBuilder::HandlePolymorphicElementAccess( checked_key = AddBoundsCheck(key, length, ALLOW_SMI_KEY); access = AddInstruction(BuildFastElementAccess( elements, checked_key, val, elements_kind_branch, - elements_kind, is_store)); + elements_kind, is_store, STANDARD_STORE)); if (!is_store) { Push(access); } @@ -7059,7 +7370,7 @@ HValue* HOptimizedGraphBuilder::HandlePolymorphicElementAccess( checked_key = AddBoundsCheck(key, length, ALLOW_SMI_KEY); access = AddInstruction(BuildFastElementAccess( elements, checked_key, val, elements_kind_branch, - elements_kind, is_store)); + elements_kind, is_store, STANDARD_STORE)); } else if (elements_kind == DICTIONARY_ELEMENTS) { if (is_store) { access = AddInstruction(BuildStoreKeyedGeneric(object, key, val)); @@ -7105,23 +7416,26 @@ HValue* HOptimizedGraphBuilder::HandleKeyedElementAccess( if (map->has_slow_elements_kind()) { instr = is_store ? BuildStoreKeyedGeneric(obj, key, val) : BuildLoadKeyedGeneric(obj, key); + AddInstruction(instr); } else { AddInstruction(new(zone()) HCheckNonSmi(obj)); - instr = BuildMonomorphicElementAccess(obj, key, val, NULL, map, is_store); + instr = BuildMonomorphicElementAccess( + obj, key, val, NULL, map, is_store, expr->GetStoreMode()); } } else if (expr->GetReceiverTypes() != NULL && !expr->GetReceiverTypes()->is_empty()) { return HandlePolymorphicElementAccess( - obj, key, val, expr, ast_id, position, is_store, has_side_effects); + obj, key, val, expr, ast_id, position, is_store, + expr->GetStoreMode(), has_side_effects); } else { if (is_store) { instr = BuildStoreKeyedGeneric(obj, key, val); } else { instr = BuildLoadKeyedGeneric(obj, key); } + AddInstruction(instr); } if (position != RelocInfo::kNoPosition) instr->set_position(position); - AddInstruction(instr); *has_side_effects = instr->HasObservableSideEffects(); return instr; } diff --git a/src/hydrogen.h b/src/hydrogen.h index 7f5326b..0072c77 100644 --- a/src/hydrogen.h +++ b/src/hydrogen.h @@ -909,7 +909,20 @@ class HGraphBuilder { HValue* val, HValue* dependency, ElementsKind elements_kind, - bool is_store); + bool is_store, + KeyedAccessStoreMode store_mode); + + HValue* BuildCheckForCapacityGrow(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length, + HValue* key, + bool is_js_array); + + HValue* BuildCopyElementsOnWrite(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length); HInstruction* BuildUncheckedMonomorphicElementAccess( HValue* object, @@ -919,6 +932,7 @@ class HGraphBuilder { bool is_js_array, ElementsKind elements_kind, bool is_store, + KeyedAccessStoreMode store_mode, Representation checked_index_representation = Representation::None()); HInstruction* BuildStoreMap(HValue* object, HValue* map, BailoutId id); @@ -931,8 +945,9 @@ class HGraphBuilder { if (!finished_) End(); } - void CheckNotUndefined(HValue* value); - void CheckIntegerEq(HValue* left, HValue* right); + HValue* CheckNotUndefined(HValue* value); + HValue* CheckIntegerCompare(HValue* left, HValue* right, Token::Value op); + HValue* CheckIntegerEq(HValue* left, HValue* right); void End(); private: @@ -952,12 +967,14 @@ class HGraphBuilder { if (!finished_) End(); } - HInstruction* BeginTrue( + HInstruction* BeginIf( HValue* left, HValue* right, Token::Value token, Representation input_representation = Representation::Integer32()); - void BeginFalse(); + HInstruction* BeginIfObjectsEqual(HValue* left, HValue* right); + HInstruction* BeginIfMapEquals(HValue* value, Handle map); + void BeginElse(); void End(); private: @@ -965,6 +982,7 @@ class HGraphBuilder { HGraphBuilder* builder_; bool finished_; + bool did_else_; HBasicBlock* first_true_block_; HBasicBlock* last_true_block_; HBasicBlock* first_false_block_; @@ -1011,16 +1029,39 @@ class HGraphBuilder { bool finished_; }; - HValue* BuildAllocateElements(HContext* context, - ElementsKind kind, - HValue* capacity); + HValue* BuildNewElementsCapacity(HValue* context, + HValue* old_capacity); + + void BuildNewSpaceArrayCheck(HValue* length, + ElementsKind kind); - void BuildCopyElements(HContext* context, + HValue* BuildAllocateElements(HValue* context, + ElementsKind kind, + HValue* capacity, + BailoutId ast_id); + + HValue* BuildGrowElementsCapacity(HValue* object, + HValue* elements, + ElementsKind kind, + HValue* length, + HValue* new_capacity, + BailoutId ast_id); + + void BuildFillElementsWithHole(HValue* context, + HValue* elements, + ElementsKind elements_kind, + HValue* from, + HValue* to, + BailoutId ast_id); + + void BuildCopyElements(HValue* context, HValue* from_elements, ElementsKind from_elements_kind, HValue* to_elements, ElementsKind to_elements_kind, - HValue* length); + HValue* length, + HValue* capacity, + BailoutId ast_id); private: HGraphBuilder(); @@ -1348,7 +1389,8 @@ class HOptimizedGraphBuilder: public HGraphBuilder, public AstVisitor { HValue* val, HValue* dependency, Handle map, - bool is_store); + bool is_store, + KeyedAccessStoreMode store_mode); HValue* HandlePolymorphicElementAccess(HValue* object, HValue* key, @@ -1357,6 +1399,7 @@ class HOptimizedGraphBuilder: public HGraphBuilder, public AstVisitor { BailoutId ast_id, int position, bool is_store, + KeyedAccessStoreMode store_mode, bool* has_side_effects); HValue* HandleKeyedElementAccess(HValue* obj, diff --git a/src/ia32/code-stubs-ia32.cc b/src/ia32/code-stubs-ia32.cc index 88207c4..b30d519 100644 --- a/src/ia32/code-stubs-ia32.cc +++ b/src/ia32/code-stubs-ia32.cc @@ -67,6 +67,17 @@ void KeyedLoadFastElementStub::InitializeInterfaceDescriptor( } +void KeyedStoreFastElementStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { edx, ecx, eax }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(KeyedStoreIC_MissFromStubFailure); +} + + void TransitionElementsKindStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { diff --git a/src/ic.cc b/src/ic.cc index da2211b..2c3a488 100644 --- a/src/ic.cc +++ b/src/ic.cc @@ -1355,7 +1355,8 @@ MaybeObject* KeyedLoadIC::Load(State state, stub = non_strict_arguments_stub(); } else if (receiver->HasIndexedInterceptor()) { stub = indexed_interceptor_stub(); - } else if (key->IsSmi() && (target() != *non_strict_arguments_stub())) { + } else if (!key->ToSmi()->IsFailure() && + (target() != *non_strict_arguments_stub())) { stub = LoadElementStub(receiver); } } @@ -1649,7 +1650,8 @@ Handle KeyedStoreIC::StoreElementStub(Handle receiver, return strict_mode == kStrictMode ? generic_stub_strict() : generic_stub(); } - if ((store_mode == STORE_NO_TRANSITION_HANDLE_COW || + if (!FLAG_compiled_keyed_stores && + (store_mode == STORE_NO_TRANSITION_HANDLE_COW || store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS)) { // TODO(danno): We'll soon handle MONOMORPHIC ICs that also support // copying COW arrays and silently ignoring some OOB stores into external @@ -1712,9 +1714,12 @@ Handle KeyedStoreIC::StoreElementStub(Handle receiver, return isolate()->stub_cache()->ComputeKeyedStoreElement( transitioned_receiver_map, strict_mode, store_mode); } else if (*previous_receiver_map == receiver->map()) { - if (IsGrowStoreMode(store_mode)) { + if (IsGrowStoreMode(store_mode) || + store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS || + store_mode == STORE_NO_TRANSITION_HANDLE_COW) { // A "normal" IC that handles stores can switch to a version that can - // grow at the end of the array and still stay MONOMORPHIC. + // grow at the end of the array, handle OOB accesses or copy COW arrays + // and still stay MONOMORPHIC. return isolate()->stub_cache()->ComputeKeyedStoreElement( receiver_map, strict_mode, store_mode); } @@ -1813,8 +1818,10 @@ bool IsOutOfBoundsAccess(Handle receiver, KeyedAccessStoreMode KeyedStoreIC::GetStoreMode(Handle receiver, Handle key, Handle value) { - ASSERT(key->IsSmi()); - int index = Smi::cast(*key)->value(); + ASSERT(!key->ToSmi()->IsFailure()); + Smi* smi_key = NULL; + key->ToSmi()->To(&smi_key); + int index = smi_key->value(); bool oob_access = IsOutOfBoundsAccess(receiver, index); bool allow_growth = receiver->IsJSArray() && oob_access; if (allow_growth) { @@ -1872,6 +1879,10 @@ KeyedAccessStoreMode KeyedStoreIC::GetStoreMode(Handle receiver, if (!FLAG_trace_external_array_abuse && receiver->map()->has_external_array_elements() && oob_access) { return STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS; + } + Heap* heap = receiver->GetHeap(); + if (receiver->elements()->map() == heap->fixed_cow_array_map()) { + return STORE_NO_TRANSITION_HANDLE_COW; } else { return STANDARD_STORE; } @@ -1910,13 +1921,20 @@ MaybeObject* KeyedStoreIC::Store(State state, if (miss_mode != MISS_FORCE_GENERIC) { if (object->IsJSObject()) { Handle receiver = Handle::cast(object); + bool key_is_smi_like = key->IsSmi() || + (FLAG_compiled_keyed_stores && !key->ToSmi()->IsFailure()); if (receiver->elements()->map() == isolate()->heap()->non_strict_arguments_elements_map()) { stub = non_strict_arguments_stub(); - } else if (key->IsSmi() && (target() != *non_strict_arguments_stub())) { + } else if (key_is_smi_like && + (target() != *non_strict_arguments_stub())) { KeyedAccessStoreMode store_mode = GetStoreMode(receiver, key, value); stub = StoreElementStub(receiver, store_mode, strict_mode); + } else { + TRACE_GENERIC_IC(isolate(), "KeyedStoreIC", "key not a number"); } + } else { + TRACE_GENERIC_IC(isolate(), "KeyedStoreIC", "not an object"); } } else { TRACE_GENERIC_IC(isolate(), "KeyedStoreIC", "force generic"); @@ -2074,7 +2092,7 @@ RUNTIME_FUNCTION(MaybeObject*, KeyedLoadIC_MissForceGeneric) { RUNTIME_FUNCTION(MaybeObject*, StoreIC_Miss) { HandleScope scope(isolate); ASSERT(args.length() == 3); - StoreIC ic(isolate); + StoreIC ic(IC::NO_EXTRA_FRAME, isolate); IC::State state = IC::StateFrom(ic.target(), args[0], args[1]); Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state(); return ic.Store(state, @@ -2150,7 +2168,22 @@ RUNTIME_FUNCTION(MaybeObject*, SharedStoreIC_ExtendStorage) { RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Miss) { HandleScope scope(isolate); ASSERT(args.length() == 3); - KeyedStoreIC ic(isolate); + KeyedStoreIC ic(IC::NO_EXTRA_FRAME, isolate); + IC::State state = IC::StateFrom(ic.target(), args[0], args[1]); + Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state(); + return ic.Store(state, + Code::GetStrictMode(extra_ic_state), + args.at(0), + args.at(1), + args.at(2), + MISS); +} + + +RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_MissFromStubFailure) { + HandleScope scope(isolate); + ASSERT(args.length() == 3); + KeyedStoreIC ic(IC::EXTRA_CALL_FRAME, isolate); IC::State state = IC::StateFrom(ic.target(), args[0], args[1]); Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state(); return ic.Store(state, @@ -2165,7 +2198,7 @@ RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Miss) { RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Slow) { NoHandleAllocation na(isolate); ASSERT(args.length() == 3); - KeyedStoreIC ic(isolate); + KeyedStoreIC ic(IC::NO_EXTRA_FRAME, isolate); Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state(); Handle object = args.at(0); Handle key = args.at(1); @@ -2183,7 +2216,7 @@ RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Slow) { RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_MissForceGeneric) { HandleScope scope(isolate); ASSERT(args.length() == 3); - KeyedStoreIC ic(isolate); + KeyedStoreIC ic(IC::NO_EXTRA_FRAME, isolate); IC::State state = IC::StateFrom(ic.target(), args[0], args[1]); Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state(); return ic.Store(state, diff --git a/src/ic.h b/src/ic.h index b225955..2b8f378 100644 --- a/src/ic.h +++ b/src/ic.h @@ -496,7 +496,7 @@ class KeyedLoadIC: public LoadIC { class StoreIC: public IC { public: - explicit StoreIC(Isolate* isolate) : IC(NO_EXTRA_FRAME, isolate) { + StoreIC(FrameDepth depth, Isolate* isolate) : IC(depth, isolate) { ASSERT(target()->is_store_stub() || target()->is_keyed_store_stub()); } @@ -585,7 +585,8 @@ enum KeyedStoreIncrementLength { class KeyedStoreIC: public StoreIC { public: - explicit KeyedStoreIC(Isolate* isolate) : StoreIC(isolate) { + KeyedStoreIC(FrameDepth depth, Isolate* isolate) + : StoreIC(depth, isolate) { ASSERT(target()->is_keyed_store_stub()); } @@ -786,6 +787,7 @@ enum InlinedSmiCheck { ENABLE_INLINED_SMI_CHECK, DISABLE_INLINED_SMI_CHECK }; void PatchInlinedSmiCode(Address address, InlinedSmiCheck check); DECLARE_RUNTIME_FUNCTION(MaybeObject*, KeyedLoadIC_MissFromStubFailure); +DECLARE_RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_MissFromStubFailure); } } // namespace v8::internal diff --git a/src/runtime.cc b/src/runtime.cc index e220d6b..934c82e 100644 --- a/src/runtime.cc +++ b/src/runtime.cc @@ -9085,8 +9085,7 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_AllocateInNewSpace) { RUNTIME_ASSERT(IsAligned(size, kPointerSize)); RUNTIME_ASSERT(size > 0); Heap* heap = isolate->heap(); - const int kMinFreeNewSpaceAfterGC = heap->InitialSemiSpaceSize() * 3/4; - RUNTIME_ASSERT(size <= kMinFreeNewSpaceAfterGC); + RUNTIME_ASSERT(size <= heap->MaxNewSpaceAllocationSize()); Object* allocation; { MaybeObject* maybe_allocation = heap->new_space()->AllocateRaw(size); if (maybe_allocation->ToObject(&allocation)) { diff --git a/src/stub-cache.cc b/src/stub-cache.cc index a0d98e2..c104d80 100644 --- a/src/stub-cache.cc +++ b/src/stub-cache.cc @@ -441,7 +441,9 @@ Handle StubCache::ComputeKeyedStoreElement( Code::KEYED_STORE_IC, extra_state); ASSERT(store_mode == STANDARD_STORE || - store_mode == STORE_AND_GROW_NO_TRANSITION); + store_mode == STORE_AND_GROW_NO_TRANSITION || + store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS || + store_mode == STORE_NO_TRANSITION_HANDLE_COW); Handle name = isolate()->factory()->KeyedStoreElementMonomorphic_string(); @@ -899,7 +901,9 @@ Handle StubCache::ComputeStoreElementPolymorphic( KeyedAccessStoreMode store_mode, StrictModeFlag strict_mode) { ASSERT(store_mode == STANDARD_STORE || - store_mode == STORE_AND_GROW_NO_TRANSITION); + store_mode == STORE_AND_GROW_NO_TRANSITION || + store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS || + store_mode == STORE_NO_TRANSITION_HANDLE_COW); Handle cache = isolate_->factory()->polymorphic_code_cache(); Code::ExtraICState extra_state = Code::ComputeExtraICState(store_mode, @@ -1660,10 +1664,19 @@ Handle KeyedStoreStubCompiler::CompileStoreElement( Handle receiver_map) { ElementsKind elements_kind = receiver_map->elements_kind(); bool is_jsarray = receiver_map->instance_type() == JS_ARRAY_TYPE; - Handle stub = - KeyedStoreElementStub(is_jsarray, - elements_kind, - store_mode_).GetCode(isolate()); + Handle stub; + if (FLAG_compiled_keyed_stores && + (receiver_map->has_fast_elements() || + receiver_map->has_external_array_elements())) { + stub = KeyedStoreFastElementStub( + is_jsarray, + elements_kind, + store_mode_).GetCode(isolate()); + } else { + stub = KeyedStoreElementStub(is_jsarray, + elements_kind, + store_mode_).GetCode(isolate()); + } __ DispatchMap(receiver(), scratch1(), receiver_map, stub, DO_SMI_CHECK); @@ -1809,10 +1822,19 @@ Handle KeyedStoreStubCompiler::CompileStoreElementPolymorphic( strict_mode(), store_mode_).GetCode(isolate()); } else { - cached_stub = KeyedStoreElementStub( - is_js_array, - elements_kind, - store_mode_).GetCode(isolate()); + if (FLAG_compiled_keyed_stores && + (receiver_map->has_fast_elements() || + receiver_map->has_external_array_elements())) { + cached_stub = KeyedStoreFastElementStub( + is_js_array, + elements_kind, + store_mode_).GetCode(isolate()); + } else { + cached_stub = KeyedStoreElementStub( + is_js_array, + elements_kind, + store_mode_).GetCode(isolate()); + } } ASSERT(!cached_stub.is_null()); handlers.Add(cached_stub); diff --git a/src/type-info.cc b/src/type-info.cc index 6ac0554..f31edb7 100644 --- a/src/type-info.cc +++ b/src/type-info.cc @@ -126,9 +126,9 @@ bool TypeFeedbackOracle::StoreIsMonomorphicNormal(TypeFeedbackId ast_id) { if (map_or_code->IsMap()) return true; if (map_or_code->IsCode()) { Handle code = Handle::cast(map_or_code); - bool standard_store = - Code::GetKeyedAccessStoreMode(code->extra_ic_state()) == - STANDARD_STORE; + bool standard_store = FLAG_compiled_keyed_stores || + (Code::GetKeyedAccessStoreMode(code->extra_ic_state()) == + STANDARD_STORE); bool preliminary_checks = code->is_keyed_store_stub() && standard_store && @@ -146,9 +146,9 @@ bool TypeFeedbackOracle::StoreIsPolymorphic(TypeFeedbackId ast_id) { Handle map_or_code = GetInfo(ast_id); if (map_or_code->IsCode()) { Handle code = Handle::cast(map_or_code); - bool standard_store = - Code::GetKeyedAccessStoreMode(code->extra_ic_state()) == - STANDARD_STORE; + bool standard_store = FLAG_compiled_keyed_stores || + (Code::GetKeyedAccessStoreMode(code->extra_ic_state()) == + STANDARD_STORE); return code->is_keyed_store_stub() && standard_store && code->ic_state() == POLYMORPHIC; } diff --git a/src/x64/code-stubs-x64.cc b/src/x64/code-stubs-x64.cc index 04c3e68..4ff1635 100644 --- a/src/x64/code-stubs-x64.cc +++ b/src/x64/code-stubs-x64.cc @@ -62,6 +62,17 @@ void KeyedLoadFastElementStub::InitializeInterfaceDescriptor( } +void KeyedStoreFastElementStub::InitializeInterfaceDescriptor( + Isolate* isolate, + CodeStubInterfaceDescriptor* descriptor) { + static Register registers[] = { rdx, rcx, rax }; + descriptor->register_param_count_ = 3; + descriptor->register_params_ = registers; + descriptor->deoptimization_handler_ = + FUNCTION_ADDR(KeyedStoreIC_MissFromStubFailure); +} + + void TransitionElementsKindStub::InitializeInterfaceDescriptor( Isolate* isolate, CodeStubInterfaceDescriptor* descriptor) { diff --git a/test/mjsunit/elements-kind.js b/test/mjsunit/elements-kind.js index d952556..0e8e209 100644 --- a/test/mjsunit/elements-kind.js +++ b/test/mjsunit/elements-kind.js @@ -28,9 +28,10 @@ // Flags: --allow-natives-syntax --smi-only-arrays --expose-gc // Flags: --notrack_allocation_sites -// Limit the number of stress runs to reduce polymorphism it defeats some of -// they assumptions made about how elements transitions work because transition -// stubs end up going generic. Flags: --stress-runs=2 +// Limit the number of stress runs to reduce polymorphism it defeats some of the +// assumptions made about how elements transitions work because transition stubs +// end up going generic. +// Flags: --stress-runs=2 // Test element kind of objects. // Since --smi-only-arrays affects builtins, its default setting at compile -- 2.7.4