From 5224c3d0f09531cdc8b2da3f7cd9818a6291e13f Mon Sep 17 00:00:00 2001 From: "mvstanton@chromium.org" Date: Tue, 18 Feb 2014 15:33:34 +0000 Subject: [PATCH] Second attempt at introducing a premonomorphic state in the call target caches. This time we don't go through the premonomorphic state for the Array call target caches to avoid losing information from allocation sites that aren't only used once, but where the resulting array is used heavily. R=mvstanton@chromium.org Review URL: https://codereview.chromium.org/169683003 Patch from Kasper Lund . git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19457 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/a64/code-stubs-a64.cc | 68 ++++++++++++++++++++++-------------------- src/arm/code-stubs-arm.cc | 72 ++++++++++++++++++++++++++------------------- src/ia32/code-stubs-ia32.cc | 67 ++++++++++++++++++++++------------------- src/objects-inl.h | 5 ++++ src/objects.h | 3 ++ src/runtime.cc | 1 + src/x64/code-stubs-x64.cc | 68 ++++++++++++++++++++++-------------------- test/cctest/test-heap.cc | 4 ++- 8 files changed, 161 insertions(+), 127 deletions(-) diff --git a/src/a64/code-stubs-a64.cc b/src/a64/code-stubs-a64.cc index a03459a..b640677 100644 --- a/src/a64/code-stubs-a64.cc +++ b/src/a64/code-stubs-a64.cc @@ -3265,12 +3265,17 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // x1 : the function to call // x2 : feedback vector // x3 : slot in feedback vector (smi) - Label initialize, done, miss, megamorphic, not_array_function; + Label check_array, initialize_array, initialize_non_array, megamorphic, done; ASSERT_EQ(*TypeFeedbackInfo::MegamorphicSentinel(masm->isolate()), masm->isolate()->heap()->undefined_value()); + Heap::RootListIndex kMegamorphicRootIndex = Heap::kUndefinedValueRootIndex; ASSERT_EQ(*TypeFeedbackInfo::UninitializedSentinel(masm->isolate()), masm->isolate()->heap()->the_hole_value()); + Heap::RootListIndex kUninitializedRootIndex = Heap::kTheHoleValueRootIndex; + ASSERT_EQ(*TypeFeedbackInfo::PremonomorphicSentinel(masm->isolate()), + masm->isolate()->heap()->null_value()); + Heap::RootListIndex kPremonomorphicRootIndex = Heap::kNullValueRootIndex; // Load the cache state. __ Add(x4, x2, Operand::UntagSmiAndScale(x3, kPointerSizeLog2)); @@ -3280,43 +3285,44 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // function without changing the state. __ Cmp(x4, x1); __ B(eq, &done); + __ JumpIfRoot(x4, kMegamorphicRootIndex, &done); - // If we came here, we need to see if we are the array function. - // If we didn't have a matching function, and we didn't find the megamorph - // sentinel, then we have in the slot either some other function or an - // AllocationSite. Do a map check on the object in ecx. - __ Ldr(x5, FieldMemOperand(x4, AllocationSite::kMapOffset)); - __ JumpIfNotRoot(x5, Heap::kAllocationSiteMapRootIndex, &miss); + // Check if we're dealing with the Array function or not. + __ LoadArrayFunction(x5); + __ Cmp(x1, x5); + __ B(eq, &check_array); + + // Non-array cache: Check the cache state. + __ JumpIfRoot(x4, kPremonomorphicRootIndex, &initialize_non_array); + __ JumpIfNotRoot(x4, kUninitializedRootIndex, &megamorphic); - // Make sure the function is the Array() function - __ LoadArrayFunction(x4); - __ Cmp(x1, x4); - __ B(ne, &megamorphic); + // Non-array cache: Uninitialized -> premonomorphic. The sentinel is an + // immortal immovable object (null) so no write-barrier is needed. + __ Add(x4, x2, Operand::UntagSmiAndScale(x3, kPointerSizeLog2)); + __ LoadRoot(x10, kPremonomorphicRootIndex); + __ Str(x10, FieldMemOperand(x4, FixedArray::kHeaderSize)); __ B(&done); - __ Bind(&miss); + // Array cache: Check the cache state to see if we're in a monomorphic + // state where the state object is an AllocationSite object. + __ Bind(&check_array); + __ Ldr(x5, FieldMemOperand(x4, AllocationSite::kMapOffset)); + __ JumpIfRoot(x5, Heap::kAllocationSiteMapRootIndex, &done); + + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ JumpIfRoot(x4, kUninitializedRootIndex, &initialize_array); + __ JumpIfRoot(x4, kPremonomorphicRootIndex, &initialize_array); - // A monomorphic miss (i.e, here the cache is not uninitialized) goes - // megamorphic. - __ JumpIfRoot(x4, Heap::kTheHoleValueRootIndex, &initialize); - // MegamorphicSentinel is an immortal immovable object (undefined) so no - // write-barrier is needed. + // Both caches: Monomorphic -> megamorphic. The sentinel is an + // immortal immovable object (undefined) so no write-barrier is needed. __ Bind(&megamorphic); __ Add(x4, x2, Operand::UntagSmiAndScale(x3, kPointerSizeLog2)); - __ LoadRoot(x10, Heap::kUndefinedValueRootIndex); + __ LoadRoot(x10, kMegamorphicRootIndex); __ Str(x10, FieldMemOperand(x4, FixedArray::kHeaderSize)); __ B(&done); - // An uninitialized cache is patched with the function or sentinel to - // indicate the ElementsKind if function is the Array constructor. - __ Bind(&initialize); - // Make sure the function is the Array() function - __ LoadArrayFunction(x4); - __ Cmp(x1, x4); - __ B(ne, ¬_array_function); - - // The target function is the Array constructor, - // Create an AllocationSite if we don't already have it, store it in the slot. + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ Bind(&initialize_array); { FrameScope scope(masm, StackFrame::INTERNAL); CreateAllocationSiteStub create_stub; @@ -3332,9 +3338,8 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { } __ B(&done); - __ Bind(¬_array_function); - // An uninitialized cache is patched with the function. - + // Non-array cache: Premonomorphic -> monomorphic. + __ Bind(&initialize_non_array); __ Add(x4, x2, Operand::UntagSmiAndScale(x3, kPointerSizeLog2)); // TODO(all): Does the value need to be left in x4? If not, FieldMemOperand // could be used to avoid this add. @@ -3347,7 +3352,6 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { __ Pop(x1, x2, x4); // TODO(all): Are x4, x2 and x1 outputs? This isn't clear. - __ Bind(&done); } diff --git a/src/arm/code-stubs-arm.cc b/src/arm/code-stubs-arm.cc index b3a3acf..501b5c7 100644 --- a/src/arm/code-stubs-arm.cc +++ b/src/arm/code-stubs-arm.cc @@ -3012,12 +3012,17 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // r1 : the function to call // r2 : Feedback vector // r3 : slot in feedback vector (Smi) - Label initialize, done, miss, megamorphic, not_array_function; + Label check_array, initialize_array, initialize_non_array, megamorphic, done; ASSERT_EQ(*TypeFeedbackInfo::MegamorphicSentinel(masm->isolate()), masm->isolate()->heap()->undefined_value()); + Heap::RootListIndex kMegamorphicRootIndex = Heap::kUndefinedValueRootIndex; ASSERT_EQ(*TypeFeedbackInfo::UninitializedSentinel(masm->isolate()), masm->isolate()->heap()->the_hole_value()); + Heap::RootListIndex kUninitializedRootIndex = Heap::kTheHoleValueRootIndex; + ASSERT_EQ(*TypeFeedbackInfo::PremonomorphicSentinel(masm->isolate()), + masm->isolate()->heap()->null_value()); + Heap::RootListIndex kPremonomorphicRootIndex = Heap::kNullValueRootIndex; // Load the cache state into r4. __ add(r4, r2, Operand::PointerOffsetFromSmiKey(r3)); @@ -3027,45 +3032,50 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // function without changing the state. __ cmp(r4, r1); __ b(eq, &done); + __ CompareRoot(r4, kMegamorphicRootIndex); + __ b(eq, &done); - // If we came here, we need to see if we are the array function. - // If we didn't have a matching function, and we didn't find the megamorph - // sentinel, then we have in the slot either some other function or an - // AllocationSite. Do a map check on the object in ecx. - __ ldr(r5, FieldMemOperand(r4, 0)); - __ CompareRoot(r5, Heap::kAllocationSiteMapRootIndex); - __ b(ne, &miss); + // Check if we're dealing with the Array function or not. + __ LoadArrayFunction(r5); + __ cmp(r1, r5); + __ b(eq, &check_array); - // Make sure the function is the Array() function - __ LoadArrayFunction(r4); - __ cmp(r1, r4); + // Non-array cache: Check the cache state. + __ CompareRoot(r4, kPremonomorphicRootIndex); + __ b(eq, &initialize_non_array); + __ CompareRoot(r4, kUninitializedRootIndex); __ b(ne, &megamorphic); + + // Non-array cache: Uninitialized -> premonomorphic. The sentinel is an + // immortal immovable object (null) so no write-barrier is needed. + __ add(r4, r2, Operand::PointerOffsetFromSmiKey(r3)); + __ LoadRoot(ip, kPremonomorphicRootIndex); + __ str(ip, FieldMemOperand(r4, FixedArray::kHeaderSize)); __ jmp(&done); - __ bind(&miss); + // Array cache: Check the cache state to see if we're in a monomorphic + // state where the state object is an AllocationSite object. + __ bind(&check_array); + __ ldr(r5, FieldMemOperand(r4, 0)); + __ CompareRoot(r5, Heap::kAllocationSiteMapRootIndex); + __ b(eq, &done); - // A monomorphic miss (i.e, here the cache is not uninitialized) goes - // megamorphic. - __ CompareRoot(r4, Heap::kTheHoleValueRootIndex); - __ b(eq, &initialize); - // MegamorphicSentinel is an immortal immovable object (undefined) so no - // write-barrier is needed. + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ CompareRoot(r4, kUninitializedRootIndex); + __ b(eq, &initialize_array); + __ CompareRoot(r4, kPremonomorphicRootIndex); + __ b(eq, &initialize_array); + + // Both caches: Monomorphic -> megamorphic. The sentinel is an + // immortal immovable object (undefined) so no write-barrier is needed. __ bind(&megamorphic); __ add(r4, r2, Operand::PointerOffsetFromSmiKey(r3)); - __ LoadRoot(ip, Heap::kUndefinedValueRootIndex); + __ LoadRoot(ip, kMegamorphicRootIndex); __ str(ip, FieldMemOperand(r4, FixedArray::kHeaderSize)); __ jmp(&done); - // An uninitialized cache is patched with the function or sentinel to - // indicate the ElementsKind if function is the Array constructor. - __ bind(&initialize); - // Make sure the function is the Array() function - __ LoadArrayFunction(r4); - __ cmp(r1, r4); - __ b(ne, ¬_array_function); - - // The target function is the Array constructor, - // Create an AllocationSite if we don't already have it, store it in the slot. + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ bind(&initialize_array); { FrameScope scope(masm, StackFrame::INTERNAL); @@ -3081,8 +3091,8 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { } __ b(&done); - __ bind(¬_array_function); - + // Non-array cache: Premonomorphic -> monomorphic. + __ bind(&initialize_non_array); __ add(r4, r2, Operand::PointerOffsetFromSmiKey(r3)); __ add(r4, r4, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); __ str(r1, MemOperand(r4, 0)); diff --git a/src/ia32/code-stubs-ia32.cc b/src/ia32/code-stubs-ia32.cc index 7882419..22709e4 100644 --- a/src/ia32/code-stubs-ia32.cc +++ b/src/ia32/code-stubs-ia32.cc @@ -2330,7 +2330,7 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // edx : slot in feedback vector (Smi) // edi : the function to call Isolate* isolate = masm->isolate(); - Label initialize, done, miss, megamorphic, not_array_function; + Label check_array, initialize_array, initialize_non_array, megamorphic, done; // Load the cache state into ecx. __ mov(ecx, FieldOperand(ebx, edx, times_half_pointer_size, @@ -2343,48 +2343,53 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { __ cmp(ecx, Immediate(TypeFeedbackInfo::MegamorphicSentinel(isolate))); __ j(equal, &done, Label::kFar); - // If we came here, we need to see if we are the array function. - // If we didn't have a matching function, and we didn't find the megamorph - // sentinel, then we have in the slot either some other function or an - // AllocationSite. Do a map check on the object in ecx. - Handle allocation_site_map = - masm->isolate()->factory()->allocation_site_map(); - __ cmp(FieldOperand(ecx, 0), Immediate(allocation_site_map)); - __ j(not_equal, &miss); - - // Load the global or builtins object from the current context + // Load the global or builtins object from the current context and check + // if we're dealing with the Array function or not. __ LoadGlobalContext(ecx); - // Make sure the function is the Array() function __ cmp(edi, Operand(ecx, Context::SlotOffset(Context::ARRAY_FUNCTION_INDEX))); + __ j(equal, &check_array); + + // Non-array cache: Reload the cache state and check it. + __ mov(ecx, FieldOperand(ebx, edx, times_half_pointer_size, + FixedArray::kHeaderSize)); + __ cmp(ecx, Immediate(TypeFeedbackInfo::PremonomorphicSentinel(isolate))); + __ j(equal, &initialize_non_array); + __ cmp(ecx, Immediate(TypeFeedbackInfo::UninitializedSentinel(isolate))); __ j(not_equal, &megamorphic); + + // Non-array cache: Uninitialized -> premonomorphic. The sentinel is an + // immortal immovable object (null) so no write-barrier is needed. + __ mov(FieldOperand(ebx, edx, times_half_pointer_size, + FixedArray::kHeaderSize), + Immediate(TypeFeedbackInfo::PremonomorphicSentinel(isolate))); __ jmp(&done, Label::kFar); - __ bind(&miss); + // Array cache: Reload the cache state and check to see if we're in a + // monomorphic state where the state object is an AllocationSite object. + __ bind(&check_array); + __ mov(ecx, FieldOperand(ebx, edx, times_half_pointer_size, + FixedArray::kHeaderSize)); + Handle allocation_site_map = isolate->factory()->allocation_site_map(); + __ cmp(FieldOperand(ecx, 0), Immediate(allocation_site_map)); + __ j(equal, &done, Label::kFar); - // A monomorphic miss (i.e, here the cache is not uninitialized) goes - // megamorphic. + // Array cache: Uninitialized or premonomorphic -> monomorphic. __ cmp(ecx, Immediate(TypeFeedbackInfo::UninitializedSentinel(isolate))); - __ j(equal, &initialize); - // MegamorphicSentinel is an immortal immovable object (undefined) so no - // write-barrier is needed. + __ j(equal, &initialize_array); + __ cmp(ecx, Immediate(TypeFeedbackInfo::PremonomorphicSentinel(isolate))); + __ j(equal, &initialize_array); + + // Both caches: Monomorphic -> megamorphic. The sentinel is an + // immortal immovable object (undefined) so no write-barrier is needed. __ bind(&megamorphic); __ mov(FieldOperand(ebx, edx, times_half_pointer_size, FixedArray::kHeaderSize), Immediate(TypeFeedbackInfo::MegamorphicSentinel(isolate))); __ jmp(&done, Label::kFar); - // An uninitialized cache is patched with the function or sentinel to - // indicate the ElementsKind if function is the Array constructor. - __ bind(&initialize); - __ LoadGlobalContext(ecx); - // Make sure the function is the Array() function - __ cmp(edi, Operand(ecx, - Context::SlotOffset(Context::ARRAY_FUNCTION_INDEX))); - __ j(not_equal, ¬_array_function); - - // The target function is the Array constructor, - // Create an AllocationSite if we don't already have it, store it in the slot. + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ bind(&initialize_array); { FrameScope scope(masm, StackFrame::INTERNAL); @@ -2406,11 +2411,11 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { } __ jmp(&done); - __ bind(¬_array_function); + // Non-array cache: Premonomorphic -> monomorphic. + __ bind(&initialize_non_array); __ mov(FieldOperand(ebx, edx, times_half_pointer_size, FixedArray::kHeaderSize), edi); - // We won't need edx or ebx anymore, just save edi __ push(edi); __ push(ebx); __ push(edx); diff --git a/src/objects-inl.h b/src/objects-inl.h index 4f16806..ffec178 100644 --- a/src/objects-inl.h +++ b/src/objects-inl.h @@ -6543,6 +6543,11 @@ Handle TypeFeedbackInfo::UninitializedSentinel(Isolate* isolate) { } +Handle TypeFeedbackInfo::PremonomorphicSentinel(Isolate* isolate) { + return isolate->factory()->null_value(); +} + + Handle TypeFeedbackInfo::MegamorphicSentinel(Isolate* isolate) { return isolate->factory()->undefined_value(); } diff --git a/src/objects.h b/src/objects.h index acd2c11..05ab695 100644 --- a/src/objects.h +++ b/src/objects.h @@ -8174,6 +8174,9 @@ class TypeFeedbackInfo: public Struct { // The object that indicates an uninitialized cache. static inline Handle UninitializedSentinel(Isolate* isolate); + // The object that indicates a cache in pre-monomorphic state. + static inline Handle PremonomorphicSentinel(Isolate* isolate); + // The object that indicates a megamorphic state. static inline Handle MegamorphicSentinel(Isolate* isolate); diff --git a/src/runtime.cc b/src/runtime.cc index 05392ca..50621a9 100644 --- a/src/runtime.cc +++ b/src/runtime.cc @@ -14774,6 +14774,7 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_ArrayConstructor) { Handle site; if (!type_info.is_null() && + *type_info != isolate->heap()->null_value() && *type_info != isolate->heap()->undefined_value()) { site = Handle::cast(type_info); ASSERT(!site->SitePointsToLiteral()); diff --git a/src/x64/code-stubs-x64.cc b/src/x64/code-stubs-x64.cc index 8c5a430..92af1f0 100644 --- a/src/x64/code-stubs-x64.cc +++ b/src/x64/code-stubs-x64.cc @@ -2169,8 +2169,7 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { // rdx : slot in feedback vector (Smi) // rdi : the function to call Isolate* isolate = masm->isolate(); - Label initialize, done, miss, megamorphic, not_array_function, - done_no_smi_convert; + Label check_array, initialize_array, initialize_non_array, megamorphic, done; // Load the cache state into rcx. __ SmiToInteger32(rdx, rdx); @@ -2184,44 +2183,49 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { __ Cmp(rcx, TypeFeedbackInfo::MegamorphicSentinel(isolate)); __ j(equal, &done); - // If we came here, we need to see if we are the array function. - // If we didn't have a matching function, and we didn't find the megamorph - // sentinel, then we have in the slot either some other function or an - // AllocationSite. Do a map check on the object in rcx. - Handle allocation_site_map = - masm->isolate()->factory()->allocation_site_map(); - __ Cmp(FieldOperand(rcx, 0), allocation_site_map); - __ j(not_equal, &miss); - - // Make sure the function is the Array() function + // Check if we're dealing with the Array function or not. __ LoadArrayFunction(rcx); __ cmpq(rdi, rcx); + __ j(equal, &check_array); + + // Non-array cache: Reload the cache state and check it. + __ movp(rcx, FieldOperand(rbx, rdx, times_pointer_size, + FixedArray::kHeaderSize)); + __ Cmp(rcx, TypeFeedbackInfo::PremonomorphicSentinel(isolate)); + __ j(equal, &initialize_non_array); + __ Cmp(rcx, TypeFeedbackInfo::UninitializedSentinel(isolate)); __ j(not_equal, &megamorphic); - __ jmp(&done); - __ bind(&miss); + // Non-array cache: Uninitialized -> premonomorphic. The sentinel is an + // immortal immovable object (null) so no write-barrier is needed. + __ Move(FieldOperand(rbx, rdx, times_pointer_size, FixedArray::kHeaderSize), + TypeFeedbackInfo::PremonomorphicSentinel(isolate)); + __ jmp(&done, Label::kFar); - // A monomorphic miss (i.e, here the cache is not uninitialized) goes - // megamorphic. + // Array cache: Reload the cache state and check to see if we're in a + // monomorphic state where the state object is an AllocationSite object. + __ bind(&check_array); + __ movp(rcx, FieldOperand(rbx, rdx, times_pointer_size, + FixedArray::kHeaderSize)); + Handle allocation_site_map = isolate->factory()->allocation_site_map(); + __ Cmp(FieldOperand(rcx, 0), allocation_site_map); + __ j(equal, &done); + + // Array cache: Uninitialized or premonomorphic -> monomorphic. __ Cmp(rcx, TypeFeedbackInfo::UninitializedSentinel(isolate)); - __ j(equal, &initialize); - // MegamorphicSentinel is an immortal immovable object (undefined) so no - // write-barrier is needed. + __ j(equal, &initialize_array); + __ Cmp(rcx, TypeFeedbackInfo::PremonomorphicSentinel(isolate)); + __ j(equal, &initialize_array); + + // Both caches: Monomorphic -> megamorphic. The sentinel is an + // immortal immovable object (undefined) so no write-barrier is needed. __ bind(&megamorphic); __ Move(FieldOperand(rbx, rdx, times_pointer_size, FixedArray::kHeaderSize), TypeFeedbackInfo::MegamorphicSentinel(isolate)); __ jmp(&done); - // An uninitialized cache is patched with the function or sentinel to - // indicate the ElementsKind if function is the Array constructor. - __ bind(&initialize); - // Make sure the function is the Array() function - __ LoadArrayFunction(rcx); - __ cmpq(rdi, rcx); - __ j(not_equal, ¬_array_function); - - // The target function is the Array constructor, - // Create an AllocationSite if we don't already have it, store it in the slot. + // Array cache: Uninitialized or premonomorphic -> monomorphic. + __ bind(&initialize_array); { FrameScope scope(masm, StackFrame::INTERNAL); @@ -2242,13 +2246,13 @@ static void GenerateRecordCallTarget(MacroAssembler* masm) { __ pop(rax); __ SmiToInteger32(rax, rax); } + Label done_no_smi_convert; __ jmp(&done_no_smi_convert); - __ bind(¬_array_function); + // Non-array cache: Premonomorphic -> monomorphic. + __ bind(&initialize_non_array); __ movp(FieldOperand(rbx, rdx, times_pointer_size, FixedArray::kHeaderSize), rdi); - - // We won't need rdx or rbx anymore, just save rdi __ push(rdi); __ push(rbx); __ push(rdx); diff --git a/test/cctest/test-heap.cc b/test/cctest/test-heap.cc index ff9e51a..2cc9eca 100644 --- a/test/cctest/test-heap.cc +++ b/test/cctest/test-heap.cc @@ -2848,7 +2848,9 @@ TEST(IncrementalMarkingClearsTypeFeedbackInfo) { // originating from two different native contexts. CcTest::global()->Set(v8_str("fun1"), fun1); CcTest::global()->Set(v8_str("fun2"), fun2); - CompileRun("function f(a, b) { a(); b(); } f(fun1, fun2);"); + CompileRun("function f(a, b) { a(); b(); }" + "f(fun1, fun2);" // Run twice to skip premonomorphic state. + "f(fun1, fun2)"); Handle f = v8::Utils::OpenHandle( -- 2.7.4