From 80f24f7a4c2037c677531db21643f20e4dffc789 Mon Sep 17 00:00:00 2001 From: "paul.lind" Date: Thu, 16 Apr 2015 09:05:49 -0700 Subject: [PATCH] Reland MIPS: Vector-ICs - speed towards the monomorphic exit as quickly as possible. Port 35a67b745d862c5b1f584c2786ebf1369dc4f1ff Original commit message: Thanks to some careful assumptions, we can examine the object found at vector[slot] and trust it's a heap object where the second field is either a map if it's a WeakCell, or definitely not a map if it's a Symbol, String or FixedArray. Use this to save a memory read. BUG= Review URL: https://codereview.chromium.org/1053843003 Cr-Commit-Position: refs/heads/master@{#27757} Review URL: https://codereview.chromium.org/1083413003 Cr-Commit-Position: refs/heads/master@{#27893} --- src/mips/code-stubs-mips.cc | 123 +++++++++++++++++---------------------- src/mips64/code-stubs-mips64.cc | 125 +++++++++++++++++----------------------- 2 files changed, 106 insertions(+), 142 deletions(-) diff --git a/src/mips/code-stubs-mips.cc b/src/mips/code-stubs-mips.cc index b05d447..6e7541f 100644 --- a/src/mips/code-stubs-mips.cc +++ b/src/mips/code-stubs-mips.cc @@ -4602,21 +4602,15 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { static void HandleArrayCases(MacroAssembler* masm, Register receiver, Register key, Register vector, Register slot, - Register feedback, Register scratch1, - Register scratch2, Register scratch3, + Register feedback, Register receiver_map, + Register scratch1, Register scratch2, bool is_polymorphic, Label* miss) { // feedback initially contains the feedback array Label next_loop, prepare_next; - Label load_smi_map, compare_map; Label start_polymorphic; - Register receiver_map = scratch1; - Register cached_map = scratch2; + Register cached_map = scratch1; - // Receiver might not be a heap object. - __ JumpIfSmi(receiver, &load_smi_map); - __ lw(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); - __ bind(&compare_map); __ lw(cached_map, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); __ lw(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); @@ -4628,7 +4622,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, __ Jump(t9); - Register length = scratch3; + Register length = scratch2; __ bind(&start_polymorphic); __ lw(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); if (!is_polymorphic) { @@ -4647,9 +4641,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, // ^ ^ // | | // pointer_reg too_far - // aka feedback scratch3 - // also need receiver_map (aka scratch1) - // use cached_map (scratch2) to look in the weak map values. + // aka feedback scratch2 + // also need receiver_map + // use cached_map (scratch1) to look in the weak map values. __ sll(at, length, kPointerSizeLog2 - kSmiTagSize); __ Addu(too_far, feedback, Operand(at)); __ Addu(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); @@ -4670,43 +4664,23 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, // We exhausted our array of map handler pairs. __ jmp(miss); - - __ bind(&load_smi_map); - __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); - __ jmp(&compare_map); } static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, - Register key, Register vector, Register slot, - Register weak_cell, Register scratch, - Label* miss) { - // feedback initially contains the feedback array - Label compare_smi_map; - Register receiver_map = scratch; - Register cached_map = weak_cell; - - // Move the weak map into the weak_cell register. - __ lw(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); - - // Receiver might not be a heap object. - __ JumpIfSmi(receiver, &compare_smi_map); + Register receiver_map, Register feedback, + Register vector, Register slot, + Register scratch, Label* compare_map, + Label* load_smi_map, Label* try_array) { + __ JumpIfSmi(receiver, load_smi_map); __ lw(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); - __ Branch(miss, ne, cached_map, Operand(receiver_map)); - - Register handler = weak_cell; - __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); - __ Addu(handler, vector, Operand(at)); - __ lw(handler, - FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); - __ Addu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); - __ Jump(t9); + __ bind(compare_map); + Register cached_map = scratch; + // Move the weak map into the weak_cell register. + __ lw(cached_map, FieldMemOperand(feedback, WeakCell::kValueOffset)); + __ Branch(try_array, ne, cached_map, Operand(receiver_map)); + Register handler = feedback; - // In microbenchmarks, it made sense to unroll this code so that the call to - // the handler is duplicated for a HeapObject receiver and a Smi receiver. - __ bind(&compare_smi_map); - __ LoadRoot(at, Heap::kHeapNumberMapRootIndex); - __ Branch(miss, ne, at, Operand(weak_cell)); __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); __ Addu(handler, vector, Operand(at)); __ lw(handler, @@ -4722,27 +4696,28 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 Register feedback = t0; - Register scratch1 = t1; + Register receiver_map = t1; + Register scratch1 = t4; __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); __ Addu(feedback, vector, Operand(at)); __ lw(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); - // Is it a weak cell? - Label try_array; - Label not_array, smi_key, key_okay, miss; - __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); - __ LoadRoot(at, Heap::kWeakCellMapRootIndex); - __ Branch(&try_array, ne, at, Operand(scratch1)); - HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, - &miss); + // Try to quickly handle the monomorphic case without knowing for sure + // if we have a weak cell in feedback. We do know it's safe to look + // at WeakCell::kValueOffset. + Label try_array, load_smi_map, compare_map; + Label not_array, miss; + HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot, + scratch1, &compare_map, &load_smi_map, &try_array); // Is it a fixed array? __ bind(&try_array); + __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); __ Branch(¬_array, ne, at, Operand(scratch1)); - HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, t4, - t5, true, &miss); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map, + scratch1, t5, true, &miss); __ bind(¬_array); __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); @@ -4751,10 +4726,14 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Code::ComputeHandlerFlags(Code::LOAD_IC)); masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, false, receiver, name, feedback, - scratch1, t4, t5); + receiver_map, scratch1, t5); __ bind(&miss); LoadIC::GenerateMiss(masm); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); } @@ -4774,24 +4753,24 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 Register feedback = t0; - Register scratch1 = t1; + Register receiver_map = t1; + Register scratch1 = t4; __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); __ Addu(feedback, vector, Operand(at)); __ lw(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); - // Is it a weak cell? - Label try_array; - Label not_array, smi_key, key_okay, miss; - __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); - __ LoadRoot(at, Heap::kWeakCellMapRootIndex); - __ Branch(&try_array, ne, at, Operand(scratch1)); - __ JumpIfNotSmi(key, &miss); - HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, - &miss); + // Try to quickly handle the monomorphic case without knowing for sure + // if we have a weak cell in feedback. We do know it's safe to look + // at WeakCell::kValueOffset. + Label try_array, load_smi_map, compare_map; + Label not_array, miss; + HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot, + scratch1, &compare_map, &load_smi_map, &try_array); __ bind(&try_array); // Is it a fixed array? + __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); __ Branch(¬_array, ne, at, Operand(scratch1)); // We have a polymorphic element handler. @@ -4799,8 +4778,8 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Label polymorphic, try_poly_name; __ bind(&polymorphic); - HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, t4, - t5, true, &miss); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map, + scratch1, t5, true, &miss); __ bind(¬_array); // Is it generic? @@ -4819,11 +4798,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { __ Addu(feedback, vector, Operand(at)); __ lw(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); - HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, t4, - t5, false, &miss); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map, + scratch1, t5, false, &miss); __ bind(&miss); KeyedLoadIC::GenerateMiss(masm); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); } diff --git a/src/mips64/code-stubs-mips64.cc b/src/mips64/code-stubs-mips64.cc index 11234e1..02acdb0 100644 --- a/src/mips64/code-stubs-mips64.cc +++ b/src/mips64/code-stubs-mips64.cc @@ -4646,21 +4646,15 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { static void HandleArrayCases(MacroAssembler* masm, Register receiver, Register key, Register vector, Register slot, - Register feedback, Register scratch1, - Register scratch2, Register scratch3, + Register feedback, Register receiver_map, + Register scratch1, Register scratch2, bool is_polymorphic, Label* miss) { // feedback initially contains the feedback array Label next_loop, prepare_next; - Label load_smi_map, compare_map; Label start_polymorphic; - Register receiver_map = scratch1; - Register cached_map = scratch2; + Register cached_map = scratch1; - // Receiver might not be a heap object. - __ JumpIfSmi(receiver, &load_smi_map); - __ ld(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); - __ bind(&compare_map); __ ld(cached_map, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); __ ld(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); @@ -4671,7 +4665,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, __ Daddu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); __ Jump(t9); - Register length = scratch3; + Register length = scratch2; __ bind(&start_polymorphic); __ ld(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); if (!is_polymorphic) { @@ -4690,9 +4684,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, // ^ ^ // | | // pointer_reg too_far - // aka feedback scratch3 - // also need receiver_map (aka scratch1) - // use cached_map (scratch2) to look in the weak map values. + // aka feedback scratch2 + // also need receiver_map + // use cached_map (scratch1) to look in the weak map values. __ SmiScale(too_far, length, kPointerSizeLog2); __ Daddu(too_far, feedback, Operand(too_far)); __ Daddu(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); @@ -4713,44 +4707,22 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, // We exhausted our array of map handler pairs. __ Branch(miss); - - __ bind(&load_smi_map); - __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); - __ Branch(&compare_map); } static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, - Register key, Register vector, Register slot, - Register weak_cell, Register scratch, - Label* miss) { - // feedback initially contains the feedback array - Label compare_smi_map; - Register receiver_map = scratch; - Register cached_map = weak_cell; - - // Move the weak map into the weak_cell register. - __ ld(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); - - // Receiver might not be a heap object. - __ JumpIfSmi(receiver, &compare_smi_map); + Register receiver_map, Register feedback, + Register vector, Register slot, + Register scratch, Label* compare_map, + Label* load_smi_map, Label* try_array) { + __ JumpIfSmi(receiver, load_smi_map); __ ld(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); - __ Branch(miss, ne, cached_map, Operand(receiver_map)); - - Register handler = weak_cell; - __ SmiScale(handler, slot, kPointerSizeLog2); - __ Daddu(handler, vector, Operand(handler)); - __ ld(handler, - FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); - __ Daddu(t9, handler, Code::kHeaderSize - kHeapObjectTag); - __ Jump(t9); - - // In microbenchmarks, it made sense to unroll this code so that the call to - // the handler is duplicated for a HeapObject receiver and a Smi receiver. - // TODO(mvstanton): does this hold on ARM? - __ bind(&compare_smi_map); - __ LoadRoot(at, Heap::kHeapNumberMapRootIndex); - __ Branch(miss, ne, weak_cell, Operand(at)); + __ bind(compare_map); + Register cached_map = scratch; + // Move the weak map into the weak_cell register. + __ ld(cached_map, FieldMemOperand(feedback, WeakCell::kValueOffset)); + __ Branch(try_array, ne, cached_map, Operand(receiver_map)); + Register handler = feedback; __ SmiScale(handler, slot, kPointerSizeLog2); __ Daddu(handler, vector, Operand(handler)); __ ld(handler, @@ -4766,27 +4738,28 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 Register feedback = a4; - Register scratch1 = a5; + Register receiver_map = a5; + Register scratch1 = a6; __ SmiScale(feedback, slot, kPointerSizeLog2); __ Daddu(feedback, vector, Operand(feedback)); __ ld(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); - // Is it a weak cell? - Label try_array; - Label not_array, smi_key, key_okay, miss; - __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); - __ LoadRoot(at, Heap::kWeakCellMapRootIndex); - __ Branch(&try_array, ne, scratch1, Operand(at)); - HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, - &miss); + // Try to quickly handle the monomorphic case without knowing for sure + // if we have a weak cell in feedback. We do know it's safe to look + // at WeakCell::kValueOffset. + Label try_array, load_smi_map, compare_map; + Label not_array, miss; + HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot, + scratch1, &compare_map, &load_smi_map, &try_array); // Is it a fixed array? __ bind(&try_array); + __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); __ Branch(¬_array, ne, scratch1, Operand(at)); - HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, a6, - a7, true, &miss); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map, + scratch1, a7, true, &miss); __ bind(¬_array); __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); @@ -4795,10 +4768,14 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Code::ComputeHandlerFlags(Code::LOAD_IC)); masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, false, receiver, name, feedback, - scratch1, a6, a7); + receiver_map, scratch1, a7); __ bind(&miss); LoadIC::GenerateMiss(masm); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ Branch(&compare_map); } @@ -4818,24 +4795,24 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 Register feedback = a4; - Register scratch1 = a5; + Register receiver_map = a5; + Register scratch1 = a6; __ SmiScale(feedback, slot, kPointerSizeLog2); __ Daddu(feedback, vector, Operand(feedback)); __ ld(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); - // Is it a weak cell? - Label try_array; - Label not_array, smi_key, key_okay, miss; - __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); - __ LoadRoot(at, Heap::kWeakCellMapRootIndex); - __ Branch(&try_array, ne, scratch1, Operand(at)); - __ JumpIfNotSmi(key, &miss); - HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, - &miss); + // Try to quickly handle the monomorphic case without knowing for sure + // if we have a weak cell in feedback. We do know it's safe to look + // at WeakCell::kValueOffset. + Label try_array, load_smi_map, compare_map; + Label not_array, miss; + HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot, + scratch1, &compare_map, &load_smi_map, &try_array); __ bind(&try_array); // Is it a fixed array? + __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); __ Branch(¬_array, ne, scratch1, Operand(at)); // We have a polymorphic element handler. @@ -4843,8 +4820,8 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { Label polymorphic, try_poly_name; __ bind(&polymorphic); - HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, a6, - a7, true, &miss); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map, + scratch1, a7, true, &miss); __ bind(¬_array); // Is it generic? @@ -4863,11 +4840,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { __ Daddu(feedback, vector, Operand(feedback)); __ ld(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); - HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, a6, - a7, false, &miss); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map, + scratch1, a7, false, &miss); __ bind(&miss); KeyedLoadIC::GenerateMiss(masm); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ Branch(&compare_map); } -- 2.7.4