1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 // VirtualCallStubCpu.hpp
7 #ifndef _VIRTUAL_CALL_STUB_ARM_H
8 #define _VIRTUAL_CALL_STUB_ARM_H
11 #include "asmconstants.h"
17 //#define STUB_LOGGING
19 #include <pshpack1.h> // Since we are placing code, we want byte packing of the structs
21 #define USES_LOOKUP_STUBS 1
23 /*********************************************************************************************
24 Stubs that contain code are all part of larger structs called Holders. There is a
25 Holder for each kind of stub, i.e XXXStub is contained with XXXHolder. Holders are
26 essentially an implementation trick that allowed rearranging the code sequences more
27 easily while trying out different alternatives, and for dealing with any alignment
28 issues in a way that was mostly immune to the actually code sequences. These Holders
29 should be revisited when the stub code sequences are fixed, since in many cases they
30 add extra space to a stub that is not really needed.
32 Stubs are placed in cache and hash tables. Since unaligned access of data in memory
33 is very slow, the keys used in those tables should be aligned. The things used as keys
34 typically also occur in the generated code, e.g. a token as an immediate part of an instruction.
35 For now, to avoid alignment computations as different code strategies are tried out, the key
36 fields are all in the Holders. Eventually, many of these fields should be dropped, and the instruction
37 streams aligned so that the immediate fields fall on aligned boundaries.
45 /*LookupStub**************************************************************************************
46 Virtual and interface call sites are initially setup to point at LookupStubs.
47 This is because the runtime type of the <this> pointer is not yet known,
48 so the target cannot be resolved. Note: if the jit is able to determine the runtime type
49 of the <this> pointer, it should be generating a direct call not a virtual or interface call.
50 This stub pushes a lookup token onto the stack to identify the sought after method, and then
51 jumps into the EE (VirtualCallStubManager::ResolveWorkerStub) to effectuate the lookup and
52 transfer of control to the appropriate target method implementation, perhaps patching of the call site
53 along the way to point to a more appropriate stub. Hence callsites that point to LookupStubs
54 get quickly changed to point to another kind of stub.
58 inline PCODE entryPoint() { LIMITED_METHOD_CONTRACT; return (PCODE)&_entryPoint[0] + THUMB_CODE; }
59 inline size_t token() { LIMITED_METHOD_CONTRACT; return _token; }
60 inline size_t size() { LIMITED_METHOD_CONTRACT; return sizeof(LookupStub); }
63 friend struct LookupHolder;
64 const static int entryPointLen = 4;
66 WORD _entryPoint[entryPointLen];
67 PCODE _resolveWorkerTarget; // xx xx xx xx target address
68 size_t _token; // xx xx xx xx 32-bit constant
71 /* LookupHolders are the containers for LookupStubs, they provide for any alignment of
72 stubs as necessary. In the case of LookupStubs, alignment is necessary since
73 LookupStubs are placed in a hash table keyed by token. */
76 static void InitializeStatic();
78 void Initialize(PCODE resolveWorkerTarget, size_t dispatchToken);
80 LookupStub* stub() { LIMITED_METHOD_CONTRACT; return &_stub; }
82 static LookupHolder* FromLookupEntry(PCODE lookupEntry);
85 friend struct LookupStub;
91 #endif // USES_LOOKUP_STUBS
94 struct DispatchHolder;
96 /*DispatchStub**************************************************************************************
97 Monomorphic and mostly monomorphic call sites eventually point to DispatchStubs.
98 A dispatch stub has an expected type (expectedMT), target address (target) and fail address (failure).
99 If the calling frame does in fact have the <this> type be of the expected type, then
100 control is transfered to the target address, the method implementation. If not,
101 then control is transfered to the fail address, a fail stub (see below) where a polymorphic
102 lookup is done to find the correct address to go to.
104 implementation note: Order, choice of instructions, and branch directions
105 should be carefully tuned since it can have an inordinate effect on performance. Particular
106 attention needs to be paid to the effects on the BTB and branch prediction, both in the small
107 and in the large, i.e. it needs to run well in the face of BTB overflow--using static predictions.
108 Note that since this stub is only used for mostly monomorphic callsites (ones that are not, get patched
109 to something else), therefore the conditional jump "jne failure" is mostly not taken, and hence it is important
110 that the branch prediction staticly predict this, which means it must be a forward jump. The alternative
111 is to reverse the order of the jumps and make sure that the resulting conditional jump "je implTarget"
112 is statically predicted as taken, i.e a backward jump. The current choice was taken since it was easier
113 to control the placement of the stubs than control the placement of the jitted code and the stubs. */
116 inline PCODE entryPoint() { LIMITED_METHOD_CONTRACT; return (PCODE)(&_entryPoint[0]) + THUMB_CODE; }
118 inline size_t expectedMT() { LIMITED_METHOD_CONTRACT; return _expectedMT; }
119 inline PCODE implTarget() { LIMITED_METHOD_CONTRACT; return _implTarget; }
120 inline PCODE failTarget() { LIMITED_METHOD_CONTRACT; return _failTarget; }
121 inline size_t size() { LIMITED_METHOD_CONTRACT; return sizeof(DispatchStub); }
124 friend struct DispatchHolder;
125 const static int entryPointLen = 12;
127 WORD _entryPoint[entryPointLen];
133 /* DispatchHolders are the containers for DispatchStubs, they provide for any alignment of
134 stubs as necessary. DispatchStubs are placed in a hashtable and in a cache. The keys for both
135 are the pair expectedMT and token. Efficiency of the of the hash table is not a big issue,
136 since lookups in it are fairly rare. Efficiency of the cache is paramount since it is accessed frequently
137 o(see ResolveStub below). Currently we are storing both of these fields in the DispatchHolder to simplify
138 alignment issues. If inlineMT in the stub itself was aligned, then it could be the expectedMT field.
139 While the token field can be logically gotten by following the failure target to the failEntryPoint
140 of the ResolveStub and then to the token over there, for perf reasons of cache access, it is duplicated here.
141 This allows us to use DispatchStubs in the cache. The alternative is to provide some other immutable struct
142 for the cache composed of the triplet (expectedMT, token, target) and some sort of reclaimation scheme when
143 they are thrown out of the cache via overwrites (since concurrency will make the obvious approaches invalid).
146 /* @workaround for ee resolution - Since the EE does not currently have a resolver function that
147 does what we want, see notes in implementation of VirtualCallStubManager::Resolver, we are
148 using dispatch stubs to siumulate what we want. That means that inlineTarget, which should be immutable
149 is in fact written. Hence we have moved target out into the holder and aligned it so we can
150 atomically update it. When we get a resolver function that does what we want, we can drop this field,
151 and live with just the inlineTarget field in the stub itself, since immutability will hold.*/
152 struct DispatchHolder
154 static void InitializeStatic();
156 void Initialize(PCODE implTarget, PCODE failTarget, size_t expectedMT);
158 DispatchStub* stub() { LIMITED_METHOD_CONTRACT; return &_stub; }
160 static DispatchHolder* FromDispatchEntry(PCODE dispatchEntry);
163 //force expectedMT to be aligned since used as key in hash tables.
168 struct ResolveHolder;
170 /*ResolveStub**************************************************************************************
171 Polymorphic call sites and monomorphic calls that fail end up in a ResolverStub. There is only
172 one resolver stub built for any given token, even though there may be many call sites that
173 use that token and many distinct <this> types that are used in the calling call frames. A resolver stub
174 actually has two entry points, one for polymorphic call sites and one for dispatch stubs that fail on their
175 expectedMT test. There is a third part of the resolver stub that enters the ee when a decision should
176 be made about changing the callsite. Therefore, we have defined the resolver stub as three distinct pieces,
177 even though they are actually allocated as a single contiguous block of memory. These pieces are:
179 A ResolveStub has two entry points:
181 FailEntry - where the dispatch stub goes if the expected MT test fails. This piece of the stub does
182 a check to see how often we are actually failing. If failures are frequent, control transfers to the
183 patch piece to cause the call site to be changed from a mostly monomorphic callsite
184 (calls dispatch stub) to a polymorphic callsize (calls resolve stub). If failures are rare, control
185 transfers to the resolve piece (see ResolveStub). The failEntryPoint decrements a counter
186 every time it is entered. The ee at various times will add a large chunk to the counter.
188 ResolveEntry - does a lookup via in a cache by hashing the actual type of the calling frame s
189 <this> and the token identifying the (contract,method) pair desired. If found, control is transfered
190 to the method implementation. If not found in the cache, the token is pushed and the ee is entered via
191 the ResolveWorkerStub to do a full lookup and eventual transfer to the correct method implementation. Since
192 there is a different resolve stub for every token, the token can be inlined and the token can be pre-hashed.
193 The effectiveness of this approach is highly sensitive to the effectiveness of the hashing algorithm used,
194 as well as its speed. It turns out it is very important to make the hash function sensitive to all
195 of the bits of the method table, as method tables are laid out in memory in a very non-random way. Before
196 making any changes to the code sequences here, it is very important to measure and tune them as perf
197 can vary greatly, in unexpected ways, with seeming minor changes.
199 Implementation note - Order, choice of instructions, and branch directions
200 should be carefully tuned since it can have an inordinate effect on performance. Particular
201 attention needs to be paid to the effects on the BTB and branch prediction, both in the small
202 and in the large, i.e. it needs to run well in the face of BTB overflow--using static predictions.
203 Note that this stub is called in highly polymorphic cases, but the cache should have been sized
204 and the hash function chosen to maximize the cache hit case. Hence the cmp/jcc instructions should
205 mostly be going down the cache hit route, and it is important that this be statically predicted as so.
206 Hence the 3 jcc instrs need to be forward jumps. As structured, there is only one jmp/jcc that typically
207 gets put in the BTB since all the others typically fall straight thru. Minimizing potential BTB entries
212 inline PCODE failEntryPoint() { LIMITED_METHOD_CONTRACT; return (PCODE)(&_failEntryPoint[0]) + THUMB_CODE; }
213 inline PCODE resolveEntryPoint() { LIMITED_METHOD_CONTRACT; return (PCODE)(&_resolveEntryPoint[0]) + THUMB_CODE; }
214 inline PCODE slowEntryPoint() { LIMITED_METHOD_CONTRACT; return (PCODE)(&_slowEntryPoint[0]) + THUMB_CODE; }
216 inline INT32* pCounter() { LIMITED_METHOD_CONTRACT; return _pCounter; }
217 inline UINT32 hashedToken() { LIMITED_METHOD_CONTRACT; return _hashedToken >> LOG2_PTRSIZE; }
218 inline size_t cacheAddress() { LIMITED_METHOD_CONTRACT; return _cacheAddress; }
219 inline size_t token() { LIMITED_METHOD_CONTRACT; return _token; }
220 inline size_t size() { LIMITED_METHOD_CONTRACT; return sizeof(ResolveStub); }
223 friend struct ResolveHolder;
224 const static int resolveEntryPointLen = 32;
225 const static int slowEntryPointLen = 4;
226 const static int failEntryPointLen = 14;
228 WORD _resolveEntryPoint[resolveEntryPointLen];
229 WORD _slowEntryPoint[slowEntryPointLen];
230 WORD _failEntryPoint[failEntryPointLen];
233 size_t _cacheAddress; // lookupCache
236 PCODE _resolveWorkerTarget;
240 /* ResolveHolders are the containers for ResolveStubs, They provide
241 for any alignment of the stubs as necessary. The stubs are placed in a hash table keyed by
242 the token for which they are built. Efficiency of access requires that this token be aligned.
243 For now, we have copied that field into the ResolveHolder itself, if the resolve stub is arranged such that
244 any of its inlined tokens (non-prehashed) is aligned, then the token field in the ResolveHolder
248 static void InitializeStatic();
250 void Initialize(PCODE resolveWorkerTarget, PCODE patcherTarget,
251 size_t dispatchToken, UINT32 hashedToken,
252 void * cacheAddr, INT32 * counterAddr);
254 ResolveStub* stub() { LIMITED_METHOD_CONTRACT; return &_stub; }
256 static ResolveHolder* FromFailEntry(PCODE failEntry);
257 static ResolveHolder* FromResolveEntry(PCODE resolveEntry);
263 /*VTableCallStub**************************************************************************************
264 These are jump stubs that perform a vtable-base virtual call. These stubs assume that an object is placed
265 in the first argument register (this pointer). From there, the stub extracts the MethodTable pointer, followed by the
266 vtable pointer, and finally jumps to the target method at a given slot in the vtable.
268 struct VTableCallStub
270 friend struct VTableCallHolder;
274 LIMITED_METHOD_CONTRACT;
276 BYTE* pStubCode = (BYTE *)this;
278 size_t cbSize = 4; // First ldr instruction
280 // If we never save r0 to the red zone, we have the short version of the stub
281 if (*(UINT32*)(&pStubCode[cbSize]) != 0x0c04f84d)
285 4 + // ldr r12,[r12+offset]
286 4 + // ldr r12,[r12+offset]
288 4; // Slot value (data storage, not a real instruction)
291 cbSize += 4; // Saving r0 into red zone
292 cbSize += (*(WORD*)(&pStubCode[cbSize]) == 0xf8dc ? 4 : 12); // Loading of vtable into r12
293 cbSize += (*(WORD*)(&pStubCode[cbSize]) == 0xf8dc ? 4 : 12); // Loading of targe address into r12
295 return cbSize + 6 /* Restore r0, bx*/ + 4 /* Slot value */;
298 inline PCODE entryPoint() const { LIMITED_METHOD_CONTRACT; return (PCODE)&_entryPoint[0] + THUMB_CODE; }
300 inline size_t token()
302 LIMITED_METHOD_CONTRACT;
303 DWORD slot = *(DWORD*)(reinterpret_cast<BYTE*>(this) + size() - 4);
304 return DispatchToken::CreateDispatchToken(slot).To_SIZE_T();
308 BYTE _entryPoint[0]; // Dynamically sized stub. See Initialize() for more details.
311 /* VTableCallHolders are the containers for VTableCallStubs, they provide for any alignment of
312 stubs as necessary. */
313 struct VTableCallHolder
315 void Initialize(unsigned slot);
317 VTableCallStub* stub() { LIMITED_METHOD_CONTRACT; return reinterpret_cast<VTableCallStub *>(this); }
319 static size_t GetHolderSize(unsigned slot)
321 STATIC_CONTRACT_WRAPPER;
322 unsigned offsetOfIndirection = MethodTable::GetVtableOffset() + MethodTable::GetIndexOfVtableIndirection(slot) * TARGET_POINTER_SIZE;
323 unsigned offsetAfterIndirection = MethodTable::GetIndexAfterVtableIndirection(slot) * TARGET_POINTER_SIZE;
325 int indirectionsSize = (offsetOfIndirection > 0xFFF ? 12 : 4) + (offsetAfterIndirection > 0xFFF ? 12 : 4);
326 if (offsetOfIndirection > 0xFFF || offsetAfterIndirection > 0xFFF)
327 indirectionsSize += 8; // Save/restore r0 using red zone
329 return 6 + indirectionsSize + 4;
332 static VTableCallHolder* VTableCallHolder::FromVTableCallEntry(PCODE entry)
334 LIMITED_METHOD_CONTRACT;
335 return (VTableCallHolder*)(entry & ~THUMB_CODE);
339 // VTableCallStub follows here. It is dynamically sized on allocation because it could
340 // use short/long instruction sizes for the mov/jmp, depending on the slot value.
348 #ifndef DACCESS_COMPILE
351 extern size_t g_lookup_inline_counter;
352 extern size_t g_mono_call_counter;
353 extern size_t g_mono_miss_counter;
354 extern size_t g_poly_call_counter;
355 extern size_t g_poly_miss_counter;
358 TADDR StubDispatchFrame_MethodFrameVPtr;
360 LookupHolder* LookupHolder::FromLookupEntry(PCODE lookupEntry)
362 lookupEntry = lookupEntry & ~THUMB_CODE;
363 return (LookupHolder*) ( lookupEntry - offsetof(LookupHolder, _stub) - offsetof(LookupStub, _entryPoint) );
367 /* Template used to generate the stub. We generate a stub by allocating a block of
368 memory and copy the template over it and just update the specific fields that need
371 DispatchStub dispatchInit;
373 DispatchHolder* DispatchHolder::FromDispatchEntry(PCODE dispatchEntry)
375 LIMITED_METHOD_CONTRACT;
376 dispatchEntry = dispatchEntry & ~THUMB_CODE;
377 DispatchHolder* dispatchHolder = (DispatchHolder*) ( dispatchEntry - offsetof(DispatchHolder, _stub) - offsetof(DispatchStub, _entryPoint) );
378 // _ASSERTE(dispatchHolder->_stub._entryPoint[0] == dispatchInit._entryPoint[0]);
379 return dispatchHolder;
383 /* Template used to generate the stub. We generate a stub by allocating a block of
384 memory and copy the template over it and just update the specific fields that need
388 ResolveStub resolveInit;
390 ResolveHolder* ResolveHolder::FromFailEntry(PCODE failEntry)
392 LIMITED_METHOD_CONTRACT;
393 failEntry = failEntry & ~THUMB_CODE;
394 ResolveHolder* resolveHolder = (ResolveHolder*) ( failEntry - offsetof(ResolveHolder, _stub) - offsetof(ResolveStub, _failEntryPoint) );
395 // _ASSERTE(resolveHolder->_stub._resolveEntryPoint[0] == resolveInit._resolveEntryPoint[0]);
396 return resolveHolder;
399 ResolveHolder* ResolveHolder::FromResolveEntry(PCODE resolveEntry)
401 LIMITED_METHOD_CONTRACT;
402 resolveEntry = resolveEntry & ~THUMB_CODE;
403 ResolveHolder* resolveHolder = (ResolveHolder*) ( resolveEntry - offsetof(ResolveHolder, _stub) - offsetof(ResolveStub, _resolveEntryPoint) );
404 // _ASSERTE(resolveHolder->_stub._resolveEntryPoint[0] == resolveInit._resolveEntryPoint[0]);
405 return resolveHolder;
408 void MovRegImm(BYTE* p, int reg, TADDR imm);
410 void VTableCallHolder::Initialize(unsigned slot)
412 unsigned offsetOfIndirection = MethodTable::GetVtableOffset() + MethodTable::GetIndexOfVtableIndirection(slot) * TARGET_POINTER_SIZE;
413 unsigned offsetAfterIndirection = MethodTable::GetIndexAfterVtableIndirection(slot) * TARGET_POINTER_SIZE;
414 _ASSERTE(MethodTable::VTableIndir_t::isRelative == false /* TODO: NYI */);
416 VTableCallStub* pStub = stub();
417 BYTE* p = (BYTE*)(pStub->entryPoint() & ~THUMB_CODE);
419 // ldr r12,[r0] : r12 = MethodTable pointer
420 *(UINT32*)p = 0xc000f8d0; p += 4;
422 if (offsetOfIndirection > 0xFFF || offsetAfterIndirection > 0xFFF)
424 // str r0, [sp, #-4]. Save r0 in the red zone
425 *(UINT32*)p = 0x0c04f84d; p += 4;
428 if (offsetOfIndirection > 0xFFF)
430 // mov r0, offsetOfIndirection
431 MovRegImm(p, 0, offsetOfIndirection); p += 8;
432 // ldr r12, [r12, r0]
433 *(UINT32*)p = 0xc000f85c; p += 4;
437 // ldr r12, [r12 + offset]
438 *(WORD *)p = 0xf8dc; p += 2;
439 *(WORD *)p = (WORD)(offsetOfIndirection | 0xc000); p += 2;
442 if (offsetAfterIndirection > 0xFFF)
444 // mov r0, offsetAfterIndirection
445 MovRegImm(p, 0, offsetAfterIndirection); p += 8;
446 // ldr r12, [r12, r0]
447 *(UINT32*)p = 0xc000f85c; p += 4;
451 // ldr r12, [r12 + offset]
452 *(WORD *)p = 0xf8dc; p += 2;
453 *(WORD *)p = (WORD)(offsetAfterIndirection | 0xc000); p += 2;
456 if (offsetOfIndirection > 0xFFF || offsetAfterIndirection > 0xFFF)
458 // ldr r0, [sp, #-4]. Restore r0 from the red zone.
459 *(UINT32*)p = 0x0c04f85d; p += 4;
463 *(UINT16*)p = 0x4760; p += 2;
465 // Store the slot value here for convenience. Not a real instruction (unreachable anyways)
466 *(UINT32*)p = slot; p += 4;
468 _ASSERT(p == (BYTE*)(stub()->entryPoint() & ~THUMB_CODE) + VTableCallHolder::GetHolderSize(slot));
469 _ASSERT(stub()->size() == VTableCallHolder::GetHolderSize(slot));
472 #endif // DACCESS_COMPILE
474 VirtualCallStubManager::StubKind VirtualCallStubManager::predictStubKind(PCODE stubStartAddress)
477 #ifdef DACCESS_COMPILE
479 return SK_BREAKPOINT; // Dac always uses the slower lookup
483 StubKind stubKind = SK_UNKNOWN;
484 TADDR pInstr = PCODEToPINSTR(stubStartAddress);
488 // If stubStartAddress is completely bogus, then this might AV,
489 // so we protect it with SEH. An AV here is OK.
490 AVInRuntimeImplOkayHolder AVOkay;
492 WORD firstWord = *((WORD*) pInstr);
494 if (*((UINT32*)pInstr) == 0xc000f8d0)
496 // Confirm the thrid word belongs to the vtable stub pattern
497 WORD thirdWord = ((WORD*)pInstr)[2];
498 if (thirdWord == 0xf84d /* Part of str r0, [sp, #-4] */ ||
499 thirdWord == 0xf8dc /* Part of ldr r12, [r12 + offset] */)
500 stubKind = SK_VTABLECALL;
503 if (stubKind == SK_UNKNOWN)
505 //Assuming that RESOLVE_STUB_FIRST_WORD & DISPATCH_STUB_FIRST_WORD have same values
506 if (firstWord == DISPATCH_STUB_FIRST_WORD)
508 WORD thirdWord = ((WORD*)pInstr)[2];
509 if (thirdWord == 0xf84d)
511 stubKind = SK_DISPATCH;
513 else if (thirdWord == 0xb460)
515 stubKind = SK_RESOLVE;
518 else if (firstWord == 0xf8df)
520 stubKind = SK_LOOKUP;
526 stubKind = SK_UNKNOWN;
528 EX_END_CATCH(SwallowAllExceptions);
532 #endif // DACCESS_COMPILE
535 #endif //DECLARE_DATA
537 #endif // _VIRTUAL_CALL_STUB_ARM_H