1 /* Copyright (C) 2008-2019 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Transactional Memory Library (libitm).
6 Libitm is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 Under Section 7 of GPL version 3, you are granted additional
17 permissions described in the GCC Runtime Library Exception, version
18 3.1, as published by the Free Software Foundation.
20 You should have received a copy of the GNU General Public License and
21 a copy of the GCC Runtime Library Exception along with this program;
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 <http://www.gnu.org/licenses/>. */
25 /* The following are internal implementation functions and definitions.
26 To distinguish them from those defined by the Intel ABI, they all
27 begin with GTM/gtm. */
39 #include "local_atomic"
41 /* Don't require libgcc_s.so for exceptions. */
42 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
47 namespace GTM HIDDEN {
51 typedef unsigned int gtm_word __attribute__((mode (word)));
53 // These values are given to GTM_restart_transaction and indicate the
54 // reason for the restart. The reason is used to decide what STM
55 // implementation should be used during the next iteration.
56 enum gtm_restart_reason
61 RESTART_VALIDATE_READ,
62 RESTART_VALIDATE_WRITE,
63 RESTART_VALIDATE_COMMIT,
66 RESTART_CLOSED_NESTING,
67 RESTART_INIT_METHOD_GROUP,
69 NO_RESTART = NUM_RESTARTS
78 #include "containers.h"
80 #ifdef __USER_LABEL_PREFIX__
81 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
82 # define UPFX1(t) UPFX2(t)
88 namespace GTM HIDDEN {
90 // A log of (de)allocation actions. We defer handling of some actions until
91 // a commit of the outermost transaction. We also rely on potentially having
92 // both an allocation and a deallocation for the same piece of memory in the
93 // log; the order in which such entries are processed does not matter because
94 // the actions are not in conflict (see below).
95 // This type is private to alloc.c, but needs to be defined so that
96 // the template used inside gtm_thread can instantiate.
97 struct gtm_alloc_action
99 // Iff free_fn_sz is nonzero, it must be used instead of free_fn, and vice
101 void (*free_fn)(void *);
102 void (*free_fn_sz)(void *, size_t);
104 // If true, this is an allocation; we discard the log entry on outermost
105 // commit, and deallocate on abort. If false, this is a deallocation and
106 // we deallocate on outermost commit and discard the log entry on abort.
112 // A transaction checkpoint: data that has to saved and restored when doing
114 struct gtm_transaction_cp
118 aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
119 size_t user_actions_size;
120 _ITM_transactionId_t id;
122 uint32_t cxa_catch_count;
123 unsigned int cxa_uncaught_count;
124 // We might want to use a different but compatible dispatch method for
125 // a nested transaction.
127 // Nesting level of this checkpoint (1 means that this is a checkpoint of
128 // the outermost transaction).
131 void save(gtm_thread* tx);
132 void commit(gtm_thread* tx);
135 // An undo log for writes.
138 vector<gtm_word> undolog;
140 // Log the previous value at a certain address.
141 // The easiest way to inline this is to just define this here.
142 void log(const void *ptr, size_t len)
144 size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
145 gtm_word *undo = undolog.push(words + 2);
146 memcpy(undo, ptr, len);
148 undo[words + 1] = (gtm_word) ptr;
151 void commit () { undolog.clear(); }
152 size_t size() const { return undolog.size(); }
155 void rollback (gtm_thread* tx, size_t until_size = 0);
158 // An entry of a read or write log. Used by multi-lock TM methods.
159 struct gtm_rwlog_entry
161 atomic<gtm_word> *orec;
165 // Contains all thread-specific data required by the entire library.
166 // This includes all data relevant to a single transaction. Because most
167 // thread-specific data is about the current transaction, we also refer to
168 // the transaction-specific parts of gtm_thread as "the transaction" (the
169 // same applies to names of variables and arguments).
170 // All but the shared part of this data structure are thread-local data.
171 // gtm_thread could be split into transaction-specific structures and other
172 // per-thread data (with those parts then nested in gtm_thread), but this
173 // would make it harder to later rearrange individual members to optimize data
174 // accesses. Thus, for now we keep one flat object, and will only split it if
175 // the code gets too messy.
181 _ITM_userCommitFunction fn;
184 _ITM_transactionId_t resuming_id;
187 // The jump buffer by which GTM_longjmp restarts the transaction.
188 // This field *must* be at the beginning of the transaction.
191 // Data used by local.c for the undo log for both local and shared memory.
194 // Read and write logs. Used by multi-lock TM methods.
195 vector<gtm_rwlog_entry> readlog;
196 vector<gtm_rwlog_entry> writelog;
198 // Data used by alloc.c for the malloc/free undo log.
199 aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
201 // Data used by useraction.c for the user-defined commit/abort handlers.
202 vector<user_action> user_actions;
204 // A numerical identifier for this transaction.
205 _ITM_transactionId_t id;
207 // The _ITM_codeProperties of this transaction as given by the compiler.
210 // The nesting depth for subsequently started transactions. This variable
211 // will be set to 1 when starting an outermost transaction.
214 // Set if this transaction owns the serial write lock.
215 // Can be reset only when restarting the outermost transaction.
216 static const uint32_t STATE_SERIAL = 0x0001;
217 // Set if the serial-irrevocable dispatch table is installed.
218 // Implies that no logging is being done, and abort is not possible.
219 // Can be reset only when restarting the outermost transaction.
220 static const uint32_t STATE_IRREVOCABLE = 0x0002;
222 // A bitmask of the above.
225 // In order to reduce cacheline contention on global_tid during
226 // beginTransaction, we allocate a block of 2**N ids to the thread
227 // all at once. This number is the next value to be allocated from
228 // the block, or 0 % 2**N if no such block is allocated.
229 _ITM_transactionId_t local_tid;
231 // Data used by eh_cpp.c for managing exceptions within the transaction.
232 uint32_t cxa_catch_count;
233 // If cxa_uncaught_count_ptr is 0, we don't need to roll back exceptions.
234 unsigned int *cxa_uncaught_count_ptr;
235 unsigned int cxa_uncaught_count;
238 // Checkpoints for closed nesting.
239 vector<gtm_transaction_cp> parent_txns;
241 // Data used by retry.c for deciding what STM implementation should
242 // be used for the next iteration of the transaction.
243 // Only restart_total is reset to zero when the transaction commits, the
244 // other counters are total values for all previously executed transactions.
245 // restart_total is also used by the HTM fastpath in a different way.
246 uint32_t restart_reason[NUM_RESTARTS];
247 uint32_t restart_total;
249 // *** The shared part of gtm_thread starts here. ***
250 // Shared state is on separate cachelines to avoid false sharing with
251 // thread-local parts of gtm_thread.
253 // Points to the next thread in the list of all threads.
254 gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
256 // If this transaction is inactive, shared_state is ~0. Otherwise, this is
257 // an active or serial transaction.
258 atomic<gtm_word> shared_state;
260 // The lock that provides access to serial mode. Non-serialized
261 // transactions acquire read locks; a serialized transaction acquires
263 // Accessed from assembly language, thus the "asm" specifier on
264 // the name, avoiding complex name mangling.
265 static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
267 // The head of the list of all threads' transactions.
268 static gtm_thread *list_of_threads;
269 // The number of all registered threads.
270 static unsigned number_of_threads;
273 void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
274 void record_allocation (void *, void (*)(void *));
275 void forget_allocation (void *, void (*)(void *));
276 void forget_allocation (void *, size_t, void (*)(void *, size_t));
277 void discard_allocation (const void *ptr)
279 alloc_actions.erase((uintptr_t) ptr);
283 void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
285 void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
291 static void *operator new(size_t);
292 static void operator delete(void *);
294 // Invoked from assembly language, thus the "asm" specifier on
295 // the name, avoiding complex name mangling.
296 static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
297 __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
299 void init_cpp_exceptions ();
300 void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
303 // Must be called outside of transactions (i.e., after rollback).
304 void decide_retry_strategy (gtm_restart_reason);
305 abi_dispatch* decide_begin_dispatch (uint32_t prop);
306 void number_of_threads_changed(unsigned previous, unsigned now);
307 // Must be called from serial mode. Does not call set_abi_disp().
308 void set_default_dispatch(abi_dispatch* disp);
310 // In method-serial.cc
311 void serialirr_mode ();
314 void rollback_user_actions (size_t until_size = 0);
315 void commit_user_actions ();
322 namespace GTM HIDDEN {
324 // An unscaled count of the number of times we should spin attempting to
325 // acquire locks before we block the current thread and defer to the OS.
326 // This variable isn't used when the standard POSIX lock implementations
328 extern uint64_t gtm_spin_count_var;
330 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
331 ITM_NORETURN ITM_REGPARM;
333 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
335 extern void GTM_error (const char *fmt, ...)
336 __attribute__((format (printf, 1, 2)));
337 extern void GTM_fatal (const char *fmt, ...)
338 __attribute__((noreturn, format (printf, 1, 2)));
340 extern abi_dispatch *dispatch_serial();
341 extern abi_dispatch *dispatch_serialirr();
342 extern abi_dispatch *dispatch_serialirr_onwrite();
343 extern abi_dispatch *dispatch_gl_wt();
344 extern abi_dispatch *dispatch_ml_wt();
345 extern abi_dispatch *dispatch_htm();