* beginend.cc (htm_fastpath): New.
(gtm_thread::begin_transaction, _ITM_commitTransaction,
_ITM_commitTransactionEH): Add HTM fastpath handling.
* config/linux/rwlock.h (gtm_rwlock.is_write_locked): New.
* config/posix/rwlock.h (gtm_rwlock.is_write_locked): New.
* config/x86/target.h (htm_available, htm_init, htm_begin_success,
htm_begin, htm_commit, htm_abort, htm_abort_should_retry): New.
* configure.tgt: Add -mrtm to XCFLAGS.
* method-serial.cc (htm_mg, o_htm_mg, htm_dispatch, dispatch_htm): New.
(gtm_thread::serialirr_mode): Add HTM fastpath handling.
* libitm_i.h (htm_fastpath, dispatch_htm): Declare.
* retry.cc (parse_default_method): Add HTM method parsing.
(gtm_thread::number_of_threads_changed): Use HTM by default if
available.
From-SVN: r193369
+2012-11-09 Torvald Riegel <triegel@redhat.com>
+
+ * beginend.cc (htm_fastpath): New.
+ (gtm_thread::begin_transaction, _ITM_commitTransaction,
+ _ITM_commitTransactionEH): Add HTM fastpath handling.
+ * config/linux/rwlock.h (gtm_rwlock.is_write_locked): New.
+ * config/posix/rwlock.h (gtm_rwlock.is_write_locked): New.
+ * config/x86/target.h (htm_available, htm_init, htm_begin_success,
+ htm_begin, htm_commit, htm_abort, htm_abort_should_retry): New.
+ * configure.tgt: Add -mrtm to XCFLAGS.
+ * method-serial.cc (htm_mg, o_htm_mg, htm_dispatch, dispatch_htm): New.
+ (gtm_thread::serialirr_mode): Add HTM fastpath handling.
+ * libitm_i.h (htm_fastpath, dispatch_htm): Declare.
+ * retry.cc (parse_default_method): Add HTM method parsing.
+ (gtm_thread::number_of_threads_changed): Use HTM by default if
+ available.
+
2012-11-04 Thomas Schwinge <thomas@codesourcery.com>
* configure: Regenerate.
static pthread_key_t thr_release_key;
static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
+// See gtm_thread::begin_transaction.
+uint32_t GTM::htm_fastpath = 0;
/* Allocate a transaction structure. */
void *
if (unlikely(prop & pr_undoLogCode))
GTM_fatal("pr_undoLogCode not supported");
+#if defined(USE_HTM_FASTPATH) && !defined(HTM_CUSTOM_FASTPATH)
+ // HTM fastpath. Only chosen in the absence of transaction_cancel to allow
+ // using an uninstrumented code path.
+ // The fastpath is enabled only by dispatch_htm's method group, which uses
+ // serial-mode methods as fallback. Serial-mode transactions cannot execute
+ // concurrently with HW transactions because the latter monitor the serial
+ // lock's writer flag and thus abort if another thread is or becomes a
+ // serial transaction. Therefore, if the fastpath is enabled, then a
+ // transaction is not executing as a HW transaction iff the serial lock is
+ // write-locked. This allows us to use htm_fastpath and the serial lock's
+ // writer flag to reliable determine whether the current thread runs a HW
+ // transaction, and thus we do not need to maintain this information in
+ // per-thread state.
+ // If an uninstrumented code path is not available, we can still run
+ // instrumented code from a HW transaction because the HTM fastpath kicks
+ // in early in both begin and commit, and the transaction is not canceled.
+ // HW transactions might get requests to switch to serial-irrevocable mode,
+ // but these can be ignored because the HTM provides all necessary
+ // correctness guarantees. Transactions cannot detect whether they are
+ // indeed in serial mode, and HW transactions should never need serial mode
+ // for any internal changes (e.g., they never abort visibly to the STM code
+ // and thus do not trigger the standard retry handling).
+ if (likely(htm_fastpath && (prop & pr_hasNoAbort)))
+ {
+ for (uint32_t t = htm_fastpath; t; t--)
+ {
+ uint32_t ret = htm_begin();
+ if (htm_begin_success(ret))
+ {
+ // We are executing a transaction now.
+ // Monitor the writer flag in the serial-mode lock, and abort
+ // if there is an active or waiting serial-mode transaction.
+ if (unlikely(serial_lock.is_write_locked()))
+ htm_abort();
+ else
+ // We do not need to set a_saveLiveVariables because of HTM.
+ return (prop & pr_uninstrumentedCode) ?
+ a_runUninstrumentedCode : a_runInstrumentedCode;
+ }
+ // The transaction has aborted. Don't retry if it's unlikely that
+ // retrying the transaction will be successful.
+ if (!htm_abort_should_retry(ret))
+ break;
+ // Wait until any concurrent serial-mode transactions have finished.
+ // This is an empty critical section, but won't be elided.
+ if (serial_lock.is_write_locked())
+ {
+ tx = gtm_thr();
+ if (unlikely(tx == NULL))
+ {
+ // See below.
+ tx = new gtm_thread();
+ set_gtm_thr(tx);
+ }
+ serial_lock.read_lock(tx);
+ serial_lock.read_unlock(tx);
+ // TODO We should probably reset the retry count t here, unless
+ // we have retried so often that we should go serial to avoid
+ // starvation.
+ }
+ }
+ }
+#endif
+
tx = gtm_thr();
if (unlikely(tx == NULL))
{
void ITM_REGPARM
_ITM_commitTransaction(void)
{
+#if defined(USE_HTM_FASTPATH)
+ // HTM fastpath. If we are not executing a HW transaction, then we will be
+ // a serial-mode transaction. If we are, then there will be no other
+ // concurrent serial-mode transaction.
+ // See gtm_thread::begin_transaction.
+ if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
tx->restart (RESTART_VALIDATE_COMMIT);
void ITM_REGPARM
_ITM_commitTransactionEH(void *exc_ptr)
{
+#if defined(USE_HTM_FASTPATH)
+ // See _ITM_commitTransaction.
+ if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
{
-/* Copyright (C) 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2011, 2012 Free Software Foundation, Inc.
Contributed by Torvald Riegel <triegel@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
bool write_upgrade (gtm_thread *tx);
void write_upgrade_finish (gtm_thread *tx);
+ // Returns true iff there is a concurrent active or waiting writer.
+ // This is primarily useful for simple HyTM approaches, and the value being
+ // checked is loaded with memory_order_relaxed.
+ bool is_write_locked()
+ {
+ return writers.load (memory_order_relaxed) != 0;
+ }
+
protected:
bool write_lock_generic (gtm_thread *tx);
};
-/* Copyright (C) 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2009, 2011, 2012 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
bool write_upgrade (gtm_thread *tx);
void write_upgrade_finish (gtm_thread *tx);
+ // Returns true iff there is a concurrent active or waiting writer.
+ // This is primarily useful for simple HyTM approaches, and the value being
+ // checked is loaded with memory_order_relaxed.
+ bool is_write_locked()
+ {
+ return summary.load (memory_order_relaxed) & (a_writer | w_writer);
+ }
+
protected:
bool write_lock_generic (gtm_thread *tx);
};
-/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2008, 2009, 2011, 2012 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
+// We'll be using some of the cpu builtins, and their associated types.
+#include <x86intrin.h>
+#include <cpuid.h>
+
namespace GTM HIDDEN {
/* ??? This doesn't work for Win64. */
__builtin_ia32_pause ();
}
-} // namespace GTM
+// Use Intel RTM if supported by the assembler.
+// See gtm_thread::begin_transaction for how these functions are used.
+#ifdef HAVE_AS_RTM
+#define USE_HTM_FASTPATH
-// We'll be using some of the cpu builtins, and their associated types.
-#ifndef __cplusplus
-/* ??? It's broken for C++. */
-#include <x86intrin.h>
-#else
-# ifdef __SSE2__
-# include <emmintrin.h>
-# elif defined(__SSE__)
-# include <xmmintrin.h>
-# endif
-# ifdef __AVX__
-# include <immintrin.h>
-# endif
+static inline bool
+htm_available ()
+{
+ const unsigned cpuid_rtm = bit_RTM;
+ if (__get_cpuid_max (0, NULL) >= 7)
+ {
+ unsigned a, b, c, d;
+ __cpuid_count (7, 0, a, b, c, d);
+ if (b & cpuid_rtm)
+ return true;
+ }
+ return false;
+}
+
+static inline uint32_t
+htm_init ()
+{
+ // Maximum number of times we try to execute a transaction as a HW
+ // transaction.
+ // ??? Why 2? Any offline or runtime tuning necessary?
+ return htm_available () ? 2 : 0;
+}
+
+static inline uint32_t
+htm_begin ()
+{
+ return _xbegin();
+}
+
+static inline bool
+htm_begin_success (uint32_t begin_ret)
+{
+ return begin_ret == _XBEGIN_STARTED;
+}
+
+static inline void
+htm_commit ()
+{
+ _xend();
+}
+
+static inline void
+htm_abort ()
+{
+ // ??? According to a yet unpublished ABI rule, 0xff is reserved and
+ // supposed to signal a busy lock. Source: andi.kleen@intel.com
+ _xabort(0xff);
+}
+
+static inline bool
+htm_abort_should_retry (uint32_t begin_ret)
+{
+ return begin_ret & _XABORT_RETRY;
+}
#endif
+
+
+} // namespace GTM
XCFLAGS="${XCFLAGS} -fomit-frame-pointer"
fi
esac
+ XCFLAGS="${XCFLAGS} -mrtm"
ARCH=x86
;;
XCFLAGS="${XCFLAGS} -fomit-frame-pointer"
;;
esac
+ XCFLAGS="${XCFLAGS} -mrtm"
ARCH=x86
;;
extern abi_dispatch *dispatch_serialirr_onwrite();
extern abi_dispatch *dispatch_gl_wt();
extern abi_dispatch *dispatch_ml_wt();
+extern abi_dispatch *dispatch_htm();
extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
+// Control variable for the HTM fastpath that uses serial mode as fallback.
+// Non-zero if the HTM fastpath is enabled. See gtm_thread::begin_transaction.
+extern uint32_t htm_fastpath;
+
} // namespace GTM
#endif // LIBITM_I_H
}
};
+// This group is pure HTM with serial mode as a fallback. There is no
+// difference to serial_mg except that we need to enable or disable the HTM
+// fastpath. See gtm_thread::begin_transaction.
+struct htm_mg : public method_group
+{
+ virtual void init()
+ {
+ // Enable the HTM fastpath if the HW is available. The fastpath is
+ // initially disabled.
+#ifdef USE_HTM_FASTPATH
+ htm_fastpath = htm_init();
+#endif
+ }
+ virtual void fini()
+ {
+ // Disable the HTM fastpath.
+ htm_fastpath = 0;
+ }
+};
+
+static htm_mg o_htm_mg;
+
+// We just need the subclass to associate it with the HTM method group that
+// sets up the HTM fast path. This will use serial_dispatch as fallback for
+// transactions that might get canceled; it has a different method group, but
+// this is harmless for serial dispatchs because they never abort.
+class htm_dispatch : public serialirr_dispatch
+{
+ public:
+ htm_dispatch() : serialirr_dispatch(false, true, false, false,
+ gtm_thread::STATE_SERIAL | gtm_thread::STATE_IRREVOCABLE, &o_htm_mg)
+ { }
+};
+
} // anon namespace
static const serialirr_dispatch o_serialirr_dispatch;
static const serial_dispatch o_serial_dispatch;
static const serialirr_onwrite_dispatch o_serialirr_onwrite_dispatch;
+static const htm_dispatch o_htm_dispatch;
abi_dispatch *
GTM::dispatch_serialirr ()
const_cast<serialirr_onwrite_dispatch *>(&o_serialirr_onwrite_dispatch);
}
+abi_dispatch *
+GTM::dispatch_htm ()
+{
+ return const_cast<htm_dispatch *>(&o_htm_dispatch);
+}
+
// Put the transaction into serial-irrevocable mode.
void
{
struct abi_dispatch *disp = abi_disp ();
+#if defined(USE_HTM_FASTPATH)
+ // HTM fastpath. If we are executing a HW transaction, don't go serial but
+ // continue. See gtm_thread::begin_transaction.
+ if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
+ return;
+#endif
+
if (this->state & STATE_SERIAL)
{
if (this->state & STATE_IRREVOCABLE)
-/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2008, 2009, 2011, 2012 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
disp = GTM::dispatch_ml_wt();
env += 5;
}
+ else if (strncmp(env, "htm", 3) == 0)
+ {
+ disp = GTM::dispatch_htm();
+ env += 3;
+ }
else
goto unknown;
set_default_dispatch(default_dispatch_user);
else
{
- abi_dispatch* a = dispatch_ml_wt();
+ // If HTM is available, use it by default with serial mode as
+ // fallback. Otherwise, use ml_wt because it probably scales best.
+ abi_dispatch* a;
+#ifdef USE_HTM_FASTPATH
+ if (htm_available())
+ a = dispatch_htm();
+ else
+#endif
+ a = dispatch_ml_wt();
if (a->supports(now))
set_default_dispatch(a);
else
projects / platform / upstream / gcc.git / commitdiff