From aee3a9dfba8152e7f5a77f9d054486ffa51f2d29 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Fri, 14 Sep 2018 13:43:20 +0300 Subject: [PATCH 01/16] [ESan] Fix typo for the testsuite. Change-Id: I4fc5c6f0978b4be6aa286c42e06b34c687574a6c --- gcc/testsuite/g++.dg/esan/{unalignet.C => unaligned.C} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gcc/testsuite/g++.dg/esan/{unalignet.C => unaligned.C} (100%) diff --git a/gcc/testsuite/g++.dg/esan/unalignet.C b/gcc/testsuite/g++.dg/esan/unaligned.C similarity index 100% rename from gcc/testsuite/g++.dg/esan/unalignet.C rename to gcc/testsuite/g++.dg/esan/unaligned.C -- 2.7.4 From 02fe217aae2dfd746e3bb4781cc304b0ad1abe7d Mon Sep 17 00:00:00 2001 From: ktkachov Date: Mon, 17 Oct 2016 16:37:38 +0000 Subject: [PATCH 02/16] [AArch64] Use new target pass registration framework for FMA steering pass * config/aarch64/aarch64.c: Delete inclusion of cortex-a57-fma-steering.h. (aarch64_override_options): Delete call to aarch64_register_fma_steering. * config/aarch64/aarch64-protos.h (make_pass_fma_steering): Declare. * config/aarch64/cortex-a57-fma-steering.h: Delete. * config/aarch64/aarch64-passes.def: New file. * config/aarch64/cortex-a57-fma-steering.c (aarch64_register_fma_steering): Delete definition. (make_pass_fma_steering): Remove static qualifier. * config/aarch64/t-aarch64 (PASSES_EXTRA): New directive. (cortex-a57-fma-steering.o): Remove dependency on cortex-a57-fma-steering.h. (backported e8ebfb45f2e320c6a1e810c9d7ca1c40e2b26e6b) Change-Id: I608f6d880296473c76c203e387c96791be9e50f2 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@241248 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 16 ++++++++++++++++ .../{cortex-a57-fma-steering.h => aarch64-passes.def} | 7 +++---- gcc/config/aarch64/aarch64-protos.h | 2 ++ gcc/config/aarch64/aarch64.c | 4 ---- gcc/config/aarch64/cortex-a57-fma-steering.c | 16 +--------------- gcc/config/aarch64/t-aarch64 | 3 ++- 6 files changed, 24 insertions(+), 24 deletions(-) rename gcc/config/aarch64/{cortex-a57-fma-steering.h => aarch64-passes.def} (78%) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 4c5f9a6..d85f9d9 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,19 @@ +2016-10-17 Kyrylo Tkachov + + * config/aarch64/aarch64.c: Delete inclusion of + cortex-a57-fma-steering.h. + (aarch64_override_options): Delete call + to aarch64_register_fma_steering. + * config/aarch64/aarch64-protos.h (make_pass_fma_steering): Declare. + * config/aarch64/cortex-a57-fma-steering.h: Delete. + * config/aarch64/aarch64-passes.def: New file. + * config/aarch64/cortex-a57-fma-steering.c + (aarch64_register_fma_steering): Delete definition. + (make_pass_fma_steering): Remove static qualifier. + * config/aarch64/t-aarch64 (PASSES_EXTRA): New directive. + (cortex-a57-fma-steering.o): Remove dependency on + cortex-a57-fma-steering.h. + 2017-07-06 Vyacheslav Barinov * tree.def: Add STATEMENT_LIST_END tree code. diff --git a/gcc/config/aarch64/cortex-a57-fma-steering.h b/gcc/config/aarch64/aarch64-passes.def similarity index 78% rename from gcc/config/aarch64/cortex-a57-fma-steering.h rename to gcc/config/aarch64/aarch64-passes.def index 65bf5ac..7fe8039 100644 --- a/gcc/config/aarch64/cortex-a57-fma-steering.h +++ b/gcc/config/aarch64/aarch64-passes.def @@ -1,6 +1,5 @@ -/* This file contains declarations for the FMA steering optimization - pass for Cortex-A57. - Copyright (C) 2015-2016 Free Software Foundation, Inc. +/* AArch64-specific passes declarations. + Copyright (C) 2016 Free Software Foundation, Inc. Contributed by ARM Ltd. This file is part of GCC. @@ -19,4 +18,4 @@ along with GCC; see the file COPYING3. If not see . */ -void aarch64_register_fma_steering (void); +INSERT_PASS_AFTER (pass_regrename, 1, pass_fma_steering); diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index f270138..23ac210 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -466,4 +466,6 @@ enum aarch64_parse_opt_result aarch64_parse_extension (const char *, std::string aarch64_get_extension_string_for_isa_flags (unsigned long, unsigned long); +rtl_opt_pass *make_pass_fma_steering (gcc::context *ctxt); + #endif /* GCC_AARCH64_PROTOS_H */ diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index bd75ef2..a1db946 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -62,7 +62,6 @@ #include "rtl-iter.h" #include "tm-constrs.h" #include "sched-int.h" -#include "cortex-a57-fma-steering.h" #include "target-globals.h" #include "common/common-target.h" @@ -8586,9 +8585,6 @@ aarch64_override_options (void) while processing functions with potential target attributes. */ target_option_default_node = target_option_current_node = build_target_option_node (&global_options); - - aarch64_register_fma_steering (); - } /* Implement targetm.override_options_after_change. */ diff --git a/gcc/config/aarch64/cortex-a57-fma-steering.c b/gcc/config/aarch64/cortex-a57-fma-steering.c index 1bf804b..b5f329f 100644 --- a/gcc/config/aarch64/cortex-a57-fma-steering.c +++ b/gcc/config/aarch64/cortex-a57-fma-steering.c @@ -35,7 +35,6 @@ #include "context.h" #include "tree-pass.h" #include "regrename.h" -#include "cortex-a57-fma-steering.h" #include "aarch64-protos.h" /* For better performance, the destination of FMADD/FMSUB instructions should @@ -1068,21 +1067,8 @@ public: /* Create a new fma steering pass instance. */ -static rtl_opt_pass * +rtl_opt_pass * make_pass_fma_steering (gcc::context *ctxt) { return new pass_fma_steering (ctxt); } - -/* Register the FMA steering pass to the pass manager. */ - -void -aarch64_register_fma_steering () -{ - opt_pass *pass_fma_steering = make_pass_fma_steering (g); - - struct register_pass_info fma_steering_info - = { pass_fma_steering, "rnreg", 1, PASS_POS_INSERT_AFTER }; - - register_pass (&fma_steering_info); -} diff --git a/gcc/config/aarch64/t-aarch64 b/gcc/config/aarch64/t-aarch64 index 1ba8a5d..b461eb5 100644 --- a/gcc/config/aarch64/t-aarch64 +++ b/gcc/config/aarch64/t-aarch64 @@ -56,12 +56,13 @@ aarch64-c.o: $(srcdir)/config/aarch64/aarch64-c.c $(CONFIG_H) $(SYSTEM_H) \ $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \ $(srcdir)/config/aarch64/aarch64-c.c +PASSES_EXTRA += $(srcdir)/config/aarch64/aarch64-passes.def + cortex-a57-fma-steering.o: $(srcdir)/config/aarch64/cortex-a57-fma-steering.c \ $(CONFIG_H) $(SYSTEM_H) $(TM_H) $(REGS_H) insn-config.h $(RTL_BASE_H) \ dominance.h cfg.h cfganal.h $(BASIC_BLOCK_H) $(INSN_ATTR_H) $(RECOG_H) \ output.h hash-map.h $(DF_H) $(OBSTACK_H) $(TARGET_H) $(RTL_H) \ $(CONTEXT_H) $(TREE_PASS_H) regrename.h \ - $(srcdir)/config/aarch64/cortex-a57-fma-steering.h \ $(srcdir)/config/aarch64/aarch64-protos.h $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \ $(srcdir)/config/aarch64/cortex-a57-fma-steering.c -- 2.7.4 From 439de83cb9c410a9dd52d4683a6b38438c80c1f7 Mon Sep 17 00:00:00 2001 From: ktkachov Date: Tue, 14 Feb 2017 10:33:50 +0000 Subject: [PATCH 03/16] [ARM] PR rtl-optimization/68664 Implement TARGET_SCHED_CAN_SPECULATE_INSN hook PR rtl-optimization/68664 * config/arm/arm.c (arm_sched_can_speculate_insn): New function. Declare prototype. (TARGET_SCHED_CAN_SPECULATE_INSN): Define. (backported a705f613f2fe0ff13b18fdf92af1dc5574c963d3) Change-Id: I1eb8f369ebf3d85662df78e9da62cf0be35d8b5a git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@245425 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 7 +++++++ gcc/config/arm/arm.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index d85f9d9..b389af4 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,10 @@ +2017-02-14 Kyrylo Tkachov + + PR rtl-optimization/68664 + * config/arm/arm.c (arm_sched_can_speculate_insn): + New function. Declare prototype. + (TARGET_SCHED_CAN_SPECULATE_INSN): Define. + 2016-10-17 Kyrylo Tkachov * config/aarch64/aarch64.c: Delete inclusion of diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c index 181c304..ae21d1a 100644 --- a/gcc/config/arm/arm.c +++ b/gcc/config/arm/arm.c @@ -238,6 +238,7 @@ static bool arm_can_inline_p (tree, tree); static void arm_relayout_function (tree); static bool arm_valid_target_attribute_p (tree, tree, tree, int); static unsigned HOST_WIDE_INT arm_shift_truncation_mask (machine_mode); +static bool arm_sched_can_speculate_insn (rtx_insn *); static bool arm_macro_fusion_p (void); static bool arm_cannot_copy_insn_p (rtx_insn *); static int arm_issue_rate (void); @@ -412,6 +413,9 @@ static const struct attribute_spec arm_attribute_table[] = #undef TARGET_COMP_TYPE_ATTRIBUTES #define TARGET_COMP_TYPE_ATTRIBUTES arm_comp_type_attributes +#undef TARGET_SCHED_CAN_SPECULATE_INSN +#define TARGET_SCHED_CAN_SPECULATE_INSN arm_sched_can_speculate_insn + #undef TARGET_SCHED_MACRO_FUSION_P #define TARGET_SCHED_MACRO_FUSION_P arm_macro_fusion_p @@ -30066,6 +30070,35 @@ arm_fusion_enabled_p (tune_params::fuse_ops op) return current_tune->fusible_ops & op; } +/* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be + scheduled for speculative execution. Reject the long-running division + and square-root instructions. */ + +static bool +arm_sched_can_speculate_insn (rtx_insn *insn) +{ + switch (get_attr_type (insn)) + { + case TYPE_SDIV: + case TYPE_UDIV: + case TYPE_FDIVS: + case TYPE_FDIVD: + case TYPE_FSQRTS: + case TYPE_FSQRTD: + case TYPE_NEON_FP_SQRT_S: + case TYPE_NEON_FP_SQRT_D: + case TYPE_NEON_FP_SQRT_S_Q: + case TYPE_NEON_FP_SQRT_D_Q: + case TYPE_NEON_FP_DIV_S: + case TYPE_NEON_FP_DIV_D: + case TYPE_NEON_FP_DIV_S_Q: + case TYPE_NEON_FP_DIV_D_Q: + return false; + default: + return true; + } +} + /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */ static unsigned HOST_WIDE_INT -- 2.7.4 From 81c87ae477be33caa487ee363c0d58a494fcd487 Mon Sep 17 00:00:00 2001 From: ktkachov Date: Fri, 2 Jun 2017 14:41:52 +0000 Subject: [PATCH 04/16] [AArch64] Emit tighter strong atomic compare-exchange loop when comparing against zero * config/aarch64/aarch64.c (aarch64_split_compare_and_swap): Emit CBNZ inside loop when doing a strong exchange and comparing against zero. Generate the CC flags after the loop. * gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c: New test. (backported c5dcf8e544c5e98f1160ec390111b28d11d73b68) git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@248832 138bc75d-0d04-0410-961f-82ee72b054a4 Change-Id: I5204c56a8d3c11b0fde6697007d601cbafdb3ccf --- gcc/ChangeLog | 6 ++++ gcc/config/aarch64/aarch64.c | 41 ++++++++++++++++++---- gcc/testsuite/ChangeLog | 4 +++ .../aarch64/atomic_cmp_exchange_zero_strong_1.c | 12 +++++++ 4 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 gcc/testsuite/gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b389af4..76eb226 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,9 @@ +2017-06-02 Kyrylo Tkachov + + * config/aarch64/aarch64.c (aarch64_split_compare_and_swap): + Emit CBNZ inside loop when doing a strong exchange and comparing + against zero. Generate the CC flags after the loop. + 2017-02-14 Kyrylo Tkachov PR rtl-optimization/68664 diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index a1db946..795033f 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -11621,6 +11621,17 @@ aarch64_split_compare_and_swap (rtx operands[]) mode = GET_MODE (mem); model = memmodel_from_int (INTVAL (model_rtx)); + /* When OLDVAL is zero and we want the strong version we can emit a tighter + loop: + .label1: + LD[A]XR rval, [mem] + CBNZ rval, .label2 + ST[L]XR scratch, newval, [mem] + CBNZ scratch, .label1 + .label2: + CMP rval, 0. */ + bool strong_zero_p = !is_weak && oldval == const0_rtx; + label1 = NULL; if (!is_weak) { @@ -11637,11 +11648,21 @@ aarch64_split_compare_and_swap (rtx operands[]) else aarch64_emit_load_exclusive (mode, rval, mem, model_rtx); - cond = aarch64_gen_compare_reg (NE, rval, oldval); - x = gen_rtx_NE (VOIDmode, cond, const0_rtx); - x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, - gen_rtx_LABEL_REF (Pmode, label2), pc_rtx); - aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); + if (strong_zero_p) + { + x = gen_rtx_NE (VOIDmode, rval, const0_rtx); + x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, + gen_rtx_LABEL_REF (Pmode, label2), pc_rtx); + aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); + } + else + { + cond = aarch64_gen_compare_reg (NE, rval, oldval); + x = gen_rtx_NE (VOIDmode, cond, const0_rtx); + x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, + gen_rtx_LABEL_REF (Pmode, label2), pc_rtx); + aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); + } aarch64_emit_store_exclusive (mode, scratch, mem, newval, model_rtx); @@ -11660,7 +11681,15 @@ aarch64_split_compare_and_swap (rtx operands[]) } emit_label (label2); - + /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL + to set the condition flags. If this is not used it will be removed by + later passes. */ + if (strong_zero_p) + { + cond = gen_rtx_REG (CCmode, CC_REGNUM); + x = gen_rtx_COMPARE (CCmode, rval, const0_rtx); + emit_insn (gen_rtx_SET (cond, x)); + } /* Emit any final barrier needed for a __sync operation. */ if (is_mm_sync (model)) aarch64_emit_post_barrier (model); diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index fcfdf84..f3714bb 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,7 @@ +2017-06-02 Kyrylo Tkachov + + * gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c: New test. + 2017-07-06 Vyacheslav Barinov * g++.dg/ext/statement-list-end.C: New. diff --git a/gcc/testsuite/gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c b/gcc/testsuite/gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c new file mode 100644 index 0000000..b14a7c2 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c @@ -0,0 +1,12 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ + +int +foo (int *a) +{ + int x = 0; + return __atomic_compare_exchange_n (a, &x, 4, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE); +} + +/* { dg-final { scan-assembler-times "cbnz\\tw\[0-9\]+" 2 } } */ -- 2.7.4 From 839e47f64150c6130cfdbf6bd43cfc04df3ac286 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:35:32 +0000 Subject: [PATCH 05/16] Add __builtin_speculation_safe_value This patch defines a new intrinsic function __builtin_speculation_safe_value. A generic default implementation is defined which will attempt to use the backend pattern "speculation_safe_barrier". If this pattern is not defined, or if it is not available, then the compiler will emit a warning, but compilation will continue. Note that the test spec-barrier-1.c will currently fail on all targets. This is deliberate, the failure will go away when appropriate action is taken for each target backend. gcc: * builtin-types.def (BT_FN_PTR_PTR_VAR): New function type. (BT_FN_I1_I1_VAR, BT_FN_I2_I2_VAR, BT_FN_I4_I4_VAR): Likewise. (BT_FN_I8_I8_VAR, BT_FN_I16_I16_VAR): Likewise. * builtin-attrs.def (ATTR_NOVOPS_NOTHROW_LEAF_LIST): New attribute list. * builtins.def (BUILT_IN_SPECULATION_SAFE_VALUE_N): New builtin. (BUILT_IN_SPECULATION_SAFE_VALUE_PTR): New internal builtin. (BUILT_IN_SPECULATION_SAFE_VALUE_1): Likewise. (BUILT_IN_SPECULATION_SAFE_VALUE_2): Likewise. (BUILT_IN_SPECULATION_SAFE_VALUE_4): Likewise. (BUILT_IN_SPECULATION_SAFE_VALUE_8): Likewise. (BUILT_IN_SPECULATION_SAFE_VALUE_16): Likewise. * builtins.c (expand_speculation_safe_value): New function. (expand_builtin): Call it. * doc/cpp.texi: Document predefine __HAVE_SPECULATION_SAFE_VALUE. * doc/extend.texi: Document __builtin_speculation_safe_value. * doc/md.texi: Document "speculation_barrier" pattern. * doc/tm.texi.in: Pull in TARGET_SPECULATION_SAFE_VALUE and TARGET_HAVE_SPECULATION_SAFE_VALUE. * doc/tm.texi: Regenerated. * target.def (have_speculation_safe_value, speculation_safe_value): New hooks. * targhooks.c (default_have_speculation_safe_value): New function. (default_speculation_safe_value): New function. * targhooks.h (default_have_speculation_safe_value): Add prototype. (default_speculation_safe_value): Add prototype. c-family: * c-common.c (speculation_safe_resolve_call): New function. (speculation_safe_resolve_params): New function. (speculation_safe_resolve_return): New function. (resolve_overloaded_builtin): Handle __builtin_speculation_safe_value. * c-cppbuiltin.c (c_cpp_builtins): Add pre-define for __HAVE_SPECULATION_SAFE_VALUE. testsuite: * c-c++-common/spec-barrier-1.c: New test. * c-c++-common/spec-barrier-2.c: New test. * gcc.dg/spec-barrier-3.c: New test. (backported 123081efd41261af8830d6c796a9b12435806bac) Change-Id: I1d0e777c8e098f5a1d30aa5fa3090c8ce6b3dbee git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263168 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 29 +++++ gcc/builtin-attrs.def | 2 + gcc/builtin-types.def | 6 + gcc/builtins.c | 60 ++++++++++ gcc/builtins.def | 22 ++++ gcc/c-family/ChangeLog | 9 ++ gcc/c-family/c-common.c | 164 ++++++++++++++++++++++++++++ gcc/c-family/c-cppbuiltin.c | 7 +- gcc/doc/cpp.texi | 4 + gcc/doc/extend.texi | 91 +++++++++++++++ gcc/doc/md.texi | 15 +++ gcc/doc/tm.texi | 31 ++++++ gcc/doc/tm.texi.in | 4 + gcc/target.def | 35 ++++++ gcc/targhooks.c | 32 ++++++ gcc/targhooks.h | 3 + gcc/testsuite/ChangeLog | 6 + gcc/testsuite/c-c++-common/spec-barrier-1.c | 38 +++++++ gcc/testsuite/c-c++-common/spec-barrier-2.c | 17 +++ gcc/testsuite/gcc.dg/spec-barrier-3.c | 13 +++ 20 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 gcc/testsuite/c-c++-common/spec-barrier-1.c create mode 100644 gcc/testsuite/c-c++-common/spec-barrier-2.c create mode 100644 gcc/testsuite/gcc.dg/spec-barrier-3.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 76eb226..9c1bbc5 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,32 @@ +2018-07-31 Richard Earnshaw + + * builtin-types.def (BT_FN_PTR_PTR_VAR): New function type. + (BT_FN_I1_I1_VAR, BT_FN_I2_I2_VAR, BT_FN_I4_I4_VAR): Likewise. + (BT_FN_I8_I8_VAR, BT_FN_I16_I16_VAR): Likewise. + * builtin-attrs.def (ATTR_NOVOPS_NOTHROW_LEAF_LIST): New attribute + list. + * builtins.def (BUILT_IN_SPECULATION_SAFE_VALUE_N): New builtin. + (BUILT_IN_SPECULATION_SAFE_VALUE_PTR): New internal builtin. + (BUILT_IN_SPECULATION_SAFE_VALUE_1): Likewise. + (BUILT_IN_SPECULATION_SAFE_VALUE_2): Likewise. + (BUILT_IN_SPECULATION_SAFE_VALUE_4): Likewise. + (BUILT_IN_SPECULATION_SAFE_VALUE_8): Likewise. + (BUILT_IN_SPECULATION_SAFE_VALUE_16): Likewise. + * builtins.c (expand_speculation_safe_value): New function. + (expand_builtin): Call it. + * doc/cpp.texi: Document predefine __HAVE_SPECULATION_SAFE_VALUE. + * doc/extend.texi: Document __builtin_speculation_safe_value. + * doc/md.texi: Document "speculation_barrier" pattern. + * doc/tm.texi.in: Pull in TARGET_SPECULATION_SAFE_VALUE and + TARGET_HAVE_SPECULATION_SAFE_VALUE. + * doc/tm.texi: Regenerated. + * target.def (have_speculation_safe_value, speculation_safe_value): New + hooks. + * targhooks.c (default_have_speculation_safe_value): New function. + (default_speculation_safe_value): New function. + * targhooks.h (default_have_speculation_safe_value): Add prototype. + (default_speculation_safe_value): Add prototype. + 2017-06-02 Kyrylo Tkachov * config/aarch64/aarch64.c (aarch64_split_compare_and_swap): diff --git a/gcc/builtin-attrs.def b/gcc/builtin-attrs.def index 089817a..80edf76 100644 --- a/gcc/builtin-attrs.def +++ b/gcc/builtin-attrs.def @@ -119,6 +119,8 @@ DEF_ATTR_TREE_LIST (ATTR_NOTHROW_LIST, ATTR_NOTHROW, ATTR_NULL, ATTR_NULL) DEF_ATTR_TREE_LIST (ATTR_NOTHROW_LEAF_LIST, ATTR_LEAF, ATTR_NULL, ATTR_NOTHROW_LIST) +DEF_ATTR_TREE_LIST (ATTR_NOVOPS_NOTHROW_LEAF_LIST, ATTR_NOVOPS, \ + ATTR_NULL, ATTR_NOTHROW_LEAF_LIST) DEF_ATTR_TREE_LIST (ATTR_CONST_NOTHROW_LIST, ATTR_CONST, \ ATTR_NULL, ATTR_NOTHROW_LIST) DEF_ATTR_TREE_LIST (ATTR_CONST_NOTHROW_LEAF_LIST, ATTR_CONST, \ diff --git a/gcc/builtin-types.def b/gcc/builtin-types.def index 7fab9f8..69c455a 100644 --- a/gcc/builtin-types.def +++ b/gcc/builtin-types.def @@ -588,6 +588,12 @@ DEF_FUNCTION_TYPE_VAR_1 (BT_FN_VOID_LONG_VAR, BT_VOID, BT_LONG) DEF_FUNCTION_TYPE_VAR_1 (BT_FN_VOID_ULL_VAR, BT_VOID, BT_ULONGLONG) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_PTR_PTR_VAR, BT_PTR, BT_PTR) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_I1_I1_VAR, BT_I1, BT_I1) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_I2_I2_VAR, BT_I2, BT_I2) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_I4_I4_VAR, BT_I4, BT_I4) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_I8_I8_VAR, BT_I8, BT_I8) +DEF_FUNCTION_TYPE_VAR_1 (BT_FN_I16_I16_VAR, BT_I16, BT_I16) DEF_FUNCTION_TYPE_VAR_2 (BT_FN_INT_FILEPTR_CONST_STRING_VAR, BT_INT, BT_FILEPTR, BT_CONST_STRING) diff --git a/gcc/builtins.c b/gcc/builtins.c index d32a1d1..cb15cb2 100644 --- a/gcc/builtins.c +++ b/gcc/builtins.c @@ -5608,6 +5608,55 @@ expand_stack_save (void) } +/* Expand a call to __builtin_speculation_safe_value_. MODE + represents the size of the first argument to that call, or VOIDmode + if the argument is a pointer. IGNORE will be true if the result + isn't used. */ +static rtx +expand_speculation_safe_value (machine_mode mode, tree exp, rtx target, + bool ignore) +{ + rtx val, failsafe; + unsigned nargs = call_expr_nargs (exp); + + tree arg0 = CALL_EXPR_ARG (exp, 0); + + if (mode == VOIDmode) + { + mode = TYPE_MODE (TREE_TYPE (arg0)); + gcc_assert (GET_MODE_CLASS (mode) == MODE_INT); + } + + val = expand_expr (arg0, NULL_RTX, mode, EXPAND_NORMAL); + + /* An optional second argument can be used as a failsafe value on + some machines. If it isn't present, then the failsafe value is + assumed to be 0. */ + if (nargs > 1) + { + tree arg1 = CALL_EXPR_ARG (exp, 1); + failsafe = expand_expr (arg1, NULL_RTX, mode, EXPAND_NORMAL); + } + else + failsafe = const0_rtx; + + /* If the result isn't used, the behavior is undefined. It would be + nice to emit a warning here, but path splitting means this might + happen with legitimate code. So simply drop the builtin + expansion in that case; we've handled any side-effects above. */ + if (ignore) + return const0_rtx; + + /* If we don't have a suitable target, create one to hold the result. */ + if (target == NULL || GET_MODE (target) != mode) + target = gen_reg_rtx (mode); + + if (GET_MODE (val) != mode && GET_MODE (val) != VOIDmode) + val = convert_modes (mode, VOIDmode, val, false); + + return targetm.speculation_safe_value (mode, target, val, failsafe); +} + /* Expand an expression EXP that calls a built-in function, with result going to TARGET if that's convenient (and in mode MODE if that's convenient). @@ -6702,6 +6751,17 @@ expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode, folding. */ break; + case BUILT_IN_SPECULATION_SAFE_VALUE_PTR: + return expand_speculation_safe_value (VOIDmode, exp, target, ignore); + + case BUILT_IN_SPECULATION_SAFE_VALUE_1: + case BUILT_IN_SPECULATION_SAFE_VALUE_2: + case BUILT_IN_SPECULATION_SAFE_VALUE_4: + case BUILT_IN_SPECULATION_SAFE_VALUE_8: + case BUILT_IN_SPECULATION_SAFE_VALUE_16: + mode = get_builtin_sync_mode (fcode - BUILT_IN_SPECULATION_SAFE_VALUE_1); + return expand_speculation_safe_value (mode, exp, target, ignore); + default: /* just do library call, if unknown builtin */ break; } diff --git a/gcc/builtins.def b/gcc/builtins.def index 10cbdbe..157ee06 100644 --- a/gcc/builtins.def +++ b/gcc/builtins.def @@ -916,6 +916,28 @@ DEF_BUILTIN (BUILT_IN_EMUTLS_REGISTER_COMMON, true, true, true, ATTR_NOTHROW_LEAF_LIST, false, !targetm.have_tls) +/* Suppressing speculation. Users are expected to use the first (N) + variant, which will be translated internally into one of the other + types. */ + +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_N, "speculation_safe_value", + BT_FN_VOID_VAR, ATTR_NOVOPS_NOTHROW_LEAF_LIST) + +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_PTR, + "speculation_safe_value_ptr", BT_FN_PTR_PTR_VAR, + ATTR_NOVOPS_NOTHROW_LEAF_LIST) +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_1, "speculation_safe_value_1", + BT_FN_I1_I1_VAR, ATTR_NOVOPS_NOTHROW_LEAF_LIST) +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_2, "speculation_safe_value_2", + BT_FN_I2_I2_VAR, ATTR_NOVOPS_NOTHROW_LEAF_LIST) +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_4, "speculation_safe_value_4", + BT_FN_I4_I4_VAR, ATTR_NOVOPS_NOTHROW_LEAF_LIST) +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_8, "speculation_safe_value_8", + BT_FN_I8_I8_VAR, ATTR_NOVOPS_NOTHROW_LEAF_LIST) +DEF_GCC_BUILTIN (BUILT_IN_SPECULATION_SAFE_VALUE_16, + "speculation_safe_value_16", BT_FN_I16_I16_VAR, + ATTR_NOVOPS_NOTHROW_LEAF_LIST) + /* Exception support. */ DEF_BUILTIN_STUB (BUILT_IN_UNWIND_RESUME, "__builtin_unwind_resume") DEF_BUILTIN_STUB (BUILT_IN_CXA_END_CLEANUP, "__builtin_cxa_end_cleanup") diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog index bdd4e9e..2b19e8f 100644 --- a/gcc/c-family/ChangeLog +++ b/gcc/c-family/ChangeLog @@ -1,3 +1,12 @@ +2018-07-31 Richard Earnshaw + + * c-common.c (speculation_safe_resolve_call): New function. + (speculation_safe_resolve_params): New function. + (speculation_safe_resolve_return): New function. + (resolve_overloaded_builtin): Handle __builtin_speculation_safe_value. + * c-cppbuiltin.c (c_cpp_builtins): Add pre-define for + __HAVE_SPECULATION_SAFE_VALUE. + 2017-01-10 Martin Liska Backport from mainline diff --git a/gcc/c-family/c-common.c b/gcc/c-family/c-common.c index b2b36b5..1a7c375 100644 --- a/gcc/c-family/c-common.c +++ b/gcc/c-family/c-common.c @@ -10787,6 +10787,122 @@ builtin_type_for_size (int size, bool unsignedp) return type ? type : error_mark_node; } +/* Work out the size of the first argument of a call to + __builtin_speculation_safe_value. Only pointers and integral types + are permitted. Return -1 if the argument type is not supported or + the size is too large; 0 if the argument type is a pointer or the + size if it is integral. */ +static enum built_in_function +speculation_safe_value_resolve_call (tree function, vec *params) +{ + /* Type of the argument. */ + tree type; + int size; + + if (vec_safe_is_empty (params)) + { + error ("too few arguments to function %qE", function); + return BUILT_IN_NONE; + } + + type = TREE_TYPE ((*params)[0]); + if (TREE_CODE (type) == ARRAY_TYPE && c_dialect_cxx ()) + { + /* Force array-to-pointer decay for C++. */ + (*params)[0] = default_conversion ((*params)[0]); + type = TREE_TYPE ((*params)[0]); + } + + if (POINTER_TYPE_P (type)) + return BUILT_IN_SPECULATION_SAFE_VALUE_PTR; + + if (!INTEGRAL_TYPE_P (type)) + goto incompatible; + + if (!COMPLETE_TYPE_P (type)) + goto incompatible; + + size = tree_to_uhwi (TYPE_SIZE_UNIT (type)); + if (size == 1 || size == 2 || size == 4 || size == 8 || size == 16) + return ((enum built_in_function) + ((int) BUILT_IN_SPECULATION_SAFE_VALUE_1 + exact_log2 (size))); + + incompatible: + /* Issue the diagnostic only if the argument is valid, otherwise + it would be redundant at best and could be misleading. */ + if (type != error_mark_node) + error ("operand type %qT is incompatible with argument %d of %qE", + type, 1, function); + + return BUILT_IN_NONE; +} + +/* Validate and coerce PARAMS, the arguments to ORIG_FUNCTION to fit + the prototype for FUNCTION. The first argument is mandatory, a second + argument, if present, must be type compatible with the first. */ +static bool +speculation_safe_value_resolve_params (location_t loc, tree orig_function, + vec *params) +{ + tree val; + + if (params->length () == 0) + { + error_at (loc, "too few arguments to function %qE", orig_function); + return false; + } + + else if (params->length () > 2) + { + error_at (loc, "too many arguments to function %qE", orig_function); + return false; + } + + val = (*params)[0]; + if (TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE) + val = default_conversion (val); + if (!(TREE_CODE (TREE_TYPE (val)) == POINTER_TYPE + || TREE_CODE (TREE_TYPE (val)) == INTEGER_TYPE)) + { + error_at (loc, + "expecting argument of type pointer or of type integer " + "for argument 1"); + return false; + } + (*params)[0] = val; + + if (params->length () == 2) + { + tree val2 = (*params)[1]; + if (TREE_CODE (TREE_TYPE (val2)) == ARRAY_TYPE) + val2 = default_conversion (val2); + if (!(TREE_TYPE (val) == TREE_TYPE (val2) + || useless_type_conversion_p (TREE_TYPE (val), TREE_TYPE (val2)))) + { + error_at (loc, "both arguments must be compatible"); + return false; + } + (*params)[1] = val2; + } + + return true; +} + +/* Cast the result of the builtin back to the type of the first argument, + preserving any qualifiers that it might have. */ +static tree +speculation_safe_value_resolve_return (tree first_param, tree result) +{ + tree ptype = TREE_TYPE (first_param); + tree rtype = TREE_TYPE (result); + ptype = TYPE_MAIN_VARIANT (ptype); + + if (tree_int_cst_equal (TYPE_SIZE (ptype), TYPE_SIZE (rtype))) + return convert (ptype, result); + + return result; +} + /* A helper function for resolve_overloaded_builtin in resolving the overloaded __sync_ builtins. Returns a positive power of 2 if the first operand of PARAMS is a pointer to a supported data type. @@ -11419,6 +11535,54 @@ resolve_overloaded_builtin (location_t loc, tree function, /* Handle BUILT_IN_NORMAL here. */ switch (orig_code) { + case BUILT_IN_SPECULATION_SAFE_VALUE_N: + { + tree new_function, first_param, result; + enum built_in_function fncode + = speculation_safe_value_resolve_call (function, params);; + + first_param = (*params)[0]; + if (fncode == BUILT_IN_NONE + || !speculation_safe_value_resolve_params (loc, function, params)) + return error_mark_node; + + if (targetm.have_speculation_safe_value (true)) + { + new_function = builtin_decl_explicit (fncode); + result = build_function_call_vec (loc, vNULL, new_function, params, + NULL); + + if (result == error_mark_node) + return result; + + return speculation_safe_value_resolve_return (first_param, result); + } + else + { + /* This target doesn't have, or doesn't need, active mitigation + against incorrect speculative execution. Simply return the + first parameter to the builtin. */ + if (!targetm.have_speculation_safe_value (false)) + /* The user has invoked __builtin_speculation_safe_value + even though __HAVE_SPECULATION_SAFE_VALUE is not + defined: emit a warning. */ + warning_at (input_location, 0, + "this target does not define a speculation barrier; " + "your program will still execute correctly, " + "but incorrect speculation may not be be " + "restricted"); + + /* If the optional second argument is present, handle any side + effects now. */ + if (params->length () == 2 + && TREE_SIDE_EFFECTS ((*params)[1])) + return build2 (COMPOUND_EXPR, TREE_TYPE (first_param), + (*params)[1], first_param); + + return first_param; + } + } + case BUILT_IN_ATOMIC_EXCHANGE: case BUILT_IN_ATOMIC_COMPARE_EXCHANGE: case BUILT_IN_ATOMIC_LOAD: diff --git a/gcc/c-family/c-cppbuiltin.c b/gcc/c-family/c-cppbuiltin.c index 3d4587e..a560fea 100644 --- a/gcc/c-family/c-cppbuiltin.c +++ b/gcc/c-family/c-cppbuiltin.c @@ -1202,7 +1202,12 @@ c_cpp_builtins (cpp_reader *pfile) cpp_define (pfile, "__WCHAR_UNSIGNED__"); cpp_atomic_builtins (pfile); - + + /* Show support for __builtin_speculation_safe_value () if the target + has been updated to fully support it. */ + if (targetm.have_speculation_safe_value (false)) + cpp_define (pfile, "__HAVE_SPECULATION_SAFE_VALUE"); + #ifdef DWARF2_UNWIND_INFO if (dwarf2out_do_cfi_asm ()) cpp_define (pfile, "__GCC_HAVE_DWARF2_CFI_ASM"); diff --git a/gcc/doc/cpp.texi b/gcc/doc/cpp.texi index 9f914b2..40e7be7 100644 --- a/gcc/doc/cpp.texi +++ b/gcc/doc/cpp.texi @@ -2381,6 +2381,10 @@ If GCC cannot determine the current date, it will emit a warning message These macros are defined when the target processor supports atomic compare and swap operations on operands 1, 2, 4, 8 or 16 bytes in length, respectively. +@item __HAVE_SPECULATION_SAFE_VALUE +This macro is defined with the value 1 to show that this version of GCC +supports @code{__builtin_speculation_safe_value}. + @item __GCC_HAVE_DWARF2_CFI_ASM This macro is defined when the compiler is emitting DWARF CFI directives to the assembler. When this is defined, it is possible to emit those same diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi index 0c91d85..7bb173f 100644 --- a/gcc/doc/extend.texi +++ b/gcc/doc/extend.texi @@ -10228,6 +10228,7 @@ in the Cilk Plus language manual which can be found at @findex __builtin_powi @findex __builtin_powif @findex __builtin_powil +@findex __builtin_speculation_safe_value @findex _Exit @findex _exit @findex abort @@ -10854,6 +10855,96 @@ an extension. @xref{Variable Length}, for details. @end deftypefn +@deftypefn {Built-in Function} @var{type} __builtin_speculation_safe_value (@var{type} val, @var{type} failval) + +This built-in function can be used to help mitigate against unsafe +speculative execution. @var{type} may be any integral type or any +pointer type. + +@enumerate +@item +If the CPU is not speculatively executing the code, then @var{val} +is returned. +@item +If the CPU is executing speculatively then either: +@itemize +@item +The function may cause execution to pause until it is known that the +code is no-longer being executed speculatively (in which case +@var{val} can be returned, as above); or +@item +The function may use target-dependent speculation tracking state to cause +@var{failval} to be returned when it is known that speculative +execution has incorrectly predicted a conditional branch operation. +@end itemize +@end enumerate + +The second argument, @var{failval}, is optional and defaults to zero +if omitted. + +GCC defines the preprocessor macro +@code{__HAVE_BUILTIN_SPECULATION_SAFE_VALUE} for targets that have been +updated to support this builtin. + +The built-in function can be used where a variable appears to be used in a +safe way, but the CPU, due to speculative execution may temporarily ignore +the bounds checks. Consider, for example, the following function: + +@smallexample +int array[500]; +int f (unsigned untrusted_index) +@{ + if (untrusted_index < 500) + return array[untrusted_index]; + return 0; +@} +@end smallexample + +If the function is called repeatedly with @code{untrusted_index} less +than the limit of 500, then a branch predictor will learn that the +block of code that returns a value stored in @code{array} will be +executed. If the function is subsequently called with an +out-of-range value it will still try to execute that block of code +first until the CPU determines that the prediction was incorrect +(the CPU will unwind any incorrect operations at that point). +However, depending on how the result of the function is used, it might be +possible to leave traces in the cache that can reveal what was stored +at the out-of-bounds location. The built-in function can be used to +provide some protection against leaking data in this way by changing +the code to: + +@smallexample +int array[500]; +int f (unsigned untrusted_index) +@{ + if (untrusted_index < 500) + return array[__builtin_speculation_safe_value (untrusted_index)]; + return 0; +@} +@end smallexample + +The built-in function will either cause execution to stall until the +conditional branch has been fully resolved, or it may permit +speculative execution to continue, but using 0 instead of +@code{untrusted_value} if that exceeds the limit. + +If accessing any memory location is potentially unsafe when speculative +execution is incorrect, then the code can be rewritten as + +@smallexample +int array[500]; +int f (unsigned untrusted_index) +@{ + if (untrusted_index < 500) + return *__builtin_speculation_safe_value (&array[untrusted_index], NULL); + return 0; +@} +@end smallexample + +which will cause a @code{NULL} pointer to be used for the unsafe case. + +@end deftypefn + @deftypefn {Built-in Function} int __builtin_types_compatible_p (@var{type1}, @var{type2}) You can use the built-in function @code{__builtin_types_compatible_p} to diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi index d12c7b1..45406c9 100644 --- a/gcc/doc/md.texi +++ b/gcc/doc/md.texi @@ -6645,6 +6645,21 @@ should be defined to an instruction that orders both loads and stores before the instruction with respect to loads and stores after the instruction. This pattern has no operands. +@cindex @code{speculation_barrier} instruction pattern +@item @samp{speculation_barrier} +If the target can support speculative execution, then this pattern should +be defined to an instruction that will block subsequent execution until +any prior speculation conditions has been resolved. The pattern must also +ensure that the compiler cannot move memory operations past the barrier, +so it needs to be an UNSPEC_VOLATILE pattern. The pattern has no +operands. + +If this pattern is not defined then the default expansion of +@code{__builtin_speculation_safe_value} will emit a warning. You can +suppress this warning by defining this pattern with a final condition +of @code{0} (zero), which tells the compiler that a speculation +barrier is not needed for this target. + @cindex @code{sync_compare_and_swap@var{mode}} instruction pattern @item @samp{sync_compare_and_swap@var{mode}} This pattern, if defined, emits code for an atomic compare-and-swap diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi index 745910f..e8166b8 100644 --- a/gcc/doc/tm.texi +++ b/gcc/doc/tm.texi @@ -11678,3 +11678,34 @@ All and all it does not take long to convert ports that the maintainer is familiar with. @end defmac + +@deftypefn {Target Hook} bool TARGET_HAVE_SPECULATION_SAFE_VALUE (bool @var{active}) +This hook is used to determine the level of target support for + @code{__builtin_speculation_safe_value}. If called with an argument + of false, it returns true if the target has been modified to support + this builtin. If called with an argument of true, it returns true + if the target requires active mitigation execution might be speculative. + + The default implementation returns false if the target does not define + a pattern named @code{speculation_barrier}. Else it returns true + for the first case and whether the pattern is enabled for the current + compilation for the second case. +@end deftypefn + +@deftypefn {Target Hook} rtx TARGET_SPECULATION_SAFE_VALUE (machine_mode @var{mode}, rtx @var{result}, rtx @var{val}, rtx @var{failval}) +This target hook can be used to generate a target-specific code + sequence that implements the @code{__builtin_speculation_safe_value} + built-in function. The function must always return @var{val} in + @var{result} in mode @var{mode} when the cpu is not executing + speculatively, but must never return that when speculating until it + is known that the speculation will not be unwound. The hook supports + two primary mechanisms for implementing the requirements. The first + is to emit a speculation barrier which forces the processor to wait + until all prior speculative operations have been resolved; the second + is to use a target-specific mechanism that can track the speculation + state and to return @var{failval} if it can determine that + speculation must be unwound at a later time. + + The default implementation simply copies @var{val} to @var{result} and + emits a @code{speculation_barrier} instruction if that is defined. +@end deftypefn diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in index f31c763..a7f5790 100644 --- a/gcc/doc/tm.texi.in +++ b/gcc/doc/tm.texi.in @@ -8262,3 +8262,7 @@ All and all it does not take long to convert ports that the maintainer is familiar with. @end defmac + +@hook TARGET_HAVE_SPECULATION_SAFE_VALUE + +@hook TARGET_SPECULATION_SAFE_VALUE diff --git a/gcc/target.def b/gcc/target.def index 20f2b32..a4f21ad 100644 --- a/gcc/target.def +++ b/gcc/target.def @@ -3942,6 +3942,41 @@ normally defined in @file{libgcc2.c}.", default_external_stack_protect_fail) DEFHOOK +(have_speculation_safe_value, +"This hook is used to determine the level of target support for\n\ + @code{__builtin_speculation_safe_value}. If called with an argument\n\ + of false, it returns true if the target has been modified to support\n\ + this builtin. If called with an argument of true, it returns true\n\ + if the target requires active mitigation execution might be speculative.\n\ + \n\ + The default implementation returns false if the target does not define\n\ + a pattern named @code{speculation_barrier}. Else it returns true\n\ + for the first case and whether the pattern is enabled for the current\n\ + compilation for the second case.", +bool, (bool active), default_have_speculation_safe_value) + +DEFHOOK +(speculation_safe_value, +"This target hook can be used to generate a target-specific code\n\ + sequence that implements the @code{__builtin_speculation_safe_value}\n\ + built-in function. The function must always return @var{val} in\n\ + @var{result} in mode @var{mode} when the cpu is not executing\n\ + speculatively, but must never return that when speculating until it\n\ + is known that the speculation will not be unwound. The hook supports\n\ + two primary mechanisms for implementing the requirements. The first\n\ + is to emit a speculation barrier which forces the processor to wait\n\ + until all prior speculative operations have been resolved; the second\n\ + is to use a target-specific mechanism that can track the speculation\n\ + state and to return @var{failval} if it can determine that\n\ + speculation must be unwound at a later time.\n\ + \n\ + The default implementation simply copies @var{val} to @var{result} and\n\ + emits a @code{speculation_barrier} instruction if that is defined.", +rtx, (machine_mode mode, rtx result, rtx val, rtx failval), + default_speculation_safe_value) + + +DEFHOOK (can_use_doloop_p, "Return true if it is possible to use low-overhead loops (@code{doloop_end}\n\ and @code{doloop_begin}) for a particular loop. @var{iterations} gives the\n\ diff --git a/gcc/targhooks.c b/gcc/targhooks.c index 6b4601b..d9db06f 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -1965,4 +1965,36 @@ default_optab_supported_p (int, machine_mode, machine_mode, optimization_type) return true; } +/* Default implementation of TARGET_HAVE_SPECULATION_SAFE_VALUE. */ +bool +default_have_speculation_safe_value (bool active) +{ +#ifdef HAVE_speculation_barrier + return active ? HAVE_speculation_barrier : true; +#else + return false; +#endif +} + +/* Default implementation of the speculation-safe-load builtin. This + implementation simply copies val to result and generates a + speculation_barrier insn, if such a pattern is defined. */ +rtx +default_speculation_safe_value (machine_mode mode ATTRIBUTE_UNUSED, + rtx result, rtx val, + rtx failval ATTRIBUTE_UNUSED) +{ + emit_move_insn (result, val); + +#ifdef HAVE_speculation_barrier + /* Assume the target knows what it is doing: if it defines a + speculation barrier, but it is not enabled, then assume that one + isn't needed. */ + if (HAVE_speculation_barrier) + emit_insn (gen_speculation_barrier ()); +#endif + + return result; +} + #include "gt-targhooks.h" diff --git a/gcc/targhooks.h b/gcc/targhooks.h index 7687c39..01501b2 100644 --- a/gcc/targhooks.h +++ b/gcc/targhooks.h @@ -254,4 +254,7 @@ extern void default_setup_incoming_vararg_bounds (cumulative_args_t ca ATTRIBUTE extern bool default_optab_supported_p (int, machine_mode, machine_mode, optimization_type); +extern bool default_have_speculation_safe_value (bool); +extern rtx default_speculation_safe_value (machine_mode, rtx, rtx, rtx); + #endif /* GCC_TARGHOOKS_H */ diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index f3714bb..9d3e242 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,9 @@ +2018-07-31 Richard Earnshaw + + * c-c++-common/spec-barrier-1.c: New test. + * c-c++-common/spec-barrier-2.c: New test. + * gcc.dg/spec-barrier-3.c: New test. + 2017-06-02 Kyrylo Tkachov * gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c: New test. diff --git a/gcc/testsuite/c-c++-common/spec-barrier-1.c b/gcc/testsuite/c-c++-common/spec-barrier-1.c new file mode 100644 index 0000000..e4b44f2 --- /dev/null +++ b/gcc/testsuite/c-c++-common/spec-barrier-1.c @@ -0,0 +1,38 @@ +/* { dg-do run } */ +/* { dg-options "-O" } */ + +/* Test that __builtin_speculation_safe_value returns the correct value. */ +/* This test will cause an unfiltered warning to be emitted on targets + that have not implemented support for speculative execution + barriers. They should fix that rather than disabling this + test. */ +char a = 1; +short b = 2; +int c = 3; +long d = 4; +long long e = 5; +int *f = (int*) &c; +#ifdef __SIZEOF_INT128__ +__int128 g = 9; +#endif + +int main () +{ + if (__builtin_speculation_safe_value (a) != 1) + __builtin_abort (); + if (__builtin_speculation_safe_value (b) != 2) + __builtin_abort (); + if (__builtin_speculation_safe_value (c) != 3) + __builtin_abort (); + if (__builtin_speculation_safe_value (d) != 4) + __builtin_abort (); + if (__builtin_speculation_safe_value (e) != 5) + __builtin_abort (); + if (__builtin_speculation_safe_value (f) != &c) + __builtin_abort (); +#ifdef __SIZEOF_INT128__ + if (__builtin_speculation_safe_value (g) != 9) + __builtin_abort (); +#endif + return 0; +} diff --git a/gcc/testsuite/c-c++-common/spec-barrier-2.c b/gcc/testsuite/c-c++-common/spec-barrier-2.c new file mode 100644 index 0000000..b09567e --- /dev/null +++ b/gcc/testsuite/c-c++-common/spec-barrier-2.c @@ -0,0 +1,17 @@ +/* { dg-do run } */ + +/* Even on targets that don't need the optional failval parameter, + side-effects on the operand should still be calculated. */ + +int x = 3; +volatile int y = 9; + +int main () +{ + int z = __builtin_speculation_safe_value (x, y++); + if (z != 3 || y != 10) + __builtin_abort (); + return 0; +} + +/* { dg-prune-output "this target does not define a speculation barrier;" } */ diff --git a/gcc/testsuite/gcc.dg/spec-barrier-3.c b/gcc/testsuite/gcc.dg/spec-barrier-3.c new file mode 100644 index 0000000..7376f3e --- /dev/null +++ b/gcc/testsuite/gcc.dg/spec-barrier-3.c @@ -0,0 +1,13 @@ +/* { dg-do compile } */ +/* { dg-options "-Wpedantic" } */ + +/* __builtin_speculation_safe_value returns a value with the same type + as its first argument. There should be a warning if that isn't + type-compatible with the use. */ +int * +f (int x) +{ + return __builtin_speculation_safe_value (x); /* { dg-warning "return makes pointer from integer without a cast" } */ +} + +/* { dg-prune-output "this target does not define a speculation barrier;" } */ -- 2.7.4 From 61e24354d3a2a9153f2a730be9d33dc9f3278cec Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:35:41 +0000 Subject: [PATCH 06/16] Arm - add speculation_barrier pattern This patch defines a speculation barrier for AArch32. * config/arm/unspecs.md (unspecv): Add VUNSPEC_SPECULATION_BARRIER. * config/arm/arm.md (speculation_barrier): New expand. (speculation_barrier_insn): New pattern. (backported be626997e0edc646c17e9e1eadabf97152e82eeb) git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263169 138bc75d-0d04-0410-961f-82ee72b054a4 Change-Id: I5f587e22dd1a92440a6a208e2baca189427557da --- gcc/ChangeLog | 6 ++++++ gcc/config/arm/arm.md | 21 +++++++++++++++++++++ gcc/config/arm/unspecs.md | 1 + 3 files changed, 28 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9c1bbc5..ab2e761 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,11 @@ 2018-07-31 Richard Earnshaw + * config/arm/unspecs.md (unspecv): Add VUNSPEC_SPECULATION_BARRIER. + * config/arm/arm.md (speculation_barrier): New expand. + (speculation_barrier_insn): New pattern. + +2018-07-31 Richard Earnshaw + * builtin-types.def (BT_FN_PTR_PTR_VAR): New function type. (BT_FN_I1_I1_VAR, BT_FN_I2_I2_VAR, BT_FN_I4_I4_VAR): Likewise. (BT_FN_I8_I8_VAR, BT_FN_I16_I16_VAR): Likewise. diff --git a/gcc/config/arm/arm.md b/gcc/config/arm/arm.md index 5cc6997..ffdd6c2 100644 --- a/gcc/config/arm/arm.md +++ b/gcc/config/arm/arm.md @@ -11921,6 +11921,27 @@ DONE; }) +(define_expand "speculation_barrier" + [(unspec_volatile [(const_int 0)] VUNSPEC_SPECULATION_BARRIER)] + "TARGET_EITHER" + " + /* Don't emit anything for Thumb1 and suppress the warning from the + generic expansion. */ + if (!TARGET_32BIT) + DONE; + " +) + +;; Generate a hard speculation barrier when we have not enabled speculation +;; tracking. +(define_insn "*speculation_barrier_insn" + [(unspec_volatile [(const_int 0)] VUNSPEC_SPECULATION_BARRIER)] + "TARGET_32BIT" + "isb\;dsb\\tsy" + [(set_attr "type" "block") + (set_attr "length" "8")] +) + ;; Vector bits common to IWMMXT and Neon (include "vec-common.md") ;; Load the Intel Wireless Multimedia Extension patterns diff --git a/gcc/config/arm/unspecs.md b/gcc/config/arm/unspecs.md index 1aa39e8..db96cf1 100644 --- a/gcc/config/arm/unspecs.md +++ b/gcc/config/arm/unspecs.md @@ -150,6 +150,7 @@ VUNSPEC_GET_FPSCR ; Represent fetch of FPSCR content. VUNSPEC_SET_FPSCR ; Represent assign of FPSCR content. VUNSPEC_PROBE_STACK_RANGE ; Represent stack range probing. + VUNSPEC_SPECULATION_BARRIER ; Represents an unconditional speculation barrier. ]) ;; Enumerators for NEON unspecs. -- 2.7.4 From 81f53356576efc2be3ba0748551ac8b282bb6491 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:35:50 +0000 Subject: [PATCH 07/16] AArch64 - add speculation barrier Similar to Arm, this adds an unconditional speculation barrier for AArch64. * config/aarch64.md (unspecv): Add UNSPECV_SPECULAION_BARRIER. (speculation_barrier): New insn. (backported f9d2906f0b6803c42b2200da5e32e331930fc306) git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263170 138bc75d-0d04-0410-961f-82ee72b054a4 Change-Id: Ic9a1826c24f8597033716cec3d02de12fdb57dba --- gcc/ChangeLog | 5 +++++ gcc/config/aarch64/aarch64.md | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index ab2e761..ec6ae81 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,10 @@ 2018-07-31 Richard Earnshaw + * config/aarch64.md (unspecv): Add UNSPECV_SPECULAION_BARRIER. + (speculation_barrier): New insn. + +2018-07-31 Richard Earnshaw + * config/arm/unspecs.md (unspecv): Add VUNSPEC_SPECULATION_BARRIER. * config/arm/arm.md (speculation_barrier): New expand. (speculation_barrier_insn): New pattern. diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md index 6fc7979..6a03262 100644 --- a/gcc/config/aarch64/aarch64.md +++ b/gcc/config/aarch64/aarch64.md @@ -144,6 +144,7 @@ UNSPECV_SET_FPSR ; Represent assign of FPSR content. UNSPECV_BLOCKAGE ; Represent a blockage UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing. + UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier. ] ) @@ -5405,6 +5406,15 @@ DONE; }) +;; Hard speculation barrier. +(define_insn "speculation_barrier" + [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)] + "" + "isb\;dsb\\tsy" + [(set_attr "length" "8") + (set_attr "type" "block")] +) + ;; AdvSIMD Stuff (include "aarch64-simd.md") -- 2.7.4 From dc1279d6da8e406489871229a4778e55f4177bef Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:36:00 +0000 Subject: [PATCH 08/16] AArch64 - Add new option -mtrack-speculation This patch doesn't do anything useful, it simply adds a new command-line option -mtrack-speculation to AArch64. Subsequent patches build on this. * config/aarch64/aarch64.opt (mtrack-speculation): New target option. (backported 97eb2b8649e777da576598448d9d68afcfe69b54) git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263171 138bc75d-0d04-0410-961f-82ee72b054a4 Change-Id: I70cf2047662fa73e3d7090d62392210bac872591 --- gcc/ChangeLog | 4 ++++ gcc/config/aarch64/aarch64.opt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index ec6ae81..0177823 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,9 @@ 2018-07-31 Richard Earnshaw + * config/aarch64/aarch64.opt (mtrack-speculation): New target option. + +2018-07-31 Richard Earnshaw + * config/aarch64.md (unspecv): Add UNSPECV_SPECULAION_BARRIER. (speculation_barrier): New insn. diff --git a/gcc/config/aarch64/aarch64.opt b/gcc/config/aarch64/aarch64.opt index 82e9f5b..1c9b705 100644 --- a/gcc/config/aarch64/aarch64.opt +++ b/gcc/config/aarch64/aarch64.opt @@ -167,3 +167,7 @@ Common Var(flag_mlow_precision_div) Optimization Enable the division approximation. Enabling this reduces precision of division results to about 16 bits for single precision and to 32 bits for double precision. + +mtrack-speculation +Target Var(aarch64_track_speculation) +Generate code to track when the CPU might be speculating incorrectly. \ No newline at end of file -- 2.7.4 From 233eedd720a30dc4552a3310792871e8156e8076 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:36:09 +0000 Subject: [PATCH 09/16] AArch64 - disable CB[N]Z TB[N]Z when tracking speculation The CB[N]Z and TB[N]Z instructions do not expose the comparison through the condition code flags. This makes it impossible to track speculative execution through such a branch. We can handle this relatively easily by simply disabling the patterns in this case. A side effect of this is that the split patterns for the atomic operations need to also avoid generating these instructions. They mostly have simple fall-backs for this already. * config/aarch64/aarch64.md (cb1): Disable when aarch64_track_speculation is true. (tb1): Likewise. * config/aarch64/aarch64.c (aarch64_split_compare_regs): Do not generate CB[N]Z when tracking speculation. (aarch64_split_compare_and_swap): Likewise. (aarch64_split_atomic_op): Likewise. (backported eaf891585b16ef84046ae73a01bb1c7189c682e9) Change-Id: I0d0a090ef7bcf074b9e4d6ee115e71faa157962f git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263172 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 10 ++++++++++ gcc/config/aarch64/aarch64.c | 33 ++++++++++++++++++++++++++++++--- gcc/config/aarch64/aarch64.md | 6 +++--- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 0177823..384627c 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,15 @@ 2018-07-31 Richard Earnshaw + * config/aarch64/aarch64.md (cb1): Disable when + aarch64_track_speculation is true. + (tb1): Likewise. + * config/aarch64/aarch64.c (aarch64_split_compare_regs): Do not + generate CB[N]Z when tracking speculation. + (aarch64_split_compare_and_swap): Likewise. + (aarch64_split_atomic_op): Likewise. + +2018-07-31 Richard Earnshaw + * config/aarch64/aarch64.opt (mtrack-speculation): New target option. 2018-07-31 Richard Earnshaw diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 795033f..975617f 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -11650,7 +11650,16 @@ aarch64_split_compare_and_swap (rtx operands[]) if (strong_zero_p) { - x = gen_rtx_NE (VOIDmode, rval, const0_rtx); + if (aarch64_track_speculation) + { + /* Emit an explicit compare instruction, so that we can correctly + track the condition codes. */ + rtx cc_reg = aarch64_gen_compare_reg (NE, rval, const0_rtx); + x = gen_rtx_NE (GET_MODE (cc_reg), cc_reg, const0_rtx); + } + else + x = gen_rtx_NE (VOIDmode, rval, const0_rtx); + x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, gen_rtx_LABEL_REF (Pmode, label2), pc_rtx); aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); @@ -11668,7 +11677,16 @@ aarch64_split_compare_and_swap (rtx operands[]) if (!is_weak) { - x = gen_rtx_NE (VOIDmode, scratch, const0_rtx); + if (aarch64_track_speculation) + { + /* Emit an explicit compare instruction, so that we can correctly + track the condition codes. */ + rtx cc_reg = aarch64_gen_compare_reg (NE, scratch, const0_rtx); + x = gen_rtx_NE (GET_MODE (cc_reg), cc_reg, const0_rtx); + } + else + x = gen_rtx_NE (VOIDmode, scratch, const0_rtx); + x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, gen_rtx_LABEL_REF (Pmode, label1), pc_rtx); aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); @@ -12004,7 +12022,16 @@ aarch64_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem, aarch64_emit_store_exclusive (mode, cond, mem, gen_lowpart (mode, new_out), model_rtx); - x = gen_rtx_NE (VOIDmode, cond, const0_rtx); + if (aarch64_track_speculation) + { + /* Emit an explicit compare instruction, so that we can correctly + track the condition codes. */ + rtx cc_reg = aarch64_gen_compare_reg (NE, cond, const0_rtx); + x = gen_rtx_NE (GET_MODE (cc_reg), cc_reg, const0_rtx); + } + else + x = gen_rtx_NE (VOIDmode, cond, const0_rtx); + x = gen_rtx_IF_THEN_ELSE (VOIDmode, x, gen_rtx_LABEL_REF (Pmode, label), pc_rtx); aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md index 6a03262..bc8d736 100644 --- a/gcc/config/aarch64/aarch64.md +++ b/gcc/config/aarch64/aarch64.md @@ -597,7 +597,7 @@ (const_int 0)) (label_ref (match_operand 1 "" "")) (pc)))] - "" + "!aarch64_track_speculation" { if (get_attr_length (insn) == 8) return aarch64_gen_far_branch (operands, 1, "Lcb", "\\t%0, "); @@ -627,7 +627,7 @@ (label_ref (match_operand 2 "" "")) (pc))) (clobber (reg:CC CC_REGNUM))] - "" + "!aarch64_track_speculation" { if (get_attr_length (insn) == 8) { @@ -663,7 +663,7 @@ (label_ref (match_operand 1 "" "")) (pc))) (clobber (reg:CC CC_REGNUM))] - "" + "!aarch64_track_speculation" { if (get_attr_length (insn) == 8) { -- 2.7.4 From 8d651ade76b02287292996ed2dff85bccaad4e2a Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:36:18 +0000 Subject: [PATCH 10/16] AArch64 - new pass to add conditional-branch speculation tracking This patch is the main part of the speculation tracking code. It adds a new target-specific pass that is run just before the final branch reorg pass (so that it can clean up any new edge insertions we make). The pass is only run with -mtrack-speculation is passed on the command line. One thing that did come to light as part of this was that the stack pointer register was not being permitted in comparision instructions. We rely on that for moving the tracking state between SP and the scratch register at function call boundaries. * config/aarch64/aarch64-speculation.cc: New file. * config/aarch64/aarch64-passes.def (pass_track_speculation): Add before pass_reorder_blocks. * config/aarch64/aarch64-protos.h (make_pass_track_speculation): Add prototype. * config/aarch64/aarch64.c (aarch64_conditional_register_usage): Fix X14 and X15 when tracking speculation. * config/aarch64/aarch64.md (register name constants): Add SPECULATION_TRACKER_REGNUM and SPECULATION_SCRATCH_REGNUM. (unspec): Add UNSPEC_SPECULATION_TRACKER. (speculation_barrier): New insn attribute. (cmp): Allow SP in comparisons. (speculation_tracker): New insn. (speculation_barrier): Add speculation_barrier attribute. * config/aarch64/t-aarch64: Add make rule for aarch64-speculation.o. * config.gcc (aarch64*-*-*): Add aarch64-speculation.o to extra_objs. * doc/invoke.texi (AArch64 Options): Document -mtrack-speculation. (backported b19562a81626f3e6adc5a5d4a185b36548539a4f) Change-Id: Icece830281b0131f2a151bb43aa1aa3f94463836 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263173 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 20 ++ gcc/config.gcc | 2 +- gcc/config/aarch64/aarch64-passes.def | 1 + gcc/config/aarch64/aarch64-protos.h | 3 +- gcc/config/aarch64/aarch64-speculation.cc | 494 ++++++++++++++++++++++++++++++ gcc/config/aarch64/aarch64.c | 13 + gcc/config/aarch64/aarch64.md | 30 +- gcc/config/aarch64/t-aarch64 | 10 + gcc/doc/invoke.texi | 11 +- 9 files changed, 579 insertions(+), 5 deletions(-) create mode 100644 gcc/config/aarch64/aarch64-speculation.cc diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 384627c..722d68e 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,25 @@ 2018-07-31 Richard Earnshaw + * config/aarch64/aarch64-speculation.cc: New file. + * config/aarch64/aarch64-passes.def (pass_track_speculation): Add + before pass_reorder_blocks. + * config/aarch64/aarch64-protos.h (make_pass_track_speculation): Add + prototype. + * config/aarch64/aarch64.c (aarch64_conditional_register_usage): Fix + X14 and X15 when tracking speculation. + * config/aarch64/aarch64.md (register name constants): Add + SPECULATION_TRACKER_REGNUM and SPECULATION_SCRATCH_REGNUM. + (unspec): Add UNSPEC_SPECULATION_TRACKER. + (speculation_barrier): New insn attribute. + (cmp): Allow SP in comparisons. + (speculation_tracker): New insn. + (speculation_barrier): Add speculation_barrier attribute. + * config/aarch64/t-aarch64: Add make rule for aarch64-speculation.o. + * config.gcc (aarch64*-*-*): Add aarch64-speculation.o to extra_objs. + * doc/invoke.texi (AArch64 Options): Document -mtrack-speculation. + +2018-07-31 Richard Earnshaw + * config/aarch64/aarch64.md (cb1): Disable when aarch64_track_speculation is true. (tb1): Likewise. diff --git a/gcc/config.gcc b/gcc/config.gcc index d801654..050d0a0 100644 --- a/gcc/config.gcc +++ b/gcc/config.gcc @@ -310,7 +310,7 @@ aarch64*-*-*) extra_headers="arm_fp16.h arm_neon.h arm_acle.h" c_target_objs="aarch64-c.o" cxx_target_objs="aarch64-c.o" - extra_objs="aarch64-builtins.o aarch-common.o cortex-a57-fma-steering.o" + extra_objs="aarch64-builtins.o aarch-common.o cortex-a57-fma-steering.o aarch64-speculation.o" target_gtfiles="\$(srcdir)/config/aarch64/aarch64-builtins.c" target_has_targetm_common=yes ;; diff --git a/gcc/config/aarch64/aarch64-passes.def b/gcc/config/aarch64/aarch64-passes.def index 7fe8039..4a95c2a 100644 --- a/gcc/config/aarch64/aarch64-passes.def +++ b/gcc/config/aarch64/aarch64-passes.def @@ -19,3 +19,4 @@ . */ INSERT_PASS_AFTER (pass_regrename, 1, pass_fma_steering); +INSERT_PASS_BEFORE (pass_reorder_blocks, 1, pass_track_speculation); diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index 23ac210..ce1eaff 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -466,6 +466,7 @@ enum aarch64_parse_opt_result aarch64_parse_extension (const char *, std::string aarch64_get_extension_string_for_isa_flags (unsigned long, unsigned long); -rtl_opt_pass *make_pass_fma_steering (gcc::context *ctxt); +rtl_opt_pass *make_pass_fma_steering (gcc::context *); +rtl_opt_pass *make_pass_track_speculation (gcc::context *); #endif /* GCC_AARCH64_PROTOS_H */ diff --git a/gcc/config/aarch64/aarch64-speculation.cc b/gcc/config/aarch64/aarch64-speculation.cc new file mode 100644 index 0000000..60e03a3 --- /dev/null +++ b/gcc/config/aarch64/aarch64-speculation.cc @@ -0,0 +1,494 @@ +/* Speculation tracking and mitigation (e.g. CVE 2017-5753) for AArch64. + Copyright (C) 2018 Free Software Foundation, Inc. + Contributed by ARM Ltd. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GCC; see the file COPYING3. If not see + . */ + +#include "config.h" +#include "system.h" +#include "coretypes.h" +#include "target.h" +#include "rtl.h" +#include "tree-pass.h" +#include "bitmap.h" +#include "cfg.h" +#include "cfgbuild.h" +#include "print-rtl.h" +#include "cfgrtl.h" +#include "function.h" +#include "basic-block.h" +#include "memmodel.h" +#include "emit-rtl.h" +#include "insn-attr.h" +#include "df.h" +#include "tm_p.h" +#include "insn-config.h" +#include "recog.h" + +/* This pass scans the RTL just before the final branch + re-organisation pass. The aim is to identify all places where + there is conditional control flow and to insert code that tracks + any speculative execution of a conditional branch. + + To do this we reserve a call-clobbered register (so that it can be + initialized very early in the function prologue) that can then be + updated each time there is a conditional branch. At each such + branch we then generate a code sequence that uses conditional + select operations that are not subject to speculation themselves + (we ignore for the moment situations where that might not always be + strictly true). For example, a branch sequence such as: + + B.EQ + ... + : + + is transformed to: + + B.EQ + CSEL tracker, tracker, XZr, ne + ... + : + CSEL tracker, tracker, XZr, eq + + Since we start with the tracker initialized to all bits one, if at any + time the predicted control flow diverges from the architectural program + behavior, then the tracker will become zero (but not otherwise). + + The tracker value can be used at any time at which a value needs + guarding against incorrect speculation. This can be done in + several ways, but they all amount to the same thing. For an + untrusted address, or an untrusted offset to a trusted address, we + can simply mask the address with the tracker with the untrusted + value. If the CPU is not speculating, or speculating correctly, + then the value will remain unchanged, otherwise it will be clamped + to zero. For more complex scenarios we can compare the tracker + against zero and use the flags to form a new selection with an + alternate safe value. + + On implementations where the data processing instructions may + themselves produce speculative values, the architecture requires + that a CSDB instruction will resolve such data speculation, so each + time we use the tracker for protecting a vulnerable value we also + emit a CSDB: we do not need to do that each time the tracker itself + is updated. + + At function boundaries, we need to communicate the speculation + tracking state with the caller or the callee. This is tricky + because there is no register available for such a purpose without + creating a new ABI. We deal with this by relying on the principle + that in all real programs the stack pointer, SP will never be NULL + at a function boundary; we can thus encode the speculation state in + SP by clearing SP if the speculation tracker itself is NULL. After + the call we recover the tracking state back from SP into the + tracker register. The results is that a function call sequence is + transformed to + + MOV tmp, SP + AND tmp, tmp, tracker + MOV SP, tmp + BL + CMP SP, #0 + CSETM tracker, ne + + The additional MOV instructions in the pre-call sequence are needed + because SP cannot be used directly with the AND instruction. + + The code inside a function body uses the post-call sequence in the + prologue to establish the tracker and the pre-call sequence in the + epilogue to re-encode the state for the return. + + The code sequences have the nice property that if called from, or + calling a function that does not track speculation then the stack pointer + will always be non-NULL and hence the tracker will be initialized to all + bits one as we need: we lose the ability to fully track speculation in that + case, but we are still architecturally safe. + + Tracking speculation in this way is quite expensive, both in code + size and execution time. We employ a number of tricks to try to + limit this: + + 1) Simple leaf functions with no conditional branches (or use of + the tracker) do not need to establish a new tracker: they simply + carry the tracking state through SP for the duration of the call. + The same is also true for leaf functions that end in a tail-call. + + 2) Back-to-back function calls in a single basic block also do not + need to re-establish the tracker between the calls. Again, we can + carry the tracking state in SP for this period of time unless the + tracker value is needed at that point in time. + + We run the pass just before the final branch reorganization pass so + that we can handle most of the conditional branch cases using the + standard edge insertion code. The reorg pass will hopefully clean + things up for afterwards so that the results aren't too + horrible. */ + +/* Generate a code sequence to clobber SP if speculating incorreclty. */ +static rtx_insn * +aarch64_speculation_clobber_sp () +{ + rtx sp = gen_rtx_REG (DImode, SP_REGNUM); + rtx tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + rtx scratch = gen_rtx_REG (DImode, SPECULATION_SCRATCH_REGNUM); + + start_sequence (); + emit_insn (gen_rtx_SET (scratch, sp)); + emit_insn (gen_anddi3 (scratch, scratch, tracker)); + emit_insn (gen_rtx_SET (sp, scratch)); + rtx_insn *seq = get_insns (); + end_sequence (); + return seq; +} + +/* Generate a code sequence to establish the tracker variable from the + contents of SP. */ +static rtx_insn * +aarch64_speculation_establish_tracker () +{ + rtx sp = gen_rtx_REG (DImode, SP_REGNUM); + rtx tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + start_sequence (); + rtx cc = aarch64_gen_compare_reg (EQ, sp, const0_rtx); + emit_insn (gen_cstoredi_neg (tracker, + gen_rtx_NE (CCmode, cc, const0_rtx), cc)); + rtx_insn *seq = get_insns (); + end_sequence (); + return seq; +} + +/* Main speculation tracking pass. */ +unsigned int +aarch64_do_track_speculation () +{ + basic_block bb; + bool needs_tracking = false; + bool need_second_pass = false; + rtx_insn *insn; + int fixups_pending = 0; + + FOR_EACH_BB_FN (bb, cfun) + { + insn = BB_END (bb); + + if (dump_file) + fprintf (dump_file, "Basic block %d:\n", bb->index); + + while (insn != BB_HEAD (bb) + && NOTE_P (insn)) + insn = PREV_INSN (insn); + + if (control_flow_insn_p (insn)) + { + if (any_condjump_p (insn)) + { + if (dump_file) + { + fprintf (dump_file, " condjump\n"); + dump_insn_slim (dump_file, insn); + } + + rtx src = SET_SRC (pc_set (insn)); + + /* Check for an inverted jump, where the fall-through edge + appears first. */ + bool inverted = GET_CODE (XEXP (src, 2)) != PC; + /* The other edge must be the PC (we assume that we don't + have conditional return instructions). */ + gcc_assert (GET_CODE (XEXP (src, 1 + !inverted)) == PC); + + rtx cond = copy_rtx (XEXP (src, 0)); + gcc_assert (COMPARISON_P (cond) + && REG_P (XEXP (cond, 0)) + && REGNO (XEXP (cond, 0)) == CC_REGNUM + && XEXP (cond, 1) == const0_rtx); + enum rtx_code inv_cond_code + = reversed_comparison_code (cond, insn); + /* We should be able to reverse all conditions. */ + gcc_assert (inv_cond_code != UNKNOWN); + rtx inv_cond = gen_rtx_fmt_ee (inv_cond_code, GET_MODE (cond), + copy_rtx (XEXP (cond, 0)), + copy_rtx (XEXP (cond, 1))); + if (inverted) + std::swap (cond, inv_cond); + + insert_insn_on_edge (gen_speculation_tracker (cond), + BRANCH_EDGE (bb)); + insert_insn_on_edge (gen_speculation_tracker (inv_cond), + FALLTHRU_EDGE (bb)); + needs_tracking = true; + } + else if (GET_CODE (PATTERN (insn)) == RETURN) + { + /* If we already know we'll need a second pass, don't put + out the return sequence now, or we might end up with + two copies. Instead, we'll do all return statements + during the second pass. However, if this is the + first return insn we've found and we already + know that we'll need to emit the code, we can save a + second pass by emitting the code now. */ + if (needs_tracking && ! need_second_pass) + { + rtx_insn *seq = aarch64_speculation_clobber_sp (); + emit_insn_before (seq, insn); + } + else + { + fixups_pending++; + need_second_pass = true; + } + } + else if (find_reg_note (insn, REG_NON_LOCAL_GOTO, NULL_RTX)) + { + rtx_insn *seq = aarch64_speculation_clobber_sp (); + emit_insn_before (seq, insn); + needs_tracking = true; + } + } + else + { + if (dump_file) + { + fprintf (dump_file, " other\n"); + dump_insn_slim (dump_file, insn); + } + } + } + + FOR_EACH_BB_FN (bb, cfun) + { + rtx_insn *end = BB_END (bb); + rtx_insn *call_insn = NULL; + + if (bb->flags & BB_NON_LOCAL_GOTO_TARGET) + { + rtx_insn *label = NULL; + /* For non-local goto targets we have to recover the + speculation state from SP. Find the last code label at + the head of the block and place the fixup sequence after + that. */ + for (insn = BB_HEAD (bb); insn != end; insn = NEXT_INSN (insn)) + { + if (LABEL_P (insn)) + label = insn; + /* Never put anything before the basic block note. */ + if (NOTE_INSN_BASIC_BLOCK_P (insn)) + label = insn; + if (INSN_P (insn)) + break; + } + + gcc_assert (label); + emit_insn_after (aarch64_speculation_establish_tracker (), label); + } + + /* Scan the insns looking for calls. We need to pass the + speculation tracking state encoded in to SP. After a call we + restore the speculation tracking into the tracker register. + To avoid unnecessary transfers we look for two or more calls + within a single basic block and eliminate, where possible, + any redundant operations. */ + for (insn = BB_HEAD (bb); ; insn = NEXT_INSN (insn)) + { + if (NONDEBUG_INSN_P (insn) + && recog_memoized (insn) >= 0 + && (get_attr_speculation_barrier (insn) + == SPECULATION_BARRIER_TRUE)) + { + if (call_insn) + { + /* This instruction requires the speculation + tracking to be in the tracker register. If there + was an earlier call in this block, we need to + copy the speculation tracking back there. */ + emit_insn_after (aarch64_speculation_establish_tracker (), + call_insn); + call_insn = NULL; + } + + needs_tracking = true; + } + + if (CALL_P (insn)) + { + bool tailcall + = (SIBLING_CALL_P (insn) + || find_reg_note (insn, REG_NORETURN, NULL_RTX)); + + /* Tailcalls are like returns, we can eliminate the + transfer between the tracker register and SP if we + know that this function does not itself need + tracking. */ + if (tailcall && (need_second_pass || !needs_tracking)) + { + /* Don't clear call_insn if it is set - needs_tracking + will be true in that case and so we will end + up putting out mitigation sequences. */ + fixups_pending++; + need_second_pass = true; + break; + } + + needs_tracking = true; + + /* We always need a transfer before the first call in a BB. */ + if (!call_insn) + emit_insn_before (aarch64_speculation_clobber_sp (), insn); + + /* Tail-calls and no-return calls don't need any post-call + reestablishment of the tracker. */ + if (! tailcall) + call_insn = insn; + else + call_insn = NULL; + } + + if (insn == end) + break; + } + + if (call_insn) + { + rtx_insn *seq = aarch64_speculation_establish_tracker (); + + /* Handle debug insns at the end of the BB. Put the extra + insns after them. This ensures that we have consistent + behaviour for the placement of the extra insns between + debug and non-debug builds. */ + for (insn = call_insn; + insn != end && DEBUG_INSN_P (NEXT_INSN (insn)); + insn = NEXT_INSN (insn)) + ; + + if (insn == end) + { + edge e = find_fallthru_edge (bb->succs); + /* We need to be very careful about some calls that + appear at the end of a basic block. If the call + involves exceptions, then the compiler may depend on + this being the last instruction in the block. The + easiest way to handle this is to commit the new + instructions on the fall-through edge and to let + commit_edge_insertions clean things up for us. + + Sometimes, eg with OMP, there may not even be an + outgoing edge after the call. In that case, there's + not much we can do, presumably the compiler has + decided that the call can never return in this + context. */ + if (e) + { + /* We need to set the location lists explicitly in + this case. */ + if (! INSN_P (seq)) + { + start_sequence (); + emit_insn (seq); + seq = get_insns (); + end_sequence (); + } + + for (rtx_insn *list = seq; list; list = NEXT_INSN (list)) + INSN_LOCATION (list) = INSN_LOCATION (call_insn); + + insert_insn_on_edge (seq, e); + } + } + else + emit_insn_after (seq, call_insn); + } + } + + if (needs_tracking) + { + if (need_second_pass) + { + /* We found a return instruction before we found out whether + or not we need to emit the tracking code, but we now + know we do. Run quickly over the basic blocks and + fix up the return insns. */ + FOR_EACH_BB_FN (bb, cfun) + { + insn = BB_END (bb); + + while (insn != BB_HEAD (bb) + && NOTE_P (insn)) + insn = PREV_INSN (insn); + + if ((control_flow_insn_p (insn) + && GET_CODE (PATTERN (insn)) == RETURN) + || (CALL_P (insn) + && (SIBLING_CALL_P (insn) + || find_reg_note (insn, REG_NORETURN, NULL_RTX)))) + { + rtx_insn *seq = aarch64_speculation_clobber_sp (); + emit_insn_before (seq, insn); + fixups_pending--; + } + } + gcc_assert (fixups_pending == 0); + } + + /* Set up the initial value of the tracker, using the incoming SP. */ + insert_insn_on_edge (aarch64_speculation_establish_tracker (), + single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))); + commit_edge_insertions (); + } + + return 0; +} + +namespace { + +const pass_data pass_data_aarch64_track_speculation = +{ + RTL_PASS, /* type. */ + "speculation", /* name. */ + OPTGROUP_NONE, /* optinfo_flags. */ + TV_MACH_DEP, /* tv_id. */ + 0, /* properties_required. */ + 0, /* properties_provided. */ + 0, /* properties_destroyed. */ + 0, /* todo_flags_start. */ + 0 /* todo_flags_finish. */ +}; + +class pass_track_speculation : public rtl_opt_pass +{ + public: + pass_track_speculation(gcc::context *ctxt) + : rtl_opt_pass(pass_data_aarch64_track_speculation, ctxt) + {} + + /* opt_pass methods: */ + virtual bool gate (function *) + { + return aarch64_track_speculation; + } + + virtual unsigned int execute (function *) + { + return aarch64_do_track_speculation (); + } +}; // class pass_track_speculation. +} // anon namespace. + +/* Create a new pass instance. */ +rtl_opt_pass * +make_pass_track_speculation (gcc::context *ctxt) +{ + return new pass_track_speculation (ctxt); +} diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 975617f..29fd4ea 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -10130,6 +10130,19 @@ aarch64_conditional_register_usage (void) call_used_regs[i] = 1; } } + + /* When tracking speculation, we need a couple of call-clobbered registers + to track the speculation state. It would be nice to just use + IP0 and IP1, but currently there are numerous places that just + assume these registers are free for other uses (eg pointer + authentication). */ + if (aarch64_track_speculation) + { + fixed_regs[SPECULATION_TRACKER_REGNUM] = 1; + call_used_regs[SPECULATION_TRACKER_REGNUM] = 1; + fixed_regs[SPECULATION_SCRATCH_REGNUM] = 1; + call_used_regs[SPECULATION_SCRATCH_REGNUM] = 1; + } } /* Walk down the type tree of TYPE counting consecutive base elements. diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md index bc8d736..3b29082 100644 --- a/gcc/config/aarch64/aarch64.md +++ b/gcc/config/aarch64/aarch64.md @@ -62,6 +62,10 @@ (SFP_REGNUM 64) (AP_REGNUM 65) (CC_REGNUM 66) + ;; A couple of call-clobbered registers that we need to reserve when + ;; tracking speculation this is not ABI, so is subject to change. + (SPECULATION_TRACKER_REGNUM 15) + (SPECULATION_SCRATCH_REGNUM 14) ] ) @@ -134,6 +138,7 @@ UNSPEC_RSQRTE UNSPEC_RSQRTS UNSPEC_NZCV + UNSPEC_SPECULATION_TRACKER ]) (define_c_enum "unspecv" [ @@ -207,6 +212,11 @@ ;; no predicated insns. (define_attr "predicated" "yes,no" (const_string "no")) +;; Set to true on an insn that requires the speculation tracking state to be +;; in the tracking register before the insn issues. Otherwise the compiler +;; may chose to hold the tracking state encoded in SP. +(define_attr "speculation_barrier" "true,false" (const_string "false")) + ;; ------------------------------------------------------------------- ;; Pipeline descriptions and scheduling ;; ------------------------------------------------------------------- @@ -2946,7 +2956,7 @@ (define_insn "cmp" [(set (reg:CC CC_REGNUM) - (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r") + (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk") (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))] "" "@ @@ -5406,13 +5416,29 @@ DONE; }) +;; Track speculation through conditional branches. We assume that +;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary. +(define_insn "speculation_tracker" + [(set (reg:DI SPECULATION_TRACKER_REGNUM) + (unspec [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)] + UNSPEC_SPECULATION_TRACKER))] + "" + { + operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands); + return ""; + } + [(set_attr "type" "csel")] +) + ;; Hard speculation barrier. (define_insn "speculation_barrier" [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)] "" "isb\;dsb\\tsy" [(set_attr "length" "8") - (set_attr "type" "block")] + (set_attr "type" "block") + (set_attr "speculation_barrier" "true")] ) ;; AdvSIMD Stuff diff --git a/gcc/config/aarch64/t-aarch64 b/gcc/config/aarch64/t-aarch64 index b461eb5..84e58fe 100644 --- a/gcc/config/aarch64/t-aarch64 +++ b/gcc/config/aarch64/t-aarch64 @@ -67,6 +67,16 @@ cortex-a57-fma-steering.o: $(srcdir)/config/aarch64/cortex-a57-fma-steering.c \ $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \ $(srcdir)/config/aarch64/cortex-a57-fma-steering.c +aarch64-speculation.o: $(srcdir)/config/aarch64/aarch64-speculation.cc \ + $(CONFIG_H) \ + $(SYSTEM_H) \ + $(TM_H) \ + $(TARGET_H) \ + $(RTL_BASE_H) \ + $(TREE_PASS_H) + $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_SPPFLAGS) $(INCLUDES) \ + $(srcdir)/config/aarch64/aarch64-speculation.cc + comma=, MULTILIB_OPTIONS = $(subst $(comma),/, $(patsubst %, mabi=%, $(subst $(comma),$(comma)mabi=,$(TM_MULTILIB_CONFIG)))) MULTILIB_DIRNAMES = $(subst $(comma), ,$(TM_MULTILIB_CONFIG)) diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi index 39b7d58..ea18841 100644 --- a/gcc/doc/invoke.texi +++ b/gcc/doc/invoke.texi @@ -578,7 +578,8 @@ Objective-C and Objective-C++ Dialects}. -mlow-precision-recip-sqrt -mno-low-precision-recip-sqrt@gol -mlow-precision-sqrt -mno-low-precision-sqrt@gol -mlow-precision-div -mno-low-precision-div @gol --march=@var{name} -mcpu=@var{name} -mtune=@var{name}} +-march=@var{name} -mcpu=@var{name} -mtune=@var{name} @gol +-mtrack-speculation} @emph{Adapteva Epiphany Options} @gccoptlist{-mhalf-reg-file -mprefer-short-insn-regs @gol @@ -13092,6 +13093,14 @@ This option only has an effect if @option{-ffast-math} or precision of division results to about 16 bits for single precision and to 32 bits for double precision. +@item -mtrack-speculation +@itemx -mno-track-speculation +Enable or disable generation of additional code to track speculative +execution through conditional branches. The tracking state can then +be used by the compiler when expanding calls to +@code{__builtin_speculation_safe_copy} to permit a more efficient code +sequence to be generated. + @item -march=@var{name} @opindex march Specify the name of the target architecture and, optionally, one or -- 2.7.4 From 5234222a86c18d534a4f5e6da8d3673140c71503 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:36:26 +0000 Subject: [PATCH 11/16] AArch64 - use CSDB based sequences if speculation tracking is enabled In this final patch, now that we can track speculation through conditional branches, we can use this information to use a less expensive CSDB based speculation barrier. * config/aarch64/iterators.md (ALLI_TI): New iterator. * config/aarch64/aarch64.md (despeculate_copy): New expand. (despeculate_copy_insn): New insn. (despeculate_copyti_insn): New insn. (despeculate_simple): New insn (despeculate_simpleti): New insn. * config/aarch64/aarch64.c (aarch64_speculation_safe_value): New function. (TARGET_SPECULATION_SAFE_VALUE): Redefine to aarch64_speculation_safe_value. (aarch64_print_operand): Handle const0_rtx in modifier 'H'. (backported ce52014acef8b301ede901792f9d25035cb727bf) Change-Id: I0ea1ea8e425820a71d252d239ffb53a82162302c git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263174 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 15 +++++++ gcc/config/aarch64/aarch64.c | 47 ++++++++++++++++++++ gcc/config/aarch64/aarch64.md | 97 +++++++++++++++++++++++++++++++++++++++++ gcc/config/aarch64/iterators.md | 3 ++ 4 files changed, 162 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 722d68e..81c7432 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,20 @@ 2018-07-31 Richard Earnshaw + * config/aarch64/iterators.md (ALLI_TI): New iterator. + * config/aarch64/aarch64.md (despeculate_copy): New + expand. + (despeculate_copy_insn): New insn. + (despeculate_copyti_insn): New insn. + (despeculate_simple): New insn + (despeculate_simpleti): New insn. + * config/aarch64/aarch64.c (aarch64_speculation_safe_value): New + function. + (TARGET_SPECULATION_SAFE_VALUE): Redefine to + aarch64_speculation_safe_value. + (aarch64_print_operand): Handle const0_rtx in modifier 'H'. + +2018-07-31 Richard Earnshaw + * config/aarch64/aarch64-speculation.cc: New file. * config/aarch64/aarch64-passes.def (pass_track_speculation): Add before pass_reorder_blocks. diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 29fd4ea..3b55526 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -4656,6 +4656,11 @@ aarch64_print_operand (FILE *f, rtx x, int code) break; case 'H': + if (x == const0_rtx) + { + asm_fprintf (f, "xzr"); + break; + } /* Print the higher numbered register of a pair (TImode) of regs. */ if (!REG_P (x) || !GP_REGNUM_P (REGNO (x) + 1)) { @@ -14212,6 +14217,45 @@ aarch64_optab_supported_p (int op, machine_mode mode1, machine_mode, } } +/* Override the default target speculation_safe_value. */ +static rtx +aarch64_speculation_safe_value (machine_mode mode, + rtx result, rtx val, rtx failval) +{ + /* Maybe we should warn if falling back to hard barriers. They are + likely to be noticably more expensive than the alternative below. */ + if (!aarch64_track_speculation) + return default_speculation_safe_value (mode, result, val, failval); + + if (!REG_P (val)) + val = copy_to_mode_reg (mode, val); + + if (!aarch64_reg_or_zero (failval, mode)) + failval = copy_to_mode_reg (mode, failval); + + switch (mode) + { + case QImode: + emit_insn (gen_despeculate_copyqi (result, val, failval)); + break; + case HImode: + emit_insn (gen_despeculate_copyhi (result, val, failval)); + break; + case SImode: + emit_insn (gen_despeculate_copysi (result, val, failval)); + break; + case DImode: + emit_insn (gen_despeculate_copydi (result, val, failval)); + break; + case TImode: + emit_insn (gen_despeculate_copyti (result, val, failval)); + break; + default: + gcc_unreachable (); + } + return result; +} + #undef TARGET_ADDRESS_COST #define TARGET_ADDRESS_COST aarch64_address_cost @@ -14549,6 +14593,9 @@ aarch64_optab_supported_p (int op, machine_mode mode1, machine_mode, #undef TARGET_OPTAB_SUPPORTED_P #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p +#undef TARGET_SPECULATION_SAFE_VALUE +#define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value + #undef TARGET_OMIT_STRUCT_RETURN_REG #define TARGET_OMIT_STRUCT_RETURN_REG true diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md index 3b29082..689a0a8 100644 --- a/gcc/config/aarch64/aarch64.md +++ b/gcc/config/aarch64/aarch64.md @@ -5441,6 +5441,103 @@ (set_attr "speculation_barrier" "true")] ) +;; Support for __builtin_speculation_safe_value when we have speculation +;; tracking enabled. Use the speculation tracker to decide whether to +;; copy operand 1 to the target, or to copy the fail value (operand 2). +(define_expand "despeculate_copy" + [(set (match_operand:ALLI_TI 0 "register_operand" "=r") + (unspec_volatile:ALLI_TI + [(match_operand:ALLI_TI 1 "register_operand" "r") + (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ") + (use (reg:DI SPECULATION_TRACKER_REGNUM)) + (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] + "" + " + { + if (operands[2] == const0_rtx) + { + rtx tracker; + if (mode == TImode) + tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + else + tracker = gen_rtx_REG (mode, SPECULATION_TRACKER_REGNUM); + + emit_insn (gen_despeculate_simple (operands[0], operands[1], + tracker)); + DONE; + } + } + " +) + +;; Patterns to match despeculate_copy. Note that "hint 0x14" is the +;; encoding for CSDB, but will work in older versions of the assembler. +(define_insn "*despeculate_copy_insn" + [(set (match_operand:ALLI 0 "register_operand" "=r") + (unspec_volatile:ALLI + [(match_operand:ALLI 1 "register_operand" "r") + (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ") + (use (reg:DI SPECULATION_TRACKER_REGNUM)) + (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] + "" + { + operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + output_asm_insn ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;hint\t0x14 // csdb", + operands); + return ""; + } + [(set_attr "length" "12") + (set_attr "type" "block") + (set_attr "speculation_barrier" "true")] +) + +;; Pattern to match despeculate_copyti +(define_insn "*despeculate_copyti_insn" + [(set (match_operand:TI 0 "register_operand" "=r") + (unspec_volatile:TI + [(match_operand:TI 1 "register_operand" "r") + (match_operand:TI 2 "aarch64_reg_or_zero" "rZ") + (use (reg:DI SPECULATION_TRACKER_REGNUM)) + (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] + "" + { + operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); + output_asm_insn + ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb", + operands); + return ""; + } + [(set_attr "length" "16") + (set_attr "type" "block") + (set_attr "speculation_barrier" "true")] +) + +(define_insn "despeculate_simple" + [(set (match_operand:ALLI 0 "register_operand" "=r") + (unspec_volatile:ALLI + [(match_operand:ALLI 1 "register_operand" "r") + (use (match_operand:ALLI 2 "register_operand" ""))] + UNSPECV_SPECULATION_BARRIER))] + "" + "and\\t%0, %1, %2\;hint\t0x14 // csdb" + [(set_attr "type" "block") + (set_attr "length" "8") + (set_attr "speculation_barrier" "true")] +) + +(define_insn "despeculate_simpleti" + [(set (match_operand:TI 0 "register_operand" "=r") + (unspec_volatile:TI + [(match_operand:TI 1 "register_operand" "r") + (use (match_operand:DI 2 "register_operand" ""))] + UNSPECV_SPECULATION_BARRIER))] + "" + "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb" + [(set_attr "type" "block") + (set_attr "length" "12") + (set_attr "speculation_barrier" "true")] +) + ;; AdvSIMD Stuff (include "aarch64-simd.md") diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index 187057f..3250ce2 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -35,6 +35,9 @@ ;; Iterator for all integer modes (up to 64-bit) (define_mode_iterator ALLI [QI HI SI DI]) +;; Iterator for all integer modes (up to 128-bit) +(define_mode_iterator ALLI_TI [QI HI SI DI TI]) + ;; Iterator for all integer modes that can be extended (up to 64-bit) (define_mode_iterator ALLX [QI HI SI]) -- 2.7.4 From 1e8f9642efb0962bf13a68670ee5ebe3d2255e02 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Tue, 31 Jul 2018 17:36:36 +0000 Subject: [PATCH 12/16] targhooks - provide an alternative hook for targets that never execute speculatively This hook adds an alternative implementation for the target hook TARGET_HAVE_SPECULATION_SAFE_VALUE; it can be used by targets that have no CPU implementations that execute code speculatively. All that is needed for such targets now is to add: #undef TARGET_HAVE_SPECULATION_SAFE_VALUE #define TARGET_HAVE_SPECULATION_SAFE_VALUE speculation_safe_value_not_needed. to where you have your other target hooks and you're done. gcc: * targhooks.h (speculation_safe_value_not_needed): New prototype. * targhooks.c (speculation_safe_value_not_needed): New function. * target.def (have_speculation_safe_value): Update documentation. * doc/tm.texi: Regenerated. (backported 3e3448a9ae57194db317a72e68a3266a4cbdb39d) Change-Id: I4532e9bc21029d5e452868dbe95c25a41d1f4a5c git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263175 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 7 +++++++ gcc/doc/tm.texi | 5 +++++ gcc/target.def | 7 ++++++- gcc/targhooks.c | 7 +++++++ gcc/targhooks.h | 1 + 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 81c7432..50fe0d4 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,12 @@ 2018-07-31 Richard Earnshaw + * targhooks.h (speculation_safe_value_not_needed): New prototype. + * targhooks.c (speculation_safe_value_not_needed): New function. + * target.def (have_speculation_safe_value): Update documentation. + * doc/tm.texi: Regenerated. + +2018-07-31 Richard Earnshaw + * config/aarch64/iterators.md (ALLI_TI): New iterator. * config/aarch64/aarch64.md (despeculate_copy): New expand. diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi index e8166b8..e3bb4d3 100644 --- a/gcc/doc/tm.texi +++ b/gcc/doc/tm.texi @@ -11690,6 +11690,11 @@ This hook is used to determine the level of target support for a pattern named @code{speculation_barrier}. Else it returns true for the first case and whether the pattern is enabled for the current compilation for the second case. + + For targets that have no processors that can execute instructions + speculatively an alternative implemenation of this hook is available: + simply redefine this hook to @code{speculation_safe_value_not_needed} + along with your other target hooks. @end deftypefn @deftypefn {Target Hook} rtx TARGET_SPECULATION_SAFE_VALUE (machine_mode @var{mode}, rtx @var{result}, rtx @var{val}, rtx @var{failval}) diff --git a/gcc/target.def b/gcc/target.def index a4f21ad..af368a3 100644 --- a/gcc/target.def +++ b/gcc/target.def @@ -3952,7 +3952,12 @@ DEFHOOK The default implementation returns false if the target does not define\n\ a pattern named @code{speculation_barrier}. Else it returns true\n\ for the first case and whether the pattern is enabled for the current\n\ - compilation for the second case.", + compilation for the second case.\n\ + \n\ + For targets that have no processors that can execute instructions\n\ + speculatively an alternative implemenation of this hook is available:\n\ + simply redefine this hook to @code{speculation_safe_value_not_needed}\n\ + along with your other target hooks.", bool, (bool active), default_have_speculation_safe_value) DEFHOOK diff --git a/gcc/targhooks.c b/gcc/targhooks.c index d9db06f..58c9068 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -1975,6 +1975,13 @@ default_have_speculation_safe_value (bool active) return false; #endif } +/* Alternative implementation of TARGET_HAVE_SPECULATION_SAFE_VALUE + that can be used on targets that never have speculative execution. */ +bool +speculation_safe_value_not_needed (bool active) +{ + return !active; +} /* Default implementation of the speculation-safe-load builtin. This implementation simply copies val to result and generates a diff --git a/gcc/targhooks.h b/gcc/targhooks.h index 01501b2..102ebb5 100644 --- a/gcc/targhooks.h +++ b/gcc/targhooks.h @@ -255,6 +255,7 @@ extern bool default_optab_supported_p (int, machine_mode, machine_mode, optimization_type); extern bool default_have_speculation_safe_value (bool); +extern bool speculation_safe_value_not_needed (bool); extern rtx default_speculation_safe_value (machine_mode, rtx, rtx, rtx); #endif /* GCC_TARGHOOKS_H */ -- 2.7.4 From 070b3cd5223758d8f65eeb75bcfdcaeed1b19106 Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 31 Jul 2018 20:51:06 +0000 Subject: [PATCH 13/16] * targhooks.c (default_have_speculation_safe_value): Add ATTRIBUTE_UNUSED. (backported 30929ff6997501127e0d09e4917eef3f42d33899) Change-Id: I59da0b5a1f07516058eacb291a436e80fc2d52d5 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263180 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 5 +++++ gcc/targhooks.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 50fe0d4..d8ad496 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,8 @@ +2018-07-31 Ian Lance Taylor + + * targhooks.c (default_have_speculation_safe_value): Add + ATTRIBUTE_UNUSED. + 2018-07-31 Richard Earnshaw * targhooks.h (speculation_safe_value_not_needed): New prototype. diff --git a/gcc/targhooks.c b/gcc/targhooks.c index 58c9068..23dd41f 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -1967,7 +1967,7 @@ default_optab_supported_p (int, machine_mode, machine_mode, optimization_type) /* Default implementation of TARGET_HAVE_SPECULATION_SAFE_VALUE. */ bool -default_have_speculation_safe_value (bool active) +default_have_speculation_safe_value (bool active ATTRIBUTE_UNUSED) { #ifdef HAVE_speculation_barrier return active ? HAVE_speculation_barrier : true; -- 2.7.4 From fa5fafbf45197bb30ddc50f477d0c8af409d413f Mon Sep 17 00:00:00 2001 From: rearnsha Date: Wed, 1 Aug 2018 08:16:38 +0000 Subject: [PATCH 14/16] x86 - add speculation_barrier pattern This patch adds a speculation barrier for x86, based on my understanding of the required mitigation for that CPU, which is to use an lfence instruction. This patch needs some review by an x86 expert and if adjustments are needed, I'd appreciate it if they could be picked up by the port maintainer. This is supposed to serve as an example of how to deploy the new __builtin_speculation_safe_value() intrinsic on this architecture. * config/i386/i386.md (unspecv): Add UNSPECV_SPECULATION_BARRIER. (speculation_barrier): New insn. (backported 5812770a029b1a49b72cabf08b2f202d2d00dc69) Change-Id: I94d1aacb04c9288a0fbeb38109aeb80a59ca47e6 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263196 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 5 +++++ gcc/config/i386/i386.md | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index d8ad496..b5383da 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,8 @@ +2018-08-01 Richard Earnshaw + + * config/i386/i386.md (unspecv): Add UNSPECV_SPECULATION_BARRIER. + (speculation_barrier): New insn. + 2018-07-31 Ian Lance Taylor * targhooks.c (default_have_speculation_safe_value): Add diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index 46987d3..593866f 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -271,6 +271,9 @@ ;; For RDPKRU and WRPKRU support UNSPECV_PKU + + ;; For Speculation Barrier support + UNSPECV_SPECULATION_BARRIER ]) ;; Constants to represent rounding modes in the ROUND instruction @@ -19634,6 +19637,13 @@ "wrpkru" [(set_attr "type" "other")]) +(define_insn "speculation_barrier" + [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)] + "" + "lfence" + [(set_attr "type" "other") + (set_attr "length" "3")]) + (include "mmx.md") (include "sse.md") (include "sync.md") -- 2.7.4 From f0815b07a172fef498eddc36b5aa0c5047952419 Mon Sep 17 00:00:00 2001 From: rearnsha Date: Thu, 23 Aug 2018 09:47:34 +0000 Subject: [PATCH 15/16] PR target/86951 arm - Handle speculation barriers on pre-armv7 CPUs The AArch32 instruction sets prior to Armv7 do not define the ISB and DSB instructions that are needed to form a speculation barrier. While I do not know of any instances of cores based on those instruction sets being vulnerable to speculative side channel attacks it is possible to run code built for those ISAs on more recent hardware where they would become vulnerable. This patch works around this by using a library call added to libgcc. That code can then take any platform-specific actions necessary to ensure safety. For the moment I've only handled two cases: the library code being built for armv7 or later anyway and running on Linux. On Linux we can handle this by calling the kernel function that will flush a small amount of cache. Such a sequence ends with a ISB+DSB sequence if running on an Armv7 or later CPU. gcc: PR target/86951 * config/arm/arm-protos.h (arm_emit_speculation_barrier): New prototype. * config/arm/arm.c (speculation_barrier_libfunc): New static variable. (arm_init_libfuncs): Initialize it. (arm_emit_speculation_barrier): New function. * config/arm/arm.md (speculation_barrier): Call arm_emit_speculation_barrier for architectures that do not have DSB or ISB. (speculation_barrier_insn): Only match on Armv7 or later. libgcc: PR target/86951 * config/arm/lib1funcs.asm (speculation_barrier): New function. * config/arm/t-arm (LIB1ASMFUNCS): Add it to list of functions to build. (backported a7852b351bded1c6885e36e3ba70c298afcd1699) Change-Id: I4b77551dad3effbe49b06ff8bdd20daae69869e3 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@263806 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 14 ++++++++++++++ gcc/config/arm/arm-protos.h | 2 ++ gcc/config/arm/arm.c | 14 ++++++++++++++ gcc/config/arm/arm.md | 16 +++++++++++----- libgcc/ChangeLog | 7 +++++++ libgcc/config/arm/lib1funcs.S | 44 +++++++++++++++++++++++++++++++++++++++++++ libgcc/config/arm/t-arm | 2 +- 7 files changed, 93 insertions(+), 6 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b5383da..ea8a549 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,17 @@ +2018-08-23 Richard Earnshaw + + PR target/86951 + * config/arm/arm-protos.h (arm_emit_speculation_barrier): New + prototype. + * config/arm/arm.c (speculation_barrier_libfunc): New static + variable. + (arm_init_libfuncs): Initialize it. + (arm_emit_speculation_barrier): New function. + * config/arm/arm.md (speculation_barrier): Call + arm_emit_speculation_barrier for architectures that do not have + DSB or ISB. + (speculation_barrier_insn): Only match on Armv7 or later. + 2018-08-01 Richard Earnshaw * config/i386/i386.md (unspecv): Add UNSPECV_SPECULATION_BARRIER. diff --git a/gcc/config/arm/arm-protos.h b/gcc/config/arm/arm-protos.h index 54ff5d8..3a96ca4 100644 --- a/gcc/config/arm/arm-protos.h +++ b/gcc/config/arm/arm-protos.h @@ -56,6 +56,8 @@ extern void arm_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update extern rtx arm_simd_vect_par_cnst_half (machine_mode mode, bool high); extern bool arm_simd_check_vect_par_cnst_half_p (rtx op, machine_mode mode, bool high); +extern void arm_emit_speculation_barrier_function (void); + #ifdef RTX_CODE extern void arm_gen_unlikely_cbranch (enum rtx_code, machine_mode cc_mode, rtx label_ref); diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c index ae21d1a..022ae1e 100644 --- a/gcc/config/arm/arm.c +++ b/gcc/config/arm/arm.c @@ -2409,6 +2409,8 @@ arm_set_fixed_conv_libfunc (convert_optab optable, machine_mode to, set_conv_libfunc (optable, to, from, buffer); } +static GTY(()) rtx speculation_barrier_libfunc; + /* Set up library functions unique to ARM. */ static void @@ -2691,6 +2693,8 @@ arm_init_libfuncs (void) if (TARGET_AAPCS_BASED) synchronize_libfunc = init_one_libfunc ("__sync_synchronize"); + + speculation_barrier_libfunc = init_one_libfunc ("__speculation_barrier"); } /* On AAPCS systems, this is the "struct __va_list". */ @@ -30717,4 +30721,14 @@ arm_gen_unlikely_cbranch (enum rtx_code code, machine_mode cc_mode, emit_unlikely_jump (gen_rtx_SET (pc_rtx, x)); } +/* Emit a speculation barrier on target architectures that do not have + DSB/ISB directly. Such systems probably don't need a barrier + themselves, but if the code is ever run on a later architecture, it + might become a problem. */ +void +arm_emit_speculation_barrier_function () +{ + emit_library_call (speculation_barrier_libfunc, LCT_NORMAL, VOIDmode, 0); +} + #include "gt-arm.h" diff --git a/gcc/config/arm/arm.md b/gcc/config/arm/arm.md index ffdd6c2..bd2f6d8 100644 --- a/gcc/config/arm/arm.md +++ b/gcc/config/arm/arm.md @@ -11925,10 +11925,16 @@ [(unspec_volatile [(const_int 0)] VUNSPEC_SPECULATION_BARRIER)] "TARGET_EITHER" " - /* Don't emit anything for Thumb1 and suppress the warning from the - generic expansion. */ - if (!TARGET_32BIT) - DONE; + /* For thumb1 (except Armv8 derivatives), and for pre-Armv7 we don't + have a usable barrier (and probably don't need one in practice). + But to be safe if such code is run on later architectures, call a + helper function in libgcc that will do the thing for the active + system. */ + if (!(arm_arch7 || arm_arch8)) + { + arm_emit_speculation_barrier_function (); + DONE; + } " ) @@ -11936,7 +11942,7 @@ ;; tracking. (define_insn "*speculation_barrier_insn" [(unspec_volatile [(const_int 0)] VUNSPEC_SPECULATION_BARRIER)] - "TARGET_32BIT" + "arm_arch7 || arm_arch8" "isb\;dsb\\tsy" [(set_attr "type" "block") (set_attr "length" "8")] diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog index 48568ef..a5c403c 100644 --- a/libgcc/ChangeLog +++ b/libgcc/ChangeLog @@ -1,3 +1,10 @@ +2018-08-23 Richard Earnshaw + + PR target/86951 + * config/arm/lib1funcs.asm (speculation_barrier): New function. + * config/arm/t-arm (LIB1ASMFUNCS): Add it to list of functions + to build. + 2017-01-09 Andreas Tobler Backport from mainline diff --git a/libgcc/config/arm/lib1funcs.S b/libgcc/config/arm/lib1funcs.S index ba52e7b..e8fc502 100644 --- a/libgcc/config/arm/lib1funcs.S +++ b/libgcc/config/arm/lib1funcs.S @@ -1576,6 +1576,50 @@ LSYM(Lover12): #error "This is only for ARM EABI GNU/Linux" #endif #endif /* L_clear_cache */ + +#ifdef L_speculation_barrier + FUNC_START speculation_barrier +#if __ARM_ARCH >= 7 + isb + dsb sy +#elif defined __ARM_EABI__ && defined __linux__ + /* We don't have a speculation barrier directly for this + platform/architecture variant. But we can use a kernel + clear_cache service routine which will emit such instructions + if run on a later version of the architecture. We don't + really want to flush the cache, but we must give it a valid + address, so just clear pc..pc+1. */ +#if defined __thumb__ && !defined __thumb2__ + push {r7} + mov r7, #0xf + lsl r7, #16 + add r7, #2 + adr r0, . + 4 + add r1, r0, #1 + mov r2, #0 + svc 0 + pop {r7} +#else + do_push {r7} +#ifdef __ARM_ARCH_6T2__ + movw r7, #2 + movt r7, #0xf +#else + mov r7, #0xf0000 + add r7, r7, #2 +#endif + add r0, pc, #0 /* ADR. */ + add r1, r0, #1 + mov r2, #0 + svc 0 + do_pop {r7} +#endif /* Thumb1 only */ +#else +#warning "No speculation barrier defined for this platform" +#endif + RET + FUNC_END speculation_barrier +#endif /* ------------------------------------------------------------------------ */ /* Dword shift operations. */ /* All the following Dword shift variants rely on the fact that diff --git a/libgcc/config/arm/t-arm b/libgcc/config/arm/t-arm index 9e85ac0..274bf2a 100644 --- a/libgcc/config/arm/t-arm +++ b/libgcc/config/arm/t-arm @@ -1,6 +1,6 @@ LIB1ASMSRC = arm/lib1funcs.S LIB1ASMFUNCS = _thumb1_case_sqi _thumb1_case_uqi _thumb1_case_shi \ - _thumb1_case_uhi _thumb1_case_si + _thumb1_case_uhi _thumb1_case_si _speculation_barrier HAVE_CMSE:=$(findstring __ARM_FEATURE_CMSE,$(shell $(gcc_compile_bare) -dM -E - /dev/null),) -- 2.7.4 From e98f28299c49093f3100bc77c5174c7f85095f3f Mon Sep 17 00:00:00 2001 From: rearnsha Date: Fri, 2 Nov 2018 13:36:31 +0000 Subject: [PATCH 16/16] Although there's no fundamental reason why shrink wrapping and speculation tracking are incompatible, a phase-ordering requirement (we need to do speculation tracking before the final basic block clean-up) means that the shrink wrapping pass can undo some of the changes the speculation tracking pass makes. The result is that the tracking, while still safe is less comprehensive than we really want. So to keep things simple, and because the tracking code is quite expensive anyway, it seems best to just disable that pass when we are tracking speculative execution. * config/aarch64/aarch64.c (aarch64_override_options): Disable shrink-wrapping when -mtrack-speculation. (backported 3024ac6836fa5a0f8ab539a0823bfb609aef0e93) Change-Id: I894aa3562b8ddfc951e2eb33c15bd9d79c423c0f git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@265747 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 5 +++++ gcc/config/aarch64/aarch64.c | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index ea8a549..12247a1 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,8 @@ +2018-11-02 Richard Earnshaw + + * config/aarch64/aarch64.c ((aarch64_override_options): Disable + shrink-wrapping when -mtrack-speculation. + 2018-08-23 Richard Earnshaw PR target/86951 diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 3b55526..16cd780 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -8584,6 +8584,12 @@ aarch64_override_options (void) || (aarch64_arch_string && valid_arch)) gcc_assert (explicit_arch != aarch64_no_arch); + /* The pass to insert speculation tracking runs before + shrink-wrapping and the latter does not know how to update the + tracking status. So disable it in this case. */ + if (aarch64_track_speculation) + flag_shrink_wrap = 0; + aarch64_override_options_internal (&global_options); /* Save these options as the default ones in case we push and pop them later -- 2.7.4