From cfb582f62791dfadc243d97d37f0b83ef77cf480 Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Tue, 4 May 2021 23:31:48 +0100 Subject: [PATCH] libstdc++: Optimize std::visit for the common case [PR 78113] GCC does not do a good job of optimizing the table of function pointers used for variant visitation. This avoids using the table for the common case of visiting a single variant with a small number of alternative types. Instead we use: switch(v.index()) { case 0: return visitor(get<0>(v)); case 1: return visitor(get<1>(v)); ... } It's not quite that simple, because get<1>(v) is ill-formed if the variant only has one alternative, and similarly for each get. We need to ensure each case only applies the visitor if the index is in range for the actual type we're dealing with, and tell the compiler that the case is unreachable otherwise. We also need to invoke the visitor via the __gen_vtable_impl::__visit_invoke function, to handle the raw visitation cases used to implement std::variant assignments and comparisons. Because that gets quite verbose and repetitive, a macro is used to stamp out the cases. We also need to handle the valueless_by_exception case, but only for raw visitation, because std::visit already checks for it before calling __do_visit. Signed-off-by: Jonathan Wakely libstdc++-v3/ChangeLog: PR libstdc++/78113 * include/std/variant (__do_visit): Use a switch when we have a single variant with a small number of alternatives. --- libstdc++-v3/include/std/variant | 93 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/libstdc++-v3/include/std/variant b/libstdc++-v3/include/std/variant index c651326..19b2158 100644 --- a/libstdc++-v3/include/std/variant +++ b/libstdc++-v3/include/std/variant @@ -485,6 +485,12 @@ namespace __variant { if constexpr (__variant::__never_valueless<_Types...>()) return true; + // It would be nice if we could just return true for -fno-exceptions. + // It's possible (but inadvisable) that a std::variant could become + // valueless in a translation unit compiled with -fexceptions and then + // be passed to functions compiled with -fno-exceptions. We would need + // some #ifdef _GLIBCXX_NO_EXCEPTIONS_GLOBALLY property to elide all + // checks for valueless_by_exception(). return this->_M_index != static_cast<__index_type>(variant_npos); } @@ -1754,12 +1760,89 @@ namespace __variant constexpr decltype(auto) __do_visit(_Visitor&& __visitor, _Variants&&... __variants) { - constexpr auto& __vtable = __detail::__variant::__gen_vtable< - _Result_type, _Visitor&&, _Variants&&...>::_S_vtable; + // Get the silly case of visiting no variants out of the way first. + if constexpr (sizeof...(_Variants) == 0) + return std::forward<_Visitor>(__visitor)(); + else + { + constexpr size_t __max = 11; // "These go to eleven." + + // The type of the first variant in the pack. + using _V0 + = typename __detail::__variant::_Nth_type<0, _Variants...>::type; + // The number of alternatives in that first variant. + constexpr auto __n = variant_size_v>; + + if constexpr (sizeof...(_Variants) > 1 || __n > __max) + { + // Use a jump table for the general case. + constexpr auto& __vtable = __detail::__variant::__gen_vtable< + _Result_type, _Visitor&&, _Variants&&...>::_S_vtable; + + auto __func_ptr = __vtable._M_access(__variants.index()...); + return (*__func_ptr)(std::forward<_Visitor>(__visitor), + std::forward<_Variants>(__variants)...); + } + else // We have a single variant with a small number of alternatives. + { + // A name for the first variant in the pack. + _V0& __v0 + = [](_V0& __v, ...) -> _V0& { return __v; }(__variants...); + + using __detail::__variant::_Multi_array; + using __detail::__variant::__gen_vtable_impl; + using _Ma = _Multi_array<_Result_type (*)(_Visitor&&, _V0&&)>; - auto __func_ptr = __vtable._M_access(__variants.index()...); - return (*__func_ptr)(std::forward<_Visitor>(__visitor), - std::forward<_Variants>(__variants)...); +#ifdef _GLIBCXX_DEBUG +# define _GLIBCXX_VISIT_UNREACHABLE __builtin_trap +#else +# define _GLIBCXX_VISIT_UNREACHABLE __builtin_unreachable +#endif + +#define _GLIBCXX_VISIT_CASE(N) \ + case N: \ + { \ + if constexpr (N < __n) \ + { \ + return __gen_vtable_impl<_Ma, index_sequence>:: \ + __visit_invoke(std::forward<_Visitor>(__visitor), \ + std::forward<_V0>(__v0)); \ + } \ + else _GLIBCXX_VISIT_UNREACHABLE(); \ + } + + switch (__v0.index()) + { + _GLIBCXX_VISIT_CASE(0) + _GLIBCXX_VISIT_CASE(1) + _GLIBCXX_VISIT_CASE(2) + _GLIBCXX_VISIT_CASE(3) + _GLIBCXX_VISIT_CASE(4) + _GLIBCXX_VISIT_CASE(5) + _GLIBCXX_VISIT_CASE(6) + _GLIBCXX_VISIT_CASE(7) + _GLIBCXX_VISIT_CASE(8) + _GLIBCXX_VISIT_CASE(9) + _GLIBCXX_VISIT_CASE(10) + case variant_npos: + using __detail::__variant::__variant_idx_cookie; + using __detail::__variant::__variant_cookie; + if constexpr (is_same_v<_Result_type, __variant_idx_cookie> + || is_same_v<_Result_type, __variant_cookie>) + { + return __gen_vtable_impl<_Ma, index_sequence<-1>>:: + __visit_invoke(std::forward<_Visitor>(__visitor), + std::forward<_V0>(__v0)); + } + else + _GLIBCXX_VISIT_UNREACHABLE(); + default: + _GLIBCXX_VISIT_UNREACHABLE(); + } +#undef _GLIBCXX_VISIT_CASE +#undef _GLIBCXX_VISIT_UNREACHABLE + } + } } /// @endcond -- 2.7.4