As of now containers key_eq might get called when rehashing happens, which is redundant for unique keys containers.
Reviewed By: #libc, philnik, Mordante
Differential Revision: https://reviews.llvm.org/D128021
}
}
+template <class Container, class GenInputs>
+static void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
+ auto in = gen(st.range(0));
+ c.max_load_factor(3.0);
+ c.insert(in.begin(), in.end());
+ benchmark::DoNotOptimize(c);
+ const auto bucket_count = c.bucket_count();
+ while (st.KeepRunning()) {
+ c.rehash(bucket_count + 1);
+ c.rehash(bucket_count);
+ benchmark::ClobberMemory();
+ }
+}
+
} // end namespace ContainerBenchmarks
#endif // BENCHMARK_CONTAINER_BENCHMARKS_H
}
};
+// The sole purpose of this comparator is to be used in BM_Rehash, where
+// we need something slow enough to be easily noticable in benchmark results.
+// The default implementation of operator== for strings seems to be a little
+// too fast for that specific benchmark to reliably show a noticeable
+// improvement, but unoptimized bytewise comparison fits just right.
+// Early return is there just for convenience, since we only compare strings
+// of equal length in BM_Rehash.
+struct SlowStringEq {
+ SlowStringEq() = default;
+ inline TEST_ALWAYS_INLINE
+ bool operator()(const std::string& lhs, const std::string& rhs) const {
+ if (lhs.size() != rhs.size()) return false;
+
+ bool eq = true;
+ for (size_t i = 0; i < lhs.size(); ++i) {
+ eq &= lhs[i] == rhs[i];
+ }
+ return eq;
+ }
+};
+
//----------------------------------------------------------------------------//
// BM_Hash
// ---------------------------------------------------------------------------//
std::unordered_set<std::string>{},
getRandomStringInputs)->Arg(TestNumInputs);
+//----------------------------------------------------------------------------//
+// BM_Rehash
+// ---------------------------------------------------------------------------//
+
+BENCHMARK_CAPTURE(BM_Rehash,
+ unordered_set_string_arg,
+ std::unordered_set<std::string, std::hash<std::string>, SlowStringEq>{},
+ getRandomStringInputs)->Arg(TestNumInputs);
+
+BENCHMARK_CAPTURE(BM_Rehash,
+ unordered_set_int_arg,
+ std::unordered_set<int>{},
+ getRandomIntegerInputs<int>)->Arg(TestNumInputs);
+
///////////////////////////////////////////////////////////////////////////////
BENCHMARK_CAPTURE(BM_InsertDuplicate,
unordered_set_int,
#endif
void clear() _NOEXCEPT;
- void rehash(size_type __n);
- _LIBCPP_INLINE_VISIBILITY void reserve(size_type __n)
- {rehash(static_cast<size_type>(ceil(__n / max_load_factor())));}
+ _LIBCPP_INLINE_VISIBILITY void __rehash_unique(size_type __n) { __rehash<true>(__n); }
+ _LIBCPP_INLINE_VISIBILITY void __rehash_multi(size_type __n) { __rehash<false>(__n); }
+ _LIBCPP_INLINE_VISIBILITY void __reserve_unique(size_type __n)
+ {
+ __rehash_unique(static_cast<size_type>(ceil(__n / max_load_factor())));
+ }
+ _LIBCPP_INLINE_VISIBILITY void __reserve_multi(size_type __n)
+ {
+ __rehash_multi(static_cast<size_type>(ceil(__n / max_load_factor())));
+ }
_LIBCPP_INLINE_VISIBILITY
size_type bucket_count() const _NOEXCEPT
#endif // _LIBCPP_ENABLE_DEBUG_MODE
private:
- void __rehash(size_type __n);
+ template <bool _UniqueKeys> void __rehash(size_type __n);
+ template <bool _UniqueKeys> void __do_rehash(size_type __n);
template <class ..._Args>
__node_holder __construct_node(_Args&& ...__args);
}
if (size()+1 > __bc * max_load_factor() || __bc == 0)
{
- rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
+ __rehash_unique(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
size_type(ceil(float(size() + 1) / max_load_factor()))));
}
return nullptr;
size_type __bc = bucket_count();
if (size()+1 > __bc * max_load_factor() || __bc == 0)
{
- rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
+ __rehash_multi(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
size_type(ceil(float(size() + 1) / max_load_factor()))));
__bc = bucket_count();
}
size_type __bc = bucket_count();
if (size()+1 > __bc * max_load_factor() || __bc == 0)
{
- rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
+ __rehash_multi(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
size_type(ceil(float(size() + 1) / max_load_factor()))));
__bc = bucket_count();
}
__node_holder __h = __construct_node_hash(__hash, _VSTD::forward<_Args>(__args)...);
if (size()+1 > __bc * max_load_factor() || __bc == 0)
{
- rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
+ __rehash_unique(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
size_type(ceil(float(size() + 1) / max_load_factor()))));
__bc = bucket_count();
__chash = __constrain_hash(__hash, __bc);
#endif // _LIBCPP_STD_VER > 14
template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <bool _UniqueKeys>
void
-__hash_table<_Tp, _Hash, _Equal, _Alloc>::rehash(size_type __n)
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n)
_LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
{
if (__n == 1)
__n = __next_prime(__n);
size_type __bc = bucket_count();
if (__n > __bc)
- __rehash(__n);
+ __do_rehash<_UniqueKeys>(__n);
else if (__n < __bc)
{
__n = _VSTD::max<size_type>
__next_prime(size_t(ceil(float(size()) / max_load_factor())))
);
if (__n < __bc)
- __rehash(__n);
+ __do_rehash<_UniqueKeys>(__n);
}
}
template <class _Tp, class _Hash, class _Equal, class _Alloc>
+template <bool _UniqueKeys>
void
-__hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __nbc)
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc)
{
std::__debug_db_invalidate_all(this);
__pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc();
else
{
__next_pointer __np = __cp;
- for (; __np->__next_ != nullptr &&
- key_eq()(__cp->__upcast()->__value_,
- __np->__next_->__upcast()->__value_);
- __np = __np->__next_)
- ;
+ if _LIBCPP_CONSTEXPR_AFTER_CXX14 (!_UniqueKeys)
+ {
+ for (; __np->__next_ != nullptr &&
+ key_eq()(__cp->__upcast()->__value_,
+ __np->__next_->__upcast()->__value_);
+ __np = __np->__next_)
+ ;
+ }
__pp->__next_ = __np->__next_;
__np->__next_ = __bucket_list_[__chash]->__next_;
__bucket_list_[__chash]->__next_ = __cp;
{return __table_.bucket_size(__n);}
_LIBCPP_INLINE_VISIBILITY
- void resize(size_type __n) {__table_.rehash(__n);}
+ void resize(size_type __n) {__table_.__rehash_unique(__n);}
private:
__node_holder __construct_node(const key_type& __k);
size_type __n, const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
const hash_map& __u)
: __table_(__u.__table_)
{
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
{return __table_.bucket_size(__n);}
_LIBCPP_INLINE_VISIBILITY
- void resize(size_type __n) {__table_.rehash(__n);}
+ void resize(size_type __n) {__table_.__rehash_multi(__n);}
};
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
size_type __n, const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
const hash_multimap& __u)
: __table_(__u.__table_)
{
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
size_type elems_in_bucket(size_type __n) const {return __table_.bucket_size(__n);}
_LIBCPP_INLINE_VISIBILITY
- void resize(size_type __n) {__table_.rehash(__n);}
+ void resize(size_type __n) {__table_.__rehash_unique(__n);}
};
template <class _Value, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
const hash_set& __u)
: __table_(__u.__table_)
{
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
size_type elems_in_bucket(size_type __n) const {return __table_.bucket_size(__n);}
_LIBCPP_INLINE_VISIBILITY
- void resize(size_type __n) {__table_.rehash(__n);}
+ void resize(size_type __n) {__table_.__rehash_multi(__n);}
};
template <class _Value, class _Hash, class _Pred, class _Alloc>
size_type __n, const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
const hasher& __hf, const key_equal& __eql)
: __table_(__hf, __eql)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
: __table_(__hf, __eql, __a)
{
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
const hash_multiset& __u)
: __table_(__u.__table_)
{
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
_LIBCPP_INLINE_VISIBILITY
void max_load_factor(float __mlf) {__table_.max_load_factor(__mlf);}
_LIBCPP_INLINE_VISIBILITY
- void rehash(size_type __n) {__table_.rehash(__n);}
+ void rehash(size_type __n) {__table_.__rehash_unique(__n);}
_LIBCPP_INLINE_VISIBILITY
- void reserve(size_type __n) {__table_.reserve(__n);}
+ void reserve(size_type __n) {__table_.__reserve_unique(__n);}
#ifdef _LIBCPP_ENABLE_DEBUG_MODE
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
: __table_(__u.__table_)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__u.__table_, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__il.begin(), __il.end());
}
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__il.begin(), __il.end());
}
_LIBCPP_INLINE_VISIBILITY
void max_load_factor(float __mlf) {__table_.max_load_factor(__mlf);}
_LIBCPP_INLINE_VISIBILITY
- void rehash(size_type __n) {__table_.rehash(__n);}
+ void rehash(size_type __n) {__table_.__rehash_multi(__n);}
_LIBCPP_INLINE_VISIBILITY
- void reserve(size_type __n) {__table_.reserve(__n);}
+ void reserve(size_type __n) {__table_.__reserve_multi(__n);}
#ifdef _LIBCPP_ENABLE_DEBUG_MODE
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
: __table_(__u.__table_)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__u.__table_, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__il.begin(), __il.end());
}
: __table_(__hf, __eql, typename __table::allocator_type(__a))
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__il.begin(), __il.end());
}
_LIBCPP_INLINE_VISIBILITY
void max_load_factor(float __mlf) {__table_.max_load_factor(__mlf);}
_LIBCPP_INLINE_VISIBILITY
- void rehash(size_type __n) {__table_.rehash(__n);}
+ void rehash(size_type __n) {__table_.__rehash_unique(__n);}
_LIBCPP_INLINE_VISIBILITY
- void reserve(size_type __n) {__table_.reserve(__n);}
+ void reserve(size_type __n) {__table_.__reserve_unique(__n);}
#ifdef _LIBCPP_ENABLE_DEBUG_MODE
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__first, __last);
}
: __table_(__u.__table_)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__u.__table_, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__il.begin(), __il.end());
}
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_unique(__n);
insert(__il.begin(), __il.end());
}
_LIBCPP_INLINE_VISIBILITY
void max_load_factor(float __mlf) {__table_.max_load_factor(__mlf);}
_LIBCPP_INLINE_VISIBILITY
- void rehash(size_type __n) {__table_.rehash(__n);}
+ void rehash(size_type __n) {__table_.__rehash_multi(__n);}
_LIBCPP_INLINE_VISIBILITY
- void reserve(size_type __n) {__table_.reserve(__n);}
+ void reserve(size_type __n) {__table_.__reserve_multi(__n);}
#ifdef _LIBCPP_ENABLE_DEBUG_MODE
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
}
template <class _Value, class _Hash, class _Pred, class _Alloc>
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__first, __last);
}
: __table_(__u.__table_)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__u.__table_, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__u.bucket_count());
+ __table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}
: __table_(__hf, __eql)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__il.begin(), __il.end());
}
: __table_(__hf, __eql, __a)
{
_VSTD::__debug_db_insert_c(this);
- __table_.rehash(__n);
+ __table_.__rehash_multi(__n);
insert(__il.begin(), __il.end());
}