diff options
| author | Tyge Løvset <[email protected]> | 2021-10-22 22:50:34 +0200 |
|---|---|---|
| committer | Tyge Løvset <[email protected]> | 2021-10-22 22:50:34 +0200 |
| commit | 63a09f6492b022eccd1583fd9b69c450ae3a8935 (patch) | |
| tree | 126d3bb60d730535505f233b70adb91eb934a2b6 | |
| parent | 4b5c7250b59401a8888b68f4b12f3ddd69f83741 (diff) | |
| download | STC-modified-63a09f6492b022eccd1583fd9b69c450ae3a8935.tar.gz STC-modified-63a09f6492b022eccd1583fd9b69c450ae3a8935.zip | |
Updated shootout2_cmap.cpp and competing hashmaps. Replaced Gregs sparsepp with his parallel_hashmap as recommended by him.
22 files changed, 18866 insertions, 12479 deletions
diff --git a/benchmarks/others/parallel_hashmap/btree.h b/benchmarks/others/parallel_hashmap/btree.h new file mode 100644 index 00000000..ae1aec5b --- /dev/null +++ b/benchmarks/others/parallel_hashmap/btree.h @@ -0,0 +1,4050 @@ +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_ +#define PHMAP_BTREE_BTREE_CONTAINER_H_ + +#ifdef _MSC_VER + #pragma warning(push) + + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4324) // structure was padded due to alignment specifier + #pragma warning(disable : 4355) // 'this': used in base member initializer list + #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member + #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list + #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted + #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted + #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#endif + + +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <limits> +#include <new> + +#include "phmap_fwd_decl.h" +#include "phmap_base.h" + +#if PHMAP_HAVE_STD_STRING_VIEW + #include <string_view> +#endif + +// MSVC constructibility traits do not detect destructor properties and so our +// implementations should not use them as a source-of-truth. +#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) + #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 +#endif + +namespace phmap { + + // Defined and documented later on in this file. + template <typename T> + struct is_trivially_destructible; + + // Defined and documented later on in this file. + template <typename T> + struct is_trivially_move_assignable; + + namespace type_traits_internal { + + // Silence MSVC warnings about the destructor being defined as deleted. +#if defined(_MSC_VER) && !defined(__GNUC__) + #pragma warning(push) + #pragma warning(disable : 4624) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + + template <class T> + union SingleMemberUnion { + T t; + }; + + // Restore the state of the destructor warning that was silenced above. +#if defined(_MSC_VER) && !defined(__GNUC__) + #pragma warning(pop) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + + template <class T> + struct IsTriviallyMoveConstructibleObject + : std::integral_constant< + bool, std::is_move_constructible< + type_traits_internal::SingleMemberUnion<T>>::value && + phmap::is_trivially_destructible<T>::value> {}; + + template <class T> + struct IsTriviallyCopyConstructibleObject + : std::integral_constant< + bool, std::is_copy_constructible< + type_traits_internal::SingleMemberUnion<T>>::value && + phmap::is_trivially_destructible<T>::value> {}; + + template <class T> + struct IsTriviallyMoveAssignableReference : std::false_type {}; + + template <class T> + struct IsTriviallyMoveAssignableReference<T&> + : phmap::is_trivially_move_assignable<T>::type {}; + + template <class T> + struct IsTriviallyMoveAssignableReference<T&&> + : phmap::is_trivially_move_assignable<T>::type {}; + + } // namespace type_traits_internal + + + template <typename... Ts> + using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; + + + template <typename T> + struct is_function + : std::integral_constant< + bool, !(std::is_reference<T>::value || + std::is_const<typename std::add_const<T>::type>::value)> {}; + + + namespace type_traits_internal { + + template <typename T> + class is_trivially_copyable_impl { + using ExtentsRemoved = typename std::remove_all_extents<T>::type; + static constexpr bool kIsCopyOrMoveConstructible = + std::is_copy_constructible<ExtentsRemoved>::value || + std::is_move_constructible<ExtentsRemoved>::value; + static constexpr bool kIsCopyOrMoveAssignable = + phmap::is_copy_assignable<ExtentsRemoved>::value || + phmap::is_move_assignable<ExtentsRemoved>::value; + + public: + static constexpr bool kValue = + (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && + (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && + (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && + is_trivially_destructible<ExtentsRemoved>::value && + // We need to check for this explicitly because otherwise we'll say + // references are trivial copyable when compiled by MSVC. + !std::is_reference<ExtentsRemoved>::value; + }; + + template <typename T> + struct is_trivially_copyable + : std::integral_constant< + bool, type_traits_internal::is_trivially_copyable_impl<T>::kValue> {}; + } // namespace type_traits_internal + + namespace swap_internal { + + // Necessary for the traits. + using std::swap; + + // This declaration prevents global `swap` and `phmap::swap` overloads from being + // considered unless ADL picks them up. + void swap(); + + template <class T> + using IsSwappableImpl = decltype(swap(std::declval<T&>(), std::declval<T&>())); + + // NOTE: This dance with the default template parameter is for MSVC. + template <class T, + class IsNoexcept = std::integral_constant< + bool, noexcept(swap(std::declval<T&>(), std::declval<T&>()))>> + using IsNothrowSwappableImpl = typename std::enable_if<IsNoexcept::value>::type; + + template <class T> + struct IsSwappable + : phmap::type_traits_internal::is_detected<IsSwappableImpl, T> {}; + + template <class T> + struct IsNothrowSwappable + : phmap::type_traits_internal::is_detected<IsNothrowSwappableImpl, T> {}; + + template <class T, phmap::enable_if_t<IsSwappable<T>::value, int> = 0> + void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable<T>::value) { + swap(lhs, rhs); + } + + using StdSwapIsUnconstrained = IsSwappable<void()>; + + } // namespace swap_internal + + namespace type_traits_internal { + + // Make the swap-related traits/function accessible from this namespace. + using swap_internal::IsNothrowSwappable; + using swap_internal::IsSwappable; + using swap_internal::Swap; + using swap_internal::StdSwapIsUnconstrained; + + } // namespace type_traits_internal + + namespace compare_internal { + + using value_type = int8_t; + + template <typename T> + struct Fail { + static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); + }; + + template <typename NullPtrT = std::nullptr_t> + struct OnlyLiteralZero { + constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT + + template < + typename T, + typename = typename std::enable_if< + std::is_same<T, std::nullptr_t>::value || + (std::is_integral<T>::value && !std::is_same<T, int>::value)>::type, + typename = typename Fail<T>::type> + OnlyLiteralZero(T); // NOLINT + }; + + enum class eq : value_type { + equal = 0, + equivalent = equal, + nonequal = 1, + nonequivalent = nonequal, + }; + + enum class ord : value_type { less = -1, greater = 1 }; + + enum class ncmp : value_type { unordered = -127 }; + +#if defined(__cpp_inline_variables) && !defined(_MSC_VER) + +#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) + +#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ + static const type name; + +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ + inline constexpr type type::name(init) + +#else // __cpp_inline_variables + +#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ + static const T name; + +#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) + +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ + template <typename T> \ + const T compare_internal::type##_base<T>::name(init) + +#endif // __cpp_inline_variables + + // These template base classes allow for defining the values of the constants + // in the header file (for performance) without using inline variables (which + // aren't available in C++11). + template <typename T> + struct weak_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) + }; + + template <typename T> + struct strong_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) + }; + + template <typename T> + struct partial_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) + }; + + template <typename T> + struct weak_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + }; + + template <typename T> + struct strong_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + }; + + } // namespace compare_internal + + class weak_equality + : public compare_internal::weak_equality_base<weak_equality> { + explicit constexpr weak_equality(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::weak_equality_base<weak_equality>; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) + + // Comparisons + friend constexpr bool operator==( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; + }; + PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, + compare_internal::eq::nonequivalent); + + class strong_equality + : public compare_internal::strong_equality_base<strong_equality> { + explicit constexpr strong_equality(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::strong_equality_base<strong_equality>; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); + PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, + compare_internal::eq::nonequal); + PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, + compare_internal::eq::nonequivalent); + + class partial_ordering + : public compare_internal::partial_ordering_base<partial_ordering> { + explicit constexpr partial_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr partial_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::partial_ordering_base<partial_ordering>; + + constexpr bool is_ordered() const noexcept { + return value_ != + compare_internal::value_type(compare_internal::ncmp::unordered); + } + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ == 0; + } + friend constexpr bool operator!=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return !v.is_ordered() || v.value_ != 0; + } + friend constexpr bool operator<( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ < 0; + } + friend constexpr bool operator<=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ <= 0; + } + friend constexpr bool operator>( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ > 0; + } + friend constexpr bool operator>=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return !v.is_ordered() || 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, + compare_internal::ord::greater); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, + compare_internal::ncmp::unordered); + + class weak_ordering + : public compare_internal::weak_ordering_base<weak_ordering> { + explicit constexpr weak_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr weak_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::weak_ordering_base<weak_ordering>; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, + compare_internal::ord::greater); + + class strong_ordering + : public compare_internal::strong_ordering_base<strong_ordering> { + explicit constexpr strong_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr strong_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::strong_ordering_base<strong_ordering>; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator strong_equality() const noexcept { // NOLINT + return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + constexpr operator weak_ordering() const noexcept { // NOLINT + return value_ == 0 + ? weak_ordering::equivalent + : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, + compare_internal::ord::greater); + +#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL +#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL +#undef PHMAP_COMPARE_INLINE_INIT + + namespace compare_internal { + // We also provide these comparator adapter functions for internal phmap use. + + // Helper functions to do a boolean comparison of two keys given a boolean + // or three-way comparator. + // SFINAE prevents implicit conversions to bool (such as from int). + template <typename BoolType, + phmap::enable_if_t<std::is_same<bool, BoolType>::value, int> = 0> + constexpr bool compare_result_as_less_than(const BoolType r) { return r; } + constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { + return r < 0; + } + + template <typename Compare, typename K, typename LK> + constexpr bool do_less_than_comparison(const Compare &compare, const K &x, + const LK &y) { + return compare_result_as_less_than(compare(x, y)); + } + + // Helper functions to do a three-way comparison of two keys given a boolean or + // three-way comparator. + // SFINAE prevents implicit conversions to int (such as from bool). + template <typename Int, + phmap::enable_if_t<std::is_same<int, Int>::value, int> = 0> + constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { + return c < 0 ? phmap::weak_ordering::less + : c == 0 ? phmap::weak_ordering::equivalent + : phmap::weak_ordering::greater; + } + constexpr phmap::weak_ordering compare_result_as_ordering( + const phmap::weak_ordering c) { + return c; + } + + template < + typename Compare, typename K, typename LK, + phmap::enable_if_t<!std::is_same<bool, phmap::invoke_result_t< + Compare, const K &, const LK &>>::value, + int> = 0> + constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare_result_as_ordering(compare(x, y)); + } + template < + typename Compare, typename K, typename LK, + phmap::enable_if_t<std::is_same<bool, phmap::invoke_result_t<Compare, + const K &, const LK &>>::value, + int> = 0> + constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare(x, y) ? phmap::weak_ordering::less + : compare(y, x) ? phmap::weak_ordering::greater + : phmap::weak_ordering::equivalent; + } + + } // namespace compare_internal +} + + +namespace phmap { + +namespace priv { + + // A helper class that indicates if the Compare parameter is a key-compare-to + // comparator. + template <typename Compare, typename T> + using btree_is_key_compare_to = + std::is_convertible<phmap::invoke_result_t<Compare, const T &, const T &>, + phmap::weak_ordering>; + + struct StringBtreeDefaultLess { + using is_transparent = void; + + StringBtreeDefaultLess() = default; + + // Compatibility constructor. + StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT +#if PHMAP_HAVE_STD_STRING_VIEW + StringBtreeDefaultLess(std::less<std::string_view>) {} // NOLINT + StringBtreeDefaultLess(phmap::Less<std::string_view>) {} // NOLINT + + phmap::weak_ordering operator()(std::string_view lhs, + std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } +#else + phmap::weak_ordering operator()(std::string lhs, + std::string rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } +#endif + }; + + struct StringBtreeDefaultGreater { + using is_transparent = void; + + StringBtreeDefaultGreater() = default; + + StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT +#if PHMAP_HAVE_STD_STRING_VIEW + StringBtreeDefaultGreater(std::greater<std::string_view>) {} // NOLINT + + phmap::weak_ordering operator()(std::string_view lhs, + std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } +#else + phmap::weak_ordering operator()(std::string lhs, + std::string rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } +#endif + }; + + // A helper class to convert a boolean comparison into a three-way "compare-to" + // comparison that returns a negative value to indicate less-than, zero to + // indicate equality and a positive value to indicate greater-than. This helper + // class is specialized for less<std::string>, greater<std::string>, + // less<std::string_view>, and greater<std::string_view>. + // + // key_compare_to_adapter is provided so that btree users + // automatically get the more efficient compare-to code when using common + // google string types with common comparison functors. + // These string-like specializations also turn on heterogeneous lookup by + // default. + template <typename Compare> + struct key_compare_to_adapter { + using type = Compare; + }; + + template <> + struct key_compare_to_adapter<std::less<std::string>> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter<phmap::Less<std::string>> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter<std::greater<std::string>> { + using type = StringBtreeDefaultGreater; + }; + +#if PHMAP_HAVE_STD_STRING_VIEW + template <> + struct key_compare_to_adapter<std::less<std::string_view>> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter<phmap::Less<std::string_view>> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter<std::greater<std::string_view>> { + using type = StringBtreeDefaultGreater; + }; +#endif + + template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool Multi, typename SlotPolicy> + struct common_params { + // If Compare is a common comparator for a std::string-like type, then we adapt it + // to use heterogeneous lookup and to be a key-compare-to comparator. + using key_compare = typename key_compare_to_adapter<Compare>::type; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; + + using allocator_type = Alloc; + using key_type = Key; + using size_type = std::size_t ; + using difference_type = ptrdiff_t; + + // True if this is a multiset or multimap. + using is_multi_container = std::integral_constant<bool, Multi>; + + using slot_policy = SlotPolicy; + using slot_type = typename slot_policy::slot_type; + using value_type = typename slot_policy::value_type; + using init_type = typename slot_policy::mutable_value_type; + using pointer = value_type *; + using const_pointer = const value_type *; + using reference = value_type &; + using const_reference = const value_type &; + + enum { + kTargetNodeSize = TargetNodeSize, + + // Upper bound for the available space for values. This is largest for leaf + // nodes, which have overhead of at least a pointer + 4 bytes (for storing + // 3 field_types and an enum). + kNodeValueSpace = + TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), + }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + using node_count_type = + phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > + (std::numeric_limits<uint8_t>::max)()), + uint16_t, uint8_t>; // NOLINT + + // The following methods are necessary for passing this struct as PolicyTraits + // for node_handle and/or are used within btree. + static value_type &element(slot_type *slot) { + return slot_policy::element(slot); + } + static const value_type &element(const slot_type *slot) { + return slot_policy::element(slot); + } + template <class... Args> + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + slot_policy::construct(alloc, slot, std::forward<Args>(args)...); + } + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + slot_policy::construct(alloc, slot, other); + } + static void destroy(Alloc *alloc, slot_type *slot) { + slot_policy::destroy(alloc, slot); + } + static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { + construct(alloc, new_slot, old_slot); + destroy(alloc, old_slot); + } + static void swap(Alloc *alloc, slot_type *a, slot_type *b) { + slot_policy::swap(alloc, a, b); + } + static void move(Alloc *alloc, slot_type *src, slot_type *dest) { + slot_policy::move(alloc, src, dest); + } + static void move(Alloc *alloc, slot_type *first, slot_type *last, + slot_type *result) { + slot_policy::move(alloc, first, last, result); + } + }; + + // A parameters structure for holding the type parameters for a btree_map. + // Compare and Alloc should be nothrow copy-constructible. + template <typename Key, typename Data, typename Compare, typename Alloc, + int TargetNodeSize, bool Multi> + struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, + phmap::priv::map_slot_policy<Key, Data>> { + using super_type = typename map_params::common_params; + using mapped_type = Data; + // This type allows us to move keys when it is safe to do so. It is safe + // for maps in which value_type and mutable_value_type are layout compatible. + using slot_policy = typename super_type::slot_policy; + using slot_type = typename super_type::slot_type; + using value_type = typename super_type::value_type; + using init_type = typename super_type::init_type; + + using key_compare = typename super_type::key_compare; + // Inherit from key_compare for empty base class optimization. + struct value_compare : private key_compare { + value_compare() = default; + explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} + + template <typename T, typename U> + auto operator()(const T &left, const U &right) const + -> decltype(std::declval<key_compare>()(left.first, right.first)) { + return key_compare::operator()(left.first, right.first); + } + }; + using is_map_container = std::true_type; + + static const Key &key(const value_type &x) { return x.first; } + static const Key &key(const init_type &x) { return x.first; } + static const Key &key(const slot_type *x) { return slot_policy::key(x); } + static mapped_type &value(value_type *value) { return value->second; } + }; + + // This type implements the necessary functions from the + // btree::priv::slot_type interface. + template <typename Key> + struct set_slot_policy { + using slot_type = Key; + using value_type = Key; + using mutable_value_type = Key; + + static value_type &element(slot_type *slot) { return *slot; } + static const value_type &element(const slot_type *slot) { return *slot; } + + template <typename Alloc, class... Args> + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + phmap::allocator_traits<Alloc>::construct(*alloc, slot, + std::forward<Args>(args)...); + } + + template <typename Alloc> + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + phmap::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); + } + + template <typename Alloc> + static void destroy(Alloc *alloc, slot_type *slot) { + phmap::allocator_traits<Alloc>::destroy(*alloc, slot); + } + + template <typename Alloc> + static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { + using std::swap; + swap(*a, *b); + } + + template <typename Alloc> + static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { + *dest = std::move(*src); + } + + template <typename Alloc> + static void move(Alloc *alloc, slot_type *first, slot_type *last, + slot_type *result) { + for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) + move(alloc, src, dest); + } + }; + + // A parameters structure for holding the type parameters for a btree_set. + // Compare and Alloc should be nothrow copy-constructible. + template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool Multi> + struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, + set_slot_policy<Key>> { + using value_type = Key; + using slot_type = typename set_params::common_params::slot_type; + using value_compare = typename set_params::common_params::key_compare; + using is_map_container = std::false_type; + + static const Key &key(const value_type &x) { return x; } + static const Key &key(const slot_type *x) { return *x; } + }; + + // An adapter class that converts a lower-bound compare into an upper-bound + // compare. Note: there is no need to make a version of this adapter specialized + // for key-compare-to functors because the upper-bound (the first value greater + // than the input) is never an exact match. + template <typename Compare> + struct upper_bound_adapter { + explicit upper_bound_adapter(const Compare &c) : comp(c) {} + template <typename K, typename LK> + bool operator()(const K &a, const LK &b) const { + // Returns true when a is not greater than b. + return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); + } + + private: + Compare comp; + }; + + enum class MatchKind : uint8_t { kEq, kNe }; + + template <typename V, bool IsCompareTo> + struct SearchResult { + V value; + MatchKind match; + + static constexpr bool HasMatch() { return true; } + bool IsEq() const { return match == MatchKind::kEq; } + }; + + // When we don't use CompareTo, `match` is not present. + // This ensures that callers can't use it accidentally when it provides no + // useful information. + template <typename V> + struct SearchResult<V, false> { + V value; + + static constexpr bool HasMatch() { return false; } + static constexpr bool IsEq() { return false; } + }; + + // A node in the btree holding. The same node type is used for both internal + // and leaf nodes in the btree, though the nodes are allocated in such a way + // that the children array is only valid in internal nodes. + template <typename Params> + class btree_node { + using is_key_compare_to = typename Params::is_key_compare_to; + using is_multi_container = typename Params::is_multi_container; + using field_type = typename Params::node_count_type; + using allocator_type = typename Params::allocator_type; + using slot_type = typename Params::slot_type; + + public: + using params_type = Params; + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using key_compare = typename Params::key_compare; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + + // Btree decides whether to use linear node search as follows: + // - If the key is arithmetic and the comparator is std::less or + // std::greater, choose linear. + // - Otherwise, choose binary. + // TODO(ezb): Might make sense to add condition(s) based on node-size. + using use_linear_search = std::integral_constant< + bool, + std::is_arithmetic<key_type>::value && + (std::is_same<phmap::Less<key_type>, key_compare>::value || + std::is_same<std::less<key_type>, key_compare>::value || + std::is_same<std::greater<key_type>, key_compare>::value)>; + + + ~btree_node() = default; + btree_node(btree_node const &) = delete; + btree_node &operator=(btree_node const &) = delete; + + // Public for EmptyNodeType. + constexpr static size_type Alignment() { + static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), + "Alignment of all nodes must be equal."); + return (size_type)InternalLayout().Alignment(); + } + + protected: + btree_node() = default; + + private: + using layout_type = phmap::priv::Layout<btree_node *, field_type, + slot_type, btree_node *>; + constexpr static size_type SizeWithNValues(size_type n) { + return (size_type)layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)n, + /*children*/ 0) + .AllocSize(); + } + // A lower bound for the overhead of fields other than values in a leaf node. + constexpr static size_type MinimumOverhead() { + return (size_type)(SizeWithNValues(1) - sizeof(value_type)); + } + + // Compute how many values we can fit onto a leaf node taking into account + // padding. + constexpr static size_type NodeTargetValues(const int begin, const int end) { + return begin == end ? begin + : SizeWithNValues((begin + end) / 2 + 1) > + params_type::kTargetNodeSize + ? NodeTargetValues(begin, (begin + end) / 2) + : NodeTargetValues((begin + end) / 2 + 1, end); + } + + enum { + kTargetNodeSize = params_type::kTargetNodeSize, + kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), + + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + // The node is internal (i.e. is not a leaf node) if and only if `max_count` + // has this value. + kInternalNodeMaxCount = 0, + }; + + // Leaves can have less than kNodeValues values. + constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)max_values, + /*children*/ 0); + } + constexpr static layout_type InternalLayout() { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ kNodeValues, + /*children*/ kNodeValues + 1); + } + constexpr static size_type LeafSize(const int max_values = kNodeValues) { + return (size_type)LeafLayout(max_values).AllocSize(); + } + constexpr static size_type InternalSize() { + return (size_type)InternalLayout().AllocSize(); + } + + // N is the index of the type in the Layout definition. + // ElementType<N> is the Nth type in the Layout definition. + template <size_type N> + inline typename layout_type::template ElementType<N> *GetField() { + // We assert that we don't read from values that aren't there. + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this)); + } + + template <size_type N> + inline const typename layout_type::template ElementType<N> *GetField() const { + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer<N>( + reinterpret_cast<const char *>(this)); + } + + void set_parent(btree_node *p) { *GetField<0>() = p; } + field_type &mutable_count() { return GetField<1>()[2]; } + slot_type *slot(size_type i) { return &GetField<2>()[i]; } + const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } + void set_position(field_type v) { GetField<1>()[0] = v; } + void set_start(field_type v) { GetField<1>()[1] = v; } + void set_count(field_type v) { GetField<1>()[2] = v; } + void set_max_count(field_type v) { GetField<1>()[3] = v; } + + public: + // Whether this is a leaf node or not. This value doesn't change after the + // node is created. + bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } + + // Getter for the position of this node in its parent. + field_type position() const { return GetField<1>()[0]; } + + // Getter for the offset of the first value in the `values` array. + field_type start() const { return GetField<1>()[1]; } + + // Getters for the number of values stored in this node. + field_type count() const { return GetField<1>()[2]; } + field_type max_count() const { + // Internal nodes have max_count==kInternalNodeMaxCount. + // Leaf nodes have max_count in [1, kNodeValues]. + const field_type max_cnt = GetField<1>()[3]; + return max_cnt == field_type{kInternalNodeMaxCount} + ? field_type{kNodeValues} + : max_cnt; + } + + // Getter for the parent of this node. + btree_node *parent() const { return *GetField<0>(); } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + assert(parent()->is_root()); + set_parent(parent()->parent()); + } + + // Getters for the key/value at position i in the node. + const key_type &key(size_type i) const { return params_type::key(slot(i)); } + reference value(size_type i) { return params_type::element(slot(i)); } + const_reference value(size_type i) const { return params_type::element(slot(i)); } + + // Getters/setter for the child at position i in the node. + btree_node *child(size_type i) const { return GetField<3>()[i]; } + btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } + void clear_child(size_type i) { + phmap::priv::SanitizerPoisonObject(&mutable_child(i)); + } + void set_child(size_type i, btree_node *c) { + phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); + mutable_child(i) = c; + c->set_position((field_type)i); + } + void init_child(int i, btree_node *c) { + set_child(i, c); + c->set_parent(this); + } + + // Returns the position of the first value whose key is not less than k. + template <typename K> + SearchResult<int, is_key_compare_to::value> lower_bound( + const K &k, const key_compare &comp) const { + return use_linear_search::value ? linear_search(k, comp) + : binary_search(k, comp); + } + // Returns the position of the first value whose key is greater than k. + template <typename K> + int upper_bound(const K &k, const key_compare &comp) const { + auto upper_compare = upper_bound_adapter<key_compare>(comp); + return use_linear_search::value ? linear_search(k, upper_compare).value + : binary_search(k, upper_compare).value; + } + + template <typename K, typename Compare> + SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> + linear_search(const K &k, const Compare &comp) const { + return linear_search_impl(k, 0, count(), comp, + btree_is_key_compare_to<Compare, key_type>()); + } + + template <typename K, typename Compare> + SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> + binary_search(const K &k, const Compare &comp) const { + return binary_search_impl(k, 0, count(), comp, + btree_is_key_compare_to<Compare, key_type>()); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template <typename K, typename Compare> + SearchResult<int, false> linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s < e) { + if (!comp(key(s), k)) { + break; + } + ++s; + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template <typename K, typename Compare> + SearchResult<int, true> linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::true_type /* IsCompareTo */) const { + while (s < e) { + const phmap::weak_ordering c = comp(key(s), k); + if (c == 0) { + return {s, MatchKind::kEq}; + } else if (c > 0) { + break; + } + ++s; + } + return {s, MatchKind::kNe}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template <typename K, typename Compare> + SearchResult<int, false> binary_search_impl( + const K &k, int s, int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s != e) { + const int mid = (s + e) >> 1; + if (comp(key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template <typename K, typename CompareTo> + SearchResult<int, true> binary_search_impl( + const K &k, int s, int e, const CompareTo &comp, + std::true_type /* IsCompareTo */) const { + if (is_multi_container::value) { + MatchKind exact_match = MatchKind::kNe; + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else { + e = mid; + if (c == 0) { + // Need to return the first value whose key is not less than k, + // which requires continuing the binary search if this is a + // multi-container. + exact_match = MatchKind::kEq; + } + } + } + return {s, exact_match}; + } else { // Not a multi-container. + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + return {mid, MatchKind::kEq}; + } + } + return {s, MatchKind::kNe}; + } + } + + // Emplaces a value at position i, shifting all existing values and + // children at positions >= i to the right by 1. + template <typename... Args> + void emplace_value(size_type i, allocator_type *alloc, Args &&... args); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i, allocator_type *alloc); + + // Removes the values at positions [i, i + to_erase), shifting all values + // after that range to the left by to_erase. Does not change children at all. + void remove_values_ignore_children(int i, size_type to_erase, + allocator_type *alloc); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(int to_move, btree_node *right, + allocator_type *alloc); + void rebalance_left_to_right(int to_move, btree_node *right, + allocator_type *alloc); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(int insert_position, btree_node *dest, allocator_type *alloc); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node *sibling, allocator_type *alloc); + + // Swap the contents of "this" and "src". + void swap(btree_node *src, allocator_type *alloc); + + // Node allocation/deletion routines. + static btree_node *init_leaf(btree_node *n, btree_node *parent, + int max_cnt) { + n->set_parent(parent); + n->set_position(0); + n->set_start(0); + n->set_count(0); + n->set_max_count((field_type)max_cnt); + phmap::priv::SanitizerPoisonMemoryRegion( + n->slot(0), max_cnt * sizeof(slot_type)); + return n; + } + static btree_node *init_internal(btree_node *n, btree_node *parent) { + init_leaf(n, parent, kNodeValues); + // Set `max_count` to a sentinel value to indicate that this node is + // internal. + n->set_max_count(kInternalNodeMaxCount); + phmap::priv::SanitizerPoisonMemoryRegion( + &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); + return n; + } + void destroy(allocator_type *alloc) { + for (int i = 0; i < count(); ++i) { + value_destroy(i, alloc); + } + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { + return use_linear_search::value; + } + + private: + template <typename... Args> + void value_init(const size_type i, allocator_type *alloc, Args &&... args) { + phmap::priv::SanitizerUnpoisonObject(slot(i)); + params_type::construct(alloc, slot(i), std::forward<Args>(args)...); + } + void value_destroy(const size_type i, allocator_type *alloc) { + params_type::destroy(alloc, slot(i)); + phmap::priv::SanitizerPoisonObject(slot(i)); + } + + // Move n values starting at value i in this node into the values starting at + // value j in node x. + void uninitialized_move_n(const size_type n, const size_type i, + const size_type j, btree_node *x, + allocator_type *alloc) { + phmap::priv::SanitizerUnpoisonMemoryRegion( + x->slot(j), n * sizeof(slot_type)); + for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); + src != end; ++src, ++dest) { + params_type::construct(alloc, dest, src); + } + } + + // Destroys a range of n values, starting at index i. + void value_destroy_n(const size_type i, const size_type n, + allocator_type *alloc) { + for (int j = 0; j < n; ++j) { + value_destroy(i + j, alloc); + } + } + + template <typename P> + friend class btree; + template <typename N, typename R, typename P> + friend struct btree_iterator; + friend class BtreeNodePeer; + }; + + template <typename Node, typename Reference, typename Pointer> + struct btree_iterator { + private: + using key_type = typename Node::key_type; + using size_type = typename Node::size_type; + using params_type = typename Node::params_type; + + using node_type = Node; + using normal_node = typename std::remove_const<Node>::type; + using const_node = const Node; + using normal_pointer = typename params_type::pointer; + using normal_reference = typename params_type::reference; + using const_pointer = typename params_type::const_pointer; + using const_reference = typename params_type::const_reference; + using slot_type = typename params_type::slot_type; + + using iterator = + btree_iterator<normal_node, normal_reference, normal_pointer>; + using const_iterator = + btree_iterator<const_node, const_reference, const_pointer>; + + public: + // These aliases are public for std::iterator_traits. + using difference_type = typename Node::difference_type; + using value_type = typename params_type::value_type; + using pointer = Pointer; + using reference = Reference; + using iterator_category = std::bidirectional_iterator_tag; + + btree_iterator() : node(nullptr), position(-1) {} + btree_iterator(Node *n, int p) : node(n), position(p) {} + + // NOTE: this SFINAE allows for implicit conversions from iterator to + // const_iterator, but it specifically avoids defining copy constructors so + // that btree_iterator can be trivially copyable. This is for performance and + // binary size reasons. + template <typename N, typename R, typename P, + phmap::enable_if_t< + std::is_same<btree_iterator<N, R, P>, iterator>::value && + std::is_same<btree_iterator, const_iterator>::value, + int> = 0> + btree_iterator(const btree_iterator<N, R, P> &x) // NOLINT + : node(x.node), position(x.position) {} + + private: + // This SFINAE allows explicit conversions from const_iterator to + // iterator, but also avoids defining a copy constructor. + // NOTE: the const_cast is safe because this constructor is only called by + // non-const methods and the container owns the nodes. + template <typename N, typename R, typename P, + phmap::enable_if_t< + std::is_same<btree_iterator<N, R, P>, const_iterator>::value && + std::is_same<btree_iterator, iterator>::value, + int> = 0> + explicit btree_iterator(const btree_iterator<N, R, P> &x) + : node(const_cast<node_type *>(x.node)), position(x.position) {} + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->count()) { + return; + } + increment_slow(); + } + void increment_slow(); + + void decrement() { + if (node->leaf() && --position >= 0) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + public: + bool operator==(const const_iterator &x) const { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator &x) const { + return node != x.node || position != x.position; + } + + // Accessors for the key/value the iterator is pointing at. + reference operator*() const { + return node->value(position); + } + pointer operator->() const { + return &node->value(position); + } + + btree_iterator& operator++() { + increment(); + return *this; + } + btree_iterator& operator--() { + decrement(); + return *this; + } + btree_iterator operator++(int) { + btree_iterator tmp = *this; + ++*this; + return tmp; + } + btree_iterator operator--(int) { + btree_iterator tmp = *this; + --*this; + return tmp; + } + + private: + template <typename Params> + friend class btree; + template <typename Tree> + friend class btree_container; + template <typename Tree> + friend class btree_set_container; + template <typename Tree> + friend class btree_map_container; + template <typename Tree> + friend class btree_multiset_container; + template <typename N, typename R, typename P> + friend struct btree_iterator; + template <typename TreeType, typename CheckerType> + friend class base_checker; + + const key_type &key() const { return node->key(position); } + slot_type *slot() { return node->slot(position); } + + // The node in the tree the iterator is pointing at. + Node *node; + // The position within the node of the tree the iterator is pointing at. + // TODO(ezb): make this a field_type + int position; + }; + + template <typename Params> + class btree { + using node_type = btree_node<Params>; + using is_key_compare_to = typename Params::is_key_compare_to; + + // We use a static empty node for the root/leftmost/rightmost of empty btrees + // in order to avoid branching in begin()/end(). + struct alignas(node_type::Alignment()) EmptyNodeType : node_type { + using field_type = typename node_type::field_type; + node_type *parent; + field_type position = 0; + field_type start = 0; + field_type count = 0; + // max_count must be != kInternalNodeMaxCount (so that this node is regarded + // as a leaf node). max_count() is never called when the tree is empty. + field_type max_count = node_type::kInternalNodeMaxCount + 1; + +#ifdef _MSC_VER + // MSVC has constexpr code generations bugs here. + EmptyNodeType() : parent(this) {} +#else + constexpr EmptyNodeType(node_type *p) : parent(p) {} +#endif + }; + + static node_type *EmptyNode() { +#ifdef _MSC_VER + static EmptyNodeType empty_node; + // This assert fails on some other construction methods. + assert(empty_node.parent == &empty_node); + return &empty_node; +#else + static constexpr EmptyNodeType empty_node( + const_cast<EmptyNodeType *>(&empty_node)); + return const_cast<EmptyNodeType *>(&empty_node); +#endif + } + + enum { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + }; + + struct node_stats { + using size_type = typename Params::size_type; + + node_stats(size_type l, size_type i) + : leaf_nodes(l), + internal_nodes(i) { + } + + node_stats& operator+=(const node_stats &x) { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; + } + + size_type leaf_nodes; + size_type internal_nodes; + }; + + public: + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + using key_compare = typename Params::key_compare; + using value_compare = typename Params::value_compare; + using allocator_type = typename Params::allocator_type; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using iterator = btree_iterator<node_type, reference, pointer>; + using const_iterator = typename iterator::const_iterator; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using node_handle_type = node_handle<Params, Params, allocator_type>; + + // Internal types made public for use by btree_container types. + using params_type = Params; + using slot_type = typename Params::slot_type; + + private: + // For use in copy_or_move_values_in_order. + const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } + value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } + + // Copies or moves (depending on the template parameter) the values in + // x into this btree in their order in x. This btree must be empty before this + // method is called. This method is used in copy construction, copy + // assignment, and move assignment. + template <typename Btree> + void copy_or_move_values_in_order(Btree *x); + + // Validates that various assumptions/requirements are true at compile time. + constexpr static bool static_assert_validation(); + + public: + btree(const key_compare &comp, const allocator_type &alloc); + + btree(const btree &x); + btree(btree &&x) noexcept + : root_(std::move(x.root_)), + rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), + size_(phmap::exchange(x.size_, 0)) { + x.mutable_root() = EmptyNode(); + } + + ~btree() { + // Put static_asserts in destructor to avoid triggering them before the type + // is complete. + static_assert(static_assert_validation(), "This call must be elided."); + clear(); + } + + // Assign the contents of x to *this. + btree &operator=(const btree &x); + btree &operator=(btree &&x) noexcept; + + iterator begin() { + return iterator(leftmost(), 0); + } + const_iterator begin() const { + return const_iterator(leftmost(), 0); + } + iterator end() { return iterator(rightmost_, rightmost_->count()); } + const_iterator end() const { + return const_iterator(rightmost_, rightmost_->count()); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + template <typename K> + iterator lower_bound(const K &key) { + return internal_end(internal_lower_bound(key)); + } + template <typename K> + const_iterator lower_bound(const K &key) const { + return internal_end(internal_lower_bound(key)); + } + + // Finds the first element whose key is greater than key. + template <typename K> + iterator upper_bound(const K &key) { + return internal_end(internal_upper_bound(key)); + } + template <typename K> + const_iterator upper_bound(const K &key) const { + return internal_end(internal_upper_bound(key)); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + template <typename K> + std::pair<iterator, iterator> equal_range(const K &key) { + return {lower_bound(key), upper_bound(key)}; + } + template <typename K> + std::pair<const_iterator, const_iterator> equal_range(const K &key) const { + return {lower_bound(key), upper_bound(key)}; + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template <typename... Args> + std::pair<iterator, bool> insert_unique(const key_type &key, Args &&... args); + + // Inserts with hint. Checks to see if the value should be placed immediately + // before `position` in the tree. If so, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique() were made. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template <typename... Args> + std::pair<iterator, bool> insert_hint_unique(iterator position, + const key_type &key, + Args &&... args); + + // Insert a range of values into the btree. + template <typename InputIterator> + void insert_iterator_unique(InputIterator b, InputIterator e); + + // Inserts a value into the btree. + template <typename ValueType> + iterator insert_multi(const key_type &key, ValueType &&v); + + // Inserts a value into the btree. + template <typename ValueType> + iterator insert_multi(ValueType &&v) { + return insert_multi(params_type::key(v), std::forward<ValueType>(v)); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + template <typename ValueType> + iterator insert_hint_multi(iterator position, ValueType &&v); + + // Insert a range of values into the btree. + template <typename InputIterator> + void insert_iterator_multi(InputIterator b, InputIterator e); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + // Requirement: does not read the value at `*iter`. + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased and an iterator pointing + // to the element after the last erased element. + std::pair<size_type, iterator> erase(iterator begin, iterator end); + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + template <typename K> + size_type erase_unique(const K &key); + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + template <typename K> + size_type erase_multi(const K &key); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + template <typename K> + iterator find(const K &key) { + return internal_end(internal_find(key)); + } + template <typename K> + const_iterator find(const K &key) const { + return internal_end(internal_find(key)); + } + + // Returns a count of the number of times the key appears in the btree. + template <typename K> + size_type count_unique(const K &key) const { + const iterator beg = internal_find(key); + if (beg.node == nullptr) { + // The key doesn't exist in the tree. + return 0; + } + return 1; + } + // Returns a count of the number of times the key appears in the btree. + template <typename K> + size_type count_multi(const K &key) const { + const auto range = equal_range(key); + return std::distance(range.first, range.second); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swap the contents of *this and x. + void swap(btree &x); + + const key_compare &key_comp() const noexcept { + return root_.template get<0>(); + } + template <typename K, typename LK> + bool compare_keys(const K &x, const LK &y) const { + return compare_internal::compare_result_as_less_than(key_comp()(x, y)); + } + + value_compare value_comp() const { return value_compare(key_comp()); } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. + size_type size() const { return size_; } + size_type max_size() const { return (std::numeric_limits<size_type>::max)(); } + bool empty() const { return size_ == 0; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (!empty()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type *n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { + return internal_stats(root()).leaf_nodes; + } + size_type internal_nodes() const { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + + node_type::LeafSize(root()->max_count()); + } else { + return sizeof(*this) + + stats.leaf_nodes * node_type::LeafSize() + + stats.internal_nodes * node_type::InternalSize(); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return node_type::LeafSize() / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + // Returns 0 for empty trees. + double fullness() const { + if (empty()) return 0.0; + return static_cast<double>(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + // Returns 0 for empty trees. + double overhead() const { + if (empty()) return 0.0; + return (bytes_used() - size() * sizeof(value_type)) / + static_cast<double>(size()); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { + return allocator(); + } + + private: + // Internal accessor routines. + node_type *root() { return root_.template get<2>(); } + const node_type *root() const { return root_.template get<2>(); } + node_type *&mutable_root() noexcept { return root_.template get<2>(); } + key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } + + // The leftmost node is stored as the parent of the root node. + node_type *leftmost() { return root()->parent(); } + const node_type *leftmost() const { return root()->parent(); } + + // Allocator routines. + allocator_type *mutable_allocator() noexcept { + return &root_.template get<1>(); + } + const allocator_type &allocator() const noexcept { + return root_.template get<1>(); + } + + // Allocates a correctly aligned node of at least size bytes using the + // allocator. + node_type *allocate(const size_type sz) { + return reinterpret_cast<node_type *>( + phmap::priv::Allocate<node_type::Alignment()>( + mutable_allocator(), (size_t)sz)); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type *parent) { + node_type *p = allocate(node_type::InternalSize()); + return node_type::init_internal(p, parent); + } + node_type* new_leaf_node(node_type *parent) { + node_type *p = allocate(node_type::LeafSize()); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type *new_leaf_root_node(const int max_count) { + node_type *p = allocate(node_type::LeafSize(max_count)); + return node_type::init_leaf(p, p, max_count); + } + + // Deletion helper routines. + void erase_same_node(iterator begin, iterator end); + iterator erase_from_leaf_node(iterator begin, size_type to_erase); + iterator rebalance_after_delete(iterator iter); + + // Deallocates a node of a certain size in bytes using the allocator. + void deallocate(const size_type sz, node_type *node) { + phmap::priv::Deallocate<node_type::Alignment()>( + mutable_allocator(), node, (size_t)sz); + } + + void delete_internal_node(node_type *node) { + node->destroy(mutable_allocator()); + deallocate(node_type::InternalSize(), node); + } + void delete_leaf_node(node_type *node) { + node->destroy(mutable_allocator()); + deallocate(node_type::LeafSize(node->max_count()), node); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator *iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type *left, node_type *right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator *iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { + return iter.node != nullptr ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const { + return iter.node != nullptr ? iter : end(); + } + + // Emplaces a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + template <typename... Args> + iterator internal_emplace(iterator iter, Args &&... args); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + // Requires: iter.node is non-null. + template <typename IterType> + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version uses a less-than comparator and is incapable of distinguishing when + // there is an exact match. The second version is for the key-compare-to + // specialization and distinguishes exact matches. The key-compare-to + // specialization allows the caller to avoid a subsequent comparison to + // determine if an exact match was made, which is important for keys with + // expensive comparison, such as strings. + template <typename K> + SearchResult<iterator, is_key_compare_to::value> internal_locate( + const K &key) const; + + template <typename K> + SearchResult<iterator, false> internal_locate_impl( + const K &key, std::false_type /* IsCompareTo */) const; + + template <typename K> + SearchResult<iterator, true> internal_locate_impl( + const K &key, std::true_type /* IsCompareTo */) const; + + // Internal routine which implements lower_bound(). + template <typename K> + iterator internal_lower_bound(const K &key) const; + + // Internal routine which implements upper_bound(). + template <typename K> + iterator internal_upper_bound(const K &key) const; + + // Internal routine which implements find(). + template <typename K> + iterator internal_find(const K &key) const; + + // Deletes a node and all of its children. + void internal_clear(node_type *node); + + // Verifies the tree structure of node. + int internal_verify(const node_type *node, + const key_type *lo, const key_type *hi) const; + + node_stats internal_stats(const node_type *node) const { + // The root can be a static empty node. + if (node == nullptr || (node == root() && empty())) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); + } + node_stats res(0, 1); + for (int i = 0; i <= node->count(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { + return node_type::testonly_uses_linear_node_search(); + } + + private: + // We use compressed tuple in order to save space because key_compare and + // allocator_type are usually empty. + phmap::priv::CompressedTuple<key_compare, allocator_type, + node_type *> + root_; + + // A pointer to the rightmost node. Note that the leftmost node is stored as + // the root's parent. + node_type *rightmost_; + + // Number of values. + size_type size_; + }; + + //// + // btree_node methods + template <typename P> + template <typename... Args> + inline void btree_node<P>::emplace_value(const size_type i, + allocator_type *alloc, + Args &&... args) { + assert(i <= count()); + // Shift old values to create space for new value and then construct it in + // place. + if (i < count()) { + value_init(count(), alloc, slot(count() - 1)); + for (size_type j = count() - 1; j > i; --j) + params_type::move(alloc, slot(j - 1), slot(j)); + value_destroy(i, alloc); + } + value_init(i, alloc, std::forward<Args>(args)...); + set_count((field_type)(count() + 1)); + + if (!leaf() && count() > i + 1) { + for (int j = count(); j > i + 1; --j) { + set_child(j, child(j - 1)); + } + clear_child(i + 1); + } + } + + template <typename P> + inline void btree_node<P>::remove_value(const int i, allocator_type *alloc) { + if (!leaf() && count() > i + 1) { + assert(child(i + 1)->count() == 0); + for (size_type j = i + 1; j < count(); ++j) { + set_child(j, child(j + 1)); + } + clear_child(count()); + } + + remove_values_ignore_children(i, /*to_erase=*/1, alloc); + } + + template <typename P> + inline void btree_node<P>::remove_values_ignore_children( + int i, size_type to_erase, allocator_type *alloc) { + params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); + value_destroy_n(count() - to_erase, to_erase, alloc); + set_count((field_type)(count() - to_erase)); + } + + template <typename P> + void btree_node<P>::rebalance_right_to_left(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(right->count() >= count()); + assert(to_move >= 1); + assert(to_move <= right->count()); + + // 1) Move the delimiting value in the parent to the left node. + value_init(count(), alloc, parent()->slot(position())); + + // 2) Move the (to_move - 1) values from the right node to the left node. + right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); + + // 3) Move the new delimiting value to the parent from the right node. + params_type::move(alloc, right->slot(to_move - 1), + parent()->slot(position())); + + // 4) Shift the values in the right node to their correct position. + params_type::move(alloc, right->slot(to_move), right->slot(right->count()), + right->slot(0)); + + // 5) Destroy the now-empty to_move entries in the right node. + right->value_destroy_n(right->count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + init_child(count() + i + 1, right->child(i)); + } + for (int i = 0; i <= right->count() - to_move; ++i) { + assert(i + to_move <= right->max_count()); + right->init_child(i, right->child(i + to_move)); + right->clear_child(i + to_move); + } + } + + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() + to_move)); + right->set_count((field_type)(right->count() - to_move)); + } + + template <typename P> + void btree_node<P>::rebalance_left_to_right(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(count() >= right->count()); + assert(to_move >= 1); + assert(to_move <= count()); + + // Values in the right node are shifted to the right to make room for the + // new to_move values. Then, the delimiting value in the parent and the + // other (to_move - 1) values in the left node are moved into the right node. + // Lastly, a new delimiting value is moved from the left node into the + // parent, and the remaining empty left node entries are destroyed. + + if (right->count() >= to_move) { + // The original location of the right->count() values are sufficient to hold + // the new to_move entries from the parent and left node. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(to_move, right->count() - to_move, + right->count(), right, alloc); + for (slot_type *src = right->slot(right->count() - to_move - 1), + *dest = right->slot(right->count() - 1), + *end = right->slot(0); + src >= end; --src, --dest) { + params_type::move(alloc, src, dest); + } + + // 2) Move the delimiting value in the parent to the right node. + params_type::move(alloc, parent()->slot(position()), + right->slot(to_move - 1)); + + // 3) Move the (to_move - 1) values from the left node to the right node. + params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), + right->slot(0)); + } else { + // The right node does not have enough initialized space to hold the new + // to_move entries, so part of them will move to uninitialized space. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); + + // 2) Move the delimiting value in the parent to the right node. + right->value_init(to_move - 1, alloc, parent()->slot(position())); + + // 3) Move the (to_move - 1) values from the left node to the right node. + const size_type uninitialized_remaining = to_move - right->count() - 1; + uninitialized_move_n(uninitialized_remaining, + count() - uninitialized_remaining, right->count(), + right, alloc); + params_type::move(alloc, slot(count() - (to_move - 1)), + slot(count() - uninitialized_remaining), right->slot(0)); + } + + // 4) Move the new delimiting value to the parent from the left node. + params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); + + // 5) Destroy the now-empty to_move entries in the left node. + value_destroy_n(count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = right->count(); i >= 0; --i) { + right->init_child(i + to_move, right->child(i)); + right->clear_child(i); + } + for (int i = 1; i <= to_move; ++i) { + right->init_child(i - 1, child(count() - to_move + i)); + clear_child(count() - to_move + i); + } + } + + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() - to_move)); + right->set_count((field_type)(right->count() + to_move)); + } + + template <typename P> + void btree_node<P>::split(const int insert_position, btree_node *dest, + allocator_type *alloc) { + assert(dest->count() == 0); + assert(max_count() == kNodeValues); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) { + dest->set_count((field_type)(count() - 1)); + } else if (insert_position == kNodeValues) { + dest->set_count(0); + } else { + dest->set_count((field_type)(count() / 2)); + } + set_count((field_type)(count() - dest->count())); + assert(count() >= 1); + + // Move values from the left sibling to the right sibling. + uninitialized_move_n(dest->count(), count(), 0, dest, alloc); + + // Destroy the now-empty entries in the left node. + value_destroy_n(count(), dest->count(), alloc); + + // The split key is the largest value in the left sibling. + set_count((field_type)(count() - 1)); + parent()->emplace_value(position(), alloc, slot(count())); + value_destroy(count(), alloc); + parent()->init_child(position() + 1, dest); + + if (!leaf()) { + for (int i = 0; i <= dest->count(); ++i) { + assert(child(count() + i + 1) != nullptr); + dest->init_child(i, child(count() + i + 1)); + clear_child(count() + i + 1); + } + } + } + + template <typename P> + void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + + // Move the delimiting value to the left node. + value_init(count(), alloc, parent()->slot(position())); + + // Move the values from the right to the left node. + src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); + + // Destroy the now-empty entries in the right node. + src->value_destroy_n(0, src->count(), alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { + init_child(count() + i + 1, src->child(i)); + src->clear_child(i); + } + } + + // Fixup the counts on the src and dest nodes. + set_count((field_type)(1 + count() + src->count())); + src->set_count(0); + + // Remove the value on the parent node. + parent()->remove_value(position(), alloc); + } + + template <typename P> + void btree_node<P>::swap(btree_node *x, allocator_type *alloc) { + using std::swap; + assert(leaf() == x->leaf()); + + // Determine which is the smaller/larger node. + btree_node *smaller = this, *larger = x; + if (smaller->count() > larger->count()) { + swap(smaller, larger); + } + + // Swap the values. + for (slot_type *a = smaller->slot(0), *b = larger->slot(0), + *end = a + smaller->count(); + a != end; ++a, ++b) { + params_type::swap(alloc, a, b); + } + + // Move values that can't be swapped. + const size_type to_move = larger->count() - smaller->count(); + larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), + smaller, alloc); + larger->value_destroy_n(smaller->count(), to_move, alloc); + + if (!leaf()) { + // Swap the child pointers. + std::swap_ranges(&smaller->mutable_child(0), + &smaller->mutable_child(smaller->count() + 1), + &larger->mutable_child(0)); + // Update swapped children's parent pointers. + int i = 0; + for (; i <= smaller->count(); ++i) { + smaller->child(i)->set_parent(smaller); + larger->child(i)->set_parent(larger); + } + // Move the child pointers that couldn't be swapped. + for (; i <= larger->count(); ++i) { + smaller->init_child(i, larger->child(i)); + larger->clear_child(i); + } + } + + // Swap the counts. + swap(mutable_count(), x->mutable_count()); + } + + //// + // btree_iterator methods + template <typename N, typename R, typename P> + void btree_iterator<N, R, P>::increment_slow() { + if (node->leaf()) { + assert(position >= node->count()); + btree_iterator save(*this); + while (position == node->count() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); + } + if (position == node->count()) { + *this = save; + } + } else { + assert(position < node->count()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->child(0); + } + position = 0; + } + } + + template <typename N, typename R, typename P> + void btree_iterator<N, R, P>::decrement_slow() { + if (node->leaf()) { + assert(position <= -1); + btree_iterator save(*this); + while (position < 0 && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); + } + if (position < 0) { + *this = save; + } + } else { + assert(position >= 0); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->count()); + } + position = node->count() - 1; + } + } + + //// + // btree methods + template <typename P> + template <typename Btree> + void btree<P>::copy_or_move_values_in_order(Btree *x) { + static_assert(std::is_same<btree, Btree>::value || + std::is_same<const btree, Btree>::value, + "Btree type must be same or const."); + assert(empty()); + + // We can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + auto iter = x->begin(); + if (iter == x->end()) return; + insert_multi(maybe_move_from_iterator(iter)); + ++iter; + for (; iter != x->end(); ++iter) { + // If the btree is not empty, we can just insert the new value at the end + // of the tree. + internal_emplace(end(), maybe_move_from_iterator(iter)); + } + } + + template <typename P> + constexpr bool btree<P>::static_assert_validation() { + static_assert(std::is_nothrow_copy_constructible<key_compare>::value, + "Key comparison must be nothrow copy constructible"); + static_assert(std::is_nothrow_copy_constructible<allocator_type>::value, + "Allocator must be nothrow copy constructible"); + static_assert(type_traits_internal::is_trivially_copyable<iterator>::value, + "iterator not trivially copyable."); + + // Note: We assert that kTargetValues, which is computed from + // Params::kTargetNodeSize, must fit the node_type::field_type. + static_assert( + kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), + "target node size too large"); + + // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. + using compare_result_type = + phmap::invoke_result_t<key_compare, key_type, key_type>; + static_assert( + std::is_same<compare_result_type, bool>::value || + std::is_convertible<compare_result_type, phmap::weak_ordering>::value, + "key comparison function must return phmap::{weak,strong}_ordering or " + "bool."); + + // Test the assumption made in setting kNodeValueSpace. + static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, + "node space assumption incorrect"); + + return true; + } + + template <typename P> + btree<P>::btree(const key_compare &comp, const allocator_type &alloc) + : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} + + template <typename P> + btree<P>::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { + copy_or_move_values_in_order(&x); + } + + template <typename P> + template <typename... Args> + auto btree<P>::insert_unique(const key_type &key, Args &&... args) + -> std::pair<iterator, bool> { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + auto res = internal_locate(key); + iterator &iter = res.value; + + if (res.HasMatch()) { + if (res.IsEq()) { + // The key already exists in the tree, do nothing. + return {iter, false}; + } + } else { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return {last, false}; + } + } + return {internal_emplace(iter, std::forward<Args>(args)...), true}; + } + + template <typename P> + template <typename... Args> + inline auto btree<P>::insert_hint_unique(iterator position, const key_type &key, + Args &&... args) + -> std::pair<iterator, bool> { + if (!empty()) { + if (position == end() || compare_keys(key, position.key())) { + iterator prev = position; + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return {internal_emplace(position, std::forward<Args>(args)...), true}; + } + } else if (compare_keys(position.key(), key)) { + ++position; + if (position == end() || compare_keys(key, position.key())) { + // {original `position`}.key() < key < {current `position`}.key() + return {internal_emplace(position, std::forward<Args>(args)...), true}; + } + } else { + // position.key() == key + return {position, false}; + } + } + return insert_unique(key, std::forward<Args>(args)...); + } + + template <typename P> + template <typename InputIterator> + void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_unique(end(), params_type::key(*b), *b); + } + } + + template <typename P> + template <typename ValueType> + auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key); + if (iter.node == nullptr) { + iter = end(); + } + return internal_emplace(iter, std::forward<ValueType>(v)); + } + + template <typename P> + template <typename ValueType> + auto btree<P>::insert_hint_multi(iterator position, ValueType &&v) -> iterator { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + iterator prev = position; + if (position == begin() || !compare_keys(key, (--prev).key())) { + // prev.key() <= key <= position.key() + return internal_emplace(position, std::forward<ValueType>(v)); + } + } else { + iterator next = position; + ++next; + if (next == end() || !compare_keys(next.key(), key)) { + // position.key() < key <= next.key() + return internal_emplace(next, std::forward<ValueType>(v)); + } + } + } + return insert_multi(std::forward<ValueType>(v)); + } + + template <typename P> + template <typename InputIterator> + void btree<P>::insert_iterator_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_multi(end(), *b); + } + } + + template <typename P> + auto btree<P>::operator=(const btree &x) -> btree & { + if (this != &x) { + clear(); + + *mutable_key_comp() = x.key_comp(); + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + *mutable_allocator() = x.allocator(); + } + + copy_or_move_values_in_order(&x); + } + return *this; + } + + template <typename P> + auto btree<P>::operator=(btree &&x) noexcept -> btree & { + if (this != &x) { + clear(); + + using std::swap; + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } else { + if (allocator() == x.allocator()) { + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } else { + // We aren't allowed to propagate the allocator and the allocator is + // different so we can't take over its memory. We must move each element + // individually. We need both `x` and `this` to have `x`s key comparator + // while moving the values so we can't swap the key comparators. + *mutable_key_comp() = x.key_comp(); + copy_or_move_values_in_order(&x); + } + } + } + return *this; + } + + template <typename P> + auto btree<P>::erase(iterator iter) -> iterator { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. First, move the largest value + // from our left child here, then delete that position (in remove_value() + // below). We can get to the largest value from our left child by + // decrementing iter. + iterator internal_iter(iter); + --iter; + assert(iter.node->leaf()); + params_type::move(mutable_allocator(), iter.node->slot(iter.position), + internal_iter.node->slot(internal_iter.position)); + internal_delete = true; + } + + // Delete the key from the leaf. + iter.node->remove_value(iter.position, mutable_allocator()); + --size_; + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + iterator res = rebalance_after_delete(iter); + + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; + } + + template <typename P> + auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + bool first_iteration = true; + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; + } + if (iter.node->count() >= kMinNodeValues) { + break; + } + bool merged = try_merge_or_rebalance(&iter); + // On the first iteration, we should update `res` with `iter` because `res` + // may have been invalidated. + if (first_iteration) { + res = iter; + first_iteration = false; + } + if (!merged) { + break; + } + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { + res.position = res.node->count() - 1; + ++res; + } + + return res; + } + + template <typename P> + auto btree<P>::erase(iterator _begin, iterator _end) + -> std::pair<size_type, iterator> { + difference_type count = std::distance(_begin, _end); + assert(count >= 0); + + if (count == 0) { + return {0, _begin}; + } + + if (count == (difference_type)size_) { + clear(); + return {count, this->end()}; + } + + if (_begin.node == _end.node) { + erase_same_node(_begin, _end); + size_ -= count; + return {count, rebalance_after_delete(_begin)}; + } + + const size_type target_size = size_ - count; + while (size_ > target_size) { + if (_begin.node->leaf()) { + const size_type remaining_to_erase = size_ - target_size; + const size_type remaining_in_node = _begin.node->count() - _begin.position; + _begin = erase_from_leaf_node( + _begin, (std::min)(remaining_to_erase, remaining_in_node)); + } else { + _begin = erase(_begin); + } + } + return {count, _begin}; + } + + template <typename P> + void btree<P>::erase_same_node(iterator _begin, iterator _end) { + assert(_begin.node == _end.node); + assert(_end.position > _begin.position); + + node_type *node = _begin.node; + size_type to_erase = _end.position - _begin.position; + if (!node->leaf()) { + // Delete all children between _begin and _end. + for (size_type i = 0; i < to_erase; ++i) { + internal_clear(node->child(_begin.position + i + 1)); + } + // Rotate children after _end into new positions. + for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { + node->set_child(i - to_erase, node->child(i)); + node->clear_child(i); + } + } + node->remove_values_ignore_children(_begin.position, to_erase, + mutable_allocator()); + + // Do not need to update rightmost_, because + // * either _end == this->end(), and therefore node == rightmost_, and still + // exists + // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since + // it wasn't covered in [_begin, _end) + } + + template <typename P> + auto btree<P>::erase_from_leaf_node(iterator _begin, size_type to_erase) + -> iterator { + node_type *node = _begin.node; + assert(node->leaf()); + assert(node->count() > _begin.position); + assert(_begin.position + to_erase <= node->count()); + + node->remove_values_ignore_children(_begin.position, to_erase, + mutable_allocator()); + + size_ -= to_erase; + + return rebalance_after_delete(_begin); + } + + template <typename P> + template <typename K> + auto btree<P>::erase_unique(const K &key) -> size_type { + const iterator iter = internal_find(key); + if (iter.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + erase(iter); + return 1; + } + + template <typename P> + template <typename K> + auto btree<P>::erase_multi(const K &key) -> size_type { + const iterator _begin = internal_lower_bound(key); + if (_begin.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + // Delete all of the keys between _begin and upper_bound(key). + const iterator _end = internal_end(internal_upper_bound(key)); + return erase(_begin, _end).first; + } + + template <typename P> + void btree<P>::clear() { + if (!empty()) { + internal_clear(root()); + } + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + size_ = 0; + } + + template <typename P> + void btree<P>::swap(btree &x) { + using std::swap; + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_swap::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + } else { + // It's undefined behavior if the allocators are unequal here. + assert(allocator() == x.allocator()); + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); + } + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } + + template <typename P> + void btree<P>::verify() const { + assert(root() != nullptr); + assert(leftmost() != nullptr); + assert(rightmost_ != nullptr); + assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost_ == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost_->leaf()); + } + + template <typename P> + void btree<P>::rebalance_or_split(iterator *iter) { + node_type *&node = iter->node; + int &insert_position = iter->position; + assert(node->count() == node->max_count()); + assert(kNodeValues == node->max_count()); + + // First try to make room on the node by rebalancing. + node_type *parent = node->parent(); + if (node != root()) { + if (node->position() > 0) { + // Try rebalancing with our left sibling. + node_type *left = parent->child(node->position() - 1); + assert(left->max_count() == kNodeValues); + if (left->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (kNodeValues - left->count()) / + (1 + (insert_position < kNodeValues)); + to_move = (std::max)(1, to_move); + + if (((insert_position - to_move) >= 0) || + ((left->count() + to_move) < kNodeValues)) { + left->rebalance_right_to_left(to_move, node, mutable_allocator()); + + assert(node->max_count() - node->count() == to_move); + insert_position = insert_position - to_move; + if (insert_position < 0) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + if (node->position() < parent->count()) { + // Try rebalancing with our right sibling. + node_type *right = parent->child(node->position() + 1); + assert(right->max_count() == kNodeValues); + if (right->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the _beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = + (kNodeValues - right->count()) / (1 + (insert_position > 0)); + to_move = (std::max)(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < kNodeValues)) { + node->rebalance_left_to_right(to_move, right, mutable_allocator()); + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = right; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + assert(parent->max_count() == kNodeValues); + if (parent->count() == kNodeValues) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + // Create a new root node and set the current root node as the child of the + // new root. + parent = new_internal_node(parent); + parent->init_child(0, root()); + mutable_root() = parent; + // If the former root was a leaf node, then it's now the rightmost node. + assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); + } + + // Split the node. + node_type *split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + if (rightmost_ == node) rightmost_ = split_node; + } else { + split_node = new_internal_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + } + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } + } + + template <typename P> + void btree<P>::merge_nodes(node_type *left, node_type *right) { + left->merge(right, mutable_allocator()); + if (right->leaf()) { + if (rightmost_ == right) rightmost_ = left; + delete_leaf_node(right); + } else { + delete_internal_node(right); + } + } + + template <typename P> + bool btree<P>::try_merge_or_rebalance(iterator *iter) { + node_type *parent = iter->node->parent(); + if (iter->node->position() > 0) { + // Try merging with our left sibling. + node_type *left = parent->child(iter->node->position() - 1); + assert(left->max_count() == kNodeValues); + if ((1 + left->count() + iter->node->count()) <= kNodeValues) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->count()) { + // Try merging with our right sibling. + node_type *right = parent->child(iter->node->position() + 1); + assert(right->max_count() == kNodeValues); + if ((1 + iter->node->count() + right->count()) <= kNodeValues) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position > 0))) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); + return false; + } + } + if (iter->node->position() > 0) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type *left = parent->child(iter->node->position() - 1); + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position < iter->node->count()))) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, left->count() - 1); + left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); + iter->position += to_move; + return false; + } + } + return false; + } + + template <typename P> + void btree<P>::try_shrink() { + if (root()->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { + assert(size() == 0); + delete_leaf_node(root()); + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + } else { + node_type *child = root()->child(0); + child->make_root(); + delete_internal_node(root()); + mutable_root() = child; + } + } + + template <typename P> + template <typename IterType> + inline IterType btree<P>::internal_last(IterType iter) { + assert(iter.node != nullptr); + while (iter.position == iter.node->count()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = nullptr; + break; + } + } + return iter; + } + + template <typename P> + template <typename... Args> + inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) + -> iterator { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + const int max_count = iter.node->max_count(); + if (iter.node->count() == max_count) { + // Make room in the leaf for the new item. + if (max_count < kNodeValues) { + // Insertion into the root where the root is smaller than the full node + // size. Simply grow the size of the root node. + assert(iter.node == root()); + iter.node = + new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count)); + iter.node->swap(root(), mutable_allocator()); + delete_leaf_node(root()); + mutable_root() = iter.node; + rightmost_ = iter.node; + } else { + rebalance_or_split(&iter); + } + } + iter.node->emplace_value(iter.position, mutable_allocator(), + std::forward<Args>(args)...); + ++size_; + return iter; + } + + template <typename P> + template <typename K> + inline auto btree<P>::internal_locate(const K &key) const + -> SearchResult<iterator, is_key_compare_to::value> { + return internal_locate_impl(key, is_key_compare_to()); + } + + template <typename P> + template <typename K> + inline auto btree<P>::internal_locate_impl( + const K &key, std::false_type /* IsCompareTo */) const + -> SearchResult<iterator, false> { + iterator iter(const_cast<node_type *>(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + // NOTE: we don't need to walk all the way down the tree if the keys are + // equal, but determining equality would require doing an extra comparison + // on each node on the way down, and we will need to go all the way to the + // leaf node in the expected case. + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter}; + } + + template <typename P> + template <typename K> + inline auto btree<P>::internal_locate_impl( + const K &key, std::true_type /* IsCompareTo */) const + -> SearchResult<iterator, true> { + iterator iter(const_cast<node_type *>(root()), 0); + for (;;) { + SearchResult<int, true> res = iter.node->lower_bound(key, key_comp()); + iter.position = res.value; + if (res.match == MatchKind::kEq) { + return {iter, MatchKind::kEq}; + } + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter, MatchKind::kNe}; + } + + template <typename P> + template <typename K> + auto btree<P>::internal_lower_bound(const K &key) const -> iterator { + iterator iter(const_cast<node_type *>(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); + } + + template <typename P> + template <typename K> + auto btree<P>::internal_upper_bound(const K &key) const -> iterator { + iterator iter(const_cast<node_type *>(root()), 0); + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); + } + + template <typename P> + template <typename K> + auto btree<P>::internal_find(const K &key) const -> iterator { + auto res = internal_locate(key); + if (res.HasMatch()) { + if (res.IsEq()) { + return res.value; + } + } else { + const iterator iter = internal_last(res.value); + if (iter.node != nullptr && !compare_keys(key, iter.key())) { + return iter; + } + } + return {nullptr, 0}; + } + + template <typename P> + void btree<P>::internal_clear(node_type *node) { + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + internal_clear(node->child(i)); + } + delete_internal_node(node); + } else { + delete_leaf_node(node); + } + } + + template <typename P> + int btree<P>::internal_verify( + const node_type *node, const key_type *lo, const key_type *hi) const { + assert(node->count() > 0); + assert(node->count() <= node->max_count()); + if (lo) { + assert(!compare_keys(node->key(0), *lo)); + } + if (hi) { + assert(!compare_keys(*hi, node->key(node->count() - 1))); + } + for (int i = 1; i < node->count(); ++i) { + assert(!compare_keys(node->key(i), node->key(i - 1))); + } + int count = node->count(); + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + assert(node->child(i) != nullptr); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); + count += internal_verify( + node->child(i), + (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + return count; + } + + // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. + // --------------------------------------------------------------------------------- + template <typename Tree> + class btree_container { + using params_type = typename Tree::params_type; + + protected: + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template <class K> + using key_arg = + typename KeyArg<IsTransparent<typename Tree::key_compare>::value>:: + template type<K, typename Tree::key_type>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using difference_type = typename Tree::difference_type; + using key_compare = typename Tree::key_compare; + using value_compare = typename Tree::value_compare; + using allocator_type = typename Tree::allocator_type; + using reference = typename Tree::reference; + using const_reference = typename Tree::const_reference; + using pointer = typename Tree::pointer; + using const_pointer = typename Tree::const_pointer; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using reverse_iterator = typename Tree::reverse_iterator; + using const_reverse_iterator = typename Tree::const_reverse_iterator; + using node_type = typename Tree::node_handle_type; + + // Constructors/assignments. + btree_container() : tree_(key_compare(), allocator_type()) {} + explicit btree_container(const key_compare &comp, + const allocator_type &alloc = allocator_type()) + : tree_(comp, alloc) {} + btree_container(const btree_container &x) = default; + btree_container(btree_container &&x) noexcept = default; + btree_container &operator=(const btree_container &x) = default; + btree_container &operator=(btree_container &&x) noexcept( + std::is_nothrow_move_assignable<Tree>::value) = default; + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + const_iterator cbegin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + const_iterator cend() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + const_reverse_iterator crbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + const_reverse_iterator crend() const { return tree_.rend(); } + + // Lookup routines. + template <typename K = key_type> + iterator find(const key_arg<K> &key) { + return tree_.find(key); + } + template <typename K = key_type> + const_iterator find(const key_arg<K> &key) const { return tree_.find(key); } + + template <typename K = key_type> + bool contains(const key_arg<K> &key) const { return find(key) != end(); } + + template <typename K = key_type> + iterator lower_bound(const key_arg<K> &key) { return tree_.lower_bound(key); } + + template <typename K = key_type> + const_iterator lower_bound(const key_arg<K> &key) const { return tree_.lower_bound(key); } + + template <typename K = key_type> + iterator upper_bound(const key_arg<K> &key) { return tree_.upper_bound(key); } + + template <typename K = key_type> + const_iterator upper_bound(const key_arg<K> &key) const { return tree_.upper_bound(key); } + + template <typename K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K> &key) { return tree_.equal_range(key); } + + template <typename K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K> &key) const { + return tree_.equal_range(key); + } + + iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } + iterator erase(iterator iter) { return tree_.erase(iter); } + iterator erase(const_iterator first, const_iterator last) { + return tree_.erase(iterator(first), iterator(last)).second; + } + + node_type extract(iterator position) { + // Use Move instead of Transfer, because the rebalancing code expects to + // have a valid object to scribble metadata bits on top of. + auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot()); + erase(position); + return node; + } + + node_type extract(const_iterator position) { + return extract(iterator(position)); + } + + public: + void clear() { tree_.clear(); } + void swap(btree_container &x) { tree_.swap(x.tree_); } + void verify() const { tree_.verify(); } + + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + + friend bool operator==(const btree_container &x, const btree_container &y) { + if (x.size() != y.size()) return false; + return std::equal(x.begin(), x.end(), y.begin()); + } + + friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } + + friend bool operator<(const btree_container &x, const btree_container &y) { + return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); + } + + friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } + + friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } + + friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } + + // The allocator used by the btree. + allocator_type get_allocator() const { return tree_.get_allocator(); } + + // The key comparator used by the btree. + key_compare key_comp() const { return tree_.key_comp(); } + value_compare value_comp() const { return tree_.value_comp(); } + + // Support absl::Hash. + template <typename State> + friend State AbslHashValue(State h, const btree_container &b) { + for (const auto &v : b) { + h = State::combine(std::move(h), v); + } + return State::combine(std::move(h), b.size()); + } + + protected: + Tree tree_; + }; + + // A common base class for btree_set and btree_map. + // ----------------------------------------------- + template <typename Tree> + class btree_set_container : public btree_container<Tree> { + using super_type = btree_container<Tree>; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + friend class BtreeNodePeer; + + protected: + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + using insert_return_type = InsertReturnType<iterator, node_type>; + using super_type::super_type; + btree_set_container() {} + + template <class InputIterator> + btree_set_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + btree_set_container(std::initializer_list<init_type> init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_set_container(init.begin(), init.end(), comp, alloc) {} + + // Lookup routines. + template <typename K = key_type> + size_type count(const key_arg<K> &key) const { + return this->tree_.count_unique(key); + } + + // Insertion routines. + std::pair<iterator, bool> insert(const value_type &x) { + return this->tree_.insert_unique(params_type::key(x), x); + } + std::pair<iterator, bool> insert(value_type &&x) { + return this->tree_.insert_unique(params_type::key(x), std::move(x)); + } + template <typename... Args> + std::pair<iterator, bool> emplace(Args &&... args) { + init_type v(std::forward<Args>(args)...); + return this->tree_.insert_unique(params_type::key(v), std::move(v)); + } + iterator insert(const_iterator position, const value_type &x) { + return this->tree_ + .insert_hint_unique(iterator(position), params_type::key(x), x) + .first; + } + iterator insert(const_iterator position, value_type &&x) { + return this->tree_ + .insert_hint_unique(iterator(position), params_type::key(x), + std::move(x)) + .first; + } + + template <typename... Args> + iterator emplace_hint(const_iterator position, Args &&... args) { + init_type v(std::forward<Args>(args)...); + return this->tree_ + .insert_hint_unique(iterator(position), params_type::key(v), + std::move(v)) + .first; + } + + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_unique(b, e); + } + + void insert(std::initializer_list<init_type> init) { + this->tree_.insert_iterator_unique(init.begin(), init.end()); + } + + insert_return_type insert(node_type &&node) { + if (!node) return {this->end(), false, node_type()}; + std::pair<iterator, bool> res = + this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) { + CommonAccess::Destroy(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + std::pair<iterator, bool> res = this->tree_.insert_hint_unique( + iterator(hint), params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) CommonAccess::Destroy(&node); + return res.first; + } + + template <typename K = key_type> + size_type erase(const key_arg<K> &key) { return this->tree_.erase_unique(key); } + using super_type::erase; + + template <typename K = key_type> + node_type extract(const key_arg<K> &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + + using super_type::extract; + + // Merge routines. + // Moves elements from `src` into `this`. If the element already exists in + // `this`, it is left unmodified in `src`. + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &src) { // NOLINT + for (auto src_it = src.begin(); src_it != src.end();) { + if (insert(std::move(*src_it)).second) { + src_it = src.erase(src_it); + } else { + ++src_it; + } + } + } + + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &&src) { + merge(src); + } + }; + + // Base class for btree_map. + // ------------------------- + template <typename Tree> + class btree_map_container : public btree_set_container<Tree> { + using super_type = btree_set_container<Tree>; + using params_type = typename Tree::params_type; + + protected: + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using mapped_type = typename params_type::mapped_type; + using value_type = typename Tree::value_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + + // Inherit constructors. + using super_type::super_type; + btree_map_container() {} + + // Insertion routines. + template <typename... Args> + std::pair<iterator, bool> try_emplace(const key_type &k, Args &&... args) { + return this->tree_.insert_unique( + k, std::piecewise_construct, std::forward_as_tuple(k), + std::forward_as_tuple(std::forward<Args>(args)...)); + } + template <typename... Args> + std::pair<iterator, bool> try_emplace(key_type &&k, Args &&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_unique guarantees that `key` is never + // referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_.insert_unique( + key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + } + template <typename... Args> + iterator try_emplace(const_iterator hint, const key_type &k, + Args &&... args) { + return this->tree_ + .insert_hint_unique(iterator(hint), k, std::piecewise_construct, + std::forward_as_tuple(k), + std::forward_as_tuple(std::forward<Args>(args)...)) + .first; + } + template <typename... Args> + iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_hint_unique guarantees that `key` is + // never referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_ + .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, + std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward<Args>(args)...)) + .first; + } + mapped_type &operator[](const key_type &k) { + return try_emplace(k).first->second; + } + mapped_type &operator[](key_type &&k) { + return try_emplace(std::move(k)).first->second; + } + + template <typename K = key_type> + mapped_type &at(const key_arg<K> &key) { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } + template <typename K = key_type> + const mapped_type &at(const key_arg<K> &key) const { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } + }; + + // A common base class for btree_multiset and btree_multimap. + template <typename Tree> + class btree_multiset_container : public btree_container<Tree> { + using super_type = btree_container<Tree>; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + + // Inherit constructors. + using super_type::super_type; + btree_multiset_container() {} + + // Range constructor. + template <class InputIterator> + btree_multiset_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Initializer list constructor. + btree_multiset_container(std::initializer_list<init_type> init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} + + // Lookup routines. + template <typename K = key_type> + size_type count(const key_arg<K> &key) const { + return this->tree_.count_multi(key); + } + + // Insertion routines. + iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } + iterator insert(value_type &&x) { + return this->tree_.insert_multi(std::move(x)); + } + iterator insert(const_iterator position, const value_type &x) { + return this->tree_.insert_hint_multi(iterator(position), x); + } + iterator insert(const_iterator position, value_type &&x) { + return this->tree_.insert_hint_multi(iterator(position), std::move(x)); + } + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_multi(b, e); + } + void insert(std::initializer_list<init_type> init) { + this->tree_.insert_iterator_multi(init.begin(), init.end()); + } + template <typename... Args> + iterator emplace(Args &&... args) { + return this->tree_.insert_multi(init_type(std::forward<Args>(args)...)); + } + template <typename... Args> + iterator emplace_hint(const_iterator position, Args &&... args) { + return this->tree_.insert_hint_multi( + iterator(position), init_type(std::forward<Args>(args)...)); + } + iterator insert(node_type &&node) { + if (!node) return this->end(); + iterator res = + this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + CommonAccess::Destroy(&node); + return res; + } + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + iterator res = this->tree_.insert_hint_multi( + iterator(hint), + std::move(params_type::element(CommonAccess::GetSlot(node)))); + CommonAccess::Destroy(&node); + return res; + } + + // Deletion routines. + template <typename K = key_type> + size_type erase(const key_arg<K> &key) { + return this->tree_.erase_multi(key); + } + using super_type::erase; + + // Node extraction routines. + template <typename K = key_type> + node_type extract(const key_arg<K> &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + using super_type::extract; + + // Merge routines. + // Moves all elements from `src` into `this`. + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &src) { // NOLINT + insert(std::make_move_iterator(src.begin()), + std::make_move_iterator(src.end())); + src.clear(); + } + + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &&src) { + merge(src); + } + }; + + // A base class for btree_multimap. + template <typename Tree> + class btree_multimap_container : public btree_multiset_container<Tree> { + using super_type = btree_multiset_container<Tree>; + using params_type = typename Tree::params_type; + + public: + using mapped_type = typename params_type::mapped_type; + + // Inherit constructors. + using super_type::super_type; + btree_multimap_container() {} + }; + +} // namespace priv + + + + // ---------------------------------------------------------------------- + // btree_set - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template <typename Key, typename Compare, typename Alloc> + class btree_set : public priv::btree_set_container< + priv::btree<priv::set_params< + Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> + { + using Base = typename btree_set::btree_set_container; + + public: + btree_set() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_set` containers. + // ------------------------------------------------------- + template <typename K, typename C, typename A> + void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template <typename K, typename C, typename A, typename Pred> + void erase_if(btree_set<K, C, A> &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } + } + + // ---------------------------------------------------------------------- + // btree_multiset - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template <typename Key, typename Compare, typename Alloc> + class btree_multiset : public priv::btree_multiset_container< + priv::btree<priv::set_params< + Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> + { + using Base = typename btree_multiset::btree_multiset_container; + + public: + btree_multiset() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_multiset` containers. + // ------------------------------------------------------------ + template <typename K, typename C, typename A> + void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template <typename K, typename C, typename A, typename Pred> + void erase_if(btree_multiset<K, C, A> &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } + } + + + // ---------------------------------------------------------------------- + // btree_map - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template <typename Key, typename Value, typename Compare, typename Alloc> + class btree_map : public priv::btree_map_container< + priv::btree<priv::map_params< + Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> + { + using Base = typename btree_map::btree_map_container; + + public: + btree_map() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_map` containers. + // ------------------------------------------------------- + template <typename K, typename V, typename C, typename A> + void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) { + return x.swap(y); + } + + // ---------------------------------------------------------------------- + template <typename K, typename V, typename C, typename A, typename Pred> + void erase_if(btree_map<K, V, C, A> &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } + } + + // ---------------------------------------------------------------------- + // btree_multimap - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template <typename Key, typename Value, typename Compare, typename Alloc> + class btree_multimap : public priv::btree_multimap_container< + priv::btree<priv::map_params< + Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> + { + using Base = typename btree_multimap::btree_multimap_container; + + public: + btree_multimap() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_multimap` containers. + // ------------------------------------------------------------ + template <typename K, typename V, typename C, typename A> + void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template <typename K, typename V, typename C, typename A, typename Pred> + void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } + } + + +} // namespace btree + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + + +#endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/benchmarks/others/parallel_hashmap/conanfile.py b/benchmarks/others/parallel_hashmap/conanfile.py new file mode 100644 index 00000000..c046377d --- /dev/null +++ b/benchmarks/others/parallel_hashmap/conanfile.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from conans import ConanFile, tools +import os + +class SparseppConan(ConanFile): + name = "parallel_hashmap" + version = "1.27" + description = "A header-only, very fast and memory-friendly hash map" + + # Indicates License type of the packaged library + license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" + + # Packages the license for the conanfile.py + exports = ["LICENSE"] + + # Custom attributes for Bincrafters recipe conventions + source_subfolder = "source_subfolder" + + def source(self): + source_url = "https://github.com/greg7mdp/parallel-hashmap" + tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) + extracted_dir = self.name + "-" + self.version + + #Rename to "source_folder" is a convention to simplify later steps + os.rename(extracted_dir, self.source_subfolder) + + + def package(self): + include_folder = os.path.join(self.source_subfolder, "parallel_hashmap") + self.copy(pattern="LICENSE") + self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder) + + def package_id(self): + self.info.header_only() diff --git a/benchmarks/others/sparsepp/spp_memory.h b/benchmarks/others/parallel_hashmap/meminfo.h index cfaa108d..872f3c69 100644 --- a/benchmarks/others/sparsepp/spp_memory.h +++ b/benchmarks/others/parallel_hashmap/meminfo.h @@ -28,6 +28,11 @@ namespace spp { + uint64_t GetSystemMemory(); + uint64_t GetTotalMemoryUsed(); + uint64_t GetProcessMemoryUsed(); + uint64_t GetPhysicalMemory(); + uint64_t GetSystemMemory() { #ifdef SPP_WIN diff --git a/benchmarks/others/parallel_hashmap/phmap.h b/benchmarks/others/parallel_hashmap/phmap.h new file mode 100644 index 00000000..653ae5ea --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap.h @@ -0,0 +1,4788 @@ +#if !defined(phmap_h_guard_) +#define phmap_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#ifdef _MSC_VER + #pragma warning(push) + + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4324) // structure was padded due to alignment specifier + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member + #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list + #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted + #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#endif + +#include <algorithm> +#include <cmath> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <tuple> +#include <type_traits> +#include <utility> +#include <array> +#include <cassert> +#include <atomic> + +#include "phmap_fwd_decl.h" +#include "phmap_utils.h" +#include "phmap_base.h" + +#if PHMAP_HAVE_STD_STRING_VIEW + #include <string_view> +#endif + +namespace phmap { + +namespace priv { + +// -------------------------------------------------------------------------- +template <size_t Width> +class probe_seq +{ +public: + probe_seq(size_t hashval, size_t mask) { + assert(((mask + 1) & mask) == 0 && "not a mask"); + mask_ = mask; + offset_ = hashval & mask_; + } + size_t offset() const { return offset_; } + size_t offset(size_t i) const { return (offset_ + i) & mask_; } + + void next() { + index_ += Width; + offset_ += index_; + offset_ &= mask_; + } + // 0-based probe index. The i-th probe in the probe sequence. + size_t getindex() const { return index_; } + +private: + size_t mask_; + size_t offset_; + size_t index_ = 0; +}; + +// -------------------------------------------------------------------------- +template <class ContainerKey, class Hash, class Eq> +struct RequireUsableKey +{ + template <class PassedKey, class... Args> + std::pair< + decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), + decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), + std::declval<const PassedKey&>()))>* + operator()(const PassedKey&, const Args&...) const; +}; + +// -------------------------------------------------------------------------- +template <class E, class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable : std::false_type {}; + +template <class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable< + phmap::void_t<decltype( + Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), + std::declval<Ts>()...))>, + Policy, Hash, Eq, Ts...> : std::true_type {}; + +// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. +// -------------------------------------------------------------------------- +template <class T> +constexpr bool IsNoThrowSwappable() { + using std::swap; + return noexcept(swap(std::declval<T&>(), std::declval<T&>())); +} + +// -------------------------------------------------------------------------- +template <typename T> +int TrailingZeros(T x) { + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountTrailingZerosNonZero64(static_cast<uint64_t>(x)); + else + return base_internal::CountTrailingZerosNonZero32(static_cast<uint32_t>(x)); +} + +// -------------------------------------------------------------------------- +template <typename T> +int LeadingZeros(T x) { + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountLeadingZeros64(static_cast<uint64_t>(x)); + else + return base_internal::CountLeadingZeros32(static_cast<uint32_t>(x)); +} + +// -------------------------------------------------------------------------- +// An abstraction over a bitmask. It provides an easy way to iterate through the +// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), +// this is a true bitmask. On non-SSE, platforms the arithematic used to +// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as +// either 0x00 or 0x80. +// +// For example: +// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 +// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 +// -------------------------------------------------------------------------- +template <class T, int SignificantBits, int Shift = 0> +class BitMask +{ + static_assert(std::is_unsigned<T>::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + +public: + // These are useful for unit tests (gunit). + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + explicit BitMask(T mask) : mask_(mask) {} + BitMask& operator++() { + mask_ &= (mask_ - 1); + return *this; + } + explicit operator bool() const { return mask_ != 0; } + int operator*() const { return LowestBitSet(); } + int LowestBitSet() const { + return priv::TrailingZeros(mask_) >> Shift; + } + int HighestBitSet() const { + return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - + 1) >> + Shift; + } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + + int TrailingZeros() const { + return priv::TrailingZeros(mask_) >> Shift; + } + + int LeadingZeros() const { + constexpr int total_significant_bits = SignificantBits << Shift; + constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; + return priv::LeadingZeros(mask_ << extra_bits) >> Shift; + } + +private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } + + T mask_; +}; + +// -------------------------------------------------------------------------- +using ctrl_t = signed char; +using h2_t = uint8_t; + +// -------------------------------------------------------------------------- +// The values here are selected for maximum performance. See the static asserts +// below for details. +// -------------------------------------------------------------------------- +enum Ctrl : ctrl_t +{ + kEmpty = -128, // 0b10000000 + kDeleted = -2, // 0b11111110 + kSentinel = -1, // 0b11111111 +}; + +static_assert( + kEmpty & kDeleted & kSentinel & 0x80, + "Special markers need to have the MSB to make checking for them efficient"); +static_assert(kEmpty < kSentinel && kDeleted < kSentinel, + "kEmpty and kDeleted must be smaller than kSentinel to make the " + "SIMD test of IsEmptyOrDeleted() efficient"); +static_assert(kSentinel == -1, + "kSentinel must be -1 to elide loading it from memory into SIMD " + "registers (pcmpeqd xmm, xmm)"); +static_assert(kEmpty == -128, + "kEmpty must be -128 to make the SIMD check for its " + "existence efficient (psignb xmm, xmm)"); +static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, + "kEmpty and kDeleted must share an unset bit that is not shared " + "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " + "efficient"); +static_assert(kDeleted == -2, + "kDeleted must be -2 to make the implementation of " + "ConvertSpecialToEmptyAndFullToDeleted efficient"); + +// -------------------------------------------------------------------------- +// A single block of empty control bytes for tables without any slots allocated. +// This enables removing a branch in the hot path of find(). +// -------------------------------------------------------------------------- +inline ctrl_t* EmptyGroup() { + alignas(16) static constexpr ctrl_t empty_group[] = { + kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, + kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; + return const_cast<ctrl_t*>(empty_group); +} + +// -------------------------------------------------------------------------- +inline size_t HashSeed(const ctrl_t* ctrl) { + // The low bits of the pointer have little or no entropy because of + // alignment. We shift the pointer to try to use higher entropy bits. A + // good number seems to be 12 bits, because that aligns with page size. + return reinterpret_cast<uintptr_t>(ctrl) >> 12; +} + +#ifdef PHMAP_NON_DETERMINISTIC + +inline size_t H1(size_t hashval, const ctrl_t* ctrl) { + // use ctrl_ pointer to add entropy to ensure + // non-deterministic iteration order. + return (hashval >> 7) ^ HashSeed(ctrl); +} + +#else + +inline size_t H1(size_t hashval, const ctrl_t* ) { + return (hashval >> 7); +} + +#endif + + +inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } + +inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } +inline bool IsFull(ctrl_t c) { return c >= 0; } +inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } +inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } + +#if PHMAP_HAVE_SSE2 + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch +#endif + +// -------------------------------------------------------------------------- +// https://github.com/abseil/abseil-cpp/issues/209 +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 +// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char +// Work around this by using the portable implementation of Group +// when using -funsigned-char under GCC. +// -------------------------------------------------------------------------- +inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { +#if defined(__GNUC__) && !defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Woverflow" + + if (std::is_unsigned<char>::value) { + const __m128i mask = _mm_set1_epi8(static_cast<char>(0x80)); + const __m128i diff = _mm_subs_epi8(b, a); + return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); + } + + #pragma GCC diagnostic pop +#endif + return _mm_cmpgt_epi8(a, b); +} + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +struct GroupSse2Impl +{ + enum { kWidth = 16 }; // the number of slots per group + + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); + } + + // Returns a bitmask representing the positions of slots that match hash. + // ---------------------------------------------------------------------- + BitMask<uint32_t, kWidth> Match(h2_t hash) const { + auto match = _mm_set1_epi8((char)hash); + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); + } + + // Returns a bitmask representing the positions of empty slots. + // ------------------------------------------------------------ + BitMask<uint32_t, kWidth> MatchEmpty() const { +#if PHMAP_HAVE_SSSE3 + // This only works because kEmpty is -128. + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); +#else + return Match(static_cast<h2_t>(kEmpty)); +#endif + } + + // Returns a bitmask representing the positions of empty or deleted slots. + // ----------------------------------------------------------------------- + BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); + } + + // Returns the number of trailing empty or deleted elements in the group. + // ---------------------------------------------------------------------- + uint32_t CountLeadingEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return TrailingZeros( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); + } + + // ---------------------------------------------------------------------- + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast<char>(-128)); + auto x126 = _mm_set1_epi8(126); +#if PHMAP_HAVE_SSSE3 + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); +#else + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } + + __m128i ctrl; +}; + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif // PHMAP_HAVE_SSE2 + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +struct GroupPortableImpl +{ + enum { kWidth = 8 }; + + explicit GroupPortableImpl(const ctrl_t* pos) + : ctrl(little_endian::Load64(pos)) {} + + BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match + // - they never occur on kEmpty, kDeleted, kSentinel + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmpty() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; + return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint64_t ctrl; +}; + +#if PHMAP_HAVE_SSE2 + using Group = GroupSse2Impl; +#else + using Group = GroupPortableImpl; +#endif + +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set; + +inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } + +// -------------------------------------------------------------------------- +// PRECONDITION: +// IsValidCapacity(capacity) +// ctrl[capacity] == kSentinel +// ctrl[i] != kSentinel for all i < capacity +// Applies mapping for every byte in ctrl: +// DELETED -> EMPTY +// EMPTY -> EMPTY +// FULL -> DELETED +// -------------------------------------------------------------------------- +inline void ConvertDeletedToEmptyAndFullToDeleted( + ctrl_t* ctrl, size_t capacity) +{ + assert(ctrl[capacity] == kSentinel); + assert(IsValidCapacity(capacity)); + for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); + } + // Copy the cloned ctrl bytes. + std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); + ctrl[capacity] = kSentinel; +} + +// -------------------------------------------------------------------------- +// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. +// -------------------------------------------------------------------------- +inline size_t NormalizeCapacity(size_t n) +{ + return n ? ~size_t{} >> LeadingZeros(n) : 1; +} + +// -------------------------------------------------------------------------- +// We use 7/8th as maximum load factor. +// For 16-wide groups, that gives an average of two empty slots per group. +// -------------------------------------------------------------------------- +inline size_t CapacityToGrowth(size_t capacity) +{ + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { + if (capacity == 7) + { + // x-x/8 does not work when x==7. + return 6; + } + } + return capacity - capacity / 8; +} + +// -------------------------------------------------------------------------- +// From desired "growth" to a lowerbound of the necessary capacity. +// Might not be a valid one and required NormalizeCapacity(). +// -------------------------------------------------------------------------- +inline size_t GrowthToLowerboundCapacity(size_t growth) +{ + // `growth*8/7` + PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { + if (growth == 7) + { + // x+(x-1)/7 does not work when x==7. + return 8; + } + } + return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); +} + +namespace hashtable_debug_internal { + +// If it is a map, call get<0>(). +using std::get; +template <typename T, typename = typename T::mapped_type> +auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { + return get<0>(pair); +} + +// If it is not a map, return the value directly. +template <typename T> +const typename T::key_type& GetKey(const typename T::key_type& key, char) { + return key; +} + +// -------------------------------------------------------------------------- +// Containers should specialize this to provide debug information for that +// container. +// -------------------------------------------------------------------------- +template <class Container, typename Enabler = void> +struct HashtableDebugAccess +{ + // Returns the number of probes required to find `key` in `c`. The "number of + // probes" is a concept that can vary by container. Implementations should + // return 0 when `key` was found in the minimum number of operations and + // should increment the result for each non-trivial operation required to find + // `key`. + // + // The default implementation uses the bucket api from the standard and thus + // works for `std::unordered_*` containers. + // -------------------------------------------------------------------------- + static size_t GetNumProbes(const Container& c, + const typename Container::key_type& key) { + if (!c.bucket_count()) return {}; + size_t num_probes = 0; + size_t bucket = c.bucket(key); + for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { + if (it == e) return num_probes; + if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes; + } + } +}; + +} // namespace hashtable_debug_internal + +// ---------------------------------------------------------------------------- +// I N F O Z S T U B S +// ---------------------------------------------------------------------------- +struct HashtablezInfo +{ + void PrepareForSampling() {} +}; + +inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} + +static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} + +static inline void RecordEraseSlow(HashtablezInfo*) {} + +static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } +static inline void UnsampleSlow(HashtablezInfo* ) {} + +class HashtablezInfoHandle +{ +public: + inline void RecordStorageChanged(size_t , size_t ) {} + inline void RecordRehash(size_t ) {} + inline void RecordInsert(size_t , size_t ) {} + inline void RecordErase() {} + friend inline void swap(HashtablezInfoHandle& , + HashtablezInfoHandle& ) noexcept {} +}; + +static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } + +class HashtablezSampler +{ +public: + // Returns a global Sampler. + static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } + HashtablezInfo* Register() { static HashtablezInfo info; return &info; } + void Unregister(HashtablezInfo* ) {} + + using DisposeCallback = void (*)(const HashtablezInfo&); + DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } + int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& ) { return 0; } +}; + +static inline void SetHashtablezEnabled(bool ) {} +static inline void SetHashtablezSampleParameter(int32_t ) {} +static inline void SetHashtablezMaxSamples(int32_t ) {} + + +namespace memory_internal { + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template <class Alloc, class T, class Tuple, size_t... I> +void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, + phmap::index_sequence<I...>) { + phmap::allocator_traits<Alloc>::construct( + *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...); +} + +template <class T, class F> +struct WithConstructedImplF { + template <class... Args> + decltype(std::declval<F>()(std::declval<T>())) operator()( + Args&&... args) const { + return std::forward<F>(f)(T(std::forward<Args>(args)...)); + } + F&& f; +}; + +template <class T, class Tuple, size_t... Is, class F> +decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl( + Tuple&& t, phmap::index_sequence<Is...>, F&& f) { + return WithConstructedImplF<T, F>{std::forward<F>(f)}( + std::get<Is>(std::forward<Tuple>(t))...); +} + +template <class T, size_t... Is> +auto TupleRefImpl(T&& t, phmap::index_sequence<Is...>) + -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) { + return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...); +} + +// Returns a tuple of references to the elements of the input tuple. T must be a +// tuple. +// ---------------------------------------------------------------------------- +template <class T> +auto TupleRef(T&& t) -> decltype( + TupleRefImpl(std::forward<T>(t), + phmap::make_index_sequence< + std::tuple_size<typename std::decay<T>::type>::value>())) { + return TupleRefImpl( + std::forward<T>(t), + phmap::make_index_sequence< + std::tuple_size<typename std::decay<T>::type>::value>()); +} + +template <class F, class K, class V> +decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct, + std::declval<std::tuple<K>>(), std::declval<V>())) +DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) { + const auto& key = std::get<0>(p.first); + return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); +} + +} // namespace memory_internal + + +// ---------------------------------------------------------------------------- +// R A W _ H A S H _ S E T +// ---------------------------------------------------------------------------- +// An open-addressing +// hashtable with quadratic probing. +// +// This is a low level hashtable on top of which different interfaces can be +// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. +// +// The table interface is similar to that of std::unordered_set. Notable +// differences are that most member functions support heterogeneous keys when +// BOTH the hash and eq functions are marked as transparent. They do so by +// providing a typedef called `is_transparent`. +// +// When heterogeneous lookup is enabled, functions that take key_type act as if +// they have an overload set like: +// +// iterator find(const key_type& key); +// template <class K> +// iterator find(const K& key); +// +// size_type erase(const key_type& key); +// template <class K> +// size_type erase(const K& key); +// +// std::pair<iterator, iterator> equal_range(const key_type& key); +// template <class K> +// std::pair<iterator, iterator> equal_range(const K& key); +// +// When heterogeneous lookup is disabled, only the explicit `key_type` overloads +// exist. +// +// find() also supports passing the hash explicitly: +// +// iterator find(const key_type& key, size_t hash); +// template <class U> +// iterator find(const U& key, size_t hash); +// +// In addition the pointer to element and iterator stability guarantees are +// weaker: all iterators and pointers are invalidated after a new element is +// inserted. +// +// IMPLEMENTATION DETAILS +// +// The table stores elements inline in a slot array. In addition to the slot +// array the table maintains some control state per slot. The extra state is one +// byte per slot and stores empty or deleted marks, or alternatively 7 bits from +// the hash of an occupied slot. The table is split into logical groups of +// slots, like so: +// +// Group 1 Group 2 Group 3 +// +---------------+---------------+---------------+ +// | | | | | | | | | | | | | | | | | | | | | | | | | +// +---------------+---------------+---------------+ +// +// On lookup the hash is split into two parts: +// - H2: 7 bits (those stored in the control bytes) +// - H1: the rest of the bits +// The groups are probed using H1. For each group the slots are matched to H2 in +// parallel. Because H2 is 7 bits (128 states) and the number of slots per group +// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// +// On insert, once the right group is found (as in lookup), its slots are +// filled in order. +// +// On erase a slot is cleared. In case the group did not have any empty slots +// before the erase, the erased slot is marked as deleted. +// +// Groups without empty slots (but maybe with deleted slots) extend the probe +// sequence. The probing algorithm is quadratic. Given N the number of groups, +// the probing function for the i'th probe is: +// +// P(0) = H1 % N +// +// P(i) = (P(i - 1) + i) % N +// +// This probing function guarantees that after N probes, all the groups of the +// table will be probed exactly once. +// ---------------------------------------------------------------------------- +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set +{ + using PolicyTraits = hash_policy_traits<Policy>; + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + +public: + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user + // code fixes! + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits<value_type>::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits<value_type>::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + +private: + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using Layout = phmap::priv::Layout<ctrl_t, slot_type>; + + static Layout MakeLayout(size_t capacity) { + assert(IsValidCapacity(capacity)); + return Layout(capacity + Group::kWidth + 1, capacity); + } + + using AllocTraits = phmap::allocator_traits<allocator_type>; + using SlotAlloc = typename phmap::allocator_traits< + allocator_type>::template rebind_alloc<slot_type>; + using SlotAllocTraits = typename phmap::allocator_traits< + allocator_type>::template rebind_traits<slot_type>; + + static_assert(std::is_lvalue_reference<reference>::value, + "Policy::element() must return a reference"); + + template <typename T> + struct SameAsElementReference + : std::is_same<typename std::remove_cv< + typename std::remove_reference<reference>::type>::type, + typename std::remove_cv< + typename std::remove_reference<T>::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + template <class T> + using RequiresInsertable = typename std::enable_if< + phmap::disjunction<std::is_convertible<T, init_type>, + SameAsElementReference<T>>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template <class T> + using RequiresNotInit = + typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; + + template <class... Ts> + using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; + +public: + static_assert(std::is_same<pointer, value_type*>::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same<const_pointer, const value_type*>::value, + "Allocators with custom pointer types are not supported"); + + class iterator + { + friend class raw_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename raw_hash_set::value_type; + using reference = + phmap::conditional_t<PolicyTraits::constant_iterators::value, + const value_type&, value_type&>; + using pointer = phmap::remove_reference_t<reference>*; + using difference_type = typename raw_hash_set::difference_type; + + iterator() {} + + // PRECONDITION: not an end() iterator. + reference operator*() const { return PolicyTraits::element(slot_); } + + // PRECONDITION: not an end() iterator. + pointer operator->() const { return &operator*(); } + + // PRECONDITION: not an end() iterator. + iterator& operator++() { + ++ctrl_; + ++slot_; + skip_empty_or_deleted(); + return *this; + } + // PRECONDITION: not an end() iterator. + iterator operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + +#if PHMAP_BIDIRECTIONAL + // PRECONDITION: not a begin() iterator. + iterator& operator--() { + assert(ctrl_); + do { + --ctrl_; + --slot_; + } while (IsEmptyOrDeleted(*ctrl_)); + return *this; + } + + // PRECONDITION: not a begin() iterator. + iterator operator--(int) { + auto tmp = *this; + --*this; + return tmp; + } +#endif + + friend bool operator==(const iterator& a, const iterator& b) { + return a.ctrl_ == b.ctrl_; + } + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: + iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() + iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} + + void skip_empty_or_deleted() { + while (IsEmptyOrDeleted(*ctrl_)) { + // ctrl is not necessarily aligned to Group::kWidth. It is also likely + // to read past the space for ctrl bytes and into slots. This is ok + // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there + // is no way to read outside the combined slot array. + uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + ctrl_ += shift; + slot_ += shift; + } + } + + ctrl_t* ctrl_ = nullptr; + // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; + }; + + class const_iterator + { + friend class raw_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename raw_hash_set::value_type; + using reference = typename raw_hash_set::const_reference; + using pointer = typename raw_hash_set::const_pointer; + using difference_type = typename raw_hash_set::difference_type; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : inner_(std::move(i)) {} + + reference operator*() const { return *inner_; } + pointer operator->() const { return inner_.operator->(); } + + const_iterator& operator++() { + ++inner_; + return *this; + } + const_iterator operator++(int) { return inner_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.inner_ == b.inner_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const ctrl_t* ctrl, const slot_type* slot) + : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} + + iterator inner_; + }; + + using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; + using insert_return_type = InsertReturnType<iterator, node_type>; + + raw_hash_set() noexcept( + std::is_nothrow_default_constructible<hasher>::value&& + std::is_nothrow_default_constructible<key_equal>::value&& + std::is_nothrow_default_constructible<allocator_type>::value) {} + + explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { + if (bucket_cnt) { + capacity_ = NormalizeCapacity(bucket_cnt); + reset_growth_left(); + initialize_slots(); + } + } + + raw_hash_set(size_t bucket_cnt, const hasher& hashfn, + const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} + + raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + + explicit raw_hash_set(const allocator_type& alloc) + : raw_hash_set(0, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { + insert(first, last); + } + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list<value_type> as the first + // argument like std::unordered_set<value_type> does, we have two overloads + // that accept std::initializer_list<T> and std::initializer_list<init_type>. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list<std::string>, then + // // copies the strings into the set. + // std::unordered_set<std::string> s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list<const char*>, then + // // copies the strings into the set. + // phmap::flat_hash_set<std::string> s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set<int> a, b{a}; + // + // RequiresNotInit<T> is a workaround for gcc prior to 7.1. + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, + const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(const raw_hash_set& that) + : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + raw_hash_set(const raw_hash_set& that, const allocator_type& a) + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + reserve(that.size()); + // Because the table is guaranteed to be empty, we can do something faster + // than a full `insert`. + for (const auto& v : that) { + const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); + auto target = find_first_non_full(hashval); + set_ctrl(target.offset, H2(hashval)); + emplace_at(target.offset, v); + infoz_.RecordInsert(hashval, target.probe_length); + } + size_ = that.size(); + growth_left() -= that.size(); + } + + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible<hasher>::value&& + std::is_nothrow_copy_constructible<key_equal>::value&& + std::is_nothrow_copy_constructible<allocator_type>::value) + : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), + slots_(phmap::exchange(that.slots_, nullptr)), + size_(phmap::exchange(that.size_, 0)), + capacity_(phmap::exchange(that.capacity_, 0)), + infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), + // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function<Key>, moving it + // would create a nullptr functor that cannot be called. + settings_(that.settings_) { + // growth_left was copied above, reset the one from `that`. + that.growth_left() = 0; + } + + raw_hash_set(raw_hash_set&& that, const allocator_type& a) + : ctrl_(EmptyGroup()), + slots_(nullptr), + size_(0), + capacity_(0), + settings_(0, that.hash_ref(), that.eq_ref(), a) { + if (a == that.alloc_ref()) { + std::swap(ctrl_, that.ctrl_); + std::swap(slots_, that.slots_); + std::swap(size_, that.size_); + std::swap(capacity_, that.capacity_); + std::swap(growth_left(), that.growth_left()); + std::swap(infoz_, that.infoz_); + } else { + reserve(that.size()); + // Note: this will copy elements of dense_set and unordered_set instead of + // moving them. This can be fixed if it ever becomes an issue. + for (auto& elem : that) insert(std::move(elem)); + } + } + + raw_hash_set& operator=(const raw_hash_set& that) { + raw_hash_set tmp(that, + AllocTraits::propagate_on_container_copy_assignment::value + ? that.alloc_ref() + : alloc_ref()); + swap(tmp); + return *this; + } + + raw_hash_set& operator=(raw_hash_set&& that) noexcept( + phmap::allocator_traits<allocator_type>::is_always_equal::value&& + std::is_nothrow_move_assignable<hasher>::value&& + std::is_nothrow_move_assignable<key_equal>::value) { + // TODO(sbenza): We should only use the operations from the noexcept clause + // to make sure we actually adhere to that contract. + return move_assign( + std::move(that), + typename AllocTraits::propagate_on_container_move_assignment()); + } + + ~raw_hash_set() { destroy_slots(); } + + iterator begin() { + auto it = iterator_at(0); + it.skip_empty_or_deleted(); + return it; + } + iterator end() + { +#if PHMAP_BIDIRECTIONAL + return iterator_at(capacity_); +#else + return {ctrl_ + capacity_}; +#endif + } + + const_iterator begin() const { + return const_cast<raw_hash_set*>(this)->begin(); + } + const_iterator end() const { return const_cast<raw_hash_set*>(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } + + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + // Iterating over this container is O(bucket_count()). When bucket_count() + // is much greater than size(), iteration becomes prohibitively expensive. + // For clear() it is more important to reuse the allocated array when the + // container is small because allocation takes comparatively long time + // compared to destruction of the elements of the container. So we pick the + // largest bucket_count() threshold for which iteration is still fast and + // past that we simply deallocate the array. + if (empty()) + return; + if (capacity_ > 127) { + destroy_slots(); + } else if (capacity_) { + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + size_ = 0; + reset_ctrl(); + reset_growth_left(); + } + assert(empty()); + infoz_.RecordStorageChanged(0, capacity_); + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map<std::string, int> m; + // m.insert(std::make_pair("abc", 42)); + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, + T* = nullptr> + std::pair<iterator, bool> insert(T&& value) { + return emplace(std::forward<T>(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set<int> s; + // s.insert(n); + // + // flat_hash_set<std::string> s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + std::pair<iterator, bool> insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set<std::string, int> s; + // s.insert({"abc", 42}); + std::pair<iterator, bool> insert(init_type&& value) { + return emplace(std::move(value)); + } + + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward<T>(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template <typename It> + using IsRandomAccess = std::is_same<typename std::iterator_traits<It>::iterator_category, + std::random_access_iterator_tag>; + + + template<typename T> + struct has_difference_operator + { + private: + using yes = std::true_type; + using no = std::false_type; + + template<typename U> static auto test(int) -> decltype(std::declval<U>() - std::declval<U>() == 1, yes()); + template<typename> static no test(...); + + public: + static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; + }; + + template <class InputIt, typename phmap::enable_if_t<has_difference_operator<InputIt>::value, int> = 0> + void insert(InputIt first, InputIt last) { + this->reserve(this->size() + (last - first)); + for (; first != last; ++first) + emplace(*first); + } + + template <class InputIt, typename phmap::enable_if_t<!has_difference_operator<InputIt>::value, int> = 0> + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) + emplace(*first); + } + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> + void insert(std::initializer_list<T> ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list<init_type> ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + insert_return_type insert(node_type&& node, size_t hashval) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlotWithHash<false>{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator, node_type&& node) { + return insert(std::move(node)).first; + } + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + template <class... Args, typename std::enable_if< + IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward<Args>(args)...); + } + + template <class... Args, typename std::enable_if<IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward<Args>(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + template <class... Args, typename std::enable_if< + !IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type + raw; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); + } + + template <class... Args, typename std::enable_if<!IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { + typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlotWithHash<true>{*this, std::move(*slot), hashval}, elem); + } + + template <class... Args> + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward<Args>(args)...).first; + } + + template <class... Args> + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward<Args>(args)...).first; + } + + // Extension API: support for lazy emplace. + // + // Looks up key in the table. If found, returns the iterator to the element. + // Otherwise calls f with one argument of type raw_hash_set::constructor. f + // MUST call raw_hash_set::constructor with arguments as if a + // raw_hash_set::value_type is constructed, otherwise the behavior is + // undefined. + // + // For example: + // + // std::unordered_set<ArenaString> s; + // // Makes ArenaStr even if "abc" is in the map. + // s.insert(ArenaString(&arena, "abc")); + // + // flat_hash_set<ArenaStr> s; + // // Makes ArenaStr only if "abc" is not in the map. + // s.lazy_emplace("abc", [&](const constructor& ctor) { + // ctor(&arena, "abc"); + // }); + // + // WARNING: This API is currently experimental. If there is a way to implement + // the same thing with the rest of the API, prefer that. + class constructor + { + friend class raw_hash_set; + + public: + template <class... Args> + void operator()(Args&&... args) const { + assert(*slot_); + PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); + *slot_ = nullptr; + } + + private: + constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} + + allocator_type* alloc_; + slot_type** slot_; + }; + + template <class K = key_type, class F> + iterator lazy_emplace(const key_arg<K>& key, F&& f) { + auto res = find_or_prepare_insert(key); + if (res.second) { + lazy_emplace_at(res.first, std::forward<F>(f)); + } + return iterator_at(res.first); + } + + template <class K = key_type, class F> + iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t &hashval, F&& f) { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + lazy_emplace_at(res.first, std::forward<F>(f)); + } + return iterator_at(res.first); + } + + template <class K = key_type, class F> + void lazy_emplace_at(size_t& idx, F&& f) { + slot_type* slot = slots_ + idx; + std::forward<F>(f)(constructor(&alloc_ref(), &slot)); + assert(!slot); + } + + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + template <class K = key_type> + size_type erase(const key_arg<K>& key) { + auto it = find(key); + if (it == end()) return 0; + _erase(it); + return 1; + } + + + iterator erase(const_iterator cit) { return erase(cit.inner_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if (<pred>) { + // m._erase(it++); + // } else { + // ++it; + // } + // } + void _erase(iterator it) { + assert(it != end()); + PolicyTraits::destroy(&alloc_ref(), it.slot_); + erase_meta_only(it); + } + void _erase(const_iterator cit) { _erase(cit.inner_); } + + // This overload is necessary because otherwise erase<K>(const K&) would be + // a better match if non-const iterator is passed as an argument. + iterator erase(iterator it) { + auto res = it; + ++res; + _erase(it); + return res; + } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); + } + return last.inner_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT + assert(this != &src); + for (auto it = src.begin(), e = src.end(); it != e; ++it) { + if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, + PolicyTraits::element(it.slot_)) + .second) { + src.erase_meta_only(it); + } + } + } + + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { + merge(src); + } + + node_type extract(const_iterator position) { + auto node = + CommonAccess::Make<node_type>(alloc_ref(), position.inner_.slot_); + erase_meta_only(position); + return node; + } + + template < + class K = key_type, + typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> + node_type extract(const key_arg<K>& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + void swap(raw_hash_set& that) noexcept( + IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable<allocator_type>())) { + using std::swap; + swap(ctrl_, that.ctrl_); + swap(slots_, that.slots_); + swap(size_, that.size_); + swap(capacity_, that.capacity_); + swap(growth_left(), that.growth_left()); + swap(hash_ref(), that.hash_ref()); + swap(eq_ref(), that.eq_ref()); + swap(infoz_, that.infoz_); + if (AllocTraits::propagate_on_container_swap::value) { + swap(alloc_ref(), that.alloc_ref()); + } else { + // If the allocators do not compare equal it is officially undefined + // behavior. We choose to do nothing. + } + } + +#ifndef PHMAP_NON_DETERMINISTIC + template<typename OutputArchive> + bool dump(OutputArchive&) const; + + template<typename InputArchive> + bool load(InputArchive&); +#endif + + void rehash(size_t n) { + if (n == 0 && capacity_ == 0) return; + if (n == 0 && size_ == 0) { + destroy_slots(); + infoz_.RecordStorageChanged(0, 0); + return; + } + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity((std::max)(n, size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || m > capacity_) { + resize(m); + } + } + + void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + template <class K = key_type> + size_t count(const key_arg<K>& key) const { + return find(key) == end() ? size_t(0) : size_t(1); + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + void prefetch_hash(size_t hashval) const { + (void)hashval; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + auto seq = probe(hashval); + _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); + _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); +#elif defined(__GNUC__) + auto seq = probe(hashval); + __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); + __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); +#endif // __GNUC__ + } + + template <class K = key_type> + void prefetch(const key_arg<K>& key) const { + prefetch_hash(this->hash(key)); + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + template <class K = key_type> + iterator find(const key_arg<K>& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return iterator_at(offset); + else + return end(); + } + + template <class K = key_type> + pointer find_ptr(const key_arg<K>& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return &PolicyTraits::element(slots_ + offset); + else + return nullptr; + } + + template <class K = key_type> + iterator find(const key_arg<K>& key) { + return find(key, this->hash(key)); + } + + template <class K = key_type> + const_iterator find(const key_arg<K>& key, size_t hashval) const { + return const_cast<raw_hash_set*>(this)->find(key, hashval); + } + template <class K = key_type> + const_iterator find(const key_arg<K>& key) const { + return find(key, this->hash(key)); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key) const { + return find(key) != end(); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key, size_t hashval) const { + return find(key, hashval) != end(); + } + + template <class K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + template <class K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K>& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { return capacity_; } + float load_factor() const { + return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; + } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { + if (a.size() != b.size()) return false; + const raw_hash_set* outer = &a; + const raw_hash_set* inner = &b; + if (outer->capacity() > inner->capacity()) + std::swap(outer, inner); + for (const value_type& elem : *outer) + if (!inner->has_element(elem)) return false; + return true; + } + + friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { + return !(a == b); + } + + friend void swap(raw_hash_set& a, + raw_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } + + template <class K> + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } + +private: + template <class Container, typename Enabler> + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; + + template <class K = key_type> + bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) { + auto seq = probe(hashval); + while (true) { + Group g{ ctrl_ + seq.offset() }; + for (int i : g.Match((h2_t)H2(hashval))) { + offset = seq.offset((size_t)i); + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + offset)))) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) + return false; + seq.next(); + } + } + + struct FindElement + { + template <class K, class... Args> + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const raw_hash_set& s; + }; + + struct HashElement + { + template <class K, class... Args> + size_t operator()(const K& key, Args&&...) const { + return phmap_mix<sizeof(size_t)>()(h(key)); + } + const hasher& h; + }; + + template <class K1> + struct EqualElement + { + template <class K2, class... Args> + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + template <class K, class... Args> + std::pair<iterator, bool> emplace_decomposable(const K& key, size_t hashval, + Args&&... args) + { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + emplace_at(res.first, std::forward<Args>(args)...); + } + return {iterator_at(res.first), res.second}; + } + + struct EmplaceDecomposable + { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, s.hash(key), std::forward<Args>(args)...); + } + raw_hash_set& s; + }; + + struct EmplaceDecomposableHashval { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, hashval, std::forward<Args>(args)...); + } + raw_hash_set& s; + size_t hashval; + }; + + template <bool do_destroy> + struct InsertSlot + { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + }; + + template <bool do_destroy> + struct InsertSlotWithHash + { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key, hashval); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + size_t &hashval; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + void erase_meta_only(const_iterator it) { + assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); + --size_; + const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); + const size_t index_before = (index - Group::kWidth) & capacity_; + const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = + empty_before && empty_after && + static_cast<size_t>(empty_after.TrailingZeros() + + empty_before.LeadingZeros()) < Group::kWidth; + + set_ctrl(index, was_never_full ? kEmpty : kDeleted); + growth_left() += was_never_full; + infoz_.RecordErase(); + } + + void initialize_slots() { + assert(capacity_); + if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && + slots_ == nullptr) { + infoz_ = Sample(); + } + + auto layout = MakeLayout(capacity_); + char* mem = static_cast<char*>( + Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize())); + ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem)); + slots_ = layout.template Pointer<1>(mem); + reset_ctrl(); + reset_growth_left(); + infoz_.RecordStorageChanged(size_, capacity_); + } + + void destroy_slots() { + if (!capacity_) return; + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + auto layout = MakeLayout(capacity_); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize()); + ctrl_ = EmptyGroup(); + slots_ = nullptr; + size_ = 0; + capacity_ = 0; + growth_left() = 0; + } + + void resize(size_t new_capacity) { + assert(IsValidCapacity(new_capacity)); + auto* old_ctrl = ctrl_; + auto* old_slots = slots_; + const size_t old_capacity = capacity_; + capacity_ = new_capacity; + initialize_slots(); + + for (size_t i = 0; i != old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(old_slots + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + } + } + if (old_capacity) { + SanitizerUnpoisonMemoryRegion(old_slots, + sizeof(slot_type) * old_capacity); + auto layout = MakeLayout(old_capacity); + Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, + layout.AllocSize()); + } + } + + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + assert(IsValidCapacity(capacity_)); + assert(!is_small()); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); + typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type + raw; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + for (size_t i = 0; i != capacity_; ++i) { + if (!IsDeleted(ctrl_[i])) continue; + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slots_ + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + + // Verify if the old and new i fall within the same group wrt the hashval. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. + const auto probe_index = [&](size_t pos) { + return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; + }; + + // Element doesn't move. + if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + set_ctrl(i, H2(hashval)); + continue; + } + if (IsEmpty(ctrl_[new_i])) { + // Transfer element to the empty spot. + // set_ctrl poisons/unpoisons the slots so we have to call it at the + // right time. + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); + set_ctrl(i, kEmpty); + } else { + assert(IsDeleted(ctrl_[new_i])); + set_ctrl(new_i, H2(hashval)); + // Until we are done rehashing, DELETED marks previously FULL slots. + // Swap i and new_i elements. + PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); + --i; // repeat + } + } + reset_growth_left(); + } + + void rehash_and_grow_if_necessary() { + if (capacity_ == 0) { + resize(1); + } else if (size() <= CapacityToGrowth(capacity()) / 2) { + // Squash DELETED without growing if there is enough capacity. + drop_deletes_without_resize(); + } else { + // Otherwise grow the container. + resize(capacity_ * 2 + 1); + } + } + + bool has_element(const value_type& elem, size_t hashval) const { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == + elem)) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); + assert(seq.getindex() < capacity_ && "full table!"); + } + return false; + } + + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + return has_element(elem, hashval); + } + + // Probes the raw_hash_set with the probe sequence for hash and returns the + // pointer to the first empty or deleted slot. + // NOTE: this function must work with tables having both kEmpty and kDelete + // in one group. Such tables appears during drop_deletes_without_resize. + // + // This function is very useful when insertions happen and: + // - the input is already a set + // - there are enough slots + // - the element with the hash is not in the table + struct FindInfo + { + size_t offset; + size_t probe_length; + }; + FindInfo find_first_non_full(size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + auto mask = g.MatchEmptyOrDeleted(); + if (mask) { + return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; + } + assert(seq.getindex() < capacity_ && "full table!"); + seq.next(); + } + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { + raw_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { + raw_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + +protected: + template <class K> + std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset((size_t)i))))) + return {seq.offset((size_t)i), false}; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; + seq.next(); + } + return {prepare_insert(hashval), true}; + } + + template <class K> + std::pair<size_t, bool> find_or_prepare_insert(const K& key) { + return find_or_prepare_insert(key, this->hash(key)); + } + + size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { + auto target = find_first_non_full(hashval); + if (PHMAP_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(ctrl_[target.offset]))) { + rehash_and_grow_if_necessary(); + target = find_first_non_full(hashval); + } + ++size_; + growth_left() -= IsEmpty(ctrl_[target.offset]); + set_ctrl(target.offset, H2(hashval)); + infoz_.RecordInsert(hashval, target.probe_length); + return target.offset; + } + + // Constructs the value in the space pointed by the iterator. This only works + // after an unsuccessful find_or_prepare_insert() and before any other + // modifications happen in the raw_hash_set. + // + // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where + // k is the key decomposed from `forward<Args>(args)...`, and the bool + // returned by find_or_prepare_insert(k) was true. + // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). + template <class... Args> + void emplace_at(size_t i, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slots_ + i, + std::forward<Args>(args)...); + + assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == + iterator_at(i) && + "constructed value does not match the lookup key"); + } + + iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } + const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + +private: + friend struct RawHashSetTestOnlyAccess; + + probe_seq<Group::kWidth> probe(size_t hashval) const { + return probe_seq<Group::kWidth>(H1(hashval, ctrl_), capacity_); + } + + // Reset all ctrl bytes back to kEmpty, except the sentinel. + void reset_ctrl() { + std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); + ctrl_[capacity_] = kSentinel; + SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + } + + void reset_growth_left() { + growth_left() = CapacityToGrowth(capacity()) - size_; + } + + // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at + // the end too. + void set_ctrl(size_t i, ctrl_t h) { + assert(i < capacity_); + + if (IsFull(h)) { + SanitizerUnpoisonObject(slots_ + i); + } else { + SanitizerPoisonObject(slots_ + i); + } + + ctrl_[i] = h; + ctrl_[((i - Group::kWidth) & capacity_) + 1 + + ((Group::kWidth - 1) & capacity_)] = h; + } + + size_t& growth_left() { return settings_.template get<0>(); } + + template <size_t N, + template <class, class, class, class> class RefSet, + class M, class P, class H, class E, class A> + friend class parallel_hash_set; + + template <size_t N, + template <class, class, class, class> class RefSet, + class M, class P, class H, class E, class A> + friend class parallel_hash_map; + + // The representation of the object has two modes: + // - small: For capacities < kWidth-1 + // - large: For the rest. + // + // Differences: + // - In small mode we are able to use the whole capacity. The extra control + // bytes give us at least one "empty" control byte to stop the iteration. + // This is important to make 1 a valid capacity. + // + // - In small mode only the first `capacity()` control bytes after the + // sentinel are valid. The rest contain dummy kEmpty values that do not + // represent a real slot. This is important to take into account on + // find_first_non_full(), where we never try ShouldInsertBackwards() for + // small tables. + bool is_small() const { return capacity_ < Group::kWidth - 1; } + + hasher& hash_ref() { return settings_.template get<1>(); } + const hasher& hash_ref() const { return settings_.template get<1>(); } + key_equal& eq_ref() { return settings_.template get<2>(); } + const key_equal& eq_ref() const { return settings_.template get<2>(); } + allocator_type& alloc_ref() { return settings_.template get<3>(); } + const allocator_type& alloc_ref() const { + return settings_.template get<3>(); + } + + // TODO(alkis): Investigate removing some of these fields: + // - ctrl/slots can be derived from each other + // - size can be moved into the slot array + ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] + slot_type* slots_ = nullptr; // [capacity * slot_type] + size_t size_ = 0; // number of full slots + size_t capacity_ = 0; // total number of slots + HashtablezInfoHandle infoz_; + phmap::priv::CompressedTuple<size_t /* growth_left */, hasher, + key_equal, allocator_type> + settings_{0, hasher{}, key_equal{}, allocator_type{}}; +}; + + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> +{ + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map<K, IncompleteType>. + // MappedReference<> may be a non-reference type. + template <class P> + using MappedReference = decltype(P::value( + std::addressof(std::declval<typename raw_hash_map::reference>()))); + + // MappedConstReference<> may be a non-reference type. + template <class P> + using MappedConstReference = decltype(P::value( + std::addressof(std::declval<typename raw_hash_map::const_reference>()))); + + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + using Base = raw_hash_set<Policy, Hash, Eq, Alloc>; + +public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + + static_assert(!std::is_reference<key_type>::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference<mapped_type>::value, ""); + + using iterator = typename raw_hash_map::raw_hash_set::iterator; + using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; + + raw_hash_map() {} + using Base::raw_hash_set; // use raw_hash_set constructor + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map<int, int> m; + // m.insert_or_assign(n, n); + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { + return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { + return insert_or_assign_impl(std::forward<K>(k), v); + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { + return insert_or_assign_impl(k, std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { + return insert_or_assign_impl(k, v); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { + return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { + return insert_or_assign(std::forward<K>(k), v).first; + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { + return insert_or_assign(k, std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { + return insert_or_assign(k, v).first; + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0, + K* = nullptr> + std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { + return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0> + std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { + return try_emplace_impl(k, std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, K* = nullptr> + iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { + return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; + } + + template <class K = key_type, class... Args> + iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { + return try_emplace(k, std::forward<Args>(args)...).first; + } + + template <class K = key_type, class P = Policy> + MappedReference<P> at(const key_arg<K>& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template <class K = key_type, class P = Policy> + MappedConstReference<P> at(const key_arg<K>& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template <class K = key_type, class P = Policy, K* = nullptr> + MappedReference<P> operator[](key_arg<K>&& key) { + return Policy::value(&*try_emplace(std::forward<K>(key)).first); + } + + template <class K = key_type, class P = Policy> + MappedReference<P> operator[](const key_arg<K>& key) { + return Policy::value(&*try_emplace(key).first); + } + +private: + template <class K, class V> + std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); + else + Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v); + return {this->iterator_at(res.first), res.second}; + } + + template <class K = key_type, class... Args> + std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + return {this->iterator_at(res.first), res.second}; + } +}; + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +// Returns "random" seed. +inline size_t RandomSeed() +{ +#if PHMAP_HAVE_THREAD_LOCAL + static thread_local size_t counter = 0; + size_t value = ++counter; +#else // PHMAP_HAVE_THREAD_LOCAL + static std::atomic<size_t> counter(0); + size_t value = counter.fetch_add(1, std::memory_order_relaxed); +#endif // PHMAP_HAVE_THREAD_LOCAL + return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); +} + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +template <size_t N, + template <class, class, class, class> class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +class parallel_hash_set +{ + using PolicyTraits = hash_policy_traits<Policy>; + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + static_assert(N <= 12, "N = 12 means 4096 hash tables!"); + constexpr static size_t num_tables = 1 << N; + constexpr static size_t mask = num_tables - 1; + +public: + using EmbeddedSet = RefSet<Policy, Hash, Eq, Alloc>; + using EmbeddedIterator= typename EmbeddedSet::iterator; + using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; + using constructor = typename EmbeddedSet::constructor; + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits<value_type>::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits<value_type>::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + // -------------------------------------------------------------------- + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + +protected: + using Lockable = phmap::LockableImpl<Mtx_>; + + // -------------------------------------------------------------------- + struct Inner : public Lockable + { + bool operator==(const Inner& o) const + { + typename Lockable::SharedLocks l(const_cast<Inner &>(*this), const_cast<Inner &>(o)); + return set_ == o.set_; + } + + EmbeddedSet set_; + }; + +private: + // Give an early error when key_type is not hashable/eq. + // -------------------------------------------------------------------- + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using AllocTraits = phmap::allocator_traits<allocator_type>; + + static_assert(std::is_lvalue_reference<reference>::value, + "Policy::element() must return a reference"); + + template <typename T> + struct SameAsElementReference : std::is_same< + typename std::remove_cv<typename std::remove_reference<reference>::type>::type, + typename std::remove_cv<typename std::remove_reference<T>::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + // -------------------------------------------------------------------- + template <class T> + using RequiresInsertable = typename std::enable_if< + phmap::disjunction<std::is_convertible<T, init_type>, + SameAsElementReference<T>>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template <class T> + using RequiresNotInit = + typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; + + template <class... Ts> + using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; + +public: + static_assert(std::is_same<pointer, value_type*>::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same<const_pointer, const value_type*>::value, + "Allocators with custom pointer types are not supported"); + + // --------------------- i t e r a t o r ------------------------------ + class iterator + { + friend class parallel_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename parallel_hash_set::value_type; + using reference = + phmap::conditional_t<PolicyTraits::constant_iterators::value, + const value_type&, value_type&>; + using pointer = phmap::remove_reference_t<reference>*; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; + using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; + using EmbeddedIterator = typename EmbeddedSet::iterator; + + iterator() {} + + reference operator*() const { return *it_; } + pointer operator->() const { return &operator*(); } + + iterator& operator++() { + assert(inner_); // null inner means we are already at the end + ++it_; + skip_empty(); + return *this; + } + + iterator operator++(int) { + assert(inner_); // null inner means we are already at the end + auto tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==(const iterator& a, const iterator& b) { + return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); + } + + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: + iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : + inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() + if (inner) + it_end_ = inner->set_.end(); + } + + void skip_empty() { + while (it_ == it_end_) { + ++inner_; + if (inner_ == inner_end_) { + inner_ = nullptr; // marks end() + break; + } + else { + it_ = inner_->set_.begin(); + it_end_ = inner_->set_.end(); + } + } + } + + Inner *inner_ = nullptr; + Inner *inner_end_ = nullptr; + EmbeddedIterator it_, it_end_; + }; + + // --------------------- c o n s t i t e r a t o r ----------------- + class const_iterator + { + friend class parallel_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename parallel_hash_set::value_type; + using reference = typename parallel_hash_set::const_reference; + using pointer = typename parallel_hash_set::const_pointer; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : iter_(std::move(i)) {} + + reference operator*() const { return *(iter_); } + pointer operator->() const { return iter_.operator->(); } + + const_iterator& operator++() { + ++iter_; + return *this; + } + const_iterator operator++(int) { return iter_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.iter_ == b.iter_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) + : iter_(const_cast<Inner**>(inner), + const_cast<Inner**>(inner_end), + const_cast<EmbeddedIterator*>(it)) {} + + iterator iter_; + }; + + using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; + using insert_return_type = InsertReturnType<iterator, node_type>; + + // ------------------------- c o n s t r u c t o r s ------------------ + + parallel_hash_set() noexcept( + std::is_nothrow_default_constructible<hasher>::value&& + std::is_nothrow_default_constructible<key_equal>::value&& + std::is_nothrow_default_constructible<allocator_type>::value) {} + + explicit parallel_hash_set(size_t bucket_cnt, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) { + for (auto& inner : sets_) + inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); + } + + parallel_hash_set(size_t bucket_cnt, + const hasher& hash_param, + const allocator_type& alloc) + : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} + + parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) + : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + + explicit parallel_hash_set(const allocator_type& alloc) + : parallel_hash_set(0, hasher(), key_equal(), alloc) {} + + template <class InputIter> + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { + insert(first, last); + } + + template <class InputIter> + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} + + template <class InputIter> + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template <class InputIter> + parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list<value_type> as the first + // argument like std::unordered_set<value_type> does, we have two overloads + // that accept std::initializer_list<T> and std::initializer_list<init_type>. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list<std::string>, then copies + // // the strings into the set. + // std::unordered_set<std::string> s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list<const char*>, then + // // copies the strings into the set. + // phmap::flat_hash_set<std::string> s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set<int> a, b{a}; + // + // RequiresNotInit<T> is a workaround for gcc prior to 7.1. + // -------------------------------------------------------------------- + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + parallel_hash_set(std::initializer_list<T> init, const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list<init_type> init, + const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(const parallel_hash_set& that) + : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) + : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + for (size_t i=0; i<num_tables; ++i) + sets_[i].set_ = { that.sets_[i].set_, a }; + } + + parallel_hash_set(parallel_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible<hasher>::value&& + std::is_nothrow_copy_constructible<key_equal>::value&& + std::is_nothrow_copy_constructible<allocator_type>::value) + : parallel_hash_set(std::move(that), that.alloc_ref()) { + } + + parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) + { + for (size_t i=0; i<num_tables; ++i) + sets_[i].set_ = { std::move(that.sets_[i]).set_, a }; + } + + parallel_hash_set& operator=(const parallel_hash_set& that) { + for (size_t i=0; i<num_tables; ++i) + sets_[i].set_ = that.sets_[i].set_; + return *this; + } + + parallel_hash_set& operator=(parallel_hash_set&& that) noexcept( + phmap::allocator_traits<allocator_type>::is_always_equal::value && + std::is_nothrow_move_assignable<hasher>::value && + std::is_nothrow_move_assignable<key_equal>::value) { + for (size_t i=0; i<num_tables; ++i) + sets_[i].set_ = std::move(that.sets_[i].set_); + return *this; + } + + ~parallel_hash_set() {} + + iterator begin() { + auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin()); + it.skip_empty(); + return it; + } + + iterator end() { return iterator(); } + const_iterator begin() const { return const_cast<parallel_hash_set *>(this)->begin(); } + const_iterator end() const { return const_cast<parallel_hash_set *>(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + + size_t size() const { + size_t sz = 0; + for (const auto& inner : sets_) + sz += inner.set_.size(); + return sz; + } + + size_t capacity() const { + size_t c = 0; + for (const auto& inner : sets_) + c += inner.set_.capacity(); + return c; + } + + size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } + + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + for (auto& inner : sets_) + { + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); + } + } + + // extension - clears only soecified submap + // ---------------------------------------- + void clear(std::size_t submap_index) { + Inner& inner = sets_[submap_index]; + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map<std::string, int> m; + // m.insert(std::make_pair("abc", 42)); + // -------------------------------------------------------------------- + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, + T* = nullptr> + std::pair<iterator, bool> insert(T&& value) { + return emplace(std::forward<T>(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set<int> s; + // s.insert(n); + // + // flat_hash_set<std::string> s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + std::pair<iterator, bool> insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set<std::pair<std::string, int>> s; + // s.insert({"abc", 42}); + // -------------------------------------------------------------------- + std::pair<iterator, bool> insert(init_type&& value) { + return emplace(std::move(value)); + } + + template <class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward<T>(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template <class InputIt> + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) insert(*first); + } + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> + void insert(std::initializer_list<T> ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list<init_type> ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) + return {end(), false, node_type()}; + auto& key = node.key(); + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + + typename Lockable::UniqueLock m(inner); + auto res = set.insert(std::move(node), hashval); + return { make_iterator(&inner, res.position), + res.inserted, + res.inserted ? node_type() : std::move(res.node) }; + } + + iterator insert(const_iterator, node_type&& node) { + return insert(std::move(node)).first; + } + + struct ReturnKey_ + { + template <class Key, class... Args> + Key operator()(Key&& k, const Args&...) const { + return std::forward<Key>(k); + } + }; + + // -------------------------------------------------------------------- + // phmap expension: emplace_with_hash + // ---------------------------------- + // same as emplace, but hashval is provided + // -------------------------------------------------------------------- + template <class K, class... Args> + std::pair<iterator, bool> emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) + { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); + } + + struct EmplaceDecomposableHashval + { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable_with_hash(key, hashval, std::forward<Args>(args)...); + } + parallel_hash_set& s; + size_t hashval; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template <class... Args, typename std::enable_if< + IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, + std::forward<Args>(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template <class... Args, typename std::enable_if< + !IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { + typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash<true> f { + inner, std::move(*slot), hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } + + template <class... Args> + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward<Args>(args)...).first; + } + + template <class K = key_type, class F> + iterator lazy_emplace_with_hash(size_t hashval, const key_arg<K>& key, F&& f) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); + } + + // -------------------------------------------------------------------- + // end of phmap expension + // -------------------------------------------------------------------- + + template <class K, class... Args> + std::pair<iterator, bool> emplace_decomposable(const K& key, Args&&... args) + { + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); + } + + struct EmplaceDecomposable + { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, std::forward<Args>(args)...); + } + parallel_hash_set& s; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template <class... Args, typename std::enable_if< + IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward<Args>(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template <class... Args, typename std::enable_if< + !IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + size_t hashval = this->hash(PolicyTraits::key(slot)); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash<true> f { + inner, std::move(*slot), hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } + + template <class... Args> + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward<Args>(args)...).first; + } + + iterator make_iterator(Inner* inner, const EmbeddedIterator it) + { + if (it == inner->set_.end()) + return iterator(); + return iterator(inner, &sets_[0] + num_tables, it); + } + + std::pair<iterator, bool> make_rv(Inner* inner, + const std::pair<EmbeddedIterator, bool>& res) + { + return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; + } + + template <class K = key_type, class F> + iterator lazy_emplace(const key_arg<K>& key, F&& f) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); + } + + template <class K = key_type, class FExists, class FEmplace> + bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(key, m); + Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.lazy_emplace_at(std::get<1>(res), std::forward<FEmplace>(fEmplace)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward<FExists>(fExists)(Policy::value(&*it)); + } + return std::get<2>(res); + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + // -------------------------------------------------------------------- + template <class K = key_type> + size_type erase(const key_arg<K>& key) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UpgradeLock m(inner); + auto it = set.find(key, hashval); + if (it == set.end()) + return 0; + + typename Lockable::UpgradeToUnique unique(m); + set._erase(it); + return 1; + } + + // -------------------------------------------------------------------- + iterator erase(const_iterator cit) { return erase(cit.iter_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if (<pred>) { + // m._erase(it++); + // } else { + // ++it; + // } + // } + // -------------------------------------------------------------------- + void _erase(iterator it) { + assert(it.inner_ != nullptr); + it.inner_->set_._erase(it.it_); + } + void _erase(const_iterator cit) { _erase(cit.iter_); } + + // This overload is necessary because otherwise erase<K>(const K&) would be + // a better match if non-const iterator is passed as an argument. + // -------------------------------------------------------------------- + iterator erase(iterator it) { _erase(it++); return it; } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); + } + return last.iter_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + // -------------------------------------------------------------------- + template <typename E = Eq> + void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>& src) { // NOLINT + assert(this != &src); + if (this != &src) + { + for (size_t i=0; i<num_tables; ++i) + { + typename Lockable::UniqueLocks l(sets_[i], src.sets_[i]); + sets_[i].set_.merge(src.sets_[i].set_); + } + } + } + + template <typename E = Eq> + void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>&& src) { + merge(src); + } + + node_type extract(const_iterator position) { + return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); + } + + template < + class K = key_type, + typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> + node_type extract(const key_arg<K>& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + void swap(parallel_hash_set& that) noexcept( + IsNoThrowSwappable<EmbeddedSet>() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable<allocator_type>())) { + using std::swap; + for (size_t i=0; i<num_tables; ++i) + { + typename Lockable::UniqueLocks l(sets_[i], that.sets_[i]); + swap(sets_[i].set_, that.sets_[i].set_); + } + } + + void rehash(size_t n) { + size_t nn = n / num_tables; + for (auto& inner : sets_) + { + typename Lockable::UniqueLock m(inner); + inner.set_.rehash(nn); + } + } + + void reserve(size_t n) + { + size_t target = GrowthToLowerboundCapacity(n); + size_t normalized = 16 * NormalizeCapacity(n / num_tables); + rehash(normalized > target ? normalized : target); + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + // -------------------------------------------------------------------- + template <class K = key_type> + size_t count(const key_arg<K>& key) const { + return find(key) == end() ? 0 : 1; + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + // -------------------------------------------------------------------- + void prefetch_hash(size_t hashval) const { + const Inner& inner = sets_[subidx(hashval)]; + const auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast<Inner&>(inner)); + set.prefetch_hash(hashval); + } + + template <class K = key_type> + void prefetch(const key_arg<K>& key) const { + prefetch_hash(this->hash(key)); + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + // -------------------------------------------------------------------- + template <class K = key_type> + iterator find(const key_arg<K>& key, size_t hashval) { + typename Lockable::SharedLock m; + return find(key, hashval, m); + } + + template <class K = key_type> + iterator find(const key_arg<K>& key) { + return find(key, this->hash(key)); + } + + template <class K = key_type> + const_iterator find(const key_arg<K>& key, size_t hashval) const { + return const_cast<parallel_hash_set*>(this)->find(key, hashval); + } + + template <class K = key_type> + const_iterator find(const key_arg<K>& key) const { + return find(key, this->hash(key)); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key) const { + return find(key) != end(); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key, size_t hashval) const { + return find(key, hashval) != end(); + } + + template <class K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + template <class K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K>& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { + size_t sz = 0; + for (const auto& inner : sets_) + { + typename Lockable::SharedLock m(const_cast<Inner&>(inner)); + sz += inner.set_.bucket_count(); + } + return sz; + } + + float load_factor() const { + size_t _capacity = bucket_count(); + return _capacity ? static_cast<float>(static_cast<double>(size()) / _capacity) : 0; + } + + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { + return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); + } + + friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { + return !(a == b); + } + + friend void swap(parallel_hash_set& a, + parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } + + template <class K> + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } + +#ifndef PHMAP_NON_DETERMINISTIC + template<typename OutputArchive> + bool dump(OutputArchive& ar) const; + + template<typename InputArchive> + bool load(InputArchive& ar); +#endif + +private: + template <class Container, typename Enabler> + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; + + struct FindElement + { + template <class K, class... Args> + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const parallel_hash_set& s; + }; + + struct HashElement + { + template <class K, class... Args> + size_t operator()(const K& key, Args&&...) const { + return phmap_mix<sizeof(size_t)>()(h(key)); + } + const hasher& h; + }; + + template <class K1> + struct EqualElement + { + template <class K2, class... Args> + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + // -------------------------------------------------------------------- + void erase_meta_only(const_iterator cit) { + auto &it = cit.iter_; + assert(it.set_ != nullptr); + it.set_.erase_meta_only(const_iterator(it.it_)); + } + + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + for (auto& inner : sets_) + { + typename Lockable::UniqueLock m(inner); + inner.set_.drop_deletes_without_resize(); + } + } + + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast<Inner&>(inner)); + return set.has_element(elem, hashval); + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + // -------------------------------------------------------------------- + parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { + parallel_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + + parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { + parallel_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + +protected: + template <class K = key_type, class L = typename Lockable::SharedLock> + pointer find_ptr(const key_arg<K>& key, size_t hashval, L& mutexlock) + { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return set.find_ptr(key, hashval); + } + + template <class K = key_type, class L = typename Lockable::SharedLock> + iterator find(const key_arg<K>& key, size_t hashval, L& mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return make_iterator(&inner, set.find(key, hashval)); + } + + template <class K> + std::tuple<Inner*, size_t, bool> + find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(typename Lockable::UniqueLock(inner)); + auto p = set.find_or_prepare_insert(key, hashval); // std::pair<size_t, bool> + return std::make_tuple(&inner, p.first, p.second); + } + + template <class K> + std::tuple<Inner*, size_t, bool> + find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { + return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock); + } + + iterator iterator_at(Inner *inner, + const EmbeddedIterator& it) { + return {inner, &sets_[0] + num_tables, it}; + } + const_iterator iterator_at(Inner *inner, + const EmbeddedIterator& it) const { + return {inner, &sets_[0] + num_tables, it}; + } + + static size_t subidx(size_t hashval) { + return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; + } + + static size_t subcnt() { + return num_tables; + } + +private: + friend struct RawHashSetTestOnlyAccess; + + size_t growth_left() { + size_t sz = 0; + for (const auto& set : sets_) + sz += set.growth_left(); + return sz; + } + + hasher& hash_ref() { return sets_[0].set_.hash_ref(); } + const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } + key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } + const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } + allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } + const allocator_type& alloc_ref() const { + return sets_[0].set_.alloc_ref(); + } + +protected: // protected in case users want to derive fromm this + std::array<Inner, num_tables> sets_; +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <size_t N, + template <class, class, class, class> class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc> +{ + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map<K, IncompleteType>. + // MappedReference<> may be a non-reference type. + template <class P> + using MappedReference = decltype(P::value( + std::addressof(std::declval<typename parallel_hash_map::reference>()))); + + // MappedConstReference<> may be a non-reference type. + template <class P> + using MappedConstReference = decltype(P::value( + std::addressof(std::declval<typename parallel_hash_map::const_reference>()))); + + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + using Base = typename parallel_hash_map::parallel_hash_set; + using Lockable = phmap::LockableImpl<Mtx_>; + +public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + + static_assert(!std::is_reference<key_type>::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference<mapped_type>::value, ""); + + using iterator = typename parallel_hash_map::parallel_hash_set::iterator; + using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; + + parallel_hash_map() {} + +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using parallel_hash_map::parallel_hash_set::parallel_hash_set; +#endif + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map<int, int> m; + // m.insert_or_assign(n, n); + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { + return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { + return insert_or_assign_impl(std::forward<K>(k), v); + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { + return insert_or_assign_impl(k, std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { + return insert_or_assign_impl(k, v); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { + return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { + return insert_or_assign(std::forward<K>(k), v).first; + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { + return insert_or_assign(k, std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { + return insert_or_assign(k, v).first; + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0, + K* = nullptr> + std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { + return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0> + std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { + return try_emplace_impl(k, std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, K* = nullptr> + iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { + return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; + } + + template <class K = key_type, class... Args> + iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { + return try_emplace(k, std::forward<Args>(args)...).first; + } + + template <class K = key_type, class P = Policy> + MappedReference<P> at(const key_arg<K>& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template <class K = key_type, class P = Policy> + MappedConstReference<P> at(const key_arg<K>& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + // ----------- phmap extensions -------------------------- + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0, + K* = nullptr> + std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, key_arg<K>&& k, Args&&... args) { + return try_emplace_impl_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0> + std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, const key_arg<K>& k, Args&&... args) { + return try_emplace_impl_with_hash(hashval, k, std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, K* = nullptr> + iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg<K>&& k, Args&&... args) { + return try_emplace_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...).first; + } + + template <class K = key_type, class... Args> + iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg<K>& k, Args&&... args) { + return try_emplace_with_hash(hashval, k, std::forward<Args>(args)...).first; + } + + // if map contains key, lambda is called with the mapped value (under read lock protection), + // and if_contains returns true. This is a const API and lambda should not modify the value + // ----------------------------------------------------------------------------------------- + template <class K = key_type, class F> + bool if_contains(const key_arg<K>& key, F&& f) const { + return const_cast<parallel_hash_map*>(this)->template + modify_if_impl<K, F, typename Lockable::SharedLock>(key, std::forward<F>(f)); + } + + // if map contains key, lambda is called with the mapped value without read lock protection, + // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value + // This should be used only if we know that no other thread may be mutating the map at the time. + // ----------------------------------------------------------------------------------------- + template <class K = key_type, class F> + bool if_contains_unsafe(const key_arg<K>& key, F&& f) const { + return const_cast<parallel_hash_map*>(this)->template + modify_if_impl<K, F, LockableBaseImpl<phmap::NullMutex>::DoNothing>(key, std::forward<F>(f)); + } + + // if map contains key, lambda is called with the mapped value (under write lock protection), + // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value + // ---------------------------------------------------------------------------------------------------- + template <class K = key_type, class F> + bool modify_if(const key_arg<K>& key, F&& f) { + return modify_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); + } + + + // if map contains key, lambda is called with the mapped value (under write lock protection). + // If the lambda returns true, the key is subsequently erased from the map (the write lock + // is only released after erase). + // returns true if key was erased, false otherwise. + // ---------------------------------------------------------------------------------------------------- + template <class K = key_type, class F> + bool erase_if(const key_arg<K>& key, F&& f) { + return erase_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); + } + + // if map does not contains key, it is inserted and the mapped value is value-constructed + // with the provided arguments (if any), as with try_emplace. + // if map already contains key, then the lambda is called with the mapped value (under + // write lock protection) and can update the mapped value. + // returns true if key was not already present, false otherwise. + // --------------------------------------------------------------------------------------- + template <class K = key_type, class F, class... Args> + bool try_emplace_l(K&& k, F&& f, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward<F>(f)(Policy::value(&*it)); + } + return std::get<2>(res); + } + + // ----------- end of phmap extensions -------------------------- + + template <class K = key_type, class P = Policy, K* = nullptr> + MappedReference<P> operator[](key_arg<K>&& key) { + return Policy::value(&*try_emplace(std::forward<K>(key)).first); + } + + template <class K = key_type, class P = Policy> + MappedReference<P> operator[](const key_arg<K>& key) { + return Policy::value(&*try_emplace(key).first); + } + +private: + template <class K = key_type, class F, class L> + bool modify_if_impl(const key_arg<K>& key, F&& f) { +#if __cplusplus >= 201703L + static_assert(std::is_invocable<F, mapped_type&>::value); +#endif + L m; + auto ptr = this->template find_ptr<K, L>(key, this->hash(key), m); + if (ptr == nullptr) + return false; + std::forward<F>(f)(Policy::value(ptr)); + return true; + } + + template <class K = key_type, class F, class L> + bool erase_if_impl(const key_arg<K>& key, F&& f) { +#if __cplusplus >= 201703L + static_assert(std::is_invocable<F, mapped_type&>::value); +#endif + L m; + auto it = this->template find<K, L>(key, this->hash(key), m); + if (it == this->end()) return false; + if (std::forward<F>(f)(Policy::value(&*it))) + { + this->erase(it); + return true; + } + return false; + } + + + template <class K, class V> + std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::forward<K>(k), std::forward<V>(v)); + else + Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward<V>(v); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + template <class K = key_type, class... Args> + std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + template <class K = key_type, class... Args> + std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + +}; + + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template <class Alloc, class T, class Tuple> +void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { + memory_internal::ConstructFromTupleImpl( + alloc, ptr, std::forward<Tuple>(t), + phmap::make_index_sequence< + std::tuple_size<typename std::decay<Tuple>::type>::value>()); +} + +// Constructs T using the args specified in the tuple and calls F with the +// constructed value. +// ---------------------------------------------------------------------------- +template <class T, class Tuple, class F> +decltype(std::declval<F>()(std::declval<T>())) WithConstructed( + Tuple&& t, F&& f) { + return memory_internal::WithConstructedImpl<T>( + std::forward<Tuple>(t), + phmap::make_index_sequence< + std::tuple_size<typename std::decay<Tuple>::type>::value>(), + std::forward<F>(f)); +} + +// ---------------------------------------------------------------------------- +// Given arguments of an std::pair's consructor, PairArgs() returns a pair of +// tuples with references to the passed arguments. The tuples contain +// constructor arguments for the first and the second elements of the pair. +// +// The following two snippets are equivalent. +// +// 1. std::pair<F, S> p(args...); +// +// 2. auto a = PairArgs(args...); +// std::pair<F, S> p(std::piecewise_construct, +// std::move(p.first), std::move(p.second)); +// ---------------------------------------------------------------------------- +inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; } + +template <class F, class S> +std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) { + return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)), + std::forward_as_tuple(std::forward<S>(s))}; +} + +template <class F, class S> +std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs( + const std::pair<F, S>& p) { + return PairArgs(p.first, p.second); +} + +template <class F, class S> +std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) { + return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second)); +} + +template <class F, class S> +auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) + -> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), + memory_internal::TupleRef(std::forward<S>(s)))) { + return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), + memory_internal::TupleRef(std::forward<S>(s))); +} + +// A helper function for implementing apply() in map policies. +// ---------------------------------------------------------------------------- +template <class F, class... Args> +auto DecomposePair(F&& f, Args&&... args) + -> decltype(memory_internal::DecomposePairImpl( + std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) { + return memory_internal::DecomposePairImpl( + std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); +} + +// A helper function for implementing apply() in set policies. +// ---------------------------------------------------------------------------- +template <class F, class Arg> +decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>())) +DecomposeValue(F&& f, Arg&& arg) { + const auto& key = arg; + return std::forward<F>(f)(key, std::forward<Arg>(arg)); +} + + +// -------------------------------------------------------------------------- +// Policy: a policy defines how to perform different operations on +// the slots of the hashtable (see hash_policy_traits.h for the full interface +// of policy). +// +// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The +// functor should accept a key and return size_t as hash. For best performance +// it is important that the hash function provides high entropy across all bits +// of the hash. +// +// Eq: a (possibly polymorphic) functor that compares two keys for equality. It +// should accept two (of possibly different type) keys and return a bool: true +// if they are equal, false if they are not. If two keys compare equal, then +// their hash values as defined by Hash MUST be equal. +// +// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which +// the storage of the hashtable will be allocated and the elements will be +// constructed and destroyed. +// -------------------------------------------------------------------------- +template <class T> +struct FlatHashSetPolicy +{ + using slot_type = T; + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + phmap::allocator_traits<Allocator>::construct(*alloc, slot, + std::forward<Args>(args)...); + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + phmap::allocator_traits<Allocator>::destroy(*alloc, slot); + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } + + static T& element(slot_type* slot) { return *slot; } + + template <class F, class... Args> + static decltype(phmap::priv::DecomposeValue( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue( + std::forward<F>(f), std::forward<Args>(args)...); + } + + static size_t space_used(const T*) { return 0; } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <class K, class V> +struct FlatHashMapPolicy +{ + using slot_policy = priv::map_slot_policy<K, V>; + using slot_type = typename slot_policy::slot_type; + using key_type = K; + using mapped_type = V; + using init_type = std::pair</*non const*/ key_type, mapped_type>; + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + slot_policy::construct(alloc, slot, std::forward<Args>(args)...); + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + slot_policy::destroy(alloc, slot); + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + slot_policy::transfer(alloc, new_slot, old_slot); + } + + template <class F, class... Args> + static decltype(phmap::priv::DecomposePair( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward<F>(f), + std::forward<Args>(args)...); + } + + static size_t space_used(const slot_type*) { return 0; } + + static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } + + static V& value(std::pair<const K, V>* kv) { return kv->second; } + static const V& value(const std::pair<const K, V>* kv) { return kv->second; } +}; + +template <class Reference, class Policy> +struct node_hash_policy { + static_assert(std::is_lvalue_reference<Reference>::value, ""); + + using slot_type = typename std::remove_cv< + typename std::remove_reference<Reference>::type>::type*; + + template <class Alloc, class... Args> + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + *slot = Policy::new_element(alloc, std::forward<Args>(args)...); + } + + template <class Alloc> + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::delete_element(alloc, *slot); + } + + template <class Alloc> + static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { + *new_slot = *old_slot; + } + + static size_t space_used(const slot_type* slot) { + if (slot == nullptr) return Policy::element_space_used(nullptr); + return Policy::element_space_used(*slot); + } + + static Reference element(slot_type* slot) { return **slot; } + + template <class T, class P = Policy> + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + template <class... Ts, class P = Policy> + static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward<Ts>(ts)...)) { + return P::apply(std::forward<Ts>(ts)...); + } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <class T> +struct NodeHashSetPolicy + : phmap::priv::node_hash_policy<T&, NodeHashSetPolicy<T>> +{ + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template <class Allocator, class... Args> + static T* new_element(Allocator* alloc, Args&&... args) { + using ValueAlloc = + typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; + ValueAlloc value_alloc(*alloc); + T* res = phmap::allocator_traits<ValueAlloc>::allocate(value_alloc, 1); + phmap::allocator_traits<ValueAlloc>::construct(value_alloc, res, + std::forward<Args>(args)...); + return res; + } + + template <class Allocator> + static void delete_element(Allocator* alloc, T* elem) { + using ValueAlloc = + typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; + ValueAlloc value_alloc(*alloc); + phmap::allocator_traits<ValueAlloc>::destroy(value_alloc, elem); + phmap::allocator_traits<ValueAlloc>::deallocate(value_alloc, elem, 1); + } + + template <class F, class... Args> + static decltype(phmap::priv::DecomposeValue( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue( + std::forward<F>(f), std::forward<Args>(args)...); + } + + static size_t element_space_used(const T*) { return sizeof(T); } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <class Key, class Value> +class NodeHashMapPolicy + : public phmap::priv::node_hash_policy< + std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>> +{ + using value_type = std::pair<const Key, Value>; + +public: + using key_type = Key; + using mapped_type = Value; + using init_type = std::pair</*non const*/ key_type, mapped_type>; + + template <class Allocator, class... Args> + static value_type* new_element(Allocator* alloc, Args&&... args) { + using PairAlloc = typename phmap::allocator_traits< + Allocator>::template rebind_alloc<value_type>; + PairAlloc pair_alloc(*alloc); + value_type* res = + phmap::allocator_traits<PairAlloc>::allocate(pair_alloc, 1); + phmap::allocator_traits<PairAlloc>::construct(pair_alloc, res, + std::forward<Args>(args)...); + return res; + } + + template <class Allocator> + static void delete_element(Allocator* alloc, value_type* pair) { + using PairAlloc = typename phmap::allocator_traits< + Allocator>::template rebind_alloc<value_type>; + PairAlloc pair_alloc(*alloc); + phmap::allocator_traits<PairAlloc>::destroy(pair_alloc, pair); + phmap::allocator_traits<PairAlloc>::deallocate(pair_alloc, pair, 1); + } + + template <class F, class... Args> + static decltype(phmap::priv::DecomposePair( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward<F>(f), + std::forward<Args>(args)...); + } + + static size_t element_space_used(const value_type*) { + return sizeof(value_type); + } + + static Value& value(value_type* elem) { return elem->second; } + static const Value& value(const value_type* elem) { return elem->second; } +}; + + +// -------------------------------------------------------------------------- +// hash_default +// -------------------------------------------------------------------------- + +#if PHMAP_HAVE_STD_STRING_VIEW + +// support char16_t wchar_t .... +template<class CharT> +struct StringHashT +{ + using is_transparent = void; + + size_t operator()(std::basic_string_view<CharT> v) const { + std::string_view bv{reinterpret_cast<const char*>(v.data()), v.size() * sizeof(CharT)}; + return std::hash<std::string_view>()(bv); + } +}; + +// Supports heterogeneous lookup for basic_string<T>-like elements. +template<class CharT> +struct StringHashEqT +{ + using Hash = StringHashT<CharT>; + + struct Eq { + using is_transparent = void; + + bool operator()(std::basic_string_view<CharT> lhs, std::basic_string_view<CharT> rhs) const { + return lhs == rhs; + } + }; +}; + +template <> +struct HashEq<std::string> : StringHashEqT<char> {}; + +template <> +struct HashEq<std::string_view> : StringHashEqT<char> {}; + +// char16_t +template <> +struct HashEq<std::u16string> : StringHashEqT<char16_t> {}; + +template <> +struct HashEq<std::u16string_view> : StringHashEqT<char16_t> {}; + +// wchar_t +template <> +struct HashEq<std::wstring> : StringHashEqT<wchar_t> {}; + +template <> +struct HashEq<std::wstring_view> : StringHashEqT<wchar_t> {}; + +#endif + +// Supports heterogeneous lookup for pointers and smart pointers. +// ------------------------------------------------------------- +template <class T> +struct HashEq<T*> +{ + struct Hash { + using is_transparent = void; + template <class U> + size_t operator()(const U& ptr) const { + return phmap::Hash<const T*>{}(HashEq::ToPtr(ptr)); + } + }; + + struct Eq { + using is_transparent = void; + template <class A, class B> + bool operator()(const A& a, const B& b) const { + return HashEq::ToPtr(a) == HashEq::ToPtr(b); + } + }; + +private: + static const T* ToPtr(const T* ptr) { return ptr; } + + template <class U, class D> + static const T* ToPtr(const std::unique_ptr<U, D>& ptr) { + return ptr.get(); + } + + template <class U> + static const T* ToPtr(const std::shared_ptr<U>& ptr) { + return ptr.get(); + } +}; + +template <class T, class D> +struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {}; + +template <class T> +struct HashEq<std::shared_ptr<T>> : HashEq<T*> {}; + +namespace hashtable_debug_internal { + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template <typename Set> +struct HashtableDebugAccess<Set, phmap::void_t<typename Set::raw_hash_set>> +{ + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + + static size_t GetNumProbes(const Set& set, + const typename Set::key_type& key) { + size_t num_probes = 0; + size_t hashval = set.hash(key); + auto seq = set.probe(hashval); + while (true) { + priv::Group g{set.ctrl_ + seq.offset()}; + for (int i : g.Match(priv::H2(hashval))) { + if (Traits::apply( + typename Set::template EqualElement<typename Set::key_type>{ + key, set.eq_ref()}, + Traits::element(set.slots_ + seq.offset((size_t)i)))) + return num_probes; + ++num_probes; + } + if (g.MatchEmpty()) return num_probes; + seq.next(); + ++num_probes; + } + } + + static size_t AllocatedByteSize(const Set& c) { + size_t capacity = c.capacity_; + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(capacity); + size_t m = layout.AllocSize(); + + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * c.size(); + } else { + for (size_t i = 0; i != capacity; ++i) { + if (priv::IsFull(c.ctrl_[i])) { + m += Traits::space_used(c.slots_ + i); + } + } + } + return m; + } + + static size_t LowerBoundAllocatedByteSize(size_t size) { + size_t capacity = GrowthToLowerboundCapacity(size); + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); + size_t m = layout.AllocSize(); + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * size; + } + return m; + } +}; + +} // namespace hashtable_debug_internal +} // namespace priv + +// ----------------------------------------------------------------------------- +// phmap::flat_hash_set +// ----------------------------------------------------------------------------- +// An `phmap::flat_hash_set<T>` is an unordered associative container which has +// been optimized for both speed and memory footprint in most common use cases. +// Its interface is similar to that of `std::unordered_set<T>` with the +// following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the set is provided a compatible heterogeneous +// hashing function and equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `_erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h +class flat_hash_set + : public phmap::priv::raw_hash_set< + phmap::priv::FlatHashSetPolicy<T>, Hash, Eq, Alloc> +{ + using Base = typename flat_hash_set::raw_hash_set; + +public: + flat_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_set; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::flat_hash_map +// ----------------------------------------------------------------------------- +// +// An `phmap::flat_hash_map<K, V>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map<K, V>` with +// the following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `_erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template <class K, class V, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h +class flat_hash_map : public phmap::priv::raw_hash_map< + phmap::priv::FlatHashMapPolicy<K, V>, + Hash, Eq, Alloc> { + using Base = typename flat_hash_map::raw_hash_map; + +public: + flat_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_map; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::node_hash_set +// ----------------------------------------------------------------------------- +// An `phmap::node_hash_set<T>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_set<T>` with the +// following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h +class node_hash_set + : public phmap::priv::raw_hash_set< + phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> +{ + using Base = typename node_hash_set::raw_hash_set; + +public: + node_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_set; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::node_hash_map +// ----------------------------------------------------------------------------- +// +// An `phmap::node_hash_map<K, V>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map<K, V>` with +// the following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template <class Key, class Value, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h +class node_hash_map + : public phmap::priv::raw_hash_map< + phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, + Alloc> +{ + using Base = typename node_hash_map::raw_hash_map; + +public: + node_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_map; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_flat_hash_set +// ----------------------------------------------------------------------------- +template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> // default values in phmap_fwd_decl.h +class parallel_flat_hash_set + : public phmap::priv::parallel_hash_set< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::FlatHashSetPolicy<T>, + Hash, Eq, Alloc> +{ + using Base = typename parallel_flat_hash_set::parallel_hash_set; + +public: + parallel_flat_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h +// ----------------------------------------------------------------------------- +template <class K, class V, class Hash, class Eq, class Alloc, size_t N, class Mtx_> +class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::FlatHashMapPolicy<K, V>, + Hash, Eq, Alloc> +{ + using Base = typename parallel_flat_hash_map::parallel_hash_map; + +public: + parallel_flat_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_map; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::try_emplace_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_node_hash_set +// ----------------------------------------------------------------------------- +template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> +class parallel_node_hash_set + : public phmap::priv::parallel_hash_set< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> +{ + using Base = typename parallel_node_hash_set::parallel_hash_set; + +public: + parallel_node_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_node_hash_map +// ----------------------------------------------------------------------------- +template <class Key, class Value, class Hash, class Eq, class Alloc, size_t N, class Mtx_> +class parallel_node_hash_map + : public phmap::priv::parallel_hash_map< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, + Alloc> +{ + using Base = typename parallel_node_hash_map::parallel_hash_map; + +public: + parallel_node_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_map; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::try_emplace_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +} // namespace phmap + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + + +#endif // phmap_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_base.h b/benchmarks/others/parallel_hashmap/phmap_base.h new file mode 100644 index 00000000..6b9ea9ee --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_base.h @@ -0,0 +1,5171 @@ +#if !defined(phmap_base_h_guard_) +#define phmap_base_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <initializer_list> +#include <iterator> +#include <string> +#include <type_traits> +#include <utility> +#include <functional> +#include <tuple> +#include <utility> +#include <memory> +#include <mutex> // for std::lock + +#include "phmap_config.h" + +#ifdef PHMAP_HAVE_SHARED_MUTEX + #include <shared_mutex> // after "phmap_config.h" +#endif + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4582) // constructor is not implicitly called + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member +#endif // _MSC_VER + +namespace phmap { + +template <class T> using Allocator = typename std::allocator<T>; + +template<class T1, class T2> using Pair = typename std::pair<T1, T2>; + +template <class T> +struct EqualTo +{ + inline bool operator()(const T& a, const T& b) const + { + return std::equal_to<T>()(a, b); + } +}; + +template <class T> +struct Less +{ + inline bool operator()(const T& a, const T& b) const + { + return std::less<T>()(a, b); + } +}; + +namespace type_traits_internal { + +template <typename... Ts> +struct VoidTImpl { + using type = void; +}; + +// This trick to retrieve a default alignment is necessary for our +// implementation of aligned_storage_t to be consistent with any implementation +// of std::aligned_storage. +// --------------------------------------------------------------------------- +template <size_t Len, typename T = std::aligned_storage<Len>> +struct default_alignment_of_aligned_storage; + +template <size_t Len, size_t Align> +struct default_alignment_of_aligned_storage<Len, + std::aligned_storage<Len, Align>> { + static constexpr size_t value = Align; +}; + +// NOTE: The `is_detected` family of templates here differ from the library +// fundamentals specification in that for library fundamentals, `Op<Args...>` is +// evaluated as soon as the type `is_detected<Op, Args...>` undergoes +// substitution, regardless of whether or not the `::value` is accessed. That +// is inconsistent with all other standard traits and prevents lazy evaluation +// in larger contexts (such as if the `is_detected` check is a trailing argument +// of a `conjunction`. This implementation opts to instead be lazy in the same +// way that the standard traits are (this "defect" of the detection idiom +// specifications has been reported). +// --------------------------------------------------------------------------- + +template <class Enabler, template <class...> class Op, class... Args> +struct is_detected_impl { + using type = std::false_type; +}; + +template <template <class...> class Op, class... Args> +struct is_detected_impl<typename VoidTImpl<Op<Args...>>::type, Op, Args...> { + using type = std::true_type; +}; + +template <template <class...> class Op, class... Args> +struct is_detected : is_detected_impl<void, Op, Args...>::type {}; + +template <class Enabler, class To, template <class...> class Op, class... Args> +struct is_detected_convertible_impl { + using type = std::false_type; +}; + +template <class To, template <class...> class Op, class... Args> +struct is_detected_convertible_impl< + typename std::enable_if<std::is_convertible<Op<Args...>, To>::value>::type, + To, Op, Args...> { + using type = std::true_type; +}; + +template <class To, template <class...> class Op, class... Args> +struct is_detected_convertible + : is_detected_convertible_impl<void, To, Op, Args...>::type {}; + +template <typename T> +using IsCopyAssignableImpl = + decltype(std::declval<T&>() = std::declval<const T&>()); + +template <typename T> +using IsMoveAssignableImpl = decltype(std::declval<T&>() = std::declval<T&&>()); + +} // namespace type_traits_internal + +template <typename T> +struct is_copy_assignable : type_traits_internal::is_detected< + type_traits_internal::IsCopyAssignableImpl, T> { +}; + +template <typename T> +struct is_move_assignable : type_traits_internal::is_detected< + type_traits_internal::IsMoveAssignableImpl, T> { +}; + +// --------------------------------------------------------------------------- +// void_t() +// +// Ignores the type of any its arguments and returns `void`. In general, this +// metafunction allows you to create a general case that maps to `void` while +// allowing specializations that map to specific types. +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::void_t` metafunction. +// +// NOTE: `phmap::void_t` does not use the standard-specified implementation so +// that it can remain compatible with gcc < 5.1. This can introduce slightly +// different behavior, such as when ordering partial specializations. +// --------------------------------------------------------------------------- +template <typename... Ts> +using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; + +// --------------------------------------------------------------------------- +// conjunction +// +// Performs a compile-time logical AND operation on the passed types (which +// must have `::value` members convertible to `bool`. Short-circuits if it +// encounters any `false` members (and does not compare the `::value` members +// of any remaining arguments). +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::conjunction` metafunction. +// --------------------------------------------------------------------------- +template <typename... Ts> +struct conjunction; + +template <typename T, typename... Ts> +struct conjunction<T, Ts...> + : std::conditional<T::value, conjunction<Ts...>, T>::type {}; + +template <typename T> +struct conjunction<T> : T {}; + +template <> +struct conjunction<> : std::true_type {}; + +// --------------------------------------------------------------------------- +// disjunction +// +// Performs a compile-time logical OR operation on the passed types (which +// must have `::value` members convertible to `bool`. Short-circuits if it +// encounters any `true` members (and does not compare the `::value` members +// of any remaining arguments). +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::disjunction` metafunction. +// --------------------------------------------------------------------------- +template <typename... Ts> +struct disjunction; + +template <typename T, typename... Ts> +struct disjunction<T, Ts...> : + std::conditional<T::value, T, disjunction<Ts...>>::type {}; + +template <typename T> +struct disjunction<T> : T {}; + +template <> +struct disjunction<> : std::false_type {}; + +template <typename T> +struct negation : std::integral_constant<bool, !T::value> {}; + +template <typename T> +struct is_trivially_destructible + : std::integral_constant<bool, __has_trivial_destructor(T) && + std::is_destructible<T>::value> {}; + +template <typename T> +struct is_trivially_default_constructible + : std::integral_constant<bool, __has_trivial_constructor(T) && + std::is_default_constructible<T>::value && + is_trivially_destructible<T>::value> {}; + +template <typename T> +struct is_trivially_copy_constructible + : std::integral_constant<bool, __has_trivial_copy(T) && + std::is_copy_constructible<T>::value && + is_trivially_destructible<T>::value> {}; + +template <typename T> +struct is_trivially_copy_assignable + : std::integral_constant< + bool, __has_trivial_assign(typename std::remove_reference<T>::type) && + phmap::is_copy_assignable<T>::value> {}; + +// ----------------------------------------------------------------------------- +// C++14 "_t" trait aliases +// ----------------------------------------------------------------------------- + +template <typename T> +using remove_cv_t = typename std::remove_cv<T>::type; + +template <typename T> +using remove_const_t = typename std::remove_const<T>::type; + +template <typename T> +using remove_volatile_t = typename std::remove_volatile<T>::type; + +template <typename T> +using add_cv_t = typename std::add_cv<T>::type; + +template <typename T> +using add_const_t = typename std::add_const<T>::type; + +template <typename T> +using add_volatile_t = typename std::add_volatile<T>::type; + +template <typename T> +using remove_reference_t = typename std::remove_reference<T>::type; + +template <typename T> +using add_lvalue_reference_t = typename std::add_lvalue_reference<T>::type; + +template <typename T> +using add_rvalue_reference_t = typename std::add_rvalue_reference<T>::type; + +template <typename T> +using remove_pointer_t = typename std::remove_pointer<T>::type; + +template <typename T> +using add_pointer_t = typename std::add_pointer<T>::type; + +template <typename T> +using make_signed_t = typename std::make_signed<T>::type; + +template <typename T> +using make_unsigned_t = typename std::make_unsigned<T>::type; + +template <typename T> +using remove_extent_t = typename std::remove_extent<T>::type; + +template <typename T> +using remove_all_extents_t = typename std::remove_all_extents<T>::type; + +template <size_t Len, size_t Align = type_traits_internal:: + default_alignment_of_aligned_storage<Len>::value> +using aligned_storage_t = typename std::aligned_storage<Len, Align>::type; + +template <typename T> +using decay_t = typename std::decay<T>::type; + +template <bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template <bool B, typename T, typename F> +using conditional_t = typename std::conditional<B, T, F>::type; + + +template <typename... T> +using common_type_t = typename std::common_type<T...>::type; + +template <typename T> +using underlying_type_t = typename std::underlying_type<T>::type; + +template< class F, class... ArgTypes> +#if PHMAP_HAVE_CC17 + using invoke_result_t = typename std::invoke_result_t<F, ArgTypes...>; +#else + using invoke_result_t = typename std::result_of<F(ArgTypes...)>::type; +#endif + +namespace type_traits_internal { + +// ---------------------------------------------------------------------- +// In MSVC we can't probe std::hash or stdext::hash because it triggers a +// static_assert instead of failing substitution. Libc++ prior to 4.0 +// also used a static_assert. +// ---------------------------------------------------------------------- +#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \ + _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) + #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 +#else + #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 +#endif + +#if !PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + template <typename Key, typename = size_t> + struct IsHashable : std::true_type {}; +#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + template <typename Key, typename = void> + struct IsHashable : std::false_type {}; + + template <typename Key> + struct IsHashable<Key, + phmap::enable_if_t<std::is_convertible< + decltype(std::declval<std::hash<Key>&>()(std::declval<Key const&>())), + std::size_t>::value>> : std::true_type {}; +#endif + +struct AssertHashEnabledHelper +{ +private: + static void Sink(...) {} + struct NAT {}; + + template <class Key> + static auto GetReturnType(int) + -> decltype(std::declval<std::hash<Key>>()(std::declval<Key const&>())); + template <class Key> + static NAT GetReturnType(...); + + template <class Key> + static std::nullptr_t DoIt() { + static_assert(IsHashable<Key>::value, + "std::hash<Key> does not provide a call operator"); + static_assert( + std::is_default_constructible<std::hash<Key>>::value, + "std::hash<Key> must be default constructible when it is enabled"); + static_assert( + std::is_copy_constructible<std::hash<Key>>::value, + "std::hash<Key> must be copy constructible when it is enabled"); + static_assert(phmap::is_copy_assignable<std::hash<Key>>::value, + "std::hash<Key> must be copy assignable when it is enabled"); + // is_destructible is unchecked as it's implied by each of the + // is_constructible checks. + using ReturnType = decltype(GetReturnType<Key>(0)); + static_assert(std::is_same<ReturnType, NAT>::value || + std::is_same<ReturnType, size_t>::value, + "std::hash<Key> must return size_t"); + return nullptr; + } + + template <class... Ts> + friend void AssertHashEnabled(); +}; + +template <class... Ts> +inline void AssertHashEnabled +() +{ + using Helper = AssertHashEnabledHelper; + Helper::Sink(Helper::DoIt<Ts>()...); +} + +} // namespace type_traits_internal + +} // namespace phmap + + +// ----------------------------------------------------------------------------- +// hash_policy_traits +// ----------------------------------------------------------------------------- +namespace phmap { +namespace priv { + +// Defines how slots are initialized/destroyed/moved. +template <class Policy, class = void> +struct hash_policy_traits +{ +private: + struct ReturnKey + { + // We return `Key` here. + // When Key=T&, we forward the lvalue reference. + // When Key=T, we return by value to avoid a dangling reference. + // eg, for string_hash_map. + template <class Key, class... Args> + Key operator()(Key&& k, const Args&...) const { + return std::forward<Key>(k); + } + }; + + template <class P = Policy, class = void> + struct ConstantIteratorsImpl : std::false_type {}; + + template <class P> + struct ConstantIteratorsImpl<P, phmap::void_t<typename P::constant_iterators>> + : P::constant_iterators {}; + +public: + // The actual object stored in the hash table. + using slot_type = typename Policy::slot_type; + + // The type of the keys stored in the hashtable. + using key_type = typename Policy::key_type; + + // The argument type for insertions into the hashtable. This is different + // from value_type for increased performance. See initializer_list constructor + // and insert() member functions for more details. + using init_type = typename Policy::init_type; + + using reference = decltype(Policy::element(std::declval<slot_type*>())); + using pointer = typename std::remove_reference<reference>::type*; + using value_type = typename std::remove_reference<reference>::type; + + // Policies can set this variable to tell raw_hash_set that all iterators + // should be constant, even `iterator`. This is useful for set-like + // containers. + // Defaults to false if not provided by the policy. + using constant_iterators = ConstantIteratorsImpl<>; + + // PRECONDITION: `slot` is UNINITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template <class Alloc, class... Args> + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + Policy::construct(alloc, slot, std::forward<Args>(args)...); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is UNINITIALIZED + template <class Alloc> + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::destroy(alloc, slot); + } + + // Transfers the `old_slot` to `new_slot`. Any memory allocated by the + // allocator inside `old_slot` to `new_slot` can be transferred. + // + // OPTIONAL: defaults to: + // + // clone(new_slot, std::move(*old_slot)); + // destroy(old_slot); + // + // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED + // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is + // UNINITIALIZED + template <class Alloc> + static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { + transfer_impl(alloc, new_slot, old_slot, 0); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template <class P = Policy> + static auto element(slot_type* slot) -> decltype(P::element(slot)) { + return P::element(slot); + } + + // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. + // + // If `slot` is nullptr, returns the constant amount of memory owned by any + // full slot or -1 if slots own variable amounts of memory. + // + // PRECONDITION: `slot` is INITIALIZED or nullptr + template <class P = Policy> + static size_t space_used(const slot_type* slot) { + return P::space_used(slot); + } + + // Provides generalized access to the key for elements, both for elements in + // the table and for elements that have not yet been inserted (or even + // constructed). We would like an API that allows us to say: `key(args...)` + // but we cannot do that for all cases, so we use this more general API that + // can be used for many things, including the following: + // + // - Given an element in a table, get its key. + // - Given an element initializer, get its key. + // - Given `emplace()` arguments, get the element key. + // + // Implementations of this must adhere to a very strict technical + // specification around aliasing and consuming arguments: + // + // Let `value_type` be the result type of `element()` without ref- and + // cv-qualifiers. The first argument is a functor, the rest are constructor + // arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where + // `k` is the element key, and `xs...` are the new constructor arguments for + // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias + // `ts...`. The key won't be touched once `xs...` are used to construct an + // element; `ts...` won't be touched at all, which allows `apply()` to consume + // any rvalues among them. + // + // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not + // trigger a hard compile error unless it originates from `f`. In other words, + // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not + // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. + // + // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, + // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. + template <class F, class... Ts, class P = Policy> + static auto apply(F&& f, Ts&&... ts) + -> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) { + return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...); + } + + // Returns the "key" portion of the slot. + // Used for node handle manipulation. + template <class P = Policy> + static auto key(slot_type* slot) + -> decltype(P::apply(ReturnKey(), element(slot))) { + return P::apply(ReturnKey(), element(slot)); + } + + // Returns the "value" (as opposed to the "key") portion of the element. Used + // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. + template <class T, class P = Policy> + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + +private: + + // Use auto -> decltype as an enabler. + template <class Alloc, class P = Policy> + static auto transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, int) + -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { + P::transfer(alloc, new_slot, old_slot); + } + + template <class Alloc> + static void transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, char) { + construct(alloc, new_slot, std::move(element(old_slot))); + destroy(alloc, old_slot); + } +}; + +} // namespace priv +} // namespace phmap + +// ----------------------------------------------------------------------------- +// file utility.h +// ----------------------------------------------------------------------------- + +// --------- identity.h +namespace phmap { +namespace internal { + +template <typename T> +struct identity { + typedef T type; +}; + +template <typename T> +using identity_t = typename identity<T>::type; + +} // namespace internal +} // namespace phmap + + +// --------- inline_variable.h + +#ifdef __cpp_inline_variables + +#if defined(__clang__) + #define PHMAP_INTERNAL_EXTERN_DECL(type, name) \ + extern const ::phmap::internal::identity_t<type> name; +#else // Otherwise, just define the macro to do nothing. + #define PHMAP_INTERNAL_EXTERN_DECL(type, name) +#endif // defined(__clang__) + +// See above comment at top of file for details. +#define PHMAP_INTERNAL_INLINE_CONSTEXPR(type, name, init) \ + PHMAP_INTERNAL_EXTERN_DECL(type, name) \ + inline constexpr ::phmap::internal::identity_t<type> name = init + +#else + +// See above comment at top of file for details. +// +// Note: +// identity_t is used here so that the const and name are in the +// appropriate place for pointer types, reference types, function pointer +// types, etc.. +#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ + template <class /*PhmapInternalDummy*/ = void> \ + struct PhmapInternalInlineVariableHolder##name { \ + static constexpr ::phmap::internal::identity_t<var_type> kInstance = init; \ + }; \ + \ + template <class PhmapInternalDummy> \ + constexpr ::phmap::internal::identity_t<var_type> \ + PhmapInternalInlineVariableHolder##name<PhmapInternalDummy>::kInstance; \ + \ + static constexpr const ::phmap::internal::identity_t<var_type>& \ + name = /* NOLINT */ \ + PhmapInternalInlineVariableHolder##name<>::kInstance; \ + static_assert(sizeof(void (*)(decltype(name))) != 0, \ + "Silence unused variable warnings.") + +#endif // __cpp_inline_variables + +// ----------- throw_delegate + +namespace phmap { +namespace base_internal { + +namespace { +template <typename T> +#ifdef PHMAP_HAVE_EXCEPTIONS +[[noreturn]] void Throw(const T& error) { + throw error; +} +#else +[[noreturn]] void Throw(const T&) { + std::abort(); +} +#endif +} // namespace + +static inline void ThrowStdLogicError(const std::string& what_arg) { + Throw(std::logic_error(what_arg)); +} +static inline void ThrowStdLogicError(const char* what_arg) { + Throw(std::logic_error(what_arg)); +} +static inline void ThrowStdInvalidArgument(const std::string& what_arg) { + Throw(std::invalid_argument(what_arg)); +} +static inline void ThrowStdInvalidArgument(const char* what_arg) { + Throw(std::invalid_argument(what_arg)); +} + +static inline void ThrowStdDomainError(const std::string& what_arg) { + Throw(std::domain_error(what_arg)); +} +static inline void ThrowStdDomainError(const char* what_arg) { + Throw(std::domain_error(what_arg)); +} + +static inline void ThrowStdLengthError(const std::string& what_arg) { + Throw(std::length_error(what_arg)); +} +static inline void ThrowStdLengthError(const char* what_arg) { + Throw(std::length_error(what_arg)); +} + +static inline void ThrowStdOutOfRange(const std::string& what_arg) { + Throw(std::out_of_range(what_arg)); +} +static inline void ThrowStdOutOfRange(const char* what_arg) { + Throw(std::out_of_range(what_arg)); +} + +static inline void ThrowStdRuntimeError(const std::string& what_arg) { + Throw(std::runtime_error(what_arg)); +} +static inline void ThrowStdRuntimeError(const char* what_arg) { + Throw(std::runtime_error(what_arg)); +} + +static inline void ThrowStdRangeError(const std::string& what_arg) { + Throw(std::range_error(what_arg)); +} +static inline void ThrowStdRangeError(const char* what_arg) { + Throw(std::range_error(what_arg)); +} + +static inline void ThrowStdOverflowError(const std::string& what_arg) { + Throw(std::overflow_error(what_arg)); +} +static inline void ThrowStdOverflowError(const char* what_arg) { + Throw(std::overflow_error(what_arg)); +} + +static inline void ThrowStdUnderflowError(const std::string& what_arg) { + Throw(std::underflow_error(what_arg)); +} +static inline void ThrowStdUnderflowError(const char* what_arg) { + Throw(std::underflow_error(what_arg)); +} + +static inline void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); } + +static inline void ThrowStdBadAlloc() { Throw(std::bad_alloc()); } + +} // namespace base_internal +} // namespace phmap + +// ----------- invoke.h + +namespace phmap { +namespace base_internal { + +template <typename Derived> +struct StrippedAccept +{ + template <typename... Args> + struct Accept : Derived::template AcceptImpl<typename std::remove_cv< + typename std::remove_reference<Args>::type>::type...> {}; +}; + +// (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T +// and t1 is an object of type T or a reference to an object of type T or a +// reference to an object of a type derived from T. +struct MemFunAndRef : StrippedAccept<MemFunAndRef> +{ + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename... Params, typename Obj, + typename... Args> + struct AcceptImpl<R (C::*)(Params...), Obj, Args...> + : std::is_base_of<C, Obj> {}; + + template <typename R, typename C, typename... Params, typename Obj, + typename... Args> + struct AcceptImpl<R (C::*)(Params...) const, Obj, Args...> + : std::is_base_of<C, Obj> {}; + + template <typename MemFun, typename Obj, typename... Args> + static decltype((std::declval<Obj>().* + std::declval<MemFun>())(std::declval<Args>()...)) + Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { + return (std::forward<Obj>(obj).* + std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); + } +}; + +// ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a +// class T and t1 is not one of the types described in the previous item. +struct MemFunAndPtr : StrippedAccept<MemFunAndPtr> +{ + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename... Params, typename Ptr, + typename... Args> + struct AcceptImpl<R (C::*)(Params...), Ptr, Args...> + : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; + + template <typename R, typename C, typename... Params, typename Ptr, + typename... Args> + struct AcceptImpl<R (C::*)(Params...) const, Ptr, Args...> + : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; + + template <typename MemFun, typename Ptr, typename... Args> + static decltype(((*std::declval<Ptr>()).* + std::declval<MemFun>())(std::declval<Args>()...)) + Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { + return ((*std::forward<Ptr>(ptr)).* + std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); + } +}; + +// t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is +// an object of type T or a reference to an object of type T or a reference +// to an object of a type derived from T. +struct DataMemAndRef : StrippedAccept<DataMemAndRef> +{ + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename Obj> + struct AcceptImpl<R C::*, Obj> : std::is_base_of<C, Obj> {}; + + template <typename DataMem, typename Ref> + static decltype(std::declval<Ref>().*std::declval<DataMem>()) Invoke( + DataMem&& data_mem, Ref&& ref) { + return std::forward<Ref>(ref).*std::forward<DataMem>(data_mem); + } +}; + +// (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 +// is not one of the types described in the previous item. +struct DataMemAndPtr : StrippedAccept<DataMemAndPtr> +{ + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename Ptr> + struct AcceptImpl<R C::*, Ptr> + : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; + + template <typename DataMem, typename Ptr> + static decltype((*std::declval<Ptr>()).*std::declval<DataMem>()) Invoke( + DataMem&& data_mem, Ptr&& ptr) { + return (*std::forward<Ptr>(ptr)).*std::forward<DataMem>(data_mem); + } +}; + +// f(t1, t2, ..., tN) in all other cases. +struct Callable +{ + // Callable doesn't have Accept because it's the last clause that gets picked + // when none of the previous clauses are applicable. + template <typename F, typename... Args> + static decltype(std::declval<F>()(std::declval<Args>()...)) Invoke( + F&& f, Args&&... args) { + return std::forward<F>(f)(std::forward<Args>(args)...); + } +}; + +// Resolves to the first matching clause. +template <typename... Args> +struct Invoker +{ + typedef typename std::conditional< + MemFunAndRef::Accept<Args...>::value, MemFunAndRef, + typename std::conditional< + MemFunAndPtr::Accept<Args...>::value, MemFunAndPtr, + typename std::conditional< + DataMemAndRef::Accept<Args...>::value, DataMemAndRef, + typename std::conditional<DataMemAndPtr::Accept<Args...>::value, + DataMemAndPtr, Callable>::type>::type>:: + type>::type type; +}; + +// The result type of Invoke<F, Args...>. +template <typename F, typename... Args> +using InvokeT = decltype(Invoker<F, Args...>::type::Invoke( + std::declval<F>(), std::declval<Args>()...)); + +// Invoke(f, args...) is an implementation of INVOKE(f, args...) from section +// [func.require] of the C++ standard. +template <typename F, typename... Args> +InvokeT<F, Args...> Invoke(F&& f, Args&&... args) { + return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), + std::forward<Args>(args)...); +} +} // namespace base_internal +} // namespace phmap + + +// ----------- utility.h + +namespace phmap { + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `phmap::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence<T, Ints...>); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence<int, 5>()); +// } +template <typename T, T... Ints> +struct integer_sequence +{ + using value_type = T; + static constexpr size_t size() noexcept { return sizeof...(Ints); } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `phmap::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template <size_t... Ints> +using index_sequence = integer_sequence<size_t, Ints...>; + +namespace utility_internal { + +template <typename Seq, size_t SeqSize, size_t Rem> +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template <typename T, T... Ints, size_t SeqSize> +struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> { + using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>; +}; + +template <typename T, T... Ints, size_t SeqSize> +struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> { + using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>; +}; + +// Recursion helper for 'make_integer_sequence<T, N>'. +// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'. +template <typename T, size_t N> +struct Gen { + using type = + typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type; +}; + +template <typename T> +struct Gen<T, 0> { + using type = integer_sequence<T>; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template <typename T, T N> +using make_integer_sequence = typename utility_internal::Gen<T, N>::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template <size_t N> +using make_index_sequence = make_integer_sequence<size_t, N>; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template <typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +// Tag types + +#ifdef PHMAP_HAVE_STD_OPTIONAL + +using std::in_place_t; +using std::in_place; + +#else // PHMAP_HAVE_STD_OPTIONAL + +// in_place_t +// +// Tag type used to specify in-place construction, such as with +// `phmap::optional`, designed to be a drop-in replacement for C++17's +// `std::in_place_t`. +struct in_place_t {}; + +PHMAP_INTERNAL_INLINE_CONSTEXPR(in_place_t, in_place, {}); + +#endif // PHMAP_HAVE_STD_OPTIONAL + +#if defined(PHMAP_HAVE_STD_ANY) || defined(PHMAP_HAVE_STD_VARIANT) +using std::in_place_type_t; +#else + +// in_place_type_t +// +// Tag type used for in-place construction when the type to construct needs to +// be specified, such as with `phmap::any`, designed to be a drop-in replacement +// for C++17's `std::in_place_type_t`. +template <typename T> +struct in_place_type_t {}; +#endif // PHMAP_HAVE_STD_ANY || PHMAP_HAVE_STD_VARIANT + +#ifdef PHMAP_HAVE_STD_VARIANT +using std::in_place_index_t; +#else + +// in_place_index_t +// +// Tag type used for in-place construction when the type to construct needs to +// be specified, such as with `phmap::any`, designed to be a drop-in replacement +// for C++17's `std::in_place_index_t`. +template <size_t I> +struct in_place_index_t {}; +#endif // PHMAP_HAVE_STD_VARIANT + +// Constexpr move and forward + +// move() +// +// A constexpr version of `std::move()`, designed to be a drop-in replacement +// for C++14's `std::move()`. +template <typename T> +constexpr phmap::remove_reference_t<T>&& move(T&& t) noexcept { + return static_cast<phmap::remove_reference_t<T>&&>(t); +} + +// forward() +// +// A constexpr version of `std::forward()`, designed to be a drop-in replacement +// for C++14's `std::forward()`. +template <typename T> +constexpr T&& forward( + phmap::remove_reference_t<T>& t) noexcept { // NOLINT(runtime/references) + return static_cast<T&&>(t); +} + +namespace utility_internal { +// Helper method for expanding tuple into a called method. +template <typename Functor, typename Tuple, std::size_t... Indexes> +auto apply_helper(Functor&& functor, Tuple&& t, index_sequence<Indexes...>) + -> decltype(phmap::base_internal::Invoke( + phmap::forward<Functor>(functor), + std::get<Indexes>(phmap::forward<Tuple>(t))...)) { + return phmap::base_internal::Invoke( + phmap::forward<Functor>(functor), + std::get<Indexes>(phmap::forward<Tuple>(t))...); +} + +} // namespace utility_internal + +// apply +// +// Invokes a Callable using elements of a tuple as its arguments. +// Each element of the tuple corresponds to an argument of the call (in order). +// Both the Callable argument and the tuple argument are perfect-forwarded. +// For member-function Callables, the first tuple element acts as the `this` +// pointer. `phmap::apply` is designed to be a drop-in replacement for C++17's +// `std::apply`. Unlike C++17's `std::apply`, this is not currently `constexpr`. +// +// Example: +// +// class Foo { +// public: +// void Bar(int); +// }; +// void user_function1(int, std::string); +// void user_function2(std::unique_ptr<Foo>); +// auto user_lambda = [](int, int) {}; +// +// int main() +// { +// std::tuple<int, std::string> tuple1(42, "bar"); +// // Invokes the first user function on int, std::string. +// phmap::apply(&user_function1, tuple1); +// +// std::tuple<std::unique_ptr<Foo>> tuple2(phmap::make_unique<Foo>()); +// // Invokes the user function that takes ownership of the unique +// // pointer. +// phmap::apply(&user_function2, std::move(tuple2)); +// +// auto foo = phmap::make_unique<Foo>(); +// std::tuple<Foo*, int> tuple3(foo.get(), 42); +// // Invokes the method Bar on foo with one argument, 42. +// phmap::apply(&Foo::Bar, tuple3); +// +// std::tuple<int, int> tuple4(8, 9); +// // Invokes a lambda. +// phmap::apply(user_lambda, tuple4); +// } +template <typename Functor, typename Tuple> +auto apply(Functor&& functor, Tuple&& t) + -> decltype(utility_internal::apply_helper( + phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), + phmap::make_index_sequence<std::tuple_size< + typename std::remove_reference<Tuple>::type>::value>{})) { + return utility_internal::apply_helper( + phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), + phmap::make_index_sequence<std::tuple_size< + typename std::remove_reference<Tuple>::type>::value>{}); +} + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch +#endif // _MSC_VER + +// exchange +// +// Replaces the value of `obj` with `new_value` and returns the old value of +// `obj`. `phmap::exchange` is designed to be a drop-in replacement for C++14's +// `std::exchange`. +// +// Example: +// +// Foo& operator=(Foo&& other) { +// ptr1_ = phmap::exchange(other.ptr1_, nullptr); +// int1_ = phmap::exchange(other.int1_, -1); +// return *this; +// } +template <typename T, typename U = T> +T exchange(T& obj, U&& new_value) +{ + T old_value = phmap::move(obj); + obj = phmap::forward<U>(new_value); + return old_value; +} + +#ifdef _MSC_VER + #pragma warning(pop) +#endif // _MSC_VER + + +} // namespace phmap + +// ----------------------------------------------------------------------------- +// memory.h +// ----------------------------------------------------------------------------- + +namespace phmap { + +template <typename T> +std::unique_ptr<T> WrapUnique(T* ptr) +{ + static_assert(!std::is_array<T>::value, "array types are unsupported"); + static_assert(std::is_object<T>::value, "non-object types are unsupported"); + return std::unique_ptr<T>(ptr); +} + +namespace memory_internal { + +// Traits to select proper overload and return type for `phmap::make_unique<>`. +template <typename T> +struct MakeUniqueResult { + using scalar = std::unique_ptr<T>; +}; +template <typename T> +struct MakeUniqueResult<T[]> { + using array = std::unique_ptr<T[]>; +}; +template <typename T, size_t N> +struct MakeUniqueResult<T[N]> { + using invalid = void; +}; + +} // namespace memory_internal + +#if (__cplusplus > 201103L || defined(_MSC_VER)) && \ + !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) + using std::make_unique; +#else + + template <typename T, typename... Args> + typename memory_internal::MakeUniqueResult<T>::scalar make_unique( + Args&&... args) { + return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); + } + + template <typename T> + typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) { + return std::unique_ptr<T>(new typename phmap::remove_extent_t<T>[n]()); + } + + template <typename T, typename... Args> + typename memory_internal::MakeUniqueResult<T>::invalid make_unique( + Args&&... /* args */) = delete; +#endif + +template <typename T> +auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) +{ + // ptr is a forwarding reference to support Ts with non-const operators. + return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; +} + +inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; } + +template <typename T, typename D> +std::shared_ptr<T> ShareUniquePtr(std::unique_ptr<T, D>&& ptr) { + return ptr ? std::shared_ptr<T>(std::move(ptr)) : std::shared_ptr<T>(); +} + +template <typename T> +std::weak_ptr<T> WeakenPtr(const std::shared_ptr<T>& ptr) { + return std::weak_ptr<T>(ptr); +} + +namespace memory_internal { + +// ExtractOr<E, O, D>::type evaluates to E<O> if possible. Otherwise, D. +template <template <typename> class Extract, typename Obj, typename Default, + typename> +struct ExtractOr { + using type = Default; +}; + +template <template <typename> class Extract, typename Obj, typename Default> +struct ExtractOr<Extract, Obj, Default, void_t<Extract<Obj>>> { + using type = Extract<Obj>; +}; + +template <template <typename> class Extract, typename Obj, typename Default> +using ExtractOrT = typename ExtractOr<Extract, Obj, Default, void>::type; + +// Extractors for the features of allocators. +template <typename T> +using GetPointer = typename T::pointer; + +template <typename T> +using GetConstPointer = typename T::const_pointer; + +template <typename T> +using GetVoidPointer = typename T::void_pointer; + +template <typename T> +using GetConstVoidPointer = typename T::const_void_pointer; + +template <typename T> +using GetDifferenceType = typename T::difference_type; + +template <typename T> +using GetSizeType = typename T::size_type; + +template <typename T> +using GetPropagateOnContainerCopyAssignment = + typename T::propagate_on_container_copy_assignment; + +template <typename T> +using GetPropagateOnContainerMoveAssignment = + typename T::propagate_on_container_move_assignment; + +template <typename T> +using GetPropagateOnContainerSwap = typename T::propagate_on_container_swap; + +template <typename T> +using GetIsAlwaysEqual = typename T::is_always_equal; + +template <typename T> +struct GetFirstArg; + +template <template <typename...> class Class, typename T, typename... Args> +struct GetFirstArg<Class<T, Args...>> { + using type = T; +}; + +template <typename Ptr, typename = void> +struct ElementType { + using type = typename GetFirstArg<Ptr>::type; +}; + +template <typename T> +struct ElementType<T, void_t<typename T::element_type>> { + using type = typename T::element_type; +}; + +template <typename T, typename U> +struct RebindFirstArg; + +template <template <typename...> class Class, typename T, typename... Args, + typename U> +struct RebindFirstArg<Class<T, Args...>, U> { + using type = Class<U, Args...>; +}; + +template <typename T, typename U, typename = void> +struct RebindPtr { + using type = typename RebindFirstArg<T, U>::type; +}; + +template <typename T, typename U> +struct RebindPtr<T, U, void_t<typename T::template rebind<U>>> { + using type = typename T::template rebind<U>; +}; + +template <typename T, typename U> +constexpr bool HasRebindAlloc(...) { + return false; +} + +template <typename T, typename U> +constexpr bool HasRebindAlloc(typename std::allocator_traits<T>::template rebind_alloc<U>*) { + return true; +} + +template <typename T, typename U, bool = HasRebindAlloc<T, U>(nullptr)> +struct RebindAlloc { + using type = typename RebindFirstArg<T, U>::type; +}; + +template <typename A, typename U> +struct RebindAlloc<A, U, true> { + using type = typename std::allocator_traits<A>::template rebind_alloc<U>; +}; + + +} // namespace memory_internal + +template <typename Ptr> +struct pointer_traits +{ + using pointer = Ptr; + + // element_type: + // Ptr::element_type if present. Otherwise T if Ptr is a template + // instantiation Template<T, Args...> + using element_type = typename memory_internal::ElementType<Ptr>::type; + + // difference_type: + // Ptr::difference_type if present, otherwise std::ptrdiff_t + using difference_type = + memory_internal::ExtractOrT<memory_internal::GetDifferenceType, Ptr, + std::ptrdiff_t>; + + // rebind: + // Ptr::rebind<U> if exists, otherwise Template<U, Args...> if Ptr is a + // template instantiation Template<T, Args...> + template <typename U> + using rebind = typename memory_internal::RebindPtr<Ptr, U>::type; + + // pointer_to: + // Calls Ptr::pointer_to(r) + static pointer pointer_to(element_type& r) { // NOLINT(runtime/references) + return Ptr::pointer_to(r); + } +}; + +// Specialization for T*. +template <typename T> +struct pointer_traits<T*> +{ + using pointer = T*; + using element_type = T; + using difference_type = std::ptrdiff_t; + + template <typename U> + using rebind = U*; + + // pointer_to: + // Calls std::addressof(r) + static pointer pointer_to( + element_type& r) noexcept { // NOLINT(runtime/references) + return std::addressof(r); + } +}; + +// ----------------------------------------------------------------------------- +// Class Template: allocator_traits +// ----------------------------------------------------------------------------- +// +// A C++11 compatible implementation of C++17's std::allocator_traits. +// +template <typename Alloc> +struct allocator_traits +{ + using allocator_type = Alloc; + + // value_type: + // Alloc::value_type + using value_type = typename Alloc::value_type; + + // pointer: + // Alloc::pointer if present, otherwise value_type* + using pointer = memory_internal::ExtractOrT<memory_internal::GetPointer, + Alloc, value_type*>; + + // const_pointer: + // Alloc::const_pointer if present, otherwise + // phmap::pointer_traits<pointer>::rebind<const value_type> + using const_pointer = + memory_internal::ExtractOrT<memory_internal::GetConstPointer, Alloc, + typename phmap::pointer_traits<pointer>:: + template rebind<const value_type>>; + + // void_pointer: + // Alloc::void_pointer if present, otherwise + // phmap::pointer_traits<pointer>::rebind<void> + using void_pointer = memory_internal::ExtractOrT< + memory_internal::GetVoidPointer, Alloc, + typename phmap::pointer_traits<pointer>::template rebind<void>>; + + // const_void_pointer: + // Alloc::const_void_pointer if present, otherwise + // phmap::pointer_traits<pointer>::rebind<const void> + using const_void_pointer = memory_internal::ExtractOrT< + memory_internal::GetConstVoidPointer, Alloc, + typename phmap::pointer_traits<pointer>::template rebind<const void>>; + + // difference_type: + // Alloc::difference_type if present, otherwise + // phmap::pointer_traits<pointer>::difference_type + using difference_type = memory_internal::ExtractOrT< + memory_internal::GetDifferenceType, Alloc, + typename phmap::pointer_traits<pointer>::difference_type>; + + // size_type: + // Alloc::size_type if present, otherwise + // std::make_unsigned<difference_type>::type + using size_type = memory_internal::ExtractOrT< + memory_internal::GetSizeType, Alloc, + typename std::make_unsigned<difference_type>::type>; + + // propagate_on_container_copy_assignment: + // Alloc::propagate_on_container_copy_assignment if present, otherwise + // std::false_type + using propagate_on_container_copy_assignment = memory_internal::ExtractOrT< + memory_internal::GetPropagateOnContainerCopyAssignment, Alloc, + std::false_type>; + + // propagate_on_container_move_assignment: + // Alloc::propagate_on_container_move_assignment if present, otherwise + // std::false_type + using propagate_on_container_move_assignment = memory_internal::ExtractOrT< + memory_internal::GetPropagateOnContainerMoveAssignment, Alloc, + std::false_type>; + + // propagate_on_container_swap: + // Alloc::propagate_on_container_swap if present, otherwise std::false_type + using propagate_on_container_swap = + memory_internal::ExtractOrT<memory_internal::GetPropagateOnContainerSwap, + Alloc, std::false_type>; + + // is_always_equal: + // Alloc::is_always_equal if present, otherwise std::is_empty<Alloc>::type + using is_always_equal = + memory_internal::ExtractOrT<memory_internal::GetIsAlwaysEqual, Alloc, + typename std::is_empty<Alloc>::type>; + + // rebind_alloc: + // Alloc::rebind<T>::other if present, otherwise Alloc<T, Args> if this Alloc + // is Alloc<U, Args> + template <typename T> + using rebind_alloc = typename memory_internal::RebindAlloc<Alloc, T>::type; + + // rebind_traits: + // phmap::allocator_traits<rebind_alloc<T>> + template <typename T> + using rebind_traits = phmap::allocator_traits<rebind_alloc<T>>; + + // allocate(Alloc& a, size_type n): + // Calls a.allocate(n) + static pointer allocate(Alloc& a, // NOLINT(runtime/references) + size_type n) { + return a.allocate(n); + } + + // allocate(Alloc& a, size_type n, const_void_pointer hint): + // Calls a.allocate(n, hint) if possible. + // If not possible, calls a.allocate(n) + static pointer allocate(Alloc& a, size_type n, // NOLINT(runtime/references) + const_void_pointer hint) { + return allocate_impl(0, a, n, hint); + } + + // deallocate(Alloc& a, pointer p, size_type n): + // Calls a.deallocate(p, n) + static void deallocate(Alloc& a, pointer p, // NOLINT(runtime/references) + size_type n) { + a.deallocate(p, n); + } + + // construct(Alloc& a, T* p, Args&&... args): + // Calls a.construct(p, std::forward<Args>(args)...) if possible. + // If not possible, calls + // ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...) + template <typename T, typename... Args> + static void construct(Alloc& a, T* p, // NOLINT(runtime/references) + Args&&... args) { + construct_impl(0, a, p, std::forward<Args>(args)...); + } + + // destroy(Alloc& a, T* p): + // Calls a.destroy(p) if possible. If not possible, calls p->~T(). + template <typename T> + static void destroy(Alloc& a, T* p) { // NOLINT(runtime/references) + destroy_impl(0, a, p); + } + + // max_size(const Alloc& a): + // Returns a.max_size() if possible. If not possible, returns + // std::numeric_limits<size_type>::max() / sizeof(value_type) + static size_type max_size(const Alloc& a) { return max_size_impl(0, a); } + + // select_on_container_copy_construction(const Alloc& a): + // Returns a.select_on_container_copy_construction() if possible. + // If not possible, returns a. + static Alloc select_on_container_copy_construction(const Alloc& a) { + return select_on_container_copy_construction_impl(0, a); + } + +private: + template <typename A> + static auto allocate_impl(int, A& a, // NOLINT(runtime/references) + size_type n, const_void_pointer hint) + -> decltype(a.allocate(n, hint)) { + return a.allocate(n, hint); + } + static pointer allocate_impl(char, Alloc& a, // NOLINT(runtime/references) + size_type n, const_void_pointer) { + return a.allocate(n); + } + + template <typename A, typename... Args> + static auto construct_impl(int, A& a, // NOLINT(runtime/references) + Args&&... args) + -> decltype(std::allocator_traits<A>::construct(a, std::forward<Args>(args)...)) { + std::allocator_traits<A>::construct(a, std::forward<Args>(args)...); + } + + template <typename T, typename... Args> + static void construct_impl(char, Alloc&, T* p, Args&&... args) { + ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...); + } + + template <typename A, typename T> + static auto destroy_impl(int, A& a, // NOLINT(runtime/references) + T* p) -> decltype(std::allocator_traits<A>::destroy(a, p)) { + std::allocator_traits<A>::destroy(a, p); + } + template <typename T> + static void destroy_impl(char, Alloc&, T* p) { + p->~T(); + } + + template <typename A> + static auto max_size_impl(int, const A& a) -> decltype(a.max_size()) { + return a.max_size(); + } + static size_type max_size_impl(char, const Alloc&) { + return (std::numeric_limits<size_type>::max)() / sizeof(value_type); + } + + template <typename A> + static auto select_on_container_copy_construction_impl(int, const A& a) + -> decltype(a.select_on_container_copy_construction()) { + return a.select_on_container_copy_construction(); + } + static Alloc select_on_container_copy_construction_impl(char, + const Alloc& a) { + return a; + } +}; + +namespace memory_internal { + +// This template alias transforms Alloc::is_nothrow into a metafunction with +// Alloc as a parameter so it can be used with ExtractOrT<>. +template <typename Alloc> +using GetIsNothrow = typename Alloc::is_nothrow; + +} // namespace memory_internal + +// PHMAP_ALLOCATOR_NOTHROW is a build time configuration macro for user to +// specify whether the default allocation function can throw or never throws. +// If the allocation function never throws, user should define it to a non-zero +// value (e.g. via `-DPHMAP_ALLOCATOR_NOTHROW`). +// If the allocation function can throw, user should leave it undefined or +// define it to zero. +// +// allocator_is_nothrow<Alloc> is a traits class that derives from +// Alloc::is_nothrow if present, otherwise std::false_type. It's specialized +// for Alloc = std::allocator<T> for any type T according to the state of +// PHMAP_ALLOCATOR_NOTHROW. +// +// default_allocator_is_nothrow is a class that derives from std::true_type +// when the default allocator (global operator new) never throws, and +// std::false_type when it can throw. It is a convenience shorthand for writing +// allocator_is_nothrow<std::allocator<T>> (T can be any type). +// NOTE: allocator_is_nothrow<std::allocator<T>> is guaranteed to derive from +// the same type for all T, because users should specialize neither +// allocator_is_nothrow nor std::allocator. +template <typename Alloc> +struct allocator_is_nothrow + : memory_internal::ExtractOrT<memory_internal::GetIsNothrow, Alloc, + std::false_type> {}; + +#if defined(PHMAP_ALLOCATOR_NOTHROW) && PHMAP_ALLOCATOR_NOTHROW + template <typename T> + struct allocator_is_nothrow<std::allocator<T>> : std::true_type {}; + struct default_allocator_is_nothrow : std::true_type {}; +#else + struct default_allocator_is_nothrow : std::false_type {}; +#endif + +namespace memory_internal { +template <typename Allocator, typename Iterator, typename... Args> +void ConstructRange(Allocator& alloc, Iterator first, Iterator last, + const Args&... args) +{ + for (Iterator cur = first; cur != last; ++cur) { + PHMAP_INTERNAL_TRY { + std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), + args...); + } + PHMAP_INTERNAL_CATCH_ANY { + while (cur != first) { + --cur; + std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); + } + PHMAP_INTERNAL_RETHROW; + } + } +} + +template <typename Allocator, typename Iterator, typename InputIterator> +void CopyRange(Allocator& alloc, Iterator destination, InputIterator first, + InputIterator last) +{ + for (Iterator cur = destination; first != last; + static_cast<void>(++cur), static_cast<void>(++first)) { + PHMAP_INTERNAL_TRY { + std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), + *first); + } + PHMAP_INTERNAL_CATCH_ANY { + while (cur != destination) { + --cur; + std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); + } + PHMAP_INTERNAL_RETHROW; + } + } +} +} // namespace memory_internal +} // namespace phmap + + +// ----------------------------------------------------------------------------- +// optional.h +// ----------------------------------------------------------------------------- +#ifdef PHMAP_HAVE_STD_OPTIONAL + +#include <optional> // IWYU pragma: export + +namespace phmap { +using std::bad_optional_access; +using std::optional; +using std::make_optional; +using std::nullopt_t; +using std::nullopt; +} // namespace phmap + +#else + +#if defined(__clang__) + #if __has_feature(cxx_inheriting_constructors) + #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 + #endif +#elif (defined(__GNUC__) && \ + (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8)) || \ + (__cpp_inheriting_constructors >= 200802) || \ + (defined(_MSC_VER) && _MSC_VER >= 1910) + + #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 +#endif + +namespace phmap { + +class bad_optional_access : public std::exception +{ +public: + bad_optional_access() = default; + ~bad_optional_access() override; + const char* what() const noexcept override; +}; + +template <typename T> +class optional; + +// -------------------------------- +struct nullopt_t +{ + struct init_t {}; + static init_t init; + + explicit constexpr nullopt_t(init_t& /*unused*/) {} +}; + +constexpr nullopt_t nullopt(nullopt_t::init); + +namespace optional_internal { + +// throw delegator +[[noreturn]] void throw_bad_optional_access(); + + +struct empty_struct {}; + +// This class stores the data in optional<T>. +// It is specialized based on whether T is trivially destructible. +// This is the specialization for non trivially destructible type. +template <typename T, bool unused = std::is_trivially_destructible<T>::value> +class optional_data_dtor_base +{ + struct dummy_type { + static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); + // Use an array to avoid GCC 6 placement-new warning. + empty_struct data[sizeof(T) / sizeof(empty_struct)]; + }; + +protected: + // Whether there is data or not. + bool engaged_; + // Data storage + union { + dummy_type dummy_; + T data_; + }; + + void destruct() noexcept { + if (engaged_) { + data_.~T(); + engaged_ = false; + } + } + + // dummy_ must be initialized for constexpr constructor. + constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} + + template <typename... Args> + constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) + : engaged_(true), data_(phmap::forward<Args>(args)...) {} + + ~optional_data_dtor_base() { destruct(); } +}; + +// Specialization for trivially destructible type. +template <typename T> +class optional_data_dtor_base<T, true> +{ + struct dummy_type { + static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); + // Use array to avoid GCC 6 placement-new warning. + empty_struct data[sizeof(T) / sizeof(empty_struct)]; + }; + +protected: + // Whether there is data or not. + bool engaged_; + // Data storage + union { + dummy_type dummy_; + T data_; + }; + void destruct() noexcept { engaged_ = false; } + + // dummy_ must be initialized for constexpr constructor. + constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} + + template <typename... Args> + constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) + : engaged_(true), data_(phmap::forward<Args>(args)...) {} +}; + +template <typename T> +class optional_data_base : public optional_data_dtor_base<T> +{ +protected: + using base = optional_data_dtor_base<T>; +#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using base::base; +#else + optional_data_base() = default; + + template <typename... Args> + constexpr explicit optional_data_base(in_place_t t, Args&&... args) + : base(t, phmap::forward<Args>(args)...) {} +#endif + + template <typename... Args> + void construct(Args&&... args) { + // Use dummy_'s address to work around casting cv-qualified T* to void*. + ::new (static_cast<void*>(&this->dummy_)) T(std::forward<Args>(args)...); + this->engaged_ = true; + } + + template <typename U> + void assign(U&& u) { + if (this->engaged_) { + this->data_ = std::forward<U>(u); + } else { + construct(std::forward<U>(u)); + } + } +}; + +// TODO: Add another class using +// std::is_trivially_move_constructible trait when available to match +// http://cplusplus.github.io/LWG/lwg-defects.html#2900, for types that +// have trivial move but nontrivial copy. +// Also, we should be checking is_trivially_copyable here, which is not +// supported now, so we use is_trivially_* traits instead. +template <typename T, + bool unused = phmap::is_trivially_copy_constructible<T>::value&& + phmap::is_trivially_copy_assignable<typename std::remove_cv< + T>::type>::value&& std::is_trivially_destructible<T>::value> +class optional_data; + +// Trivially copyable types +template <typename T> +class optional_data<T, true> : public optional_data_base<T> +{ +protected: +#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using optional_data_base<T>::optional_data_base; +#else + optional_data() = default; + + template <typename... Args> + constexpr explicit optional_data(in_place_t t, Args&&... args) + : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} +#endif +}; + +template <typename T> +class optional_data<T, false> : public optional_data_base<T> +{ +protected: +#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using optional_data_base<T>::optional_data_base; +#else + template <typename... Args> + constexpr explicit optional_data(in_place_t t, Args&&... args) + : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} +#endif + + optional_data() = default; + + optional_data(const optional_data& rhs) : optional_data_base<T>() { + if (rhs.engaged_) { + this->construct(rhs.data_); + } + } + + optional_data(optional_data&& rhs) noexcept( + phmap::default_allocator_is_nothrow::value || + std::is_nothrow_move_constructible<T>::value) + : optional_data_base<T>() { + if (rhs.engaged_) { + this->construct(std::move(rhs.data_)); + } + } + + optional_data& operator=(const optional_data& rhs) { + if (rhs.engaged_) { + this->assign(rhs.data_); + } else { + this->destruct(); + } + return *this; + } + + optional_data& operator=(optional_data&& rhs) noexcept( + std::is_nothrow_move_assignable<T>::value&& + std::is_nothrow_move_constructible<T>::value) { + if (rhs.engaged_) { + this->assign(std::move(rhs.data_)); + } else { + this->destruct(); + } + return *this; + } +}; + +// Ordered by level of restriction, from low to high. +// Copyable implies movable. +enum class copy_traits { copyable = 0, movable = 1, non_movable = 2 }; + +// Base class for enabling/disabling copy/move constructor. +template <copy_traits> +class optional_ctor_base; + +template <> +class optional_ctor_base<copy_traits::copyable> +{ +public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = default; + optional_ctor_base(optional_ctor_base&&) = default; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +template <> +class optional_ctor_base<copy_traits::movable> +{ +public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = delete; + optional_ctor_base(optional_ctor_base&&) = default; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +template <> +class optional_ctor_base<copy_traits::non_movable> +{ +public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = delete; + optional_ctor_base(optional_ctor_base&&) = delete; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +// Base class for enabling/disabling copy/move assignment. +template <copy_traits> +class optional_assign_base; + +template <> +class optional_assign_base<copy_traits::copyable> +{ +public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = default; + optional_assign_base& operator=(optional_assign_base&&) = default; +}; + +template <> +class optional_assign_base<copy_traits::movable> +{ +public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = delete; + optional_assign_base& operator=(optional_assign_base&&) = default; +}; + +template <> +class optional_assign_base<copy_traits::non_movable> +{ +public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = delete; + optional_assign_base& operator=(optional_assign_base&&) = delete; +}; + +template <typename T> +constexpr copy_traits get_ctor_copy_traits() +{ + return std::is_copy_constructible<T>::value + ? copy_traits::copyable + : std::is_move_constructible<T>::value ? copy_traits::movable + : copy_traits::non_movable; +} + +template <typename T> +constexpr copy_traits get_assign_copy_traits() +{ + return phmap::is_copy_assignable<T>::value && + std::is_copy_constructible<T>::value + ? copy_traits::copyable + : phmap::is_move_assignable<T>::value && + std::is_move_constructible<T>::value + ? copy_traits::movable + : copy_traits::non_movable; +} + +// Whether T is constructible or convertible from optional<U>. +template <typename T, typename U> +struct is_constructible_convertible_from_optional + : std::integral_constant< + bool, std::is_constructible<T, optional<U>&>::value || + std::is_constructible<T, optional<U>&&>::value || + std::is_constructible<T, const optional<U>&>::value || + std::is_constructible<T, const optional<U>&&>::value || + std::is_convertible<optional<U>&, T>::value || + std::is_convertible<optional<U>&&, T>::value || + std::is_convertible<const optional<U>&, T>::value || + std::is_convertible<const optional<U>&&, T>::value> {}; + +// Whether T is constructible or convertible or assignable from optional<U>. +template <typename T, typename U> +struct is_constructible_convertible_assignable_from_optional + : std::integral_constant< + bool, is_constructible_convertible_from_optional<T, U>::value || + std::is_assignable<T&, optional<U>&>::value || + std::is_assignable<T&, optional<U>&&>::value || + std::is_assignable<T&, const optional<U>&>::value || + std::is_assignable<T&, const optional<U>&&>::value> {}; + +// Helper function used by [optional.relops], [optional.comp_with_t], +// for checking whether an expression is convertible to bool. +bool convertible_to_bool(bool); + +// Base class for std::hash<phmap::optional<T>>: +// If std::hash<std::remove_const_t<T>> is enabled, it provides operator() to +// compute the hash; Otherwise, it is disabled. +// Reference N4659 23.14.15 [unord.hash]. +template <typename T, typename = size_t> +struct optional_hash_base +{ + optional_hash_base() = delete; + optional_hash_base(const optional_hash_base&) = delete; + optional_hash_base(optional_hash_base&&) = delete; + optional_hash_base& operator=(const optional_hash_base&) = delete; + optional_hash_base& operator=(optional_hash_base&&) = delete; +}; + +template <typename T> +struct optional_hash_base<T, decltype(std::hash<phmap::remove_const_t<T> >()( + std::declval<phmap::remove_const_t<T> >()))> +{ + using argument_type = phmap::optional<T>; + using result_type = size_t; + size_t operator()(const phmap::optional<T>& opt) const { + phmap::type_traits_internal::AssertHashEnabled<phmap::remove_const_t<T>>(); + if (opt) { + return std::hash<phmap::remove_const_t<T> >()(*opt); + } else { + return static_cast<size_t>(0x297814aaad196e6dULL); + } + } +}; + +} // namespace optional_internal + + +// ----------------------------------------------------------------------------- +// phmap::optional class definition +// ----------------------------------------------------------------------------- + +template <typename T> +class optional : private optional_internal::optional_data<T>, + private optional_internal::optional_ctor_base< + optional_internal::get_ctor_copy_traits<T>()>, + private optional_internal::optional_assign_base< + optional_internal::get_assign_copy_traits<T>()> +{ + using data_base = optional_internal::optional_data<T>; + +public: + typedef T value_type; + + // Constructors + + // Constructs an `optional` holding an empty value, NOT a default constructed + // `T`. + constexpr optional() noexcept {} + + // Constructs an `optional` initialized with `nullopt` to hold an empty value. + constexpr optional(nullopt_t) noexcept {} // NOLINT(runtime/explicit) + + // Copy constructor, standard semantics + optional(const optional& src) = default; + + // Move constructor, standard semantics + optional(optional&& src) = default; + + // Constructs a non-empty `optional` direct-initialized value of type `T` from + // the arguments `std::forward<Args>(args)...` within the `optional`. + // (The `in_place_t` is a tag used to indicate that the contained object + // should be constructed in-place.) + template <typename InPlaceT, typename... Args, + phmap::enable_if_t<phmap::conjunction< + std::is_same<InPlaceT, in_place_t>, + std::is_constructible<T, Args&&...> >::value>* = nullptr> + constexpr explicit optional(InPlaceT, Args&&... args) + : data_base(in_place_t(), phmap::forward<Args>(args)...) {} + + // Constructs a non-empty `optional` direct-initialized value of type `T` from + // the arguments of an initializer_list and `std::forward<Args>(args)...`. + // (The `in_place_t` is a tag used to indicate that the contained object + // should be constructed in-place.) + template <typename U, typename... Args, + typename = typename std::enable_if<std::is_constructible< + T, std::initializer_list<U>&, Args&&...>::value>::type> + constexpr explicit optional(in_place_t, std::initializer_list<U> il, + Args&&... args) + : data_base(in_place_t(), il, phmap::forward<Args>(args)...) { + } + + // Value constructor (implicit) + template < + typename U = T, + typename std::enable_if< + phmap::conjunction<phmap::negation<std::is_same< + in_place_t, typename std::decay<U>::type> >, + phmap::negation<std::is_same< + optional<T>, typename std::decay<U>::type> >, + std::is_convertible<U&&, T>, + std::is_constructible<T, U&&> >::value, + bool>::type = false> + constexpr optional(U&& v) : data_base(in_place_t(), phmap::forward<U>(v)) {} + + // Value constructor (explicit) + template < + typename U = T, + typename std::enable_if< + phmap::conjunction<phmap::negation<std::is_same< + in_place_t, typename std::decay<U>::type>>, + phmap::negation<std::is_same< + optional<T>, typename std::decay<U>::type>>, + phmap::negation<std::is_convertible<U&&, T>>, + std::is_constructible<T, U&&>>::value, + bool>::type = false> + explicit constexpr optional(U&& v) + : data_base(in_place_t(), phmap::forward<U>(v)) {} + + // Converting copy constructor (implicit) + template <typename U, + typename std::enable_if< + phmap::conjunction< + phmap::negation<std::is_same<T, U> >, + std::is_constructible<T, const U&>, + phmap::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U> >, + std::is_convertible<const U&, T> >::value, + bool>::type = false> + optional(const optional<U>& rhs) { + if (rhs) { + this->construct(*rhs); + } + } + + // Converting copy constructor (explicit) + template <typename U, + typename std::enable_if< + phmap::conjunction< + phmap::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, + phmap::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U>>, + phmap::negation<std::is_convertible<const U&, T>>>::value, + bool>::type = false> + explicit optional(const optional<U>& rhs) { + if (rhs) { + this->construct(*rhs); + } + } + + // Converting move constructor (implicit) + template <typename U, + typename std::enable_if< + phmap::conjunction< + phmap::negation<std::is_same<T, U> >, + std::is_constructible<T, U&&>, + phmap::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U> >, + std::is_convertible<U&&, T> >::value, + bool>::type = false> + optional(optional<U>&& rhs) { + if (rhs) { + this->construct(std::move(*rhs)); + } + } + + // Converting move constructor (explicit) + template < + typename U, + typename std::enable_if< + phmap::conjunction< + phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, + phmap::negation< + optional_internal::is_constructible_convertible_from_optional< + T, U>>, + phmap::negation<std::is_convertible<U&&, T>>>::value, + bool>::type = false> + explicit optional(optional<U>&& rhs) { + if (rhs) { + this->construct(std::move(*rhs)); + } + } + + // Destructor. Trivial if `T` is trivially destructible. + ~optional() = default; + + // Assignment Operators + + // Assignment from `nullopt` + // + // Example: + // + // struct S { int value; }; + // optional<S> opt = phmap::nullopt; // Could also use opt = { }; + optional& operator=(nullopt_t) noexcept { + this->destruct(); + return *this; + } + + // Copy assignment operator, standard semantics + optional& operator=(const optional& src) = default; + + // Move assignment operator, standard semantics + optional& operator=(optional&& src) = default; + + // Value assignment operators + template < + typename U = T, + typename = typename std::enable_if<phmap::conjunction< + phmap::negation< + std::is_same<optional<T>, typename std::decay<U>::type>>, + phmap::negation< + phmap::conjunction<std::is_scalar<T>, + std::is_same<T, typename std::decay<U>::type>>>, + std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type> + optional& operator=(U&& v) { + this->assign(std::forward<U>(v)); + return *this; + } + + template < + typename U, + typename = typename std::enable_if<phmap::conjunction< + phmap::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>, + phmap::negation< + optional_internal:: + is_constructible_convertible_assignable_from_optional< + T, U>>>::value>::type> + optional& operator=(const optional<U>& rhs) { + if (rhs) { + this->assign(*rhs); + } else { + this->destruct(); + } + return *this; + } + + template <typename U, + typename = typename std::enable_if<phmap::conjunction< + phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U>, + std::is_assignable<T&, U>, + phmap::negation< + optional_internal:: + is_constructible_convertible_assignable_from_optional< + T, U>>>::value>::type> + optional& operator=(optional<U>&& rhs) { + if (rhs) { + this->assign(std::move(*rhs)); + } else { + this->destruct(); + } + return *this; + } + + // Modifiers + + // optional::reset() + // + // Destroys the inner `T` value of an `phmap::optional` if one is present. + PHMAP_ATTRIBUTE_REINITIALIZES void reset() noexcept { this->destruct(); } + + // optional::emplace() + // + // (Re)constructs the underlying `T` in-place with the given forwarded + // arguments. + // + // Example: + // + // optional<Foo> opt; + // opt.emplace(arg1,arg2,arg3); // Constructs Foo(arg1,arg2,arg3) + // + // If the optional is non-empty, and the `args` refer to subobjects of the + // current object, then behaviour is undefined, because the current object + // will be destructed before the new object is constructed with `args`. + template <typename... Args, + typename = typename std::enable_if< + std::is_constructible<T, Args&&...>::value>::type> + T& emplace(Args&&... args) { + this->destruct(); + this->construct(std::forward<Args>(args)...); + return reference(); + } + + // Emplace reconstruction overload for an initializer list and the given + // forwarded arguments. + // + // Example: + // + // struct Foo { + // Foo(std::initializer_list<int>); + // }; + // + // optional<Foo> opt; + // opt.emplace({1,2,3}); // Constructs Foo({1,2,3}) + template <typename U, typename... Args, + typename = typename std::enable_if<std::is_constructible< + T, std::initializer_list<U>&, Args&&...>::value>::type> + T& emplace(std::initializer_list<U> il, Args&&... args) { + this->destruct(); + this->construct(il, std::forward<Args>(args)...); + return reference(); + } + + // Swaps + + // Swap, standard semantics + void swap(optional& rhs) noexcept( + std::is_nothrow_move_constructible<T>::value&& + std::is_trivial<T>::value) { + if (*this) { + if (rhs) { + using std::swap; + swap(**this, *rhs); + } else { + rhs.construct(std::move(**this)); + this->destruct(); + } + } else { + if (rhs) { + this->construct(std::move(*rhs)); + rhs.destruct(); + } else { + // No effect (swap(disengaged, disengaged)). + } + } + } + + // Observers + + // optional::operator->() + // + // Accesses the underlying `T` value's member `m` of an `optional`. If the + // `optional` is empty, behavior is undefined. + // + // If you need myOpt->foo in constexpr, use (*myOpt).foo instead. + const T* operator->() const { + assert(this->engaged_); + return std::addressof(this->data_); + } + T* operator->() { + assert(this->engaged_); + return std::addressof(this->data_); + } + + // optional::operator*() + // + // Accesses the underlying `T` value of an `optional`. If the `optional` is + // empty, behavior is undefined. + constexpr const T& operator*() const & { return reference(); } + T& operator*() & { + assert(this->engaged_); + return reference(); + } + constexpr const T&& operator*() const && { + return phmap::move(reference()); + } + T&& operator*() && { + assert(this->engaged_); + return std::move(reference()); + } + + // optional::operator bool() + // + // Returns false if and only if the `optional` is empty. + // + // if (opt) { + // // do something with opt.value(); + // } else { + // // opt is empty. + // } + // + constexpr explicit operator bool() const noexcept { return this->engaged_; } + + // optional::has_value() + // + // Determines whether the `optional` contains a value. Returns `false` if and + // only if `*this` is empty. + constexpr bool has_value() const noexcept { return this->engaged_; } + +// Suppress bogus warning on MSVC: MSVC complains call to reference() after +// throw_bad_optional_access() is unreachable. +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4702) +#endif // _MSC_VER + // optional::value() + // + // Returns a reference to an `optional`s underlying value. The constness + // and lvalue/rvalue-ness of the `optional` is preserved to the view of + // the `T` sub-object. Throws `phmap::bad_optional_access` when the `optional` + // is empty. + constexpr const T& value() const & { + return static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference()); + } + T& value() & { + return static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference()); + } + T&& value() && { // NOLINT(build/c++11) + return std::move( + static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference())); + } + constexpr const T&& value() const && { // NOLINT(build/c++11) + return phmap::move( + static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference())); + } +#ifdef _MSC_VER + #pragma warning(pop) +#endif // _MSC_VER + + // optional::value_or() + // + // Returns either the value of `T` or a passed default `v` if the `optional` + // is empty. + template <typename U> + constexpr T value_or(U&& v) const& { + static_assert(std::is_copy_constructible<value_type>::value, + "optional<T>::value_or: T must by copy constructible"); + static_assert(std::is_convertible<U&&, value_type>::value, + "optional<T>::value_or: U must be convertible to T"); + return static_cast<bool>(*this) + ? **this + : static_cast<T>(phmap::forward<U>(v)); + } + template <typename U> + T value_or(U&& v) && { // NOLINT(build/c++11) + static_assert(std::is_move_constructible<value_type>::value, + "optional<T>::value_or: T must by move constructible"); + static_assert(std::is_convertible<U&&, value_type>::value, + "optional<T>::value_or: U must be convertible to T"); + return static_cast<bool>(*this) ? std::move(**this) + : static_cast<T>(std::forward<U>(v)); + } + +private: + // Private accessors for internal storage viewed as reference to T. + constexpr const T& reference() const { return this->data_; } + T& reference() { return this->data_; } + + // T constraint checks. You can't have an optional of nullopt_t, in_place_t + // or a reference. + static_assert( + !std::is_same<nullopt_t, typename std::remove_cv<T>::type>::value, + "optional<nullopt_t> is not allowed."); + static_assert( + !std::is_same<in_place_t, typename std::remove_cv<T>::type>::value, + "optional<in_place_t> is not allowed."); + static_assert(!std::is_reference<T>::value, + "optional<reference> is not allowed."); +}; + +// Non-member functions + +// swap() +// +// Performs a swap between two `phmap::optional` objects, using standard +// semantics. +// +// NOTE: we assume `is_swappable()` is always `true`. A compile error will +// result if this is not the case. +template <typename T, + typename std::enable_if<std::is_move_constructible<T>::value, + bool>::type = false> +void swap(optional<T>& a, optional<T>& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); +} + +// make_optional() +// +// Creates a non-empty `optional<T>` where the type of `T` is deduced. An +// `phmap::optional` can also be explicitly instantiated with +// `make_optional<T>(v)`. +// +// Note: `make_optional()` constructions may be declared `constexpr` for +// trivially copyable types `T`. Non-trivial types require copy elision +// support in C++17 for `make_optional` to support `constexpr` on such +// non-trivial types. +// +// Example: +// +// constexpr phmap::optional<int> opt = phmap::make_optional(1); +// static_assert(opt.value() == 1, ""); +template <typename T> +constexpr optional<typename std::decay<T>::type> make_optional(T&& v) { + return optional<typename std::decay<T>::type>(phmap::forward<T>(v)); +} + +template <typename T, typename... Args> +constexpr optional<T> make_optional(Args&&... args) { + return optional<T>(in_place_t(), phmap::forward<Args>(args)...); +} + +template <typename T, typename U, typename... Args> +constexpr optional<T> make_optional(std::initializer_list<U> il, + Args&&... args) { + return optional<T>(in_place_t(), il, + phmap::forward<Args>(args)...); +} + +// Relational operators [optional.relops] + +// Empty optionals are considered equal to each other and less than non-empty +// optionals. Supports relations between optional<T> and optional<U>, between +// optional<T> and U, and between optional<T> and nullopt. +// +// Note: We're careful to support T having non-bool relationals. + +// Requires: The expression, e.g. "*x == *y" shall be well-formed and its result +// shall be convertible to bool. +// The C++17 (N4606) "Returns:" statements are translated into +// code in an obvious way here, and the original text retained as function docs. +// Returns: If bool(x) != bool(y), false; otherwise if bool(x) == false, true; +// otherwise *x == *y. +template <typename T, typename U> +constexpr auto operator==(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x == *y)) { + return static_cast<bool>(x) != static_cast<bool>(y) + ? false + : static_cast<bool>(x) == false ? true + : static_cast<bool>(*x == *y); +} + +// Returns: If bool(x) != bool(y), true; otherwise, if bool(x) == false, false; +// otherwise *x != *y. +template <typename T, typename U> +constexpr auto operator!=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x != *y)) { + return static_cast<bool>(x) != static_cast<bool>(y) + ? true + : static_cast<bool>(x) == false ? false + : static_cast<bool>(*x != *y); +} +// Returns: If !y, false; otherwise, if !x, true; otherwise *x < *y. +template <typename T, typename U> +constexpr auto operator<(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x < *y)) { + return !y ? false : !x ? true : static_cast<bool>(*x < *y); +} +// Returns: If !x, false; otherwise, if !y, true; otherwise *x > *y. +template <typename T, typename U> +constexpr auto operator>(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x > *y)) { + return !x ? false : !y ? true : static_cast<bool>(*x > *y); +} +// Returns: If !x, true; otherwise, if !y, false; otherwise *x <= *y. +template <typename T, typename U> +constexpr auto operator<=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x <= *y)) { + return !x ? true : !y ? false : static_cast<bool>(*x <= *y); +} +// Returns: If !y, true; otherwise, if !x, false; otherwise *x >= *y. +template <typename T, typename U> +constexpr auto operator>=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x >= *y)) { + return !y ? true : !x ? false : static_cast<bool>(*x >= *y); +} + +// Comparison with nullopt [optional.nullops] +// The C++17 (N4606) "Returns:" statements are used directly here. +template <typename T> +constexpr bool operator==(const optional<T>& x, nullopt_t) noexcept { + return !x; +} +template <typename T> +constexpr bool operator==(nullopt_t, const optional<T>& x) noexcept { + return !x; +} +template <typename T> +constexpr bool operator!=(const optional<T>& x, nullopt_t) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator!=(nullopt_t, const optional<T>& x) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator<(const optional<T>&, nullopt_t) noexcept { + return false; +} +template <typename T> +constexpr bool operator<(nullopt_t, const optional<T>& x) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator<=(const optional<T>& x, nullopt_t) noexcept { + return !x; +} +template <typename T> +constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept { + return true; +} +template <typename T> +constexpr bool operator>(const optional<T>& x, nullopt_t) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator>(nullopt_t, const optional<T>&) noexcept { + return false; +} +template <typename T> +constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept { + return true; +} +template <typename T> +constexpr bool operator>=(nullopt_t, const optional<T>& x) noexcept { + return !x; +} + +// Comparison with T [optional.comp_with_t] + +// Requires: The expression, e.g. "*x == v" shall be well-formed and its result +// shall be convertible to bool. +// The C++17 (N4606) "Equivalent to:" statements are used directly here. +template <typename T, typename U> +constexpr auto operator==(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x == v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x == v) : false; +} +template <typename T, typename U> +constexpr auto operator==(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v == *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v == *x) : false; +} +template <typename T, typename U> +constexpr auto operator!=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x != v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x != v) : true; +} +template <typename T, typename U> +constexpr auto operator!=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v != *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v != *x) : true; +} +template <typename T, typename U> +constexpr auto operator<(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x < v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x < v) : true; +} +template <typename T, typename U> +constexpr auto operator<(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v < *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v < *x) : false; +} +template <typename T, typename U> +constexpr auto operator<=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x <= v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x <= v) : true; +} +template <typename T, typename U> +constexpr auto operator<=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v <= *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v <= *x) : false; +} +template <typename T, typename U> +constexpr auto operator>(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x > v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x > v) : false; +} +template <typename T, typename U> +constexpr auto operator>(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v > *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v > *x) : true; +} +template <typename T, typename U> +constexpr auto operator>=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x >= v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x >= v) : false; +} +template <typename T, typename U> +constexpr auto operator>=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v >= *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v >= *x) : true; +} + +} // namespace phmap + +namespace std { + +// std::hash specialization for phmap::optional. +template <typename T> +struct hash<phmap::optional<T> > + : phmap::optional_internal::optional_hash_base<T> {}; + +} // namespace std + +#endif + +// ----------------------------------------------------------------------------- +// common.h +// ----------------------------------------------------------------------------- +namespace phmap { +namespace priv { + +template <class, class = void> +struct IsTransparent : std::false_type {}; +template <class T> +struct IsTransparent<T, phmap::void_t<typename T::is_transparent>> + : std::true_type {}; + +template <bool is_transparent> +struct KeyArg +{ + // Transparent. Forward `K`. + template <typename K, typename key_type> + using type = K; +}; + +template <> +struct KeyArg<false> +{ + // Not transparent. Always use `key_type`. + template <typename K, typename key_type> + using type = key_type; +}; + +#ifdef _MSC_VER + #pragma warning(push) + // warning C4820: '6' bytes padding added after data member + #pragma warning(disable : 4820) +#endif + +// The node_handle concept from C++17. +// We specialize node_handle for sets and maps. node_handle_base holds the +// common API of both. +// ----------------------------------------------------------------------- +template <typename PolicyTraits, typename Alloc> +class node_handle_base +{ +protected: + using slot_type = typename PolicyTraits::slot_type; + +public: + using allocator_type = Alloc; + + constexpr node_handle_base() {} + + node_handle_base(node_handle_base&& other) noexcept { + *this = std::move(other); + } + + ~node_handle_base() { destroy(); } + + node_handle_base& operator=(node_handle_base&& other) noexcept { + destroy(); + if (!other.empty()) { + alloc_ = other.alloc_; + PolicyTraits::transfer(alloc(), slot(), other.slot()); + other.reset(); + } + return *this; + } + + bool empty() const noexcept { return !alloc_; } + explicit operator bool() const noexcept { return !empty(); } + allocator_type get_allocator() const { return *alloc_; } + +protected: + friend struct CommonAccess; + + struct transfer_tag_t {}; + node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s) + : alloc_(a) { + PolicyTraits::transfer(alloc(), slot(), s); + } + + struct move_tag_t {}; + node_handle_base(move_tag_t, const allocator_type& a, slot_type* s) + : alloc_(a) { + PolicyTraits::construct(alloc(), slot(), s); + } + + node_handle_base(const allocator_type& a, slot_type* s) : alloc_(a) { + PolicyTraits::transfer(alloc(), slot(), s); + } + + //node_handle_base(const node_handle_base&) = delete; + //node_handle_base& operator=(const node_handle_base&) = delete; + + void destroy() { + if (!empty()) { + PolicyTraits::destroy(alloc(), slot()); + reset(); + } + } + + void reset() { + assert(alloc_.has_value()); + alloc_ = phmap::nullopt; + } + + slot_type* slot() const { + assert(!empty()); + return reinterpret_cast<slot_type*>(std::addressof(slot_space_)); + } + + allocator_type* alloc() { return std::addressof(*alloc_); } + +private: + phmap::optional<allocator_type> alloc_; + mutable phmap::aligned_storage_t<sizeof(slot_type), alignof(slot_type)> slot_space_; +}; + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +// For sets. +// --------- +template <typename Policy, typename PolicyTraits, typename Alloc, + typename = void> +class node_handle : public node_handle_base<PolicyTraits, Alloc> +{ + using Base = node_handle_base<PolicyTraits, Alloc>; + +public: + using value_type = typename PolicyTraits::value_type; + + constexpr node_handle() {} + + value_type& value() const { return PolicyTraits::element(this->slot()); } + + value_type& key() const { return PolicyTraits::element(this->slot()); } + +private: + friend struct CommonAccess; + + using Base::Base; +}; + +// For maps. +// --------- +template <typename Policy, typename PolicyTraits, typename Alloc> +class node_handle<Policy, PolicyTraits, Alloc, + phmap::void_t<typename Policy::mapped_type>> + : public node_handle_base<PolicyTraits, Alloc> +{ + using Base = node_handle_base<PolicyTraits, Alloc>; + +public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + + constexpr node_handle() {} + + auto key() const -> decltype(PolicyTraits::key(this->slot())) { + return PolicyTraits::key(this->slot()); + } + + mapped_type& mapped() const { + return PolicyTraits::value(&PolicyTraits::element(this->slot())); + } + +private: + friend struct CommonAccess; + + using Base::Base; +}; + +// Provide access to non-public node-handle functions. +struct CommonAccess +{ + template <typename Node> + static auto GetSlot(const Node& node) -> decltype(node.slot()) { + return node.slot(); + } + + template <typename Node> + static void Destroy(Node* node) { + node->destroy(); + } + + template <typename Node> + static void Reset(Node* node) { + node->reset(); + } + + template <typename T, typename... Args> + static T Make(Args&&... args) { + return T(std::forward<Args>(args)...); + } + + template <typename T, typename... Args> + static T Transfer(Args&&... args) { + return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...); + } + + template <typename T, typename... Args> + static T Move(Args&&... args) { + return T(typename T::move_tag_t{}, std::forward<Args>(args)...); + } +}; + +// Implement the insert_return_type<> concept of C++17. +template <class Iterator, class NodeType> +struct InsertReturnType +{ + Iterator position; + bool inserted; + NodeType node; +}; + +} // namespace priv +} // namespace phmap + + +#ifdef ADDRESS_SANITIZER + #include <sanitizer/asan_interface.h> +#endif + +// --------------------------------------------------------------------------- +// span.h +// --------------------------------------------------------------------------- + +namespace phmap { + +template <typename T> +class Span; + +namespace span_internal { +// A constexpr min function +constexpr size_t Min(size_t a, size_t b) noexcept { return a < b ? a : b; } + +// Wrappers for access to container data pointers. +template <typename C> +constexpr auto GetDataImpl(C& c, char) noexcept // NOLINT(runtime/references) + -> decltype(c.data()) { + return c.data(); +} + +// Before C++17, std::string::data returns a const char* in all cases. +inline char* GetDataImpl(std::string& s, // NOLINT(runtime/references) + int) noexcept { + return &s[0]; +} + +template <typename C> +constexpr auto GetData(C& c) noexcept // NOLINT(runtime/references) + -> decltype(GetDataImpl(c, 0)) { + return GetDataImpl(c, 0); +} + +// Detection idioms for size() and data(). +template <typename C> +using HasSize = + std::is_integral<phmap::decay_t<decltype(std::declval<C&>().size())>>; + +// We want to enable conversion from vector<T*> to Span<const T* const> but +// disable conversion from vector<Derived> to Span<Base>. Here we use +// the fact that U** is convertible to Q* const* if and only if Q is the same +// type or a more cv-qualified version of U. We also decay the result type of +// data() to avoid problems with classes which have a member function data() +// which returns a reference. +template <typename T, typename C> +using HasData = + std::is_convertible<phmap::decay_t<decltype(GetData(std::declval<C&>()))>*, + T* const*>; + +// Extracts value type from a Container +template <typename C> +struct ElementType { + using type = typename phmap::remove_reference_t<C>::value_type; +}; + +template <typename T, size_t N> +struct ElementType<T (&)[N]> { + using type = T; +}; + +template <typename C> +using ElementT = typename ElementType<C>::type; + +template <typename T> +using EnableIfMutable = + typename std::enable_if<!std::is_const<T>::value, int>::type; + +template <typename T> +bool EqualImpl(Span<T> a, Span<T> b) { + static_assert(std::is_const<T>::value, ""); + return std::equal(a.begin(), a.end(), b.begin(), b.end()); +} + +template <typename T> +bool LessThanImpl(Span<T> a, Span<T> b) { + static_assert(std::is_const<T>::value, ""); + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); +} + +// The `IsConvertible` classes here are needed because of the +// `std::is_convertible` bug in libcxx when compiled with GCC. This build +// configuration is used by Android NDK toolchain. Reference link: +// https://bugs.llvm.org/show_bug.cgi?id=27538. +template <typename From, typename To> +struct IsConvertibleHelper { + static std::true_type testval(To); + static std::false_type testval(...); + + using type = decltype(testval(std::declval<From>())); +}; + +template <typename From, typename To> +struct IsConvertible : IsConvertibleHelper<From, To>::type {}; + +// TODO(zhangxy): replace `IsConvertible` with `std::is_convertible` once the +// older version of libcxx is not supported. +template <typename From, typename To> +using EnableIfConvertibleToSpanConst = + typename std::enable_if<IsConvertible<From, Span<const To>>::value>::type; +} // namespace span_internal + +//------------------------------------------------------------------------------ +// Span +//------------------------------------------------------------------------------ +// +// A `Span` is an "array view" type for holding a view of a contiguous data +// array; the `Span` object does not and cannot own such data itself. A span +// provides an easy way to provide overloads for anything operating on +// contiguous sequences without needing to manage pointers and array lengths +// manually. + +// A span is conceptually a pointer (ptr) and a length (size) into an already +// existing array of contiguous memory; the array it represents references the +// elements "ptr[0] .. ptr[size-1]". Passing a properly-constructed `Span` +// instead of raw pointers avoids many issues related to index out of bounds +// errors. +// +// Spans may also be constructed from containers holding contiguous sequences. +// Such containers must supply `data()` and `size() const` methods (e.g +// `std::vector<T>`, `phmap::InlinedVector<T, N>`). All implicit conversions to +// `phmap::Span` from such containers will create spans of type `const T`; +// spans which can mutate their values (of type `T`) must use explicit +// constructors. +// +// A `Span<T>` is somewhat analogous to an `phmap::string_view`, but for an array +// of elements of type `T`. A user of `Span` must ensure that the data being +// pointed to outlives the `Span` itself. +// +// You can construct a `Span<T>` in several ways: +// +// * Explicitly from a reference to a container type +// * Explicitly from a pointer and size +// * Implicitly from a container type (but only for spans of type `const T`) +// * Using the `MakeSpan()` or `MakeConstSpan()` factory functions. +// +// Examples: +// +// // Construct a Span explicitly from a container: +// std::vector<int> v = {1, 2, 3, 4, 5}; +// auto span = phmap::Span<const int>(v); +// +// // Construct a Span explicitly from a C-style array: +// int a[5] = {1, 2, 3, 4, 5}; +// auto span = phmap::Span<const int>(a); +// +// // Construct a Span implicitly from a container +// void MyRoutine(phmap::Span<const int> a) { +// ... +// } +// std::vector v = {1,2,3,4,5}; +// MyRoutine(v) // convert to Span<const T> +// +// Note that `Span` objects, in addition to requiring that the memory they +// point to remains alive, must also ensure that such memory does not get +// reallocated. Therefore, to avoid undefined behavior, containers with +// associated span views should not invoke operations that may reallocate memory +// (such as resizing) or invalidate iterators into the container. +// +// One common use for a `Span` is when passing arguments to a routine that can +// accept a variety of array types (e.g. a `std::vector`, `phmap::InlinedVector`, +// a C-style array, etc.). Instead of creating overloads for each case, you +// can simply specify a `Span` as the argument to such a routine. +// +// Example: +// +// void MyRoutine(phmap::Span<const int> a) { +// ... +// } +// +// std::vector v = {1,2,3,4,5}; +// MyRoutine(v); +// +// phmap::InlinedVector<int, 4> my_inline_vector; +// MyRoutine(my_inline_vector); +// +// // Explicit constructor from pointer,size +// int* my_array = new int[10]; +// MyRoutine(phmap::Span<const int>(my_array, 10)); +template <typename T> +class Span +{ +private: + // Used to determine whether a Span can be constructed from a container of + // type C. + template <typename C> + using EnableIfConvertibleFrom = + typename std::enable_if<span_internal::HasData<T, C>::value && + span_internal::HasSize<C>::value>::type; + + // Used to SFINAE-enable a function when the slice elements are const. + template <typename U> + using EnableIfConstView = + typename std::enable_if<std::is_const<T>::value, U>::type; + + // Used to SFINAE-enable a function when the slice elements are mutable. + template <typename U> + using EnableIfMutableView = + typename std::enable_if<!std::is_const<T>::value, U>::type; + +public: + using value_type = phmap::remove_cv_t<T>; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using size_type = size_t; + using difference_type = ptrdiff_t; + + static const size_type npos = ~(size_type(0)); + + constexpr Span() noexcept : Span(nullptr, 0) {} + constexpr Span(pointer array, size_type lgth) noexcept + : ptr_(array), len_(lgth) {} + + // Implicit conversion constructors + template <size_t N> + constexpr Span(T (&a)[N]) noexcept // NOLINT(runtime/explicit) + : Span(a, N) {} + + // Explicit reference constructor for a mutable `Span<T>` type. Can be + // replaced with MakeSpan() to infer the type parameter. + template <typename V, typename = EnableIfConvertibleFrom<V>, + typename = EnableIfMutableView<V>> + explicit Span(V& v) noexcept // NOLINT(runtime/references) + : Span(span_internal::GetData(v), v.size()) {} + + // Implicit reference constructor for a read-only `Span<const T>` type + template <typename V, typename = EnableIfConvertibleFrom<V>, + typename = EnableIfConstView<V>> + constexpr Span(const V& v) noexcept // NOLINT(runtime/explicit) + : Span(span_internal::GetData(v), v.size()) {} + + // Implicit constructor from an initializer list, making it possible to pass a + // brace-enclosed initializer list to a function expecting a `Span`. Such + // spans constructed from an initializer list must be of type `Span<const T>`. + // + // void Process(phmap::Span<const int> x); + // Process({1, 2, 3}); + // + // Note that as always the array referenced by the span must outlive the span. + // Since an initializer list constructor acts as if it is fed a temporary + // array (cf. C++ standard [dcl.init.list]/5), it's safe to use this + // constructor only when the `std::initializer_list` itself outlives the span. + // In order to meet this requirement it's sufficient to ensure that neither + // the span nor a copy of it is used outside of the expression in which it's + // created: + // + // // Assume that this function uses the array directly, not retaining any + // // copy of the span or pointer to any of its elements. + // void Process(phmap::Span<const int> ints); + // + // // Okay: the std::initializer_list<int> will reference a temporary array + // // that isn't destroyed until after the call to Process returns. + // Process({ 17, 19 }); + // + // // Not okay: the storage used by the std::initializer_list<int> is not + // // allowed to be referenced after the first line. + // phmap::Span<const int> ints = { 17, 19 }; + // Process(ints); + // + // // Not okay for the same reason as above: even when the elements of the + // // initializer list expression are not temporaries the underlying array + // // is, so the initializer list must still outlive the span. + // const int foo = 17; + // phmap::Span<const int> ints = { foo }; + // Process(ints); + // + template <typename LazyT = T, + typename = EnableIfConstView<LazyT>> + Span( + std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) + : Span(v.begin(), v.size()) {} + + // Accessors + + // Span::data() + // + // Returns a pointer to the span's underlying array of data (which is held + // outside the span). + constexpr pointer data() const noexcept { return ptr_; } + + // Span::size() + // + // Returns the size of this span. + constexpr size_type size() const noexcept { return len_; } + + // Span::length() + // + // Returns the length (size) of this span. + constexpr size_type length() const noexcept { return size(); } + + // Span::empty() + // + // Returns a boolean indicating whether or not this span is considered empty. + constexpr bool empty() const noexcept { return size() == 0; } + + // Span::operator[] + // + // Returns a reference to the i'th element of this span. + constexpr reference operator[](size_type i) const noexcept { + // MSVC 2015 accepts this as constexpr, but not ptr_[i] + return *(data() + i); + } + + // Span::at() + // + // Returns a reference to the i'th element of this span. + constexpr reference at(size_type i) const { + return PHMAP_PREDICT_TRUE(i < size()) // + ? *(data() + i) + : (base_internal::ThrowStdOutOfRange( + "Span::at failed bounds check"), + *(data() + i)); + } + + // Span::front() + // + // Returns a reference to the first element of this span. + constexpr reference front() const noexcept { + return PHMAP_ASSERT(size() > 0), *data(); + } + + // Span::back() + // + // Returns a reference to the last element of this span. + constexpr reference back() const noexcept { + return PHMAP_ASSERT(size() > 0), *(data() + size() - 1); + } + + // Span::begin() + // + // Returns an iterator to the first element of this span. + constexpr iterator begin() const noexcept { return data(); } + + // Span::cbegin() + // + // Returns a const iterator to the first element of this span. + constexpr const_iterator cbegin() const noexcept { return begin(); } + + // Span::end() + // + // Returns an iterator to the last element of this span. + constexpr iterator end() const noexcept { return data() + size(); } + + // Span::cend() + // + // Returns a const iterator to the last element of this span. + constexpr const_iterator cend() const noexcept { return end(); } + + // Span::rbegin() + // + // Returns a reverse iterator starting at the last element of this span. + constexpr reverse_iterator rbegin() const noexcept { + return reverse_iterator(end()); + } + + // Span::crbegin() + // + // Returns a reverse const iterator starting at the last element of this span. + constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); } + + // Span::rend() + // + // Returns a reverse iterator starting at the first element of this span. + constexpr reverse_iterator rend() const noexcept { + return reverse_iterator(begin()); + } + + // Span::crend() + // + // Returns a reverse iterator starting at the first element of this span. + constexpr const_reverse_iterator crend() const noexcept { return rend(); } + + // Span mutations + + // Span::remove_prefix() + // + // Removes the first `n` elements from the span. + void remove_prefix(size_type n) noexcept { + assert(size() >= n); + ptr_ += n; + len_ -= n; + } + + // Span::remove_suffix() + // + // Removes the last `n` elements from the span. + void remove_suffix(size_type n) noexcept { + assert(size() >= n); + len_ -= n; + } + + // Span::subspan() + // + // Returns a `Span` starting at element `pos` and of length `len`. Both `pos` + // and `len` are of type `size_type` and thus non-negative. Parameter `pos` + // must be <= size(). Any `len` value that points past the end of the span + // will be trimmed to at most size() - `pos`. A default `len` value of `npos` + // ensures the returned subspan continues until the end of the span. + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // phmap::MakeSpan(vec).subspan(1, 2); // {11, 12} + // phmap::MakeSpan(vec).subspan(2, 8); // {12, 13} + // phmap::MakeSpan(vec).subspan(1); // {11, 12, 13} + // phmap::MakeSpan(vec).subspan(4); // {} + // phmap::MakeSpan(vec).subspan(5); // throws std::out_of_range + constexpr Span subspan(size_type pos = 0, size_type len = npos) const { + return (pos <= size()) + ? Span(data() + pos, span_internal::Min(size() - pos, len)) + : (base_internal::ThrowStdOutOfRange("pos > size()"), Span()); + } + + // Span::first() + // + // Returns a `Span` containing first `len` elements. Parameter `len` is of + // type `size_type` and thus non-negative. `len` value must be <= size(). + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // phmap::MakeSpan(vec).first(1); // {10} + // phmap::MakeSpan(vec).first(3); // {10, 11, 12} + // phmap::MakeSpan(vec).first(5); // throws std::out_of_range + constexpr Span first(size_type len) const { + return (len <= size()) + ? Span(data(), len) + : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); + } + + // Span::last() + // + // Returns a `Span` containing last `len` elements. Parameter `len` is of + // type `size_type` and thus non-negative. `len` value must be <= size(). + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // phmap::MakeSpan(vec).last(1); // {13} + // phmap::MakeSpan(vec).last(3); // {11, 12, 13} + // phmap::MakeSpan(vec).last(5); // throws std::out_of_range + constexpr Span last(size_type len) const { + return (len <= size()) + ? Span(size() - len + data(), len) + : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); + } + + // Support for phmap::Hash. + template <typename H> + friend H AbslHashValue(H h, Span v) { + return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), + v.size()); + } + +private: + pointer ptr_; + size_type len_; +}; + +template <typename T> +const typename Span<T>::size_type Span<T>::npos; + +// Span relationals + +// Equality is compared element-by-element, while ordering is lexicographical. +// We provide three overloads for each operator to cover any combination on the +// left or right hand side of mutable Span<T>, read-only Span<const T>, and +// convertible-to-read-only Span<T>. +// TODO(zhangxy): Due to MSVC overload resolution bug with partial ordering +// template functions, 5 overloads per operator is needed as a workaround. We +// should update them to 3 overloads per operator using non-deduced context like +// string_view, i.e. +// - (Span<T>, Span<T>) +// - (Span<T>, non_deduced<Span<const T>>) +// - (non_deduced<Span<const T>>, Span<T>) + +// operator== +template <typename T> +bool operator==(Span<T> a, Span<T> b) { + return span_internal::EqualImpl<const T>(a, b); +} + +template <typename T> +bool operator==(Span<const T> a, Span<T> b) { + return span_internal::EqualImpl<const T>(a, b); +} + +template <typename T> +bool operator==(Span<T> a, Span<const T> b) { + return span_internal::EqualImpl<const T>(a, b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator==(const U& a, Span<T> b) { + return span_internal::EqualImpl<const T>(a, b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator==(Span<T> a, const U& b) { + return span_internal::EqualImpl<const T>(a, b); +} + +// operator!= +template <typename T> +bool operator!=(Span<T> a, Span<T> b) { + return !(a == b); +} + +template <typename T> +bool operator!=(Span<const T> a, Span<T> b) { + return !(a == b); +} + +template <typename T> +bool operator!=(Span<T> a, Span<const T> b) { + return !(a == b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator!=(const U& a, Span<T> b) { + return !(a == b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator!=(Span<T> a, const U& b) { + return !(a == b); +} + +// operator< +template <typename T> +bool operator<(Span<T> a, Span<T> b) { + return span_internal::LessThanImpl<const T>(a, b); +} + +template <typename T> +bool operator<(Span<const T> a, Span<T> b) { + return span_internal::LessThanImpl<const T>(a, b); +} + +template <typename T> +bool operator<(Span<T> a, Span<const T> b) { + return span_internal::LessThanImpl<const T>(a, b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator<(const U& a, Span<T> b) { + return span_internal::LessThanImpl<const T>(a, b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator<(Span<T> a, const U& b) { + return span_internal::LessThanImpl<const T>(a, b); +} + +// operator> +template <typename T> +bool operator>(Span<T> a, Span<T> b) { + return b < a; +} + +template <typename T> +bool operator>(Span<const T> a, Span<T> b) { + return b < a; +} + +template <typename T> +bool operator>(Span<T> a, Span<const T> b) { + return b < a; +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator>(const U& a, Span<T> b) { + return b < a; +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator>(Span<T> a, const U& b) { + return b < a; +} + +// operator<= +template <typename T> +bool operator<=(Span<T> a, Span<T> b) { + return !(b < a); +} + +template <typename T> +bool operator<=(Span<const T> a, Span<T> b) { + return !(b < a); +} + +template <typename T> +bool operator<=(Span<T> a, Span<const T> b) { + return !(b < a); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator<=(const U& a, Span<T> b) { + return !(b < a); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator<=(Span<T> a, const U& b) { + return !(b < a); +} + +// operator>= +template <typename T> +bool operator>=(Span<T> a, Span<T> b) { + return !(a < b); +} + +template <typename T> +bool operator>=(Span<const T> a, Span<T> b) { + return !(a < b); +} + +template <typename T> +bool operator>=(Span<T> a, Span<const T> b) { + return !(a < b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator>=(const U& a, Span<T> b) { + return !(a < b); +} + +template <typename T, typename U, + typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> +bool operator>=(Span<T> a, const U& b) { + return !(a < b); +} + +// MakeSpan() +// +// Constructs a mutable `Span<T>`, deducing `T` automatically from either a +// container or pointer+size. +// +// Because a read-only `Span<const T>` is implicitly constructed from container +// types regardless of whether the container itself is a const container, +// constructing mutable spans of type `Span<T>` from containers requires +// explicit constructors. The container-accepting version of `MakeSpan()` +// deduces the type of `T` by the constness of the pointer received from the +// container's `data()` member. Similarly, the pointer-accepting version returns +// a `Span<const T>` if `T` is `const`, and a `Span<T>` otherwise. +// +// Examples: +// +// void MyRoutine(phmap::Span<MyComplicatedType> a) { +// ... +// }; +// // my_vector is a container of non-const types +// std::vector<MyComplicatedType> my_vector; +// +// // Constructing a Span implicitly attempts to create a Span of type +// // `Span<const T>` +// MyRoutine(my_vector); // error, type mismatch +// +// // Explicitly constructing the Span is verbose +// MyRoutine(phmap::Span<MyComplicatedType>(my_vector)); +// +// // Use MakeSpan() to make an phmap::Span<T> +// MyRoutine(phmap::MakeSpan(my_vector)); +// +// // Construct a span from an array ptr+size +// phmap::Span<T> my_span() { +// return phmap::MakeSpan(&array[0], num_elements_); +// } +// +template <int&... ExplicitArgumentBarrier, typename T> +constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { + return Span<T>(ptr, size); +} + +template <int&... ExplicitArgumentBarrier, typename T> +Span<T> MakeSpan(T* begin, T* end) noexcept { + return PHMAP_ASSERT(begin <= end), Span<T>(begin, end - begin); +} + +template <int&... ExplicitArgumentBarrier, typename C> +constexpr auto MakeSpan(C& c) noexcept // NOLINT(runtime/references) + -> decltype(phmap::MakeSpan(span_internal::GetData(c), c.size())) { + return MakeSpan(span_internal::GetData(c), c.size()); +} + +template <int&... ExplicitArgumentBarrier, typename T, size_t N> +constexpr Span<T> MakeSpan(T (&array)[N]) noexcept { + return Span<T>(array, N); +} + +// MakeConstSpan() +// +// Constructs a `Span<const T>` as with `MakeSpan`, deducing `T` automatically, +// but always returning a `Span<const T>`. +// +// Examples: +// +// void ProcessInts(phmap::Span<const int> some_ints); +// +// // Call with a pointer and size. +// int array[3] = { 0, 0, 0 }; +// ProcessInts(phmap::MakeConstSpan(&array[0], 3)); +// +// // Call with a [begin, end) pair. +// ProcessInts(phmap::MakeConstSpan(&array[0], &array[3])); +// +// // Call directly with an array. +// ProcessInts(phmap::MakeConstSpan(array)); +// +// // Call with a contiguous container. +// std::vector<int> some_ints = ...; +// ProcessInts(phmap::MakeConstSpan(some_ints)); +// ProcessInts(phmap::MakeConstSpan(std::vector<int>{ 0, 0, 0 })); +// +template <int&... ExplicitArgumentBarrier, typename T> +constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept { + return Span<const T>(ptr, size); +} + +template <int&... ExplicitArgumentBarrier, typename T> +Span<const T> MakeConstSpan(T* begin, T* end) noexcept { + return PHMAP_ASSERT(begin <= end), Span<const T>(begin, end - begin); +} + +template <int&... ExplicitArgumentBarrier, typename C> +constexpr auto MakeConstSpan(const C& c) noexcept -> decltype(MakeSpan(c)) { + return MakeSpan(c); +} + +template <int&... ExplicitArgumentBarrier, typename T, size_t N> +constexpr Span<const T> MakeConstSpan(const T (&array)[N]) noexcept { + return Span<const T>(array, N); +} +} // namespace phmap + +// --------------------------------------------------------------------------- +// layout.h +// --------------------------------------------------------------------------- +#if defined(__GXX_RTTI) + #define PHMAP_INTERNAL_HAS_CXA_DEMANGLE +#endif + +#ifdef PHMAP_INTERNAL_HAS_CXA_DEMANGLE + #include <cxxabi.h> +#endif + +namespace phmap { +namespace priv { + +// A type wrapper that instructs `Layout` to use the specific alignment for the +// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API +// and behavior as `Layout<..., T, ...>` except that the first element of the +// array of `T` is aligned to `N` (the rest of the elements follow without +// padding). +// +// Requires: `N >= alignof(T)` and `N` is a power of 2. +template <class T, size_t N> +struct Aligned; + +namespace internal_layout { + +template <class T> +struct NotAligned {}; + +template <class T, size_t N> +struct NotAligned<const Aligned<T, N>> { + static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified"); +}; + +template <size_t> +using IntToSize = size_t; + +template <class> +using TypeToSize = size_t; + +template <class T> +struct Type : NotAligned<T> { + using type = T; +}; + +template <class T, size_t N> +struct Type<Aligned<T, N>> { + using type = T; +}; + +template <class T> +struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {}; + +template <class T, size_t N> +struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {}; + +// Note: workaround for https://gcc.gnu.org/PR88115 +template <class T> +struct AlignOf : NotAligned<T> { + static constexpr size_t value = alignof(T); +}; + +template <class T, size_t N> +struct AlignOf<Aligned<T, N>> { + static_assert(N % alignof(T) == 0, + "Custom alignment can't be lower than the type's alignment"); + static constexpr size_t value = N; +}; + +// Does `Ts...` contain `T`? +template <class T, class... Ts> +using Contains = phmap::disjunction<std::is_same<T, Ts>...>; + +template <class From, class To> +using CopyConst = + typename std::conditional<std::is_const<From>::value, const To, To>::type; + +// Note: We're not qualifying this with phmap:: because it doesn't compile under +// MSVC. +template <class T> +using SliceType = Span<T>; + +// This namespace contains no types. It prevents functions defined in it from +// being found by ADL. +namespace adl_barrier { + +template <class Needle, class... Ts> +constexpr size_t Find(Needle, Needle, Ts...) { + static_assert(!Contains<Needle, Ts...>(), "Duplicate element type"); + return 0; +} + +template <class Needle, class T, class... Ts> +constexpr size_t Find(Needle, T, Ts...) { + return adl_barrier::Find(Needle(), Ts()...) + 1; +} + +constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); } + +// Returns `q * m` for the smallest `q` such that `q * m >= n`. +// Requires: `m` is a power of two. It's enforced by IsLegalElementType below. +constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } + +constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; } + +constexpr size_t Max(size_t a) { return a; } + +template <class... Ts> +constexpr size_t Max(size_t a, size_t b, Ts... rest) { + return adl_barrier::Max(b < a ? a : b, rest...); +} + +} // namespace adl_barrier + +template <bool C> +using EnableIf = typename std::enable_if<C, int>::type; + +// Can `T` be a template argument of `Layout`? +// --------------------------------------------------------------------------- +template <class T> +using IsLegalElementType = std::integral_constant< + bool, !std::is_reference<T>::value && !std::is_volatile<T>::value && + !std::is_reference<typename Type<T>::type>::value && + !std::is_volatile<typename Type<T>::type>::value && + adl_barrier::IsPow2(AlignOf<T>::value)>; + +template <class Elements, class SizeSeq, class OffsetSeq> +class LayoutImpl; + +// --------------------------------------------------------------------------- +// Public base class of `Layout` and the result type of `Layout::Partial()`. +// +// `Elements...` contains all template arguments of `Layout` that created this +// instance. +// +// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments +// passed to `Layout::Partial()` or `Layout::Layout()`. +// +// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is +// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we +// can compute offsets). +// --------------------------------------------------------------------------- +template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> +class LayoutImpl<std::tuple<Elements...>, phmap::index_sequence<SizeSeq...>, + phmap::index_sequence<OffsetSeq...>> +{ +private: + static_assert(sizeof...(Elements) > 0, "At least one field is required"); + static_assert(phmap::conjunction<IsLegalElementType<Elements>...>::value, + "Invalid element type (see IsLegalElementType)"); + + enum { + NumTypes = sizeof...(Elements), + NumSizes = sizeof...(SizeSeq), + NumOffsets = sizeof...(OffsetSeq), + }; + + // These are guaranteed by `Layout`. + static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), + "Internal error"); + static_assert(NumTypes > 0, "Internal error"); + + // Returns the index of `T` in `Elements...`. Results in a compilation error + // if `Elements...` doesn't contain exactly one instance of `T`. + template <class T> + static constexpr size_t ElementIndex() { + static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(), + "Type not found"); + return adl_barrier::Find(Type<T>(), + Type<typename Type<Elements>::type>()...); + } + + template <size_t N> + using ElementAlignment = + AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>; + +public: + // Element types of all arrays packed in a tuple. + using ElementTypes = std::tuple<typename Type<Elements>::type...>; + + // Element type of the Nth array. + template <size_t N> + using ElementType = typename std::tuple_element<N, ElementTypes>::type; + + constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) + : size_{sizes...} {} + + // Alignment of the layout, equal to the strictest alignment of all elements. + // All pointers passed to the methods of layout must be aligned to this value. + static constexpr size_t Alignment() { + return adl_barrier::Max(AlignOf<Elements>::value...); + } + + // Offset in bytes of the Nth array. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Offset<0>() == 0); // The ints starts from 0. + // assert(x.Offset<1>() == 16); // The doubles starts from 16. + // + // Requires: `N <= NumSizes && N < sizeof...(Ts)`. + template <size_t N, EnableIf<N == 0> = 0> + constexpr size_t Offset() const { + return 0; + } + + template <size_t N, EnableIf<N != 0> = 0> + constexpr size_t Offset() const { + static_assert(N < NumOffsets, "Index out of bounds"); + return adl_barrier::Align( + Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1], + ElementAlignment<N>::value); + } + + // Offset in bytes of the array with the specified element type. There must + // be exactly one such array and its zero-based index must be at most + // `NumSizes`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Offset<int>() == 0); // The ints starts from 0. + // assert(x.Offset<double>() == 16); // The doubles starts from 16. + template <class T> + constexpr size_t Offset() const { + return Offset<ElementIndex<T>()>(); + } + + // Offsets in bytes of all arrays for which the offsets are known. + constexpr std::array<size_t, NumOffsets> Offsets() const { + return {{Offset<OffsetSeq>()...}}; + } + + // The number of elements in the Nth array. This is the Nth argument of + // `Layout::Partial()` or `Layout::Layout()` (zero-based). + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Size<0>() == 3); + // assert(x.Size<1>() == 4); + // + // Requires: `N < NumSizes`. + template <size_t N> + constexpr size_t Size() const { + static_assert(N < NumSizes, "Index out of bounds"); + return size_[N]; + } + + // The number of elements in the array with the specified element type. + // There must be exactly one such array and its zero-based index must be + // at most `NumSizes`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Size<int>() == 3); + // assert(x.Size<double>() == 4); + template <class T> + constexpr size_t Size() const { + return Size<ElementIndex<T>()>(); + } + + // The number of elements of all arrays for which they are known. + constexpr std::array<size_t, NumSizes> Sizes() const { + return {{Size<SizeSeq>()...}}; + } + + // Pointer to the beginning of the Nth array. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // int* ints = x.Pointer<0>(p); + // double* doubles = x.Pointer<1>(p); + // + // Requires: `N <= NumSizes && N < sizeof...(Ts)`. + // Requires: `p` is aligned to `Alignment()`. + template <size_t N, class Char> + CopyConst<Char, ElementType<N>>* Pointer(Char* p) const { + using C = typename std::remove_const<Char>::type; + static_assert( + std::is_same<C, char>() || std::is_same<C, unsigned char>() || + std::is_same<C, signed char>(), + "The argument must be a pointer to [const] [signed|unsigned] char"); + constexpr size_t alignment = Alignment(); + (void)alignment; + assert(reinterpret_cast<uintptr_t>(p) % alignment == 0); + return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>()); + } + + // Pointer to the beginning of the array with the specified element type. + // There must be exactly one such array and its zero-based index must be at + // most `NumSizes`. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // int* ints = x.Pointer<int>(p); + // double* doubles = x.Pointer<double>(p); + // + // Requires: `p` is aligned to `Alignment()`. + template <class T, class Char> + CopyConst<Char, T>* Pointer(Char* p) const { + return Pointer<ElementIndex<T>()>(p); + } + + // Pointers to all arrays for which pointers are known. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // + // int* ints; + // double* doubles; + // std::tie(ints, doubles) = x.Pointers(p); + // + // Requires: `p` is aligned to `Alignment()`. + // + // Note: We're not using ElementType alias here because it does not compile + // under MSVC. + template <class Char> + std::tuple<CopyConst< + Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> + Pointers(Char* p) const { + return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( + Pointer<OffsetSeq>(p)...); + } + + // The Nth array. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // Span<int> ints = x.Slice<0>(p); + // Span<double> doubles = x.Slice<1>(p); + // + // Requires: `N < NumSizes`. + // Requires: `p` is aligned to `Alignment()`. + template <size_t N, class Char> + SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const { + return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>()); + } + + // The array with the specified element type. There must be exactly one + // such array and its zero-based index must be less than `NumSizes`. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // Span<int> ints = x.Slice<int>(p); + // Span<double> doubles = x.Slice<double>(p); + // + // Requires: `p` is aligned to `Alignment()`. + template <class T, class Char> + SliceType<CopyConst<Char, T>> Slice(Char* p) const { + return Slice<ElementIndex<T>()>(p); + } + + // All arrays with known sizes. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // + // Span<int> ints; + // Span<double> doubles; + // std::tie(ints, doubles) = x.Slices(p); + // + // Requires: `p` is aligned to `Alignment()`. + // + // Note: We're not using ElementType alias here because it does not compile + // under MSVC. + template <class Char> + std::tuple<SliceType<CopyConst< + Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> + Slices(Char* p) const { + // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed + // in 6.1). + (void)p; + return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( + Slice<SizeSeq>(p)...); + } + + // The size of the allocation that fits all arrays. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes + // + // Requires: `NumSizes == sizeof...(Ts)`. + constexpr size_t AllocSize() const { + static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); + return Offset<NumTypes - 1>() + + SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1]; + } + + // If built with --config=asan, poisons padding bytes (if any) in the + // allocation. The pointer must point to a memory block at least + // `AllocSize()` bytes in length. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // Requires: `p` is aligned to `Alignment()`. + template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0> + void PoisonPadding(const Char* p) const { + Pointer<0>(p); // verify the requirements on `Char` and `p` + } + + template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0> + void PoisonPadding(const Char* p) const { + static_assert(N < NumOffsets, "Index out of bounds"); + (void)p; +#ifdef ADDRESS_SANITIZER + PoisonPadding<Char, N - 1>(p); + // The `if` is an optimization. It doesn't affect the observable behaviour. + if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { + size_t start = + Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1]; + ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); + } +#endif + } + +private: + // Arguments of `Layout::Partial()` or `Layout::Layout()`. + size_t size_[NumSizes > 0 ? NumSizes : 1]; +}; + +template <size_t NumSizes, class... Ts> +using LayoutType = LayoutImpl< + std::tuple<Ts...>, phmap::make_index_sequence<NumSizes>, + phmap::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; + +} // namespace internal_layout + +// --------------------------------------------------------------------------- +// Descriptor of arrays of various types and sizes laid out in memory one after +// another. See the top of the file for documentation. +// +// Check out the public API of internal_layout::LayoutImpl above. The type is +// internal to the library but its methods are public, and they are inherited +// by `Layout`. +// --------------------------------------------------------------------------- +template <class... Ts> +class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> +{ +public: + static_assert(sizeof...(Ts) > 0, "At least one field is required"); + static_assert( + phmap::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, + "Invalid element type (see IsLegalElementType)"); + + template <size_t NumSizes> + using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; + + template <class... Sizes> + static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { + static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); + return PartialType<sizeof...(Sizes)>(phmap::forward<Sizes>(sizes)...); + } + + // Creates a layout with the sizes of all arrays specified. If you know + // only the sizes of the first N arrays (where N can be zero), you can use + // `Partial()` defined above. The constructor is essentially equivalent to + // calling `Partial()` and passing in all array sizes; the constructor is + // provided as a convenient abbreviation. + // + // Note: The sizes of the arrays must be specified in number of elements, + // not in bytes. + constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) + : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} +}; + +} // namespace priv +} // namespace phmap + +// --------------------------------------------------------------------------- +// compressed_tuple.h +// --------------------------------------------------------------------------- + +#ifdef _MSC_VER + // We need to mark these classes with this declspec to ensure that + // CompressedTuple happens. + #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases) +#else // _MSC_VER + #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC +#endif // _MSC_VER + +namespace phmap { +namespace priv { + +template <typename... Ts> +class CompressedTuple; + +namespace internal_compressed_tuple { + +template <typename D, size_t I> +struct Elem; +template <typename... B, size_t I> +struct Elem<CompressedTuple<B...>, I> + : std::tuple_element<I, std::tuple<B...>> {}; +template <typename D, size_t I> +using ElemT = typename Elem<D, I>::type; + +// --------------------------------------------------------------------------- +// Use the __is_final intrinsic if available. Where it's not available, classes +// declared with the 'final' specifier cannot be used as CompressedTuple +// elements. +// TODO(sbenza): Replace this with std::is_final in C++14. +// --------------------------------------------------------------------------- +template <typename T> +constexpr bool IsFinal() { +#if defined(__clang__) || defined(__GNUC__) + return __is_final(T); +#else + return false; +#endif +} + +template <typename T> +constexpr bool ShouldUseBase() { +#ifdef __INTEL_COMPILER + // avoid crash in Intel compiler + // assertion failed at: "shared/cfe/edgcpfe/lower_init.c", line 7013 + return false; +#else + return std::is_class<T>::value && std::is_empty<T>::value && !IsFinal<T>(); +#endif +} + +// The storage class provides two specializations: +// - For empty classes, it stores T as a base class. +// - For everything else, it stores T as a member. +// ------------------------------------------------ +template <typename D, size_t I, bool = ShouldUseBase<ElemT<D, I>>()> +struct Storage +{ + using T = ElemT<D, I>; + T value; + constexpr Storage() = default; + explicit constexpr Storage(T&& v) : value(phmap::forward<T>(v)) {} + constexpr const T& get() const& { return value; } + T& get() & { return value; } + constexpr const T&& get() const&& { return phmap::move(*this).value; } + T&& get() && { return std::move(*this).value; } +}; + +template <typename D, size_t I> +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<D, I, true> + : ElemT<D, I> +{ + using T = internal_compressed_tuple::ElemT<D, I>; + constexpr Storage() = default; + explicit constexpr Storage(T&& v) : T(phmap::forward<T>(v)) {} + constexpr const T& get() const& { return *this; } + T& get() & { return *this; } + constexpr const T&& get() const&& { return phmap::move(*this); } + T&& get() && { return std::move(*this); } +}; + +template <typename D, typename I> +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl; + +template <typename... Ts, size_t... I> +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC + CompressedTupleImpl<CompressedTuple<Ts...>, phmap::index_sequence<I...>> + // We use the dummy identity function through std::integral_constant to + // convince MSVC of accepting and expanding I in that context. Without it + // you would get: + // error C3548: 'I': parameter pack cannot be used in this context + : Storage<CompressedTuple<Ts...>, + std::integral_constant<size_t, I>::value>... +{ + constexpr CompressedTupleImpl() = default; + explicit constexpr CompressedTupleImpl(Ts&&... args) + : Storage<CompressedTuple<Ts...>, I>(phmap::forward<Ts>(args))... {} +}; + +} // namespace internal_compressed_tuple + +// --------------------------------------------------------------------------- +// Helper class to perform the Empty Base Class Optimization. +// Ts can contain classes and non-classes, empty or not. For the ones that +// are empty classes, we perform the CompressedTuple. If all types in Ts are +// empty classes, then CompressedTuple<Ts...> is itself an empty class. +// +// To access the members, use member .get<N>() function. +// +// Eg: +// phmap::priv::CompressedTuple<int, T1, T2, T3> value(7, t1, t2, +// t3); +// assert(value.get<0>() == 7); +// T1& t1 = value.get<1>(); +// const T2& t2 = value.get<2>(); +// ... +// +// https://en.cppreference.com/w/cpp/language/ebo +// --------------------------------------------------------------------------- +template <typename... Ts> +class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple + : private internal_compressed_tuple::CompressedTupleImpl< + CompressedTuple<Ts...>, phmap::index_sequence_for<Ts...>> +{ +private: + template <int I> + using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>; + +public: + constexpr CompressedTuple() = default; + explicit constexpr CompressedTuple(Ts... base) + : CompressedTuple::CompressedTupleImpl(phmap::forward<Ts>(base)...) {} + + template <int I> + ElemT<I>& get() & { + return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); + } + + template <int I> + constexpr const ElemT<I>& get() const& { + return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); + } + + template <int I> + ElemT<I>&& get() && { + return std::move(*this) + .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); + } + + template <int I> + constexpr const ElemT<I>&& get() const&& { + return phmap::move(*this) + .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); + } +}; + +// Explicit specialization for a zero-element tuple +// (needed to avoid ambiguous overloads for the default constructor). +// --------------------------------------------------------------------------- +template <> +class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {}; + +} // namespace priv +} // namespace phmap + + +namespace phmap { +namespace priv { + +#ifdef _MSC_VER + #pragma warning(push) + // warning warning C4324: structure was padded due to alignment specifier + #pragma warning(disable : 4324) +#endif + + +// ---------------------------------------------------------------------------- +// Allocates at least n bytes aligned to the specified alignment. +// Alignment must be a power of 2. It must be positive. +// +// Note that many allocators don't honor alignment requirements above certain +// threshold (usually either alignof(std::max_align_t) or alignof(void*)). +// Allocate() doesn't apply alignment corrections. If the underlying allocator +// returns insufficiently alignment pointer, that's what you are going to get. +// ---------------------------------------------------------------------------- +template <size_t Alignment, class Alloc> +void* Allocate(Alloc* alloc, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + struct alignas(Alignment) M {}; + using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; + using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; + A mem_alloc(*alloc); + void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); + assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 && + "allocator does not respect alignment"); + return p; +} + +// ---------------------------------------------------------------------------- +// The pointer must have been previously obtained by calling +// Allocate<Alignment>(alloc, n). +// ---------------------------------------------------------------------------- +template <size_t Alignment, class Alloc> +void Deallocate(Alloc* alloc, void* p, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + struct alignas(Alignment) M {}; + using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; + using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; + A mem_alloc(*alloc); + AT::deallocate(mem_alloc, static_cast<M*>(p), + (n + sizeof(M) - 1) / sizeof(M)); +} + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +// Helper functions for asan and msan. +// ---------------------------------------------------------------------------- +inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) { +#ifdef ADDRESS_SANITIZER + ASAN_POISON_MEMORY_REGION(m, s); +#endif +#ifdef MEMORY_SANITIZER + __msan_poison(m, s); +#endif + (void)m; + (void)s; +} + +inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) { +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(m, s); +#endif +#ifdef MEMORY_SANITIZER + __msan_unpoison(m, s); +#endif + (void)m; + (void)s; +} + +template <typename T> +inline void SanitizerPoisonObject(const T* object) { + SanitizerPoisonMemoryRegion(object, sizeof(T)); +} + +template <typename T> +inline void SanitizerUnpoisonObject(const T* object) { + SanitizerUnpoisonMemoryRegion(object, sizeof(T)); +} + +} // namespace priv +} // namespace phmap + + +// --------------------------------------------------------------------------- +// thread_annotations.h +// --------------------------------------------------------------------------- + +#if defined(__clang__) + #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else + #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +#define PHMAP_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) +#define PHMAP_PT_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) + +#define PHMAP_ACQUIRED_AFTER(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + +#define PHMAP_ACQUIRED_BEFORE(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + +#define PHMAP_EXCLUSIVE_LOCKS_REQUIRED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) + +#define PHMAP_SHARED_LOCKS_REQUIRED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) + +#define PHMAP_LOCKS_EXCLUDED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) + +#define PHMAP_LOCK_RETURNED(x) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) + +#define PHMAP_LOCKABLE \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lockable) + +#define PHMAP_SCOPED_LOCKABLE \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) + +#define PHMAP_EXCLUSIVE_LOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) + +#define PHMAP_SHARED_LOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) + +#define PHMAP_UNLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) + +#define PHMAP_EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) + +#define PHMAP_SHARED_TRYLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) + +#define PHMAP_ASSERT_EXCLUSIVE_LOCK(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) + +#define PHMAP_ASSERT_SHARED_LOCK(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) + +#define PHMAP_NO_THREAD_SAFETY_ANALYSIS \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + +//------------------------------------------------------------------------------ +// Tool-Supplied Annotations +//------------------------------------------------------------------------------ + +// TS_UNCHECKED should be placed around lock expressions that are not valid +// C++ syntax, but which are present for documentation purposes. These +// annotations will be ignored by the analysis. +#define PHMAP_TS_UNCHECKED(x) "" + +// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. +// It is used by automated tools to mark and disable invalid expressions. +// The annotation should either be fixed, or changed to TS_UNCHECKED. +#define PHMAP_TS_FIXME(x) "" + +// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of +// a particular function. However, this attribute is used to mark functions +// that are incorrect and need to be fixed. It is used by automated tools to +// avoid breaking the build when the analysis is updated. +// Code owners are expected to eventually fix the routine. +#define PHMAP_NO_THREAD_SAFETY_ANALYSIS_FIXME PHMAP_NO_THREAD_SAFETY_ANALYSIS + +// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY +// annotation that needs to be fixed, because it is producing thread safety +// warning. It disables the GUARDED_BY. +#define PHMAP_GUARDED_BY_FIXME(x) + +// Disables warnings for a single read operation. This can be used to avoid +// warnings when it is known that the read is not actually involved in a race, +// but the compiler cannot confirm that. +#define PHMAP_TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) + + +namespace phmap { +namespace thread_safety_analysis { + +// Takes a reference to a guarded data member, and returns an unguarded +// reference. +template <typename T> +inline const T& ts_unchecked_read(const T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +template <typename T> +inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +} // namespace thread_safety_analysis + +namespace priv { + +namespace memory_internal { + +// ---------------------------------------------------------------------------- +// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and +// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and +// offsetof(Pair, second) respectively. Otherwise they are -1. +// +// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout +// type, which is non-portable. +// ---------------------------------------------------------------------------- +template <class Pair, class = std::true_type> +struct OffsetOf { + static constexpr size_t kFirst = (size_t)-1; + static constexpr size_t kSecond = (size_t)-1; +}; + +template <class Pair> +struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type> +{ + static constexpr size_t kFirst = offsetof(Pair, first); + static constexpr size_t kSecond = offsetof(Pair, second); +}; + +// ---------------------------------------------------------------------------- +template <class K, class V> +struct IsLayoutCompatible +{ +private: + struct Pair { + K first; + V second; + }; + + // Is P layout-compatible with Pair? + template <class P> + static constexpr bool LayoutCompatible() { + return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) && + alignof(P) == alignof(Pair) && + memory_internal::OffsetOf<P>::kFirst == + memory_internal::OffsetOf<Pair>::kFirst && + memory_internal::OffsetOf<P>::kSecond == + memory_internal::OffsetOf<Pair>::kSecond; + } + +public: + // Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are, + // then it is safe to store them in a union and read from either. + static constexpr bool value = std::is_standard_layout<K>() && + std::is_standard_layout<Pair>() && + memory_internal::OffsetOf<Pair>::kFirst == 0 && + LayoutCompatible<std::pair<K, V>>() && + LayoutCompatible<std::pair<const K, V>>(); +}; + +} // namespace memory_internal + +// ---------------------------------------------------------------------------- +// The internal storage type for key-value containers like flat_hash_map. +// +// It is convenient for the value_type of a flat_hash_map<K, V> to be +// pair<const K, V>; the "const K" prevents accidental modification of the key +// when dealing with the reference returned from find() and similar methods. +// However, this creates other problems; we want to be able to emplace(K, V) +// efficiently with move operations, and similarly be able to move a +// pair<K, V> in insert(). +// +// The solution is this union, which aliases the const and non-const versions +// of the pair. This also allows flat_hash_map<const K, V> to work, even though +// that has the same efficiency issues with move in emplace() and insert() - +// but people do it anyway. +// +// If kMutableKeys is false, only the value member can be accessed. +// +// If kMutableKeys is true, key can be accessed through all slots while value +// and mutable_value must be accessed only via INITIALIZED slots. Slots are +// created and destroyed via mutable_value so that the key can be moved later. +// +// Accessing one of the union fields while the other is active is safe as +// long as they are layout-compatible, which is guaranteed by the definition of +// kMutableKeys. For C++11, the relevant section of the standard is +// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19) +// ---------------------------------------------------------------------------- +template <class K, class V> +union map_slot_type +{ + map_slot_type() {} + ~map_slot_type() = delete; + map_slot_type(const map_slot_type&) = delete; + map_slot_type& operator=(const map_slot_type&) = delete; + + using value_type = std::pair<const K, V>; + using mutable_value_type = std::pair<K, V>; + + value_type value; + mutable_value_type mutable_value; + K key; +}; + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +template <class K, class V> +struct map_slot_policy +{ + using slot_type = map_slot_type<K, V>; + using value_type = std::pair<const K, V>; + using mutable_value_type = std::pair<K, V>; + +private: + static void emplace(slot_type* slot) { + // The construction of union doesn't do anything at runtime but it allows us + // to access its members without violating aliasing rules. + new (slot) slot_type; + } + // If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one + // or the other via slot_type. We are also free to access the key via + // slot_type::key in this case. + using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>; + +public: + static value_type& element(slot_type* slot) { return slot->value; } + static const value_type& element(const slot_type* slot) { + return slot->value; + } + + static const K& key(const slot_type* slot) { + return kMutableKeys::value ? slot->key : slot->value.first; + } + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + emplace(slot); + if (kMutableKeys::value) { + phmap::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value, + std::forward<Args>(args)...); + } else { + phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, + std::forward<Args>(args)...); + } + } + + // Construct this slot by moving from another slot. + template <class Allocator> + static void construct(Allocator* alloc, slot_type* slot, slot_type* other) { + emplace(slot); + if (kMutableKeys::value) { + phmap::allocator_traits<Allocator>::construct( + *alloc, &slot->mutable_value, std::move(other->mutable_value)); + } else { + phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, + std::move(other->value)); + } + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + if (kMutableKeys::value) { + phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value); + } else { + phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->value); + } + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + emplace(new_slot); + if (kMutableKeys::value) { + phmap::allocator_traits<Allocator>::construct( + *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); + } else { + phmap::allocator_traits<Allocator>::construct(*alloc, &new_slot->value, + std::move(old_slot->value)); + } + destroy(alloc, old_slot); + } + + template <class Allocator> + static void swap(Allocator* alloc, slot_type* a, slot_type* b) { + if (kMutableKeys::value) { + using std::swap; + swap(a->mutable_value, b->mutable_value); + } else { + value_type tmp = std::move(a->value); + phmap::allocator_traits<Allocator>::destroy(*alloc, &a->value); + phmap::allocator_traits<Allocator>::construct(*alloc, &a->value, + std::move(b->value)); + phmap::allocator_traits<Allocator>::destroy(*alloc, &b->value); + phmap::allocator_traits<Allocator>::construct(*alloc, &b->value, + std::move(tmp)); + } + } + + template <class Allocator> + static void move(Allocator* alloc, slot_type* src, slot_type* dest) { + if (kMutableKeys::value) { + dest->mutable_value = std::move(src->mutable_value); + } else { + phmap::allocator_traits<Allocator>::destroy(*alloc, &dest->value); + phmap::allocator_traits<Allocator>::construct(*alloc, &dest->value, + std::move(src->value)); + } + } + + template <class Allocator> + static void move(Allocator* alloc, slot_type* first, slot_type* last, + slot_type* result) { + for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) + move(alloc, src, dest); + } +}; + +} // namespace priv +} // phmap + + +namespace phmap { + +#ifdef BOOST_THREAD_LOCK_OPTIONS_HPP + using defer_lock_t = boost::defer_lock_t; + using try_to_lock_t = boost::try_to_lock_t; + using adopt_lock_t = boost::adopt_lock_t; +#else + struct adopt_lock_t { explicit adopt_lock_t() = default; }; + struct defer_lock_t { explicit defer_lock_t() = default; }; + struct try_to_lock_t { explicit try_to_lock_t() = default; }; +#endif + +// ----------------------------------------------------------------------------- +// NullMutex +// ----------------------------------------------------------------------------- +// A class that implements the Mutex interface, but does nothing. This is to be +// used as a default template parameters for classes who provide optional +// internal locking (like phmap::parallel_flat_hash_map). +// ----------------------------------------------------------------------------- +class NullMutex { +public: + NullMutex() {} + ~NullMutex() {} + void lock() {} + void unlock() {} + bool try_lock() { return true; } + void lock_shared() {} + void unlock_shared() {} + bool try_lock_shared() { return true; } +}; + +// ------------------------ lockable object used internally ------------------------- +template <class MutexType> +class LockableBaseImpl +{ +public: + // ---------------------------------------------------- + struct DoNothing + { + using mutex_type = MutexType; + DoNothing() noexcept {} + explicit DoNothing(mutex_type& ) noexcept {} + explicit DoNothing(mutex_type& , mutex_type&) noexcept {} + DoNothing(mutex_type&, phmap::adopt_lock_t) noexcept {} + DoNothing(mutex_type&, phmap::defer_lock_t) noexcept {} + DoNothing(mutex_type&, phmap::try_to_lock_t) {} + template<class T> explicit DoNothing(T&&) {} + DoNothing& operator=(const DoNothing&) { return *this; } + DoNothing& operator=(DoNothing&&) { return *this; } + void swap(DoNothing &) {} + bool owns_lock() const noexcept { return true; } + }; + + // ---------------------------------------------------- + class WriteLock + { + public: + using mutex_type = MutexType; + + WriteLock() : m_(nullptr), locked_(false) {} + + explicit WriteLock(mutex_type &m) : m_(&m) { + m_->lock(); + locked_ = true; + } + + WriteLock(mutex_type& m, adopt_lock_t) noexcept : + m_(&m), locked_(true) + {} + + WriteLock(mutex_type& m, defer_lock_t) noexcept : + m_(&m), locked_(false) + {} + + WriteLock(mutex_type& m, try_to_lock_t) : + m_(&m), locked_(false) { + m_->try_lock(); + } + + WriteLock(WriteLock &&o) : + m_(std::move(o.m_)), locked_(std::move(o.locked_)) { + o.locked_ = false; + o.m_ = nullptr; + } + + WriteLock& operator=(WriteLock&& other) { + WriteLock temp(std::move(other)); + swap(temp); + return *this; + } + + ~WriteLock() { + if (locked_) + m_->unlock(); + } + + void lock() { + if (!locked_) { + m_->lock(); + locked_ = true; + } + } + + void unlock() { + if (locked_) { + m_->unlock(); + locked_ = false; + } + } + + bool try_lock() { + if (locked_) + return true; + locked_ = m_->try_lock(); + return locked_; + } + + bool owns_lock() const noexcept { return locked_; } + + void swap(WriteLock &o) noexcept { + std::swap(m_, o.m_); + std::swap(locked_, o.locked_); + } + + mutex_type *mutex() const noexcept { return m_; } + + private: + mutex_type *m_; + bool locked_; + }; + + // ---------------------------------------------------- + class ReadLock + { + public: + using mutex_type = MutexType; + + ReadLock() : m_(nullptr), locked_(false) {} + + explicit ReadLock(mutex_type &m) : m_(&m) { + m_->lock_shared(); + locked_ = true; + } + + ReadLock(mutex_type& m, adopt_lock_t) noexcept : + m_(&m), locked_(true) + {} + + ReadLock(mutex_type& m, defer_lock_t) noexcept : + m_(&m), locked_(false) + {} + + ReadLock(mutex_type& m, try_to_lock_t) : + m_(&m), locked_(false) { + m_->try_lock_shared(); + } + + ReadLock(ReadLock &&o) : + m_(std::move(o.m_)), locked_(std::move(o.locked_)) { + o.locked_ = false; + o.m_ = nullptr; + } + + ReadLock& operator=(ReadLock&& other) { + ReadLock temp(std::move(other)); + swap(temp); + return *this; + } + + ~ReadLock() { + if (locked_) + m_->unlock_shared(); + } + + void lock() { + if (!locked_) { + m_->lock_shared(); + locked_ = true; + } + } + + void unlock() { + if (locked_) { + m_->unlock_shared(); + locked_ = false; + } + } + + bool try_lock() { + if (locked_) + return true; + locked_ = m_->try_lock_shared(); + return locked_; + } + + bool owns_lock() const noexcept { return locked_; } + + void swap(ReadLock &o) noexcept { + std::swap(m_, o.m_); + std::swap(locked_, o.locked_); + } + + mutex_type *mutex() const noexcept { return m_; } + + private: + mutex_type *m_; + bool locked_; + }; + + // ---------------------------------------------------- + class WriteLocks + { + public: + using mutex_type = MutexType; + + explicit WriteLocks(mutex_type& m1, mutex_type& m2) : + _m1(m1), _m2(m2) + { + std::lock(m1, m2); + } + + WriteLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : + _m1(m1), _m2(m2) + { // adopt means we already own the mutexes + } + + ~WriteLocks() + { + _m1.unlock(); + _m2.unlock(); + } + + WriteLocks(WriteLocks const&) = delete; + WriteLocks& operator=(WriteLocks const&) = delete; + private: + mutex_type& _m1; + mutex_type& _m2; + }; + + // ---------------------------------------------------- + class ReadLocks + { + public: + using mutex_type = MutexType; + + explicit ReadLocks(mutex_type& m1, mutex_type& m2) : + _m1(m1), _m2(m2) + { + _m1.lock_shared(); + _m2.lock_shared(); + } + + ReadLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : + _m1(m1), _m2(m2) + { // adopt means we already own the mutexes + } + + ~ReadLocks() + { + _m1.unlock_shared(); + _m2.unlock_shared(); + } + + ReadLocks(ReadLocks const&) = delete; + ReadLocks& operator=(ReadLocks const&) = delete; + private: + mutex_type& _m1; + mutex_type& _m2; + }; +}; + +// ------------------------ holds a mutex ------------------------------------ +// Default implementation for Lockable, should work fine for std::mutex +// ----------------------------------- +// use as: +// using Lockable = phmap::LockableImpl<mutex_type>; +// Lockable m; +// +// Lockable::UpgradeLock read_lock(m); // take a upgradable lock +// +// { +// Lockable::UpgradeToUnique unique_lock(read_lock); +// // now locked for write +// } +// +// --------------------------------------------------------------------------- +// Generic mutex support (always write locks) +// -------------------------------------------------------------------------- +template <class Mtx_> +class LockableImpl : public Mtx_ +{ +public: + using mutex_type = Mtx_; + using Base = LockableBaseImpl<Mtx_>; + using SharedLock = typename Base::WriteLock; + using UpgradeLock = typename Base::WriteLock; + using UniqueLock = typename Base::WriteLock; + using SharedLocks = typename Base::WriteLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership +}; + +// --------------------------------------------------------------------------- +// Null mutex (no-op) - when we don't want internal synchronization +// --------------------------------------------------------------------------- +template <> +class LockableImpl<phmap::NullMutex>: public phmap::NullMutex +{ +public: + using mutex_type = phmap::NullMutex; + using Base = LockableBaseImpl<phmap::NullMutex>; + using SharedLock = typename Base::DoNothing; + using UpgradeLock = typename Base::DoNothing; + using UniqueLock = typename Base::DoNothing; + using UpgradeToUnique = typename Base::DoNothing; + using SharedLocks = typename Base::DoNothing; + using UniqueLocks = typename Base::DoNothing; +}; + +// -------------------------------------------------------------------------- +// Abseil Mutex support (read and write lock support) +// -------------------------------------------------------------------------- +#ifdef ABSL_SYNCHRONIZATION_MUTEX_H_ + + struct AbslMutex : protected absl::Mutex + { + void lock() { this->Lock(); } + void unlock() { this->Unlock(); } + void try_lock() { this->TryLock(); } + void lock_shared() { this->ReaderLock(); } + void unlock_shared() { this->ReaderUnlock(); } + void try_lock_shared() { this->ReaderTryLock(); } + }; + + template <> + class LockableImpl<absl::Mutex> : public AbslMutex + { + public: + using mutex_type = phmap::AbslMutex; + using Base = LockableBaseImpl<phmap::AbslMutex>; + using SharedLock = typename Base::ReadLock; + using UpgradeLock = typename Base::WriteLock; + using UniqueLock = typename Base::WriteLock; + using SharedLocks = typename Base::ReadLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership + }; + +#endif + +// -------------------------------------------------------------------------- +// Boost shared_mutex support (read and write lock support) +// -------------------------------------------------------------------------- +#ifdef BOOST_THREAD_SHARED_MUTEX_HPP + +#if 1 + // --------------------------------------------------------------------------- + template <> + class LockableImpl<boost::shared_mutex> : public boost::shared_mutex + { + public: + using mutex_type = boost::shared_mutex; + using Base = LockableBaseImpl<boost::shared_mutex>; + using SharedLock = boost::shared_lock<mutex_type>; + using UpgradeLock = boost::unique_lock<mutex_type>; // assume can't upgrade + using UniqueLock = boost::unique_lock<mutex_type>; + using SharedLocks = typename Base::ReadLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership + }; +#else + // --------------------------------------------------------------------------- + template <> + class LockableImpl<boost::upgrade_mutex> : public boost::upgrade_mutex + { + public: + using mutex_type = boost::upgrade_mutex; + using SharedLock = boost::shared_lock<mutex_type>; + using UpgradeLock = boost::upgrade_lock<mutex_type>; + using UniqueLock = boost::unique_lock<mutex_type>; + using SharedLocks = typename Base::ReadLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = boost::upgrade_to_unique_lock<mutex_type>; + }; +#endif + +#endif // BOOST_THREAD_SHARED_MUTEX_HPP + +// -------------------------------------------------------------------------- +// std::shared_mutex support (read and write lock support) +// -------------------------------------------------------------------------- +#ifdef PHMAP_HAVE_SHARED_MUTEX + + // --------------------------------------------------------------------------- + template <> + class LockableImpl<std::shared_mutex> : public std::shared_mutex + { + public: + using mutex_type = std::shared_mutex; + using Base = LockableBaseImpl<std::shared_mutex>; + using SharedLock = std::shared_lock<mutex_type>; + using UpgradeLock = std::unique_lock<mutex_type>; // assume can't upgrade + using UniqueLock = std::unique_lock<mutex_type>; + using SharedLocks = typename Base::ReadLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership + }; +#endif // PHMAP_HAVE_SHARED_MUTEX + + +} // phmap + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + + +#endif // phmap_base_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_bits.h b/benchmarks/others/parallel_hashmap/phmap_bits.h new file mode 100644 index 00000000..6b765fff --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_bits.h @@ -0,0 +1,663 @@ +#if !defined(phmap_bits_h_guard_) +#define phmap_bits_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +// The following guarantees declaration of the byte swap functions +#ifdef _MSC_VER + #include <stdlib.h> // NOLINT(build/include) +#elif defined(__APPLE__) + // Mac OS X / Darwin features + #include <libkern/OSByteOrder.h> +#elif defined(__FreeBSD__) + #include <sys/endian.h> +#elif defined(__GLIBC__) + #include <byteswap.h> // IWYU pragma: export +#endif + +#include <string.h> +#include <cstdint> +#include "phmap_config.h" + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4514) // unreferenced inline function has been removed +#endif + +// ----------------------------------------------------------------------------- +// unaligned APIs +// ----------------------------------------------------------------------------- +// Portable handling of unaligned loads, stores, and copies. +// On some platforms, like ARM, the copy functions can be more efficient +// then a load and a store. +// ----------------------------------------------------------------------------- + +#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ + defined(MEMORY_SANITIZER) +#include <stdint.h> + +extern "C" { + uint16_t __sanitizer_unaligned_load16(const void *p); + uint32_t __sanitizer_unaligned_load32(const void *p); + uint64_t __sanitizer_unaligned_load64(const void *p); + void __sanitizer_unaligned_store16(void *p, uint16_t v); + void __sanitizer_unaligned_store32(void *p, uint32_t v); + void __sanitizer_unaligned_store64(void *p, uint64_t v); +} // extern "C" + +namespace phmap { +namespace bits { + +inline uint16_t UnalignedLoad16(const void *p) { + return __sanitizer_unaligned_load16(p); +} + +inline uint32_t UnalignedLoad32(const void *p) { + return __sanitizer_unaligned_load32(p); +} + +inline uint64_t UnalignedLoad64(const void *p) { + return __sanitizer_unaligned_load64(p); +} + +inline void UnalignedStore16(void *p, uint16_t v) { + __sanitizer_unaligned_store16(p, v); +} + +inline void UnalignedStore32(void *p, uint32_t v) { + __sanitizer_unaligned_store32(p, v); +} + +inline void UnalignedStore64(void *p, uint64_t v) { + __sanitizer_unaligned_store64(p, v); +} + +} // namespace bits +} // namespace phmap + +#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) +#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) +#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) + +#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) +#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) +#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) + +#else + +namespace phmap { +namespace bits { + +inline uint16_t UnalignedLoad16(const void *p) { + uint16_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint32_t UnalignedLoad32(const void *p) { + uint32_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint64_t UnalignedLoad64(const void *p) { + uint64_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); } + +inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } + +inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } + +} // namespace bits +} // namespace phmap + +#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) +#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) +#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) + +#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) +#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) +#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) + +#endif + +// ----------------------------------------------------------------------------- +// File: optimization.h +// ----------------------------------------------------------------------------- + +#if defined(__pnacl__) + #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } +#elif defined(__clang__) + // Clang will not tail call given inline volatile assembly. + #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") +#elif defined(__GNUC__) + // GCC will not tail call given inline volatile assembly. + #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") +#elif defined(_MSC_VER) + #include <intrin.h> + // The __nop() intrinsic blocks the optimisation. + #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __nop() +#else + #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } +#endif + +#if defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" +#endif + +#ifdef PHMAP_HAVE_INTRINSIC_INT128 + __extension__ typedef unsigned __int128 phmap_uint128; + inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) + { + auto result = static_cast<phmap_uint128>(a) * static_cast<phmap_uint128>(b); + *high = static_cast<uint64_t>(result >> 64); + return static_cast<uint64_t>(result); + } + #define PHMAP_HAS_UMUL128 1 +#elif (defined(_MSC_VER)) + #if defined(_M_X64) + #pragma intrinsic(_umul128) + inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) + { + return _umul128(a, b, high); + } + #define PHMAP_HAS_UMUL128 1 + #endif +#endif + +#if defined(__GNUC__) + #pragma GCC diagnostic pop +#endif + +#if defined(__GNUC__) + // Cache line alignment + #if defined(__i386__) || defined(__x86_64__) + #define PHMAP_CACHELINE_SIZE 64 + #elif defined(__powerpc64__) + #define PHMAP_CACHELINE_SIZE 128 + #elif defined(__aarch64__) + // We would need to read special register ctr_el0 to find out L1 dcache size. + // This value is a good estimate based on a real aarch64 machine. + #define PHMAP_CACHELINE_SIZE 64 + #elif defined(__arm__) + // Cache line sizes for ARM: These values are not strictly correct since + // cache line sizes depend on implementations, not architectures. There + // are even implementations with cache line sizes configurable at boot + // time. + #if defined(__ARM_ARCH_5T__) + #define PHMAP_CACHELINE_SIZE 32 + #elif defined(__ARM_ARCH_7A__) + #define PHMAP_CACHELINE_SIZE 64 + #endif + #endif + + #ifndef PHMAP_CACHELINE_SIZE + // A reasonable default guess. Note that overestimates tend to waste more + // space, while underestimates tend to waste more time. + #define PHMAP_CACHELINE_SIZE 64 + #endif + + #define PHMAP_CACHELINE_ALIGNED __attribute__((aligned(PHMAP_CACHELINE_SIZE))) +#elif defined(_MSC_VER) + #define PHMAP_CACHELINE_SIZE 64 + #define PHMAP_CACHELINE_ALIGNED __declspec(align(PHMAP_CACHELINE_SIZE)) +#else + #define PHMAP_CACHELINE_SIZE 64 + #define PHMAP_CACHELINE_ALIGNED +#endif + + +#if PHMAP_HAVE_BUILTIN(__builtin_expect) || \ + (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_PREDICT_FALSE(x) (__builtin_expect(x, 0)) + #define PHMAP_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else + #define PHMAP_PREDICT_FALSE(x) (x) + #define PHMAP_PREDICT_TRUE(x) (x) +#endif + +// ----------------------------------------------------------------------------- +// File: bits.h +// ----------------------------------------------------------------------------- + +#if defined(_MSC_VER) + // We can achieve something similar to attribute((always_inline)) with MSVC by + // using the __forceinline keyword, however this is not perfect. MSVC is + // much less aggressive about inlining, and even with the __forceinline keyword. + #define PHMAP_BASE_INTERNAL_FORCEINLINE __forceinline +#else + // Use default attribute inline. + #define PHMAP_BASE_INTERNAL_FORCEINLINE inline PHMAP_ATTRIBUTE_ALWAYS_INLINE +#endif + + +namespace phmap { +namespace base_internal { + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { + int zeroes = 60; + if (n >> 32) zeroes -= 32, n >>= 32; + if (n >> 16) zeroes -= 16, n >>= 16; + if (n >> 8) zeroes -= 8, n >>= 8; + if (n >> 4) zeroes -= 4, n >>= 4; + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { +#if defined(_MSC_VER) && defined(_M_X64) + // MSVC does not have __buitin_clzll. Use _BitScanReverse64. + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse64(&result, n)) { + return (int)(63 - result); + } + return 64; +#elif defined(_MSC_VER) && !defined(__clang__) + // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse + unsigned long result = 0; // NOLINT(runtime/int) + if ((n >> 32) && _BitScanReverse(&result, (unsigned long)(n >> 32))) { + return 31 - result; + } + if (_BitScanReverse(&result, (unsigned long)n)) { + return 63 - result; + } + return 64; +#elif defined(__GNUC__) || defined(__clang__) + // Use __builtin_clzll, which uses the following instructions: + // x86: bsr + // ARM64: clz + // PPC: cntlzd + static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) + "__builtin_clzll does not take 64-bit arg"); + + // Handle 0 as a special case because __builtin_clzll(0) is undefined. + if (n == 0) { + return 64; + } + return __builtin_clzll(n); +#else + return CountLeadingZeros64Slow(n); +#endif +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { + int zeroes = 28; + if (n >> 16) zeroes -= 16, n >>= 16; + if (n >> 8) zeroes -= 8, n >>= 8; + if (n >> 4) zeroes -= 4, n >>= 4; + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse(&result, n)) { + return (int)(31 - result); + } + return 32; +#elif defined(__GNUC__) || defined(__clang__) + // Use __builtin_clz, which uses the following instructions: + // x86: bsr + // ARM64: clz + // PPC: cntlzd + static_assert(sizeof(int) == sizeof(n), + "__builtin_clz does not take 32-bit arg"); + + // Handle 0 as a special case because __builtin_clz(0) is undefined. + if (n == 0) { + return 32; + } + return __builtin_clz(n); +#else + return CountLeadingZeros32Slow(n); +#endif +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) { + int c = 63; + n &= ~n + 1; + if (n & 0x00000000FFFFFFFF) c -= 32; + if (n & 0x0000FFFF0000FFFF) c -= 16; + if (n & 0x00FF00FF00FF00FF) c -= 8; + if (n & 0x0F0F0F0F0F0F0F0F) c -= 4; + if (n & 0x3333333333333333) c -= 2; + if (n & 0x5555555555555555) c -= 1; + return c; +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) { +#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) + unsigned long result = 0; // NOLINT(runtime/int) + _BitScanForward64(&result, n); + return (int)result; +#elif defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + if (static_cast<uint32_t>(n) == 0) { + _BitScanForward(&result, (unsigned long)(n >> 32)); + return result + 32; + } + _BitScanForward(&result, (unsigned long)n); + return result; +#elif defined(__GNUC__) || defined(__clang__) + static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) + "__builtin_ctzll does not take 64-bit arg"); + return __builtin_ctzll(n); +#else + return CountTrailingZerosNonZero64Slow(n); +#endif +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) { + int c = 31; + n &= ~n + 1; + if (n & 0x0000FFFF) c -= 16; + if (n & 0x00FF00FF) c -= 8; + if (n & 0x0F0F0F0F) c -= 4; + if (n & 0x33333333) c -= 2; + if (n & 0x55555555) c -= 1; + return c; +} + +PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + _BitScanForward(&result, n); + return (int)result; +#elif defined(__GNUC__) || defined(__clang__) + static_assert(sizeof(int) == sizeof(n), + "__builtin_ctz does not take 32-bit arg"); + return __builtin_ctz(n); +#else + return CountTrailingZerosNonZero32Slow(n); +#endif +} + +#undef PHMAP_BASE_INTERNAL_FORCEINLINE + +} // namespace base_internal +} // namespace phmap + +// ----------------------------------------------------------------------------- +// File: endian.h +// ----------------------------------------------------------------------------- + +namespace phmap { + +// Use compiler byte-swapping intrinsics if they are available. 32-bit +// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. +// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. +// For simplicity, we enable them all only for GCC 4.8.0 or later. +#if defined(__clang__) || \ + (defined(__GNUC__) && \ + ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) + + inline uint64_t gbswap_64(uint64_t host_int) { + return __builtin_bswap64(host_int); + } + inline uint32_t gbswap_32(uint32_t host_int) { + return __builtin_bswap32(host_int); + } + inline uint16_t gbswap_16(uint16_t host_int) { + return __builtin_bswap16(host_int); + } + +#elif defined(_MSC_VER) + + inline uint64_t gbswap_64(uint64_t host_int) { + return _byteswap_uint64(host_int); + } + inline uint32_t gbswap_32(uint32_t host_int) { + return _byteswap_ulong(host_int); + } + inline uint16_t gbswap_16(uint16_t host_int) { + return _byteswap_ushort(host_int); + } + +#elif defined(__APPLE__) + + inline uint64_t gbswap_64(uint64_t host_int) { return OSSwapInt16(host_int); } + inline uint32_t gbswap_32(uint32_t host_int) { return OSSwapInt32(host_int); } + inline uint16_t gbswap_16(uint16_t host_int) { return OSSwapInt64(host_int); } + +#else + + inline uint64_t gbswap_64(uint64_t host_int) { +#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) + // Adapted from /usr/include/byteswap.h. Not available on Mac. + if (__builtin_constant_p(host_int)) { + return __bswap_constant_64(host_int); + } else { + uint64_t result; + __asm__("bswap %0" : "=r"(result) : "0"(host_int)); + return result; + } +#elif defined(__GLIBC__) + return bswap_64(host_int); +#else + return (((host_int & uint64_t{0xFF}) << 56) | + ((host_int & uint64_t{0xFF00}) << 40) | + ((host_int & uint64_t{0xFF0000}) << 24) | + ((host_int & uint64_t{0xFF000000}) << 8) | + ((host_int & uint64_t{0xFF00000000}) >> 8) | + ((host_int & uint64_t{0xFF0000000000}) >> 24) | + ((host_int & uint64_t{0xFF000000000000}) >> 40) | + ((host_int & uint64_t{0xFF00000000000000}) >> 56)); +#endif // bswap_64 + } + + inline uint32_t gbswap_32(uint32_t host_int) { +#if defined(__GLIBC__) + return bswap_32(host_int); +#else + return (((host_int & uint32_t{0xFF}) << 24) | + ((host_int & uint32_t{0xFF00}) << 8) | + ((host_int & uint32_t{0xFF0000}) >> 8) | + ((host_int & uint32_t{0xFF000000}) >> 24)); +#endif + } + + inline uint16_t gbswap_16(uint16_t host_int) { +#if defined(__GLIBC__) + return bswap_16(host_int); +#else + return (((host_int & uint16_t{0xFF}) << 8) | + ((host_int & uint16_t{0xFF00}) >> 8)); +#endif + } + +#endif // intrinics available + +#ifdef PHMAP_IS_LITTLE_ENDIAN + + // Definitions for ntohl etc. that don't require us to include + // netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather + // than just #defining them because in debug mode, gcc doesn't + // correctly handle the (rather involved) definitions of bswap_32. + // gcc guarantees that inline functions are as fast as macros, so + // this isn't a performance hit. + inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } + inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } + inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } + +#elif defined PHMAP_IS_BIG_ENDIAN + + // These definitions are simpler on big-endian machines + // These are functions instead of macros to avoid self-assignment warnings + // on calls such as "i = ghtnol(i);". This also provides type checking. + inline uint16_t ghtons(uint16_t x) { return x; } + inline uint32_t ghtonl(uint32_t x) { return x; } + inline uint64_t ghtonll(uint64_t x) { return x; } + +#else + #error \ + "Unsupported byte order: Either PHMAP_IS_BIG_ENDIAN or " \ + "PHMAP_IS_LITTLE_ENDIAN must be defined" +#endif // byte order + +inline uint16_t gntohs(uint16_t x) { return ghtons(x); } +inline uint32_t gntohl(uint32_t x) { return ghtonl(x); } +inline uint64_t gntohll(uint64_t x) { return ghtonll(x); } + +// Utilities to convert numbers between the current hosts's native byte +// order and little-endian byte order +// +// Load/Store methods are alignment safe +namespace little_endian { +// Conversion functions. +#ifdef PHMAP_IS_LITTLE_ENDIAN + + inline uint16_t FromHost16(uint16_t x) { return x; } + inline uint16_t ToHost16(uint16_t x) { return x; } + + inline uint32_t FromHost32(uint32_t x) { return x; } + inline uint32_t ToHost32(uint32_t x) { return x; } + + inline uint64_t FromHost64(uint64_t x) { return x; } + inline uint64_t ToHost64(uint64_t x) { return x; } + + inline constexpr bool IsLittleEndian() { return true; } + +#elif defined PHMAP_IS_BIG_ENDIAN + + inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } + inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } + + inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } + inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } + + inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } + inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } + + inline constexpr bool IsLittleEndian() { return false; } + +#endif /* ENDIAN */ + +// Functions to do unaligned loads and stores in little-endian order. +inline uint16_t Load16(const void *p) { + return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); +} + +inline void Store16(void *p, uint16_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); +} + +inline uint32_t Load32(const void *p) { + return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); +} + +inline void Store32(void *p, uint32_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); +} + +inline uint64_t Load64(const void *p) { + return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); +} + +inline void Store64(void *p, uint64_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); +} + +} // namespace little_endian + +// Utilities to convert numbers between the current hosts's native byte +// order and big-endian byte order (same as network byte order) +// +// Load/Store methods are alignment safe +namespace big_endian { +#ifdef PHMAP_IS_LITTLE_ENDIAN + + inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } + inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } + + inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } + inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } + + inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } + inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } + + inline constexpr bool IsLittleEndian() { return true; } + +#elif defined PHMAP_IS_BIG_ENDIAN + + inline uint16_t FromHost16(uint16_t x) { return x; } + inline uint16_t ToHost16(uint16_t x) { return x; } + + inline uint32_t FromHost32(uint32_t x) { return x; } + inline uint32_t ToHost32(uint32_t x) { return x; } + + inline uint64_t FromHost64(uint64_t x) { return x; } + inline uint64_t ToHost64(uint64_t x) { return x; } + + inline constexpr bool IsLittleEndian() { return false; } + +#endif /* ENDIAN */ + +// Functions to do unaligned loads and stores in big-endian order. +inline uint16_t Load16(const void *p) { + return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); +} + +inline void Store16(void *p, uint16_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); +} + +inline uint32_t Load32(const void *p) { + return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); +} + +inline void Store32(void *p, uint32_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); +} + +inline uint64_t Load64(const void *p) { + return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); +} + +inline void Store64(void *p, uint64_t v) { + PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); +} + +} // namespace big_endian + +} // namespace phmap + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif // phmap_bits_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_config.h b/benchmarks/others/parallel_hashmap/phmap_config.h new file mode 100644 index 00000000..fa515025 --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_config.h @@ -0,0 +1,771 @@ +#if !defined(phmap_config_h_guard_) +#define phmap_config_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#define PHMAP_VERSION_MAJOR 1 +#define PHMAP_VERSION_MINOR 0 +#define PHMAP_VERSION_PATCH 0 + +// Included for the __GLIBC__ macro (or similar macros on other systems). +#include <limits.h> + +#ifdef __cplusplus + // Included for __GLIBCXX__, _LIBCPP_VERSION + #include <cstddef> +#endif // __cplusplus + +#if defined(__APPLE__) + // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, + // __IPHONE_8_0. + #include <Availability.h> + #include <TargetConditionals.h> +#endif + +#define PHMAP_XSTR(x) PHMAP_STR(x) +#define PHMAP_STR(x) #x +#define PHMAP_VAR_NAME_VALUE(var) #var "=" PHMAP_STR(var) + +// ----------------------------------------------------------------------------- +// Some sanity checks +// ----------------------------------------------------------------------------- +//#if defined(__CYGWIN__) +// #error "Cygwin is not supported." +//#endif + +#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023918 && !defined(__clang__) + #error "phmap requires Visual Studio 2015 Update 2 or higher." +#endif + +// We support gcc 4.7 and later. +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) + #error "phmap requires gcc 4.7 or higher." + #endif +#endif + +// We support Apple Xcode clang 4.2.1 (version 421.11.65) and later. +// This corresponds to Apple Xcode version 4.5. +#if defined(__apple_build_version__) && __apple_build_version__ < 4211165 + #error "phmap requires __apple_build_version__ of 4211165 or higher." +#endif + +// Enforce C++11 as the minimum. +#if defined(__cplusplus) && !defined(_MSC_VER) + #if __cplusplus < 201103L + #error "C++ versions less than C++11 are not supported." + #endif +#endif + +// We have chosen glibc 2.12 as the minimum +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) + #if !__GLIBC_PREREQ(2, 12) + #error "Minimum required version of glibc is 2.12." + #endif +#endif + +#if defined(_STLPORT_VERSION) + #error "STLPort is not supported." +#endif + +#if CHAR_BIT != 8 + #error "phmap assumes CHAR_BIT == 8." +#endif + +// phmap currently assumes that an int is 4 bytes. +#if INT_MAX < 2147483647 + #error "phmap assumes that int is at least 4 bytes. " +#endif + + + +// ----------------------------------------------------------------------------- +// Compiler Feature Checks +// ----------------------------------------------------------------------------- + +#ifdef __has_builtin + #define PHMAP_HAVE_BUILTIN(x) __has_builtin(x) +#else + #define PHMAP_HAVE_BUILTIN(x) 0 +#endif + +#if (defined(_MSVC_LANG) && _MSVC_LANG >= 201703) || __cplusplus >= 201703 + #define PHMAP_HAVE_CC17 1 +#else + #define PHMAP_HAVE_CC17 0 +#endif + +#define PHMAP_BRANCHLESS 1 + +// ---------------------------------------------------------------- +// Checks whether `std::is_trivially_destructible<T>` is supported. +// ---------------------------------------------------------------- +#ifdef PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE + #error PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set +#elif defined(_LIBCPP_VERSION) || \ + (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ + defined(_MSC_VER) + #define PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 +#endif + +// -------------------------------------------------------------- +// Checks whether `std::is_trivially_default_constructible<T>` is +// supported. +// -------------------------------------------------------------- +#if defined(PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) + #error PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set +#elif defined(PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) + #error PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set +#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ + (!defined(__clang__) && defined(__GNUC__) && \ + (__GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)) && \ + (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ + (defined(_MSC_VER) && !defined(__NVCC__)) + #define PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 + #define PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 +#endif + +// ------------------------------------------------------------------- +// Checks whether C++11's `thread_local` storage duration specifier is +// supported. +// ------------------------------------------------------------------- +#ifdef PHMAP_HAVE_THREAD_LOCAL + #error PHMAP_HAVE_THREAD_LOCAL cannot be directly set +#elif defined(__APPLE__) + #if __has_feature(cxx_thread_local) && \ + !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0) + #define PHMAP_HAVE_THREAD_LOCAL 1 + #endif +#else // !defined(__APPLE__) + #define PHMAP_HAVE_THREAD_LOCAL 1 +#endif + +#if defined(__ANDROID__) && defined(__clang__) + + #if __has_include(<android/ndk-version.h>) + #include <android/ndk-version.h> + #endif // __has_include(<android/ndk-version.h>) + + #if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ + defined(__NDK_MINOR__) && \ + ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) + #undef PHMAP_HAVE_TLS + #undef PHMAP_HAVE_THREAD_LOCAL + #endif +#endif + +// ------------------------------------------------------------ +// Checks whether the __int128 compiler extension for a 128-bit +// integral type is supported. +// ------------------------------------------------------------ +#ifdef PHMAP_HAVE_INTRINSIC_INT128 + #error PHMAP_HAVE_INTRINSIC_INT128 cannot be directly set +#elif defined(__SIZEOF_INT128__) + #if (defined(__clang__) && !defined(_WIN32) && !defined(__aarch64__)) || \ + (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ + (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)) + #define PHMAP_HAVE_INTRINSIC_INT128 1 + #elif defined(__CUDACC__) + #if __CUDACC_VER__ >= 70000 + #define PHMAP_HAVE_INTRINSIC_INT128 1 + #endif // __CUDACC_VER__ >= 70000 + #endif // defined(__CUDACC__) +#endif + +// ------------------------------------------------------------------ +// Checks whether the compiler both supports and enables exceptions. +// ------------------------------------------------------------------ +#ifdef PHMAP_HAVE_EXCEPTIONS + #error PHMAP_HAVE_EXCEPTIONS cannot be directly set. +#elif defined(__clang__) + #if defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) + #define PHMAP_HAVE_EXCEPTIONS 1 + #endif // defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) +#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ + !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ + !(defined(_MSC_VER) && !defined(_CPPUNWIND)) + #define PHMAP_HAVE_EXCEPTIONS 1 +#endif + + +// ----------------------------------------------------------------------- +// Checks whether the platform has an mmap(2) implementation as defined in +// POSIX.1-2001. +// ----------------------------------------------------------------------- +#ifdef PHMAP_HAVE_MMAP + #error PHMAP_HAVE_MMAP cannot be directly set +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \ + defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \ + defined(__ASYLO__) + #define PHMAP_HAVE_MMAP 1 +#endif + +// ----------------------------------------------------------------------- +// Checks the endianness of the platform. +// ----------------------------------------------------------------------- +#if defined(PHMAP_IS_BIG_ENDIAN) + #error "PHMAP_IS_BIG_ENDIAN cannot be directly set." +#endif + +#if defined(PHMAP_IS_LITTLE_ENDIAN) + #error "PHMAP_IS_LITTLE_ENDIAN cannot be directly set." +#endif + +#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define PHMAP_IS_LITTLE_ENDIAN 1 +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define PHMAP_IS_BIG_ENDIAN 1 +#elif defined(_WIN32) + #define PHMAP_IS_LITTLE_ENDIAN 1 +#else + #error "phmap endian detection needs to be set up for your compiler" +#endif + +#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ + defined(__MAC_OS_X_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400 + #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 1 +#else + #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 0 +#endif + +// --------------------------------------------------------------------------- +// Checks whether C++17 std::any is available by checking whether <any> exists. +// --------------------------------------------------------------------------- +#ifdef PHMAP_HAVE_STD_ANY + #error "PHMAP_HAVE_STD_ANY cannot be directly set." +#endif + +#ifdef __has_include + #if __has_include(<any>) && __cplusplus >= 201703L && \ + !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE + #define PHMAP_HAVE_STD_ANY 1 + #endif +#endif + +#ifdef PHMAP_HAVE_STD_OPTIONAL + #error "PHMAP_HAVE_STD_OPTIONAL cannot be directly set." +#endif + +#ifdef __has_include + #if __has_include(<optional>) && __cplusplus >= 201703L && \ + !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE + #define PHMAP_HAVE_STD_OPTIONAL 1 + #endif +#endif + +#ifdef PHMAP_HAVE_STD_VARIANT + #error "PHMAP_HAVE_STD_VARIANT cannot be directly set." +#endif + +#ifdef __has_include + #if __has_include(<variant>) && __cplusplus >= 201703L && \ + !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE + #define PHMAP_HAVE_STD_VARIANT 1 + #endif +#endif + +#ifdef PHMAP_HAVE_STD_STRING_VIEW + #error "PHMAP_HAVE_STD_STRING_VIEW cannot be directly set." +#endif + +#ifdef __has_include + #if __has_include(<string_view>) && __cplusplus >= 201703L && \ + (!defined(_MSC_VER) || _MSC_VER >= 1920) // vs2019 + #define PHMAP_HAVE_STD_STRING_VIEW 1 + #endif +#endif + +// #pragma message(PHMAP_VAR_NAME_VALUE(_MSVC_LANG)) + +#if defined(_MSC_VER) && _MSC_VER >= 1910 && PHMAP_HAVE_CC17 + // #define PHMAP_HAVE_STD_ANY 1 + #define PHMAP_HAVE_STD_OPTIONAL 1 + #define PHMAP_HAVE_STD_VARIANT 1 + #if !defined(PHMAP_HAVE_STD_STRING_VIEW) && _MSC_VER >= 1920 + #define PHMAP_HAVE_STD_STRING_VIEW 1 + #endif +#endif + +#if PHMAP_HAVE_CC17 + #define PHMAP_HAVE_SHARED_MUTEX 1 +#endif + +#ifndef PHMAP_HAVE_STD_STRING_VIEW + #define PHMAP_HAVE_STD_STRING_VIEW 0 +#endif + +// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION +// SEH exception from emplace for variant<SomeStruct> when constructing the +// struct can throw. This defeats some of variant_test and +// variant_exception_safety_test. +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG) + #define PHMAP_INTERNAL_MSVC_2017_DBG_MODE +#endif + +// --------------------------------------------------------------------------- +// Checks whether wchar_t is treated as a native type +// (MSVC: /Zc:wchar_t- treats wchar_t as unsigned short) +// --------------------------------------------------------------------------- +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +#define PHMAP_HAS_NATIVE_WCHAR_T +#endif + +// ----------------------------------------------------------------------------- +// Sanitizer Attributes +// ----------------------------------------------------------------------------- +// +// Sanitizer-related attributes are not "defined" in this file (and indeed +// are not defined as such in any file). To utilize the following +// sanitizer-related attributes within your builds, define the following macros +// within your build using a `-D` flag, along with the given value for +// `-fsanitize`: +// +// * `ADDRESS_SANITIZER` + `-fsanitize=address` (Clang, GCC 4.8) +// * `MEMORY_SANITIZER` + `-fsanitize=memory` (Clang-only) +// * `THREAD_SANITIZER + `-fsanitize=thread` (Clang, GCC 4.8+) +// * `UNDEFINED_BEHAVIOR_SANITIZER` + `-fsanitize=undefined` (Clang, GCC 4.9+) +// * `CONTROL_FLOW_INTEGRITY` + -fsanitize=cfi (Clang-only) +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// A function-like feature checking macro that is a wrapper around +// `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a +// nonzero constant integer if the attribute is supported or 0 if not. +// +// It evaluates to zero if `__has_attribute` is not defined by the compiler. +// ----------------------------------------------------------------------------- +#ifdef __has_attribute + #define PHMAP_HAVE_ATTRIBUTE(x) __has_attribute(x) +#else + #define PHMAP_HAVE_ATTRIBUTE(x) 0 +#endif + +// ----------------------------------------------------------------------------- +// A function-like feature checking macro that accepts C++11 style attributes. +// It's a wrapper around `__has_cpp_attribute`, defined by ISO C++ SD-6 +// (https://en.cppreference.com/w/cpp/experimental/feature_test). If we don't +// find `__has_cpp_attribute`, will evaluate to 0. +// ----------------------------------------------------------------------------- +#if defined(__cplusplus) && defined(__has_cpp_attribute) + #define PHMAP_HAVE_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else + #define PHMAP_HAVE_CPP_ATTRIBUTE(x) 0 +#endif + +// ----------------------------------------------------------------------------- +// Function Attributes +// ----------------------------------------------------------------------------- +#if PHMAP_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) + #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) \ + __attribute__((__format__(__scanf__, string_index, first_to_check))) +#else + #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) + #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) +#endif + +#if PHMAP_HAVE_ATTRIBUTE(always_inline) || \ + (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) + #define PHMAP_HAVE_ATTRIBUTE_ALWAYS_INLINE 1 +#else + #define PHMAP_ATTRIBUTE_ALWAYS_INLINE +#endif + +#if !defined(__INTEL_COMPILER) && (PHMAP_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__))) + #define PHMAP_ATTRIBUTE_NOINLINE __attribute__((noinline)) + #define PHMAP_HAVE_ATTRIBUTE_NOINLINE 1 +#else + #define PHMAP_ATTRIBUTE_NOINLINE +#endif + +#if PHMAP_HAVE_ATTRIBUTE(disable_tail_calls) + #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 + #define PHMAP_ATTRIBUTE_NO_TAIL_CALL __attribute__((disable_tail_calls)) +#elif defined(__GNUC__) && !defined(__clang__) + #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 + #define PHMAP_ATTRIBUTE_NO_TAIL_CALL \ + __attribute__((optimize("no-optimize-sibling-calls"))) +#else + #define PHMAP_ATTRIBUTE_NO_TAIL_CALL + #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 0 +#endif + +#if (PHMAP_HAVE_ATTRIBUTE(weak) || \ + (defined(__GNUC__) && !defined(__clang__))) && \ + !(defined(__llvm__) && defined(_WIN32)) + #undef PHMAP_ATTRIBUTE_WEAK + #define PHMAP_ATTRIBUTE_WEAK __attribute__((weak)) + #define PHMAP_HAVE_ATTRIBUTE_WEAK 1 +#else + #define PHMAP_ATTRIBUTE_WEAK + #define PHMAP_HAVE_ATTRIBUTE_WEAK 0 +#endif + +#if PHMAP_HAVE_ATTRIBUTE(nonnull) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_NONNULL(arg_index) __attribute__((nonnull(arg_index))) +#else + #define PHMAP_ATTRIBUTE_NONNULL(...) +#endif + +#if PHMAP_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_NORETURN __attribute__((noreturn)) +#elif defined(_MSC_VER) + #define PHMAP_ATTRIBUTE_NORETURN __declspec(noreturn) +#else + #define PHMAP_ATTRIBUTE_NORETURN +#endif + +#if defined(__GNUC__) && defined(ADDRESS_SANITIZER) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS +#endif + +#if defined(__GNUC__) && defined(MEMORY_SANITIZER) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY +#endif + +#if defined(__GNUC__) && defined(THREAD_SANITIZER) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD +#endif + +#if defined(__GNUC__) && \ + (defined(UNDEFINED_BEHAVIOR_SANITIZER) || defined(ADDRESS_SANITIZER)) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED \ + __attribute__((no_sanitize("undefined"))) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED +#endif + +#if defined(__GNUC__) && defined(CONTROL_FLOW_INTEGRITY) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI __attribute__((no_sanitize("cfi"))) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI +#endif + +#if defined(__GNUC__) && defined(SAFESTACK_SANITIZER) + #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK \ + __attribute__((no_sanitize("safe-stack"))) +#else + #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK +#endif + +#if PHMAP_HAVE_ATTRIBUTE(returns_nonnull) || \ + (defined(__GNUC__) && \ + (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ + !defined(__clang__)) + #define PHMAP_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) +#else + #define PHMAP_ATTRIBUTE_RETURNS_NONNULL +#endif + +#ifdef PHMAP_HAVE_ATTRIBUTE_SECTION + #error PHMAP_HAVE_ATTRIBUTE_SECTION cannot be directly set +#elif (PHMAP_HAVE_ATTRIBUTE(section) || \ + (defined(__GNUC__) && !defined(__clang__))) && \ + !defined(__APPLE__) && PHMAP_HAVE_ATTRIBUTE_WEAK + #define PHMAP_HAVE_ATTRIBUTE_SECTION 1 + #ifndef PHMAP_ATTRIBUTE_SECTION + #define PHMAP_ATTRIBUTE_SECTION(name) \ + __attribute__((section(#name))) __attribute__((noinline)) + #endif + #ifndef PHMAP_ATTRIBUTE_SECTION_VARIABLE + #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) + #endif + #ifndef PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS + #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ + extern char __start_##name[] PHMAP_ATTRIBUTE_WEAK; \ + extern char __stop_##name[] PHMAP_ATTRIBUTE_WEAK + #endif + #ifndef PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS + #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) + #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) + #endif + #define PHMAP_ATTRIBUTE_SECTION_START(name) \ + (reinterpret_cast<void *>(__start_##name)) + #define PHMAP_ATTRIBUTE_SECTION_STOP(name) \ + (reinterpret_cast<void *>(__stop_##name)) +#else // !PHMAP_HAVE_ATTRIBUTE_SECTION + #define PHMAP_HAVE_ATTRIBUTE_SECTION 0 + #define PHMAP_ATTRIBUTE_SECTION(name) + #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) + #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) + #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) + #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) + #define PHMAP_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0)) + #define PHMAP_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0)) +#endif // PHMAP_ATTRIBUTE_SECTION + +#if PHMAP_HAVE_ATTRIBUTE(force_align_arg_pointer) || \ + (defined(__GNUC__) && !defined(__clang__)) + #if defined(__i386__) + #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC \ + __attribute__((force_align_arg_pointer)) + #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) + #elif defined(__x86_64__) + #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (1) + #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC + #else // !__i386__ && !__x86_64 + #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) + #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC + #endif // __i386__ +#else + #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC + #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) +#endif + +#if PHMAP_HAVE_ATTRIBUTE(nodiscard) + #define PHMAP_MUST_USE_RESULT [[nodiscard]] +#elif defined(__clang__) && PHMAP_HAVE_ATTRIBUTE(warn_unused_result) + #define PHMAP_MUST_USE_RESULT __attribute__((warn_unused_result)) +#else + #define PHMAP_MUST_USE_RESULT +#endif + +#if PHMAP_HAVE_ATTRIBUTE(hot) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_HOT __attribute__((hot)) +#else + #define PHMAP_ATTRIBUTE_HOT +#endif + +#if PHMAP_HAVE_ATTRIBUTE(cold) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_COLD __attribute__((cold)) +#else + #define PHMAP_ATTRIBUTE_COLD +#endif + +#if defined(__clang__) + #if PHMAP_HAVE_CPP_ATTRIBUTE(clang::reinitializes) + #define PHMAP_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] + #else + #define PHMAP_ATTRIBUTE_REINITIALIZES + #endif +#else + #define PHMAP_ATTRIBUTE_REINITIALIZES +#endif + +#if PHMAP_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) + #undef PHMAP_ATTRIBUTE_UNUSED + #define PHMAP_ATTRIBUTE_UNUSED __attribute__((__unused__)) +#else + #define PHMAP_ATTRIBUTE_UNUSED +#endif + +#if PHMAP_HAVE_ATTRIBUTE(tls_model) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_INITIAL_EXEC __attribute__((tls_model("initial-exec"))) +#else + #define PHMAP_ATTRIBUTE_INITIAL_EXEC +#endif + +#if PHMAP_HAVE_ATTRIBUTE(packed) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_PACKED __attribute__((__packed__)) +#else + #define PHMAP_ATTRIBUTE_PACKED +#endif + +#if PHMAP_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__)) + #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes))) +#else + #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) +#endif + +// ---------------------------------------------------------------------- +// Figure out SSE support +// ---------------------------------------------------------------------- +#ifndef PHMAP_HAVE_SSE2 + #if defined(__SSE2__) || \ + (defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) + #define PHMAP_HAVE_SSE2 1 + #else + #define PHMAP_HAVE_SSE2 0 + #endif +#endif + +#ifndef PHMAP_HAVE_SSSE3 + #if defined(__SSSE3__) || defined(__AVX2__) + #define PHMAP_HAVE_SSSE3 1 + #else + #define PHMAP_HAVE_SSSE3 0 + #endif +#endif + +#if PHMAP_HAVE_SSSE3 && !PHMAP_HAVE_SSE2 + #error "Bad configuration!" +#endif + +#if PHMAP_HAVE_SSE2 + #include <emmintrin.h> +#endif + +#if PHMAP_HAVE_SSSE3 + #include <tmmintrin.h> +#endif + + +// ---------------------------------------------------------------------- +// constexpr if +// ---------------------------------------------------------------------- +#if PHMAP_HAVE_CC17 + #define PHMAP_IF_CONSTEXPR(expr) if constexpr ((expr)) +#else + #define PHMAP_IF_CONSTEXPR(expr) if ((expr)) +#endif + +// ---------------------------------------------------------------------- +// base/macros.h +// ---------------------------------------------------------------------- + +// PHMAP_ARRAYSIZE() +// +// Returns the number of elements in an array as a compile-time constant, which +// can be used in defining new arrays. If you use this macro on a pointer by +// mistake, you will get a compile-time error. +#define PHMAP_ARRAYSIZE(array) \ + (sizeof(::phmap::macros_internal::ArraySizeHelper(array))) + +namespace phmap { +namespace macros_internal { + // Note: this internal template function declaration is used by PHMAP_ARRAYSIZE. + // The function doesn't need a definition, as we only use its type. + template <typename T, size_t N> + auto ArraySizeHelper(const T (&array)[N]) -> char (&)[N]; +} // namespace macros_internal +} // namespace phmap + +// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported. +#if defined(__clang__) && defined(__has_warning) + #if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") + #define PHMAP_FALLTHROUGH_INTENDED [[clang::fallthrough]] + #endif +#elif defined(__GNUC__) && __GNUC__ >= 7 + #define PHMAP_FALLTHROUGH_INTENDED [[gnu::fallthrough]] +#endif + +#ifndef PHMAP_FALLTHROUGH_INTENDED + #define PHMAP_FALLTHROUGH_INTENDED \ + do { } while (0) +#endif + +// PHMAP_DEPRECATED() +// +// Marks a deprecated class, struct, enum, function, method and variable +// declarations. The macro argument is used as a custom diagnostic message (e.g. +// suggestion of a better alternative). +// +// Example: +// +// class PHMAP_DEPRECATED("Use Bar instead") Foo {...}; +// PHMAP_DEPRECATED("Use Baz instead") void Bar() {...} +// +// Every usage of a deprecated entity will trigger a warning when compiled with +// clang's `-Wdeprecated-declarations` option. This option is turned off by +// default, but the warnings will be reported by clang-tidy. +#if defined(__clang__) && __cplusplus >= 201103L + #define PHMAP_DEPRECATED(message) __attribute__((deprecated(message))) +#endif + +#ifndef PHMAP_DEPRECATED + #define PHMAP_DEPRECATED(message) +#endif + +// PHMAP_BAD_CALL_IF() +// +// Used on a function overload to trap bad calls: any call that matches the +// overload will cause a compile-time error. This macro uses a clang-specific +// "enable_if" attribute, as described at +// http://clang.llvm.org/docs/AttributeReference.html#enable-if +// +// Overloads which use this macro should be bracketed by +// `#ifdef PHMAP_BAD_CALL_IF`. +// +// Example: +// +// int isdigit(int c); +// #ifdef PHMAP_BAD_CALL_IF +// int isdigit(int c) +// PHMAP_BAD_CALL_IF(c <= -1 || c > 255, +// "'c' must have the value of an unsigned char or EOF"); +// #endif // PHMAP_BAD_CALL_IF + +#if defined(__clang__) + #if __has_attribute(enable_if) + #define PHMAP_BAD_CALL_IF(expr, msg) \ + __attribute__((enable_if(expr, "Bad call trap"), unavailable(msg))) + #endif +#endif + +// PHMAP_ASSERT() +// +// In C++11, `assert` can't be used portably within constexpr functions. +// PHMAP_ASSERT functions as a runtime assert but works in C++11 constexpr +// functions. Example: +// +// constexpr double Divide(double a, double b) { +// return PHMAP_ASSERT(b != 0), a / b; +// } +// +// This macro is inspired by +// https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ +#if defined(NDEBUG) + #define PHMAP_ASSERT(expr) (false ? (void)(expr) : (void)0) +#else + #define PHMAP_ASSERT(expr) \ + (PHMAP_PREDICT_TRUE((expr)) ? (void)0 \ + : [] { assert(false && #expr); }()) // NOLINT +#endif + +#ifdef PHMAP_HAVE_EXCEPTIONS + #define PHMAP_INTERNAL_TRY try + #define PHMAP_INTERNAL_CATCH_ANY catch (...) + #define PHMAP_INTERNAL_RETHROW do { throw; } while (false) +#else // PHMAP_HAVE_EXCEPTIONS + #define PHMAP_INTERNAL_TRY if (true) + #define PHMAP_INTERNAL_CATCH_ANY else if (false) + #define PHMAP_INTERNAL_RETHROW do {} while (false) +#endif // PHMAP_HAVE_EXCEPTIONS + + +#endif // phmap_config_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_dump.h b/benchmarks/others/parallel_hashmap/phmap_dump.h new file mode 100644 index 00000000..0f2018ef --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_dump.h @@ -0,0 +1,227 @@ +#if !defined(phmap_dump_h_guard_) +#define phmap_dump_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// providing dump/load/mmap_load +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include <iostream> +#include <fstream> +#include <sstream> +#include "phmap.h" +namespace phmap +{ + +namespace type_traits_internal { + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 + template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {}; +#else + template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {}; +#endif + +template <class T1, class T2> +struct IsTriviallyCopyable<std::pair<T1, T2>> { + static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value; +}; +} + +namespace priv { + +// ------------------------------------------------------------------------ +// dump/load for raw_hash_set +// ------------------------------------------------------------------------ +template <class Policy, class Hash, class Eq, class Alloc> +template<typename OutputArchive> +bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { + static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, + "value_type should be trivially copyable"); + + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast<char*>(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast<char*>(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template <class Policy, class Hash, class Eq, class Alloc> +template<typename InputArchive> +bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, + "value_type should be trivially copyable"); + raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content + if (!ar.load(&size_)) { + std::cerr << "Failed to load size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; + return false; + } + + // allocate memory for ctrl_ and slots_ + initialize_slots(); + if (!ar.load(reinterpret_cast<char*>(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast<char*>(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } + return true; +} + +// ------------------------------------------------------------------------ +// dump/load for parallel_hash_set +// ------------------------------------------------------------------------ +template <size_t N, + template <class, class, class, class> class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template<typename OutputArchive> +bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { + static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, + "value_type should be trivially copyable"); + + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); + if (!inner.set_.dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template <size_t N, + template <class, class, class, class> class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template<typename InputArchive> +bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, + "value_type should be trivially copyable"); + + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); + if (!inner.set_.load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} +} // namespace priv + + + +// ------------------------------------------------------------------------ +// BinaryArchive +// File is closed when archive object is destroyed +// ------------------------------------------------------------------------ + +// ------------------------------------------------------------------------ +// ------------------------------------------------------------------------ +class BinaryOutputArchive { +public: + BinaryOutputArchive(const char *file_path) { + ofs_.open(file_path, std::ios_base::binary); + } + + bool dump(const char *p, size_t sz) { + ofs_.write(p, sz); + return true; + } + + template<typename V> + typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type + dump(const V& v) { + ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V)); + return true; + } + +private: + std::ofstream ofs_; +}; + + +class BinaryInputArchive { +public: + BinaryInputArchive(const char * file_path) { + ifs_.open(file_path, std::ios_base::binary); + } + + bool load(char* p, size_t sz) { + ifs_.read(p, sz); + return true; + } + + template<typename V> + typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type + load(V* v) { + ifs_.read(reinterpret_cast<char *>(v), sizeof(V)); + return true; + } + +private: + std::ifstream ifs_; +}; + +} // namespace phmap + +#endif // phmap_dump_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h b/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h new file mode 100644 index 00000000..a7719c49 --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h @@ -0,0 +1,154 @@ +#if !defined(phmap_fwd_decl_h_guard_) +#define phmap_fwd_decl_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// --------------------------------------------------------------------------- + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion +#endif + +#include <memory> +#include <utility> + +#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_) + namespace absl { template <class T> struct Hash; }; +#endif + +namespace phmap { + +#if defined(PHMAP_USE_ABSL_HASH) + template <class T> using Hash = ::absl::Hash<T>; +#else + template <class T> struct Hash; +#endif + + template <class T> struct EqualTo; + template <class T> struct Less; + template <class T> using Allocator = typename std::allocator<T>; + template<class T1, class T2> using Pair = typename std::pair<T1, T2>; + + class NullMutex; + + namespace priv { + + // The hash of an object of type T is computed by using phmap::Hash. + template <class T, class E = void> + struct HashEq + { + using Hash = phmap::Hash<T>; + using Eq = phmap::EqualTo<T>; + }; + + template <class T> + using hash_default_hash = typename priv::HashEq<T>::Hash; + + template <class T> + using hash_default_eq = typename priv::HashEq<T>::Eq; + + // type alias for std::allocator so we can forward declare without including other headers + template <class T> + using Allocator = typename phmap::Allocator<T>; + + // type alias for std::pair so we can forward declare without including other headers + template<class T1, class T2> + using Pair = typename phmap::Pair<T1, T2>; + + } // namespace priv + + // ------------- forward declarations for hash containers ---------------------------------- + template <class T, + class Hash = phmap::priv::hash_default_hash<T>, + class Eq = phmap::priv::hash_default_eq<T>, + class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator + class flat_hash_set; + + template <class K, class V, + class Hash = phmap::priv::hash_default_hash<K>, + class Eq = phmap::priv::hash_default_eq<K>, + class Alloc = phmap::priv::Allocator< + phmap::priv::Pair<const K, V>>> // alias for std::allocator + class flat_hash_map; + + template <class T, + class Hash = phmap::priv::hash_default_hash<T>, + class Eq = phmap::priv::hash_default_eq<T>, + class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator + class node_hash_set; + + template <class Key, class Value, + class Hash = phmap::priv::hash_default_hash<Key>, + class Eq = phmap::priv::hash_default_eq<Key>, + class Alloc = phmap::priv::Allocator< + phmap::priv::Pair<const Key, Value>>> // alias for std::allocator + class node_hash_map; + + template <class T, + class Hash = phmap::priv::hash_default_hash<T>, + class Eq = phmap::priv::hash_default_eq<T>, + class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator + size_t N = 4, // 2**N submaps + class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks + class parallel_flat_hash_set; + + template <class K, class V, + class Hash = phmap::priv::hash_default_hash<K>, + class Eq = phmap::priv::hash_default_eq<K>, + class Alloc = phmap::priv::Allocator< + phmap::priv::Pair<const K, V>>, // alias for std::allocator + size_t N = 4, // 2**N submaps + class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks + class parallel_flat_hash_map; + + template <class T, + class Hash = phmap::priv::hash_default_hash<T>, + class Eq = phmap::priv::hash_default_eq<T>, + class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator + size_t N = 4, // 2**N submaps + class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks + class parallel_node_hash_set; + + template <class Key, class Value, + class Hash = phmap::priv::hash_default_hash<Key>, + class Eq = phmap::priv::hash_default_eq<Key>, + class Alloc = phmap::priv::Allocator< + phmap::priv::Pair<const Key, Value>>, // alias for std::allocator + size_t N = 4, // 2**N submaps + class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks + class parallel_node_hash_map; + + // ------------- forward declarations for btree containers ---------------------------------- + template <typename Key, typename Compare = phmap::Less<Key>, + typename Alloc = phmap::Allocator<Key>> + class btree_set; + + template <typename Key, typename Compare = phmap::Less<Key>, + typename Alloc = phmap::Allocator<Key>> + class btree_multiset; + + template <typename Key, typename Value, typename Compare = phmap::Less<Key>, + typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> + class btree_map; + + template <typename Key, typename Value, typename Compare = phmap::Less<Key>, + typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> + class btree_multimap; + +} // namespace phmap + + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif // phmap_fwd_decl_h_guard_ diff --git a/benchmarks/others/parallel_hashmap/phmap_utils.h b/benchmarks/others/parallel_hashmap/phmap_utils.h new file mode 100644 index 00000000..1d0c4728 --- /dev/null +++ b/benchmarks/others/parallel_hashmap/phmap_utils.h @@ -0,0 +1,378 @@ +#if !defined(phmap_utils_h_guard_) +#define phmap_utils_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - [email protected] +// +// minimal header providing phmap::HashState +// +// use as: phmap::HashState().combine(0, _first_name, _last_name, _age); +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion +#endif + +#include <cstdint> +#include <functional> +#include <tuple> +#include "phmap_bits.h" + +// --------------------------------------------------------------- +// Absl forward declaration requires global scope. +// --------------------------------------------------------------- +#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) && !defined(ABSL_HASH_HASH_H_) + namespace absl { template <class T> struct Hash; }; +#endif + +namespace phmap +{ + +// --------------------------------------------------------------- +// --------------------------------------------------------------- +template<int n> +struct phmap_mix +{ + inline size_t operator()(size_t) const; +}; + +template<> +struct phmap_mix<4> +{ + inline size_t operator()(size_t a) const + { + static constexpr uint64_t kmul = 0xcc9e2d51UL; + // static constexpr uint64_t kmul = 0x3B9ACB93UL; // [greg] my own random prime + uint64_t l = a * kmul; + return static_cast<size_t>(l ^ (l >> 32)); + } +}; + +#if defined(PHMAP_HAS_UMUL128) + template<> + struct phmap_mix<8> + { + // Very fast mixing (similar to Abseil) + inline size_t operator()(size_t a) const + { + static constexpr uint64_t k = 0xde5fb9d2630458e9ULL; + // static constexpr uint64_t k = 0x7C9D0BF0567102A5ULL; // [greg] my own random prime + uint64_t h; + uint64_t l = umul128(a, k, &h); + return static_cast<size_t>(h + l); + } + }; +#else + template<> + struct phmap_mix<8> + { + inline size_t operator()(size_t a) const + { + a = (~a) + (a << 21); // a = (a << 21) - a - 1; + a = a ^ (a >> 24); + a = (a + (a << 3)) + (a << 8); // a * 265 + a = a ^ (a >> 14); + a = (a + (a << 2)) + (a << 4); // a * 21 + a = a ^ (a >> 28); + a = a + (a << 31); + return static_cast<size_t>(a); + } + }; +#endif + +// -------------------------------------------- +template<int n> +struct fold_if_needed +{ + inline size_t operator()(uint64_t) const; +}; + +template<> +struct fold_if_needed<4> +{ + inline size_t operator()(uint64_t a) const + { + return static_cast<size_t>(a ^ (a >> 32)); + } +}; + +template<> +struct fold_if_needed<8> +{ + inline size_t operator()(uint64_t a) const + { + return static_cast<size_t>(a); + } +}; + +// --------------------------------------------------------------- +// see if class T has a hash_value() friend method +// --------------------------------------------------------------- +template<typename T> +struct has_hash_value +{ +private: + typedef std::true_type yes; + typedef std::false_type no; + + template<typename U> static auto test(int) -> decltype(hash_value(std::declval<const U&>()) == 1, yes()); + + template<typename> static no test(...); + +public: + static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; +}; + +#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) + template <class T> using Hash = ::absl::Hash<T>; +#elif !defined(PHMAP_USE_ABSL_HASH) +// --------------------------------------------------------------- +// phmap::Hash +// --------------------------------------------------------------- +template <class T> +struct Hash +{ + template <class U, typename std::enable_if<has_hash_value<U>::value, int>::type = 0> + size_t _hash(const T& val) const + { + return hash_value(val); + } + + template <class U, typename std::enable_if<!has_hash_value<U>::value, int>::type = 0> + size_t _hash(const T& val) const + { + return std::hash<T>()(val); + } + + inline size_t operator()(const T& val) const + { + return _hash<T>(val); + } +}; + +template <class T> +struct Hash<T *> +{ + inline size_t operator()(const T *val) const noexcept + { + return static_cast<size_t>(reinterpret_cast<const uintptr_t>(val)); + } +}; + +template<class ArgumentType, class ResultType> +struct phmap_unary_function +{ + typedef ArgumentType argument_type; + typedef ResultType result_type; +}; + +template <> +struct Hash<bool> : public phmap_unary_function<bool, size_t> +{ + inline size_t operator()(bool val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<char> : public phmap_unary_function<char, size_t> +{ + inline size_t operator()(char val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<signed char> : public phmap_unary_function<signed char, size_t> +{ + inline size_t operator()(signed char val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<unsigned char> : public phmap_unary_function<unsigned char, size_t> +{ + inline size_t operator()(unsigned char val) const noexcept + { return static_cast<size_t>(val); } +}; + +#ifdef PHMAP_HAS_NATIVE_WCHAR_T +template <> +struct Hash<wchar_t> : public phmap_unary_function<wchar_t, size_t> +{ + inline size_t operator()(wchar_t val) const noexcept + { return static_cast<size_t>(val); } +}; +#endif + +template <> +struct Hash<int16_t> : public phmap_unary_function<int16_t, size_t> +{ + inline size_t operator()(int16_t val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<uint16_t> : public phmap_unary_function<uint16_t, size_t> +{ + inline size_t operator()(uint16_t val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<int32_t> : public phmap_unary_function<int32_t, size_t> +{ + inline size_t operator()(int32_t val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<uint32_t> : public phmap_unary_function<uint32_t, size_t> +{ + inline size_t operator()(uint32_t val) const noexcept + { return static_cast<size_t>(val); } +}; + +template <> +struct Hash<int64_t> : public phmap_unary_function<int64_t, size_t> +{ + inline size_t operator()(int64_t val) const noexcept + { return fold_if_needed<sizeof(size_t)>()(static_cast<uint64_t>(val)); } +}; + +template <> +struct Hash<uint64_t> : public phmap_unary_function<uint64_t, size_t> +{ + inline size_t operator()(uint64_t val) const noexcept + { return fold_if_needed<sizeof(size_t)>()(val); } +}; + +template <> +struct Hash<float> : public phmap_unary_function<float, size_t> +{ + inline size_t operator()(float val) const noexcept + { + // -0.0 and 0.0 should return same hash + uint32_t *as_int = reinterpret_cast<uint32_t *>(&val); + return (val == 0) ? static_cast<size_t>(0) : + static_cast<size_t>(*as_int); + } +}; + +template <> +struct Hash<double> : public phmap_unary_function<double, size_t> +{ + inline size_t operator()(double val) const noexcept + { + // -0.0 and 0.0 should return same hash + uint64_t *as_int = reinterpret_cast<uint64_t *>(&val); + return (val == 0) ? static_cast<size_t>(0) : + fold_if_needed<sizeof(size_t)>()(*as_int); + } +}; + +#endif + +template <class H, int sz> struct Combiner +{ + H operator()(H seed, size_t value); +}; + +template <class H> struct Combiner<H, 4> +{ + H operator()(H seed, size_t value) + { + return seed ^ (value + 0x9e3779b9 + (seed << 6) + (seed >> 2)); + } +}; + +template <class H> struct Combiner<H, 8> +{ + H operator()(H seed, size_t value) + { + return seed ^ (value + size_t(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2)); + } +}; + +// define HashState to combine member hashes... see example below +// ----------------------------------------------------------------------------- +template <typename H> +class HashStateBase { +public: + template <typename T, typename... Ts> + static H combine(H state, const T& value, const Ts&... values); + + static H combine(H state) { return state; } +}; + +template <typename H> +template <typename T, typename... Ts> +H HashStateBase<H>::combine(H seed, const T& v, const Ts&... vs) +{ + return HashStateBase<H>::combine(Combiner<H, sizeof(H)>()( + seed, phmap::Hash<T>()(v)), + vs...); +} + +using HashState = HashStateBase<size_t>; + +// ----------------------------------------------------------------------------- + +#if !defined(PHMAP_USE_ABSL_HASH) + +// define Hash for std::pair +// ------------------------- +template<class T1, class T2> +struct Hash<std::pair<T1, T2>> { + size_t operator()(std::pair<T1, T2> const& p) const noexcept { + return phmap::HashState().combine(phmap::Hash<T1>()(p.first), p.second); + } +}; + +// define Hash for std::tuple +// -------------------------- +template<class... T> +struct Hash<std::tuple<T...>> { + size_t operator()(std::tuple<T...> const& t) const noexcept { + return _hash_helper(t); + } + +private: + template<size_t I = 0, class ...P> + typename std::enable_if<I == sizeof...(P), size_t>::type + _hash_helper(const std::tuple<P...> &) const noexcept { return 0; } + + template<size_t I = 0, class ...P> + typename std::enable_if<I < sizeof...(P), size_t>::type + _hash_helper(const std::tuple<P...> &t) const noexcept { + const auto &el = std::get<I>(t); + using el_type = typename std::remove_cv<typename std::remove_reference<decltype(el)>::type>::type; + return Combiner<size_t, sizeof(size_t)>()( + phmap::Hash<el_type>()(el), _hash_helper<I + 1>(t)); + } +}; + + +#endif + + +} // namespace phmap + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif // phmap_utils_h_guard_ diff --git a/benchmarks/others/robin_hood.hpp b/benchmarks/others/robin_hood.hpp index a602dac9..ec2af8c6 100644 --- a/benchmarks/others/robin_hood.hpp +++ b/benchmarks/others/robin_hood.hpp @@ -1,2465 +1,2533 @@ -// ______ _____ ______ _________
-// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
-// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
-// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
-// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
-// _/_____/
-//
-// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
-// https://github.com/martinus/robin-hood-hashing
-//
-// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2018-2020 Martin Ankerl <http://martin.ankerl.com>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#ifndef ROBIN_HOOD_H_INCLUDED
-#define ROBIN_HOOD_H_INCLUDED
-
-// see https://semver.org/
-#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
-#define ROBIN_HOOD_VERSION_MINOR 10 // for adding functionality in a backwards-compatible manner
-#define ROBIN_HOOD_VERSION_PATCH 0 // for backwards-compatible bug fixes
-
-#include <algorithm>
-#include <cstdlib>
-#include <cstring>
-#include <functional>
-#include <memory> // only to support hash of smart pointers
-#include <stdexcept>
-#include <string>
-#include <type_traits>
-#include <utility>
-#if __cplusplus >= 201703L
-# include <string_view>
-#endif
-
-// #define ROBIN_HOOD_LOG_ENABLED
-#ifdef ROBIN_HOOD_LOG_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_LOG(...) \
- std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
-#else
-# define ROBIN_HOOD_LOG(x)
-#endif
-
-// #define ROBIN_HOOD_TRACE_ENABLED
-#ifdef ROBIN_HOOD_TRACE_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_TRACE(...) \
- std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
-#else
-# define ROBIN_HOOD_TRACE(x)
-#endif
-
-// #define ROBIN_HOOD_COUNT_ENABLED
-#ifdef ROBIN_HOOD_COUNT_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_COUNT(x) ++counts().x;
-namespace robin_hood {
-struct Counts {
- uint64_t shiftUp{};
- uint64_t shiftDown{};
-};
-inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
- return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
-}
-
-static Counts& counts() {
- static Counts counts{};
- return counts;
-}
-} // namespace robin_hood
-#else
-# define ROBIN_HOOD_COUNT(x)
-#endif
-
-// all non-argument macros should use this facility. See
-// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
-#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
-
-// mark unused members with this macro
-#define ROBIN_HOOD_UNUSED(identifier)
-
-// bitness
-#if SIZE_MAX == UINT32_MAX
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
-#elif SIZE_MAX == UINT64_MAX
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
-#else
-# error Unsupported bitness
-#endif
-
-// endianess
-#ifdef _MSC_VER
-# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
- (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-#endif
-
-// inline
-#ifdef _MSC_VER
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
-#endif
-
-// exceptions
-#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
-#endif
-
-// count leading/trailing bits
-#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
-# ifdef _MSC_VER
-# if ROBIN_HOOD(BITNESS) == 32
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
-# endif
-# include <intrin.h>
-# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
-# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
- [](size_t mask) noexcept -> int { \
- unsigned long index; \
- return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
- : ROBIN_HOOD(BITNESS); \
- }(x)
-# else
-# if ROBIN_HOOD(BITNESS) == 32
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
-# endif
-# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
-# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
-# endif
-#endif
-
-// fallthrough
-#ifndef __has_cpp_attribute // For backwards compatibility
-# define __has_cpp_attribute(x) 0
-#endif
-#if __has_cpp_attribute(clang::fallthrough)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
-#elif __has_cpp_attribute(gnu::fallthrough)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
-#endif
-
-// likely/unlikely
-#ifdef _MSC_VER
-# define ROBIN_HOOD_LIKELY(condition) condition
-# define ROBIN_HOOD_UNLIKELY(condition) condition
-#else
-# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
-# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
-#endif
-
-// detect if native wchar_t type is availiable in MSVC
-#ifdef _MSC_VER
-# ifdef _NATIVE_WCHAR_T_DEFINED
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
-# endif
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
-#endif
-
-// workaround missing "is_trivially_copyable" in g++ < 5.0
-// See https://stackoverflow.com/a/31798726/48181
-#if defined(__GNUC__) && __GNUC__ < 5
-# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
-#else
-# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
-#endif
-
-// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
-#endif
-
-namespace robin_hood {
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
-# define ROBIN_HOOD_STD std
-#else
-
-// c++11 compatibility layer
-namespace ROBIN_HOOD_STD {
-template <class T>
-struct alignment_of
- : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
-
-template <class T, T... Ints>
-class integer_sequence {
-public:
- using value_type = T;
- static_assert(std::is_integral<value_type>::value, "not integral type");
- static constexpr std::size_t size() noexcept {
- return sizeof...(Ints);
- }
-};
-template <std::size_t... Inds>
-using index_sequence = integer_sequence<std::size_t, Inds...>;
-
-namespace detail_ {
-template <class T, T Begin, T End, bool>
-struct IntSeqImpl {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
-
- template <class, class>
- struct IntSeqCombiner;
-
- template <TValue... Inds0, TValue... Inds1>
- struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> {
- using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
- };
-
- using TResult =
- typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
- (End - Begin) / 2 == 1>::TResult,
- typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
- (End - Begin + 1) / 2 == 1>::TResult>::TResult;
-};
-
-template <class T, T Begin>
-struct IntSeqImpl<T, Begin, Begin, false> {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0, "unexpected argument (Begin<0)");
- using TResult = integer_sequence<TValue>;
-};
-
-template <class T, T Begin, T End>
-struct IntSeqImpl<T, Begin, End, true> {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0, "unexpected argument (Begin<0)");
- using TResult = integer_sequence<TValue, Begin>;
-};
-} // namespace detail_
-
-template <class T, T N>
-using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
-
-template <std::size_t N>
-using make_index_sequence = make_integer_sequence<std::size_t, N>;
-
-template <class... T>
-using index_sequence_for = make_index_sequence<sizeof...(T)>;
-
-} // namespace ROBIN_HOOD_STD
-
-#endif
-
-namespace detail {
-
-// make sure we static_cast to the correct type for hash_int
-#if ROBIN_HOOD(BITNESS) == 64
-using SizeT = uint64_t;
-#else
-using SizeT = uint32_t;
-#endif
-
-template <typename T>
-T rotr(T x, unsigned k) {
- return (x >> k) | (x << (8U * sizeof(T) - k));
-}
-
-// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
-// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
-// care!
-template <typename T>
-inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
- return reinterpret_cast<T>(ptr);
-}
-
-template <typename T>
-inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
- return reinterpret_cast<T>(ptr);
-}
-
-// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
-// inlinings more difficult. Throws are also generally the slow path.
-template <typename E, typename... Args>
-[[noreturn]] ROBIN_HOOD(NOINLINE)
-#if ROBIN_HOOD(HAS_EXCEPTIONS)
- void doThrow(Args&&... args) {
- // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
- throw E(std::forward<Args>(args)...);
-}
-#else
- void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
- abort();
-}
-#endif
-
-template <typename E, typename T, typename... Args>
-T* assertNotNull(T* t, Args&&... args) {
- if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
- doThrow<E>(std::forward<Args>(args)...);
- }
- return t;
-}
-
-template <typename T>
-inline T unaligned_load(void const* ptr) noexcept {
- // using memcpy so we don't get into unaligned load problems.
- // compiler should optimize this very well anyways.
- T t;
- std::memcpy(&t, ptr, sizeof(T));
- return t;
-}
-
-// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
-// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
-// pointer.
-template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
-class BulkPoolAllocator {
-public:
- BulkPoolAllocator() noexcept = default;
-
- // does not copy anything, just creates a new allocator.
- BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept
- : mHead(nullptr)
- , mListForFree(nullptr) {}
-
- BulkPoolAllocator(BulkPoolAllocator&& o) noexcept
- : mHead(o.mHead)
- , mListForFree(o.mListForFree) {
- o.mListForFree = nullptr;
- o.mHead = nullptr;
- }
-
- BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept {
- reset();
- mHead = o.mHead;
- mListForFree = o.mListForFree;
- o.mListForFree = nullptr;
- o.mHead = nullptr;
- return *this;
- }
-
- BulkPoolAllocator&
- // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
- operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
- // does not do anything
- return *this;
- }
-
- ~BulkPoolAllocator() noexcept {
- reset();
- }
-
- // Deallocates all allocated memory.
- void reset() noexcept {
- while (mListForFree) {
- T* tmp = *mListForFree;
- ROBIN_HOOD_LOG("std::free")
- std::free(mListForFree);
- mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
- }
- mHead = nullptr;
- }
-
- // allocates, but does NOT initialize. Use in-place new constructor, e.g.
- // T* obj = pool.allocate();
- // ::new (static_cast<void*>(obj)) T();
- T* allocate() {
- T* tmp = mHead;
- if (!tmp) {
- tmp = performAllocation();
- }
-
- mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
- return tmp;
- }
-
- // does not actually deallocate but puts it in store.
- // make sure you have already called the destructor! e.g. with
- // obj->~T();
- // pool.deallocate(obj);
- void deallocate(T* obj) noexcept {
- *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
- mHead = obj;
- }
-
- // Adds an already allocated block of memory to the allocator. This allocator is from now on
- // responsible for freeing the data (with free()). If the provided data is not large enough to
- // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
- void addOrFree(void* ptr, const size_t numBytes) noexcept {
- // calculate number of available elements in ptr
- if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
- // not enough data for at least one element. Free and return.
- ROBIN_HOOD_LOG("std::free")
- std::free(ptr);
- } else {
- ROBIN_HOOD_LOG("add to buffer")
- add(ptr, numBytes);
- }
- }
-
- void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs>& other) noexcept {
- using std::swap;
- swap(mHead, other.mHead);
- swap(mListForFree, other.mListForFree);
- }
-
-private:
- // iterates the list of allocated memory to calculate how many to alloc next.
- // Recalculating this each time saves us a size_t member.
- // This ignores the fact that memory blocks might have been added manually with addOrFree. In
- // practice, this should not matter much.
- ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
- auto tmp = mListForFree;
- size_t numAllocs = MinNumAllocs;
-
- while (numAllocs * 2 <= MaxNumAllocs && tmp) {
- auto x = reinterpret_cast<T***>(tmp);
- tmp = *x;
- numAllocs *= 2;
- }
-
- return numAllocs;
- }
-
- // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
- void add(void* ptr, const size_t numBytes) noexcept {
- const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
-
- auto data = reinterpret_cast<T**>(ptr);
-
- // link free list
- auto x = reinterpret_cast<T***>(data);
- *x = mListForFree;
- mListForFree = data;
-
- // create linked list for newly allocated data
- auto* const headT =
- reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
-
- auto* const head = reinterpret_cast<char*>(headT);
-
- // Visual Studio compiler automatically unrolls this loop, which is pretty cool
- for (size_t i = 0; i < numElements; ++i) {
- *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
- head + (i + 1) * ALIGNED_SIZE;
- }
-
- // last one points to 0
- *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
- mHead;
- mHead = headT;
- }
-
- // Called when no memory is available (mHead == 0).
- // Don't inline this slow path.
- ROBIN_HOOD(NOINLINE) T* performAllocation() {
- size_t const numElementsToAlloc = calcNumElementsToAlloc();
-
- // alloc new memory: [prev |T, T, ... T]
- size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
- ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
- << " * " << numElementsToAlloc)
- add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
- return mHead;
- }
-
- // enforce byte alignment of the T's
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
- static constexpr size_t ALIGNMENT =
- (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
-#else
- static const size_t ALIGNMENT =
- (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value)
- ? ROBIN_HOOD_STD::alignment_of<T>::value
- : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
-#endif
-
- static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
-
- static_assert(MinNumAllocs >= 1, "MinNumAllocs");
- static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
- static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
- static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
- static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
-
- T* mHead{nullptr};
- T** mListForFree{nullptr};
-};
-
-template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
-struct NodeAllocator;
-
-// dummy allocator that does nothing
-template <typename T, size_t MinSize, size_t MaxSize>
-struct NodeAllocator<T, MinSize, MaxSize, true> {
-
- // we are not using the data, so just free it.
- void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
- ROBIN_HOOD_LOG("std::free")
- std::free(ptr);
- }
-};
-
-template <typename T, size_t MinSize, size_t MaxSize>
-struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
-
-// dummy hash, unsed as mixer when robin_hood::hash is already used
-template <typename T>
-struct identity_hash {
- constexpr size_t operator()(T const& obj) const noexcept {
- return static_cast<size_t>(obj);
- }
-};
-
-// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
-// my own here.
-namespace swappable {
-#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
-using std::swap;
-template <typename T>
-struct nothrow {
- static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
-};
-#else
-template <typename T>
-struct nothrow {
- static const bool value = std::is_nothrow_swappable<T>::value;
-};
-#endif
-} // namespace swappable
-
-} // namespace detail
-
-struct is_transparent_tag {};
-
-// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
-// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is
-// also tested.
-template <typename T1, typename T2>
-struct pair {
- using first_type = T1;
- using second_type = T2;
-
- template <typename U1 = T1, typename U2 = T2,
- typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
- std::is_default_constructible<U2>::value>::type>
- constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
- : first()
- , second() {}
-
- // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
- explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
- noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
- : first(o.first)
- , second(o.second) {}
-
- // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
- explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
- T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
- : first(std::move(o.first))
- , second(std::move(o.second)) {}
-
- constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
- T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
- : first(std::move(a))
- , second(std::move(b)) {}
-
- template <typename U1, typename U2>
- constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
- std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
- : first(std::forward<U1>(a))
- , second(std::forward<U2>(b)) {}
-
- template <typename... U1, typename... U2>
- constexpr pair(
- std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
- std::tuple<U2...> b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
- std::declval<std::tuple<U2...>&>(),
- ROBIN_HOOD_STD::index_sequence_for<U1...>(),
- ROBIN_HOOD_STD::index_sequence_for<U2...>())))
- : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(),
- ROBIN_HOOD_STD::index_sequence_for<U2...>()) {}
-
- // constructor called from the std::piecewise_construct_t ctor
- template <typename... U1, size_t... I1, typename... U2, size_t... I2>
- pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
- noexcept(T1(std::forward<U1>(std::get<I1>(
- std::declval<std::tuple<
- U1...>&>()))...)) && noexcept(T2(std::
- forward<U2>(std::get<I2>(
- std::declval<std::tuple<U2...>&>()))...)))
- : first(std::forward<U1>(std::get<I1>(a))...)
- , second(std::forward<U2>(std::get<I2>(b))...) {
- // make visual studio compiler happy about warning about unused a & b.
- // Visual studio's pair implementation disables warning 4100.
- (void)a;
- (void)b;
- }
-
- void swap(pair<T1, T2>& o) noexcept((detail::swappable::nothrow<T1>::value) &&
- (detail::swappable::nothrow<T2>::value)) {
- using std::swap;
- swap(first, o.first);
- swap(second, o.second);
- }
-
- T1 first; // NOLINT(misc-non-private-member-variables-in-classes)
- T2 second; // NOLINT(misc-non-private-member-variables-in-classes)
-};
-
-template <typename A, typename B>
-inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
- noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
- a.swap(b);
-}
-
-template <typename A, typename B>
-inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
- return (x.first == y.first) && (x.second == y.second);
-}
-template <typename A, typename B>
-inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x == y);
-}
-template <typename A, typename B>
-inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
- std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
- std::declval<B const&>())) {
- return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
-}
-template <typename A, typename B>
-inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
- return y < x;
-}
-template <typename A, typename B>
-inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x > y);
-}
-template <typename A, typename B>
-inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x < y);
-}
-
-inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
- static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
- static constexpr uint64_t seed = UINT64_C(0xe17a1465);
- static constexpr unsigned int r = 47;
-
- auto const* const data64 = static_cast<uint64_t const*>(ptr);
- uint64_t h = seed ^ (len * m);
-
- size_t const n_blocks = len / 8;
- for (size_t i = 0; i < n_blocks; ++i) {
- auto k = detail::unaligned_load<uint64_t>(data64 + i);
-
- k *= m;
- k ^= k >> r;
- k *= m;
-
- h ^= k;
- h *= m;
- }
-
- auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
- switch (len & 7U) {
- case 7:
- h ^= static_cast<uint64_t>(data8[6]) << 48U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 6:
- h ^= static_cast<uint64_t>(data8[5]) << 40U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 5:
- h ^= static_cast<uint64_t>(data8[4]) << 32U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 4:
- h ^= static_cast<uint64_t>(data8[3]) << 24U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 3:
- h ^= static_cast<uint64_t>(data8[2]) << 16U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 2:
- h ^= static_cast<uint64_t>(data8[1]) << 8U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 1:
- h ^= static_cast<uint64_t>(data8[0]);
- h *= m;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- default:
- break;
- }
-
- h ^= h >> r;
- h *= m;
- h ^= h >> r;
- return static_cast<size_t>(h);
-}
-
-inline size_t hash_int(uint64_t x) noexcept {
- // inspired by lemire's strongly universal hashing
- // https://lemire.me/blog/2018/08/15/fast-strongly-universal-64-bit-hashing-everywhere/
- //
- // Instead of shifts, we use rotations so we don't lose any bits.
- //
- // Added a final multiplcation with a constant for more mixing. It is most important that
- // the lower bits are well mixed.
- auto h1 = x * UINT64_C(0xA24BAED4963EE407);
- auto h2 = detail::rotr(x, 32U) * UINT64_C(0x9FB21C651E98DF25);
- auto h = detail::rotr(h1 + h2, 32U);
- return static_cast<size_t>(h);
-}
-
-// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
-template <typename T, typename Enable = void>
-struct hash : public std::hash<T> {
- size_t operator()(T const& obj) const
- noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
- // call base hash
- auto result = std::hash<T>::operator()(obj);
- // return mixed of that, to be save against identity has
- return hash_int(static_cast<detail::SizeT>(result));
- }
-};
-
-template <typename CharT>
-struct hash<std::basic_string<CharT>> {
- size_t operator()(std::basic_string<CharT> const& str) const noexcept {
- return hash_bytes(str.data(), sizeof(CharT) * str.size());
- }
-};
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
-template <typename CharT>
-struct hash<std::basic_string_view<CharT>> {
- size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
- return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
- }
-};
-#endif
-
-template <class T>
-struct hash<T*> {
- size_t operator()(T* ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr));
- }
-};
-
-template <class T>
-struct hash<std::unique_ptr<T>> {
- size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
- }
-};
-
-template <class T>
-struct hash<std::shared_ptr<T>> {
- size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
- }
-};
-
-template <typename Enum>
-struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
- size_t operator()(Enum e) const noexcept {
- using Underlying = typename std::underlying_type<Enum>::type;
- return hash<Underlying>{}(static_cast<Underlying>(e));
- }
-};
-
-#define ROBIN_HOOD_HASH_INT(T) \
- template <> \
- struct hash<T> { \
- size_t operator()(T const& obj) const noexcept { \
- return hash_int(static_cast<uint64_t>(obj)); \
- } \
- }
-
-#if defined(__GNUC__) && !defined(__clang__)
-# pragma GCC diagnostic push
-# pragma GCC diagnostic ignored "-Wuseless-cast"
-#endif
-// see https://en.cppreference.com/w/cpp/utility/hash
-ROBIN_HOOD_HASH_INT(bool);
-ROBIN_HOOD_HASH_INT(char);
-ROBIN_HOOD_HASH_INT(signed char);
-ROBIN_HOOD_HASH_INT(unsigned char);
-ROBIN_HOOD_HASH_INT(char16_t);
-ROBIN_HOOD_HASH_INT(char32_t);
-#if ROBIN_HOOD(HAS_NATIVE_WCHART)
-ROBIN_HOOD_HASH_INT(wchar_t);
-#endif
-ROBIN_HOOD_HASH_INT(short);
-ROBIN_HOOD_HASH_INT(unsigned short);
-ROBIN_HOOD_HASH_INT(int);
-ROBIN_HOOD_HASH_INT(unsigned int);
-ROBIN_HOOD_HASH_INT(long);
-ROBIN_HOOD_HASH_INT(long long);
-ROBIN_HOOD_HASH_INT(unsigned long);
-ROBIN_HOOD_HASH_INT(unsigned long long);
-#if defined(__GNUC__) && !defined(__clang__)
-# pragma GCC diagnostic pop
-#endif
-namespace detail {
-
-template <typename T>
-struct void_type {
- using type = void;
-};
-
-template <typename T, typename = void>
-struct has_is_transparent : public std::false_type {};
-
-template <typename T>
-struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
- : public std::true_type {};
-
-// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
-// is used. see https://stackoverflow.com/a/28771920/48181
-template <typename T>
-struct WrapHash : public T {
- WrapHash() = default;
- explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
- : T(o) {}
-};
-
-template <typename T>
-struct WrapKeyEqual : public T {
- WrapKeyEqual() = default;
- explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
- : T(o) {}
-};
-
-// A highly optimized hashmap implementation, using the Robin Hood algorithm.
-//
-// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
-// be about 2x faster in most cases and require much less allocations.
-//
-// This implementation uses the following memory layout:
-//
-// [Node, Node, ... Node | info, info, ... infoSentinel ]
-//
-// * Node: either a DataNode that directly has the std::pair<key, val> as member,
-// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
-// depends on how fast the swap() operation is. Heuristically, this is automatically choosen
-// based on sizeof(). there are always 2^n Nodes.
-//
-// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
-// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
-// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
-// actually belongs to the previous position and was pushed out because that place is already
-// taken.
-//
-// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
-// need for a idx variable.
-//
-// According to STL, order of templates has effect on throughput. That's why I've moved the
-// boolean to the front.
-// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
-template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
- typename KeyEqual>
-class Table
- : public WrapHash<Hash>,
- public WrapKeyEqual<KeyEqual>,
- detail::NodeAllocator<
- typename std::conditional<
- std::is_void<T>::value, Key,
- robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
- 4, 16384, IsFlat> {
-public:
- static constexpr bool is_flat = IsFlat;
- static constexpr bool is_map = !std::is_void<T>::value;
- static constexpr bool is_set = !is_map;
- static constexpr bool is_transparent =
- has_is_transparent<Hash>::value && has_is_transparent<KeyEqual>::value;
-
- using key_type = Key;
- using mapped_type = T;
- using value_type = typename std::conditional<
- is_set, Key,
- robin_hood::pair<typename std::conditional<is_flat, Key, Key const>::type, T>>::type;
- using size_type = size_t;
- using hasher = Hash;
- using key_equal = KeyEqual;
- using Self = Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
-
-private:
- static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
- "MaxLoadFactor100 needs to be >10 && < 100");
-
- using WHash = WrapHash<Hash>;
- using WKeyEqual = WrapKeyEqual<KeyEqual>;
-
- // configuration defaults
-
- // make sure we have 8 elements, needed to quickly rehash mInfo
- static constexpr size_t InitialNumElements = sizeof(uint64_t);
- static constexpr uint32_t InitialInfoNumBits = 5;
- static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
- static constexpr size_t InfoMask = InitialInfoInc - 1U;
- static constexpr uint8_t InitialInfoHashShift = 0;
- using DataPool = detail::NodeAllocator<value_type, 4, 16384, IsFlat>;
-
- // type needs to be wider than uint8_t.
- using InfoType = uint32_t;
-
- // DataNode ////////////////////////////////////////////////////////
-
- // Primary template for the data node. We have special implementations for small and big
- // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
- // on the heap so swap merely swaps a pointer.
- template <typename M, bool>
- class DataNode {};
-
- // Small: just allocate on the stack.
- template <typename M>
- class DataNode<M, true> final {
- public:
- template <typename... Args>
- explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
- noexcept(value_type(std::forward<Args>(args)...)))
- : mData(std::forward<Args>(args)...) {}
-
- DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
- std::is_nothrow_move_constructible<value_type>::value)
- : mData(std::move(n.mData)) {}
-
- // doesn't do anything
- void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
- void destroyDoNotDeallocate() noexcept {}
-
- value_type const* operator->() const noexcept {
- return &mData;
- }
- value_type* operator->() noexcept {
- return &mData;
- }
-
- const value_type& operator*() const noexcept {
- return mData;
- }
-
- value_type& operator*() noexcept {
- return mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
- return mData.first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
- return mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type const&>::type
- getFirst() const noexcept {
- return mData.first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
- return mData;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
- return mData.second;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
- return mData.second;
- }
-
- void swap(DataNode<M, true>& o) noexcept(
- noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
- mData.swap(o.mData);
- }
-
- private:
- value_type mData;
- };
-
- // big object: allocate on heap.
- template <typename M>
- class DataNode<M, false> {
- public:
- template <typename... Args>
- explicit DataNode(M& map, Args&&... args)
- : mData(map.allocate()) {
- ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
- }
-
- DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
- : mData(std::move(n.mData)) {}
-
- void destroy(M& map) noexcept {
- // don't deallocate, just put it into list of datapool.
- mData->~value_type();
- map.deallocate(mData);
- }
-
- void destroyDoNotDeallocate() noexcept {
- mData->~value_type();
- }
-
- value_type const* operator->() const noexcept {
- return mData;
- }
-
- value_type* operator->() noexcept {
- return mData;
- }
-
- const value_type& operator*() const {
- return *mData;
- }
-
- value_type& operator*() {
- return *mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
- return mData->first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
- return *mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type const&>::type
- getFirst() const noexcept {
- return mData->first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
- return *mData;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
- return mData->second;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
- return mData->second;
- }
-
- void swap(DataNode<M, false>& o) noexcept {
- using std::swap;
- swap(mData, o.mData);
- }
-
- private:
- value_type* mData;
- };
-
- using Node = DataNode<Self, IsFlat>;
-
- // helpers for doInsert: extract first entry (only const required)
- ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
- return n.getFirst();
- }
-
- // in case we have void mapped_type, we are not using a pair, thus we just route k through.
- // No need to disable this because it's just not used if not applicable.
- ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
- return k;
- }
-
- // in case we have non-void mapped_type, we have a standard robin_hood::pair
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
- getFirstConst(value_type const& vt) const noexcept {
- return vt.first;
- }
-
- // Cloner //////////////////////////////////////////////////////////
-
- template <typename M, bool UseMemcpy>
- struct Cloner;
-
- // fast path: Just copy data, without allocating anything.
- template <typename M>
- struct Cloner<M, true> {
- void operator()(M const& source, M& target) const {
- auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
- auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
- auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
- std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
- }
- };
-
- template <typename M>
- struct Cloner<M, false> {
- void operator()(M const& s, M& t) const {
- auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
- std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
-
- for (size_t i = 0; i < numElementsWithBuffer; ++i) {
- if (t.mInfo[i]) {
- ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
- }
- }
- }
- };
-
- // Destroyer ///////////////////////////////////////////////////////
-
- template <typename M, bool IsFlatAndTrivial>
- struct Destroyer {};
-
- template <typename M>
- struct Destroyer<M, true> {
- void nodes(M& m) const noexcept {
- m.mNumElements = 0;
- }
-
- void nodesDoNotDeallocate(M& m) const noexcept {
- m.mNumElements = 0;
- }
- };
-
- template <typename M>
- struct Destroyer<M, false> {
- void nodes(M& m) const noexcept {
- m.mNumElements = 0;
- // clear also resets mInfo to 0, that's sometimes not necessary.
- auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
-
- for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
- if (0 != m.mInfo[idx]) {
- Node& n = m.mKeyVals[idx];
- n.destroy(m);
- n.~Node();
- }
- }
- }
-
- void nodesDoNotDeallocate(M& m) const noexcept {
- m.mNumElements = 0;
- // clear also resets mInfo to 0, that's sometimes not necessary.
- auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
- for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
- if (0 != m.mInfo[idx]) {
- Node& n = m.mKeyVals[idx];
- n.destroyDoNotDeallocate();
- n.~Node();
- }
- }
- }
- };
-
- // Iter ////////////////////////////////////////////////////////////
-
- struct fast_forward_tag {};
-
- // generic iterator for both const_iterator and iterator.
- template <bool IsConst>
- // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
- class Iter {
- private:
- using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
-
- public:
- using difference_type = std::ptrdiff_t;
- using value_type = typename Self::value_type;
- using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
- using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
- using iterator_category = std::forward_iterator_tag;
-
- // default constructed iterator can be compared to itself, but WON'T return true when
- // compared to end().
- Iter() = default;
-
- // Rule of zero: nothing specified. The conversion constructor is only enabled for
- // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
-
- // Conversion constructor from iterator to const_iterator.
- template <bool OtherIsConst,
- typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
- // NOLINTNEXTLINE(hicpp-explicit-conversions)
- Iter(Iter<OtherIsConst> const& other) noexcept
- : mKeyVals(other.mKeyVals)
- , mInfo(other.mInfo) {}
-
- Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
- : mKeyVals(valPtr)
- , mInfo(infoPtr) {}
-
- Iter(NodePtr valPtr, uint8_t const* infoPtr,
- fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
- : mKeyVals(valPtr)
- , mInfo(infoPtr) {
- fastForward();
- }
-
- template <bool OtherIsConst,
- typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
- Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
- mKeyVals = other.mKeyVals;
- mInfo = other.mInfo;
- return *this;
- }
-
- // prefix increment. Undefined behavior if we are at end()!
- Iter& operator++() noexcept {
- mInfo++;
- mKeyVals++;
- fastForward();
- return *this;
- }
-
- Iter operator++(int) noexcept {
- Iter tmp = *this;
- ++(*this);
- return tmp;
- }
-
- reference operator*() const {
- return **mKeyVals;
- }
-
- pointer operator->() const {
- return &**mKeyVals;
- }
-
- template <bool O>
- bool operator==(Iter<O> const& o) const noexcept {
- return mKeyVals == o.mKeyVals;
- }
-
- template <bool O>
- bool operator!=(Iter<O> const& o) const noexcept {
- return mKeyVals != o.mKeyVals;
- }
-
- private:
- // fast forward to the next non-free info byte
- // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
- // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
- void fastForward() noexcept {
- size_t n = 0;
- while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
- mInfo += sizeof(size_t);
- mKeyVals += sizeof(size_t);
- }
-#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
- // we know for certain that within the next 8 bytes we'll find a non-zero one.
- if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
- mInfo += 4;
- mKeyVals += 4;
- }
- if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
- mInfo += 2;
- mKeyVals += 2;
- }
- if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
- mInfo += 1;
- mKeyVals += 1;
- }
-#else
-# if ROBIN_HOOD(LITTLE_ENDIAN)
- auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
-# else
- auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
-# endif
- mInfo += inc;
- mKeyVals += inc;
-#endif
- }
-
- friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
- NodePtr mKeyVals{nullptr};
- uint8_t const* mInfo{nullptr};
- };
-
- ////////////////////////////////////////////////////////////////////
-
- // highly performance relevant code.
- // Lower bits are used for indexing into the array (2^n size)
- // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
- template <typename HashKey>
- void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
- // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as
- // an additional mixing step. This serves as a bad hash prevention, if the given data is
- // badly mixed.
- using Mix =
- typename std::conditional<std::is_same<::robin_hood::hash<key_type>, hasher>::value,
- ::robin_hood::detail::identity_hash<size_t>,
- ::robin_hood::hash<size_t>>::type;
-
- // the lower InitialInfoNumBits are reserved for info.
- auto h = Mix{}(WHash::operator()(key));
- *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
- *idx = (h >> InitialInfoNumBits) & mMask;
- }
-
- // forwards the index by one, wrapping around at the end
- void next(InfoType* info, size_t* idx) const noexcept {
- *idx = *idx + 1;
- *info += mInfoInc;
- }
-
- void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
- // unrolling this by hand did not bring any speedups.
- while (*info < mInfo[*idx]) {
- next(info, idx);
- }
- }
-
- // Shift everything up by one element. Tries to move stuff around.
- void
- shiftUp(size_t startIdx,
- size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
- auto idx = startIdx;
- ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
- while (--idx != insertion_idx) {
- mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
- }
-
- idx = startIdx;
- while (idx != insertion_idx) {
- ROBIN_HOOD_COUNT(shiftUp)
- mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
- if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
- --idx;
- }
- }
-
- void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
- // until we find one that is either empty or has zero offset.
- // TODO(martinus) we don't need to move everything, just the last one for the same
- // bucket.
- mKeyVals[idx].destroy(*this);
-
- // until we find one that is either empty or has zero offset.
- while (mInfo[idx + 1] >= 2 * mInfoInc) {
- ROBIN_HOOD_COUNT(shiftDown)
- mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
- mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
- ++idx;
- }
-
- mInfo[idx] = 0;
- // don't destroy, we've moved it
- // mKeyVals[idx].destroy(*this);
- mKeyVals[idx].~Node();
- }
-
- // copy of find(), except that it returns iterator instead of const_iterator.
- template <typename Other>
- ROBIN_HOOD(NODISCARD)
- size_t findIdx(Other const& key) const {
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
-
- do {
- // unrolling this twice gives a bit of a speedup. More unrolling did not help.
- if (info == mInfo[idx] &&
- ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
- return idx;
- }
- next(&info, &idx);
- if (info == mInfo[idx] &&
- ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
- return idx;
- }
- next(&info, &idx);
- } while (info <= mInfo[idx]);
-
- // nothing found!
- return mMask == 0 ? 0
- : static_cast<size_t>(std::distance(
- mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
- }
-
- void cloneData(const Table& o) {
- Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
- }
-
- // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
- // @return index where the element was created
- size_t insert_move(Node&& keyval) {
- // we don't retry, fail if overflowing
- // don't need to check max num elements
- if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
- throwOverflowError(); // impossible to reach LCOV_EXCL_LINE
- }
-
- size_t idx{};
- InfoType info{};
- keyToIdx(keyval.getFirst(), &idx, &info);
-
- // skip forward. Use <= because we are certain that the element is not there.
- while (info <= mInfo[idx]) {
- idx = idx + 1;
- info += mInfoInc;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = static_cast<uint8_t>(info);
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- ::new (static_cast<void*>(&l)) Node(std::move(keyval));
- } else {
- shiftUp(idx, insertion_idx);
- l = std::move(keyval);
- }
-
- // put at empty spot
- mInfo[insertion_idx] = insertion_info;
-
- ++mNumElements;
- return insertion_idx;
- }
-
-public:
- using iterator = Iter<false>;
- using const_iterator = Iter<true>;
-
- Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
- : WHash()
- , WKeyEqual() {
- ROBIN_HOOD_TRACE(this)
- }
-
- // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
- // This tremendously speeds up ctor & dtor of a map that never receives an element. The
- // penalty is payed at the first insert, and not before. Lookup of this empty map works
- // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
- // standard, but we can ignore it.
- explicit Table(
- size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
- const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- }
-
- template <typename Iter>
- Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
- const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- insert(first, last);
- }
-
- Table(std::initializer_list<value_type> initlist,
- size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
- const KeyEqual& equal = KeyEqual{})
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- insert(initlist.begin(), initlist.end());
- }
-
- Table(Table&& o) noexcept
- : WHash(std::move(static_cast<WHash&>(o)))
- , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
- , DataPool(std::move(static_cast<DataPool&>(o))) {
- ROBIN_HOOD_TRACE(this)
- if (o.mMask) {
- mKeyVals = std::move(o.mKeyVals);
- mInfo = std::move(o.mInfo);
- mNumElements = std::move(o.mNumElements);
- mMask = std::move(o.mMask);
- mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
- mInfoInc = std::move(o.mInfoInc);
- mInfoHashShift = std::move(o.mInfoHashShift);
- // set other's mask to 0 so its destructor won't do anything
- o.init();
- }
- }
-
- Table& operator=(Table&& o) noexcept {
- ROBIN_HOOD_TRACE(this)
- if (&o != this) {
- if (o.mMask) {
- // only move stuff if the other map actually has some data
- destroy();
- mKeyVals = std::move(o.mKeyVals);
- mInfo = std::move(o.mInfo);
- mNumElements = std::move(o.mNumElements);
- mMask = std::move(o.mMask);
- mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
- mInfoInc = std::move(o.mInfoInc);
- mInfoHashShift = std::move(o.mInfoHashShift);
- WHash::operator=(std::move(static_cast<WHash&>(o)));
- WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
- DataPool::operator=(std::move(static_cast<DataPool&>(o)));
-
- o.init();
-
- } else {
- // nothing in the other map => just clear us.
- clear();
- }
- }
- return *this;
- }
-
- Table(const Table& o)
- : WHash(static_cast<const WHash&>(o))
- , WKeyEqual(static_cast<const WKeyEqual&>(o))
- , DataPool(static_cast<const DataPool&>(o)) {
- ROBIN_HOOD_TRACE(this)
- if (!o.empty()) {
- // not empty: create an exact copy. it is also possible to just iterate through all
- // elements and insert them, but copying is probably faster.
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
-
- ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = static_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
- // no need for calloc because clonData does memcpy
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
- mNumElements = o.mNumElements;
- mMask = o.mMask;
- mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
- mInfoInc = o.mInfoInc;
- mInfoHashShift = o.mInfoHashShift;
- cloneData(o);
- }
- }
-
- // Creates a copy of the given map. Copy constructor of each entry is used.
- // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
- // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
- Table& operator=(Table const& o) {
- ROBIN_HOOD_TRACE(this)
- if (&o == this) {
- // prevent assigning of itself
- return *this;
- }
-
- // we keep using the old allocator and not assign the new one, because we want to keep
- // the memory available. when it is the same size.
- if (o.empty()) {
- if (0 == mMask) {
- // nothing to do, we are empty too
- return *this;
- }
-
- // not empty: destroy what we have there
- // clear also resets mInfo to 0, that's sometimes not necessary.
- destroy();
- init();
- WHash::operator=(static_cast<const WHash&>(o));
- WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
- DataPool::operator=(static_cast<DataPool const&>(o));
-
- return *this;
- }
-
- // clean up old stuff
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
-
- if (mMask != o.mMask) {
- // no luck: we don't have the same array size allocated, so we need to realloc.
- if (0 != mMask) {
- // only deallocate if we actually have data!
- ROBIN_HOOD_LOG("std::free")
- std::free(mKeyVals);
- }
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
- ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = static_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
-
- // no need for calloc here because cloneData performs a memcpy.
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
- // sentinel is set in cloneData
- }
- WHash::operator=(static_cast<const WHash&>(o));
- WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
- DataPool::operator=(static_cast<DataPool const&>(o));
- mNumElements = o.mNumElements;
- mMask = o.mMask;
- mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
- mInfoInc = o.mInfoInc;
- mInfoHashShift = o.mInfoHashShift;
- cloneData(o);
-
- return *this;
- }
-
- // Swaps everything between the two maps.
- void swap(Table& o) {
- ROBIN_HOOD_TRACE(this)
- using std::swap;
- swap(o, *this);
- }
-
- // Clears all data, without resizing.
- void clear() {
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- // don't do anything! also important because we don't want to write to
- // DummyInfoByte::b, even though we would just write 0 to it.
- return;
- }
-
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
- // clear everything, then set the sentinel again
- uint8_t const z = 0;
- std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
- mInfo[numElementsWithBuffer] = 1;
-
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- // Destroys the map and all it's contents.
- ~Table() {
- ROBIN_HOOD_TRACE(this)
- destroy();
- }
-
- // Checks if both tables contain the same entries. Order is irrelevant.
- bool operator==(const Table& other) const {
- ROBIN_HOOD_TRACE(this)
- if (other.size() != size()) {
- return false;
- }
- for (auto const& otherEntry : other) {
- if (!has(otherEntry)) {
- return false;
- }
- }
-
- return true;
- }
-
- bool operator!=(const Table& other) const {
- ROBIN_HOOD_TRACE(this)
- return !operator==(other);
- }
-
- template <typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- return doCreateByKey(key);
- }
-
- template <typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
- ROBIN_HOOD_TRACE(this)
- return doCreateByKey(std::move(key));
- }
-
- template <typename Iter>
- void insert(Iter first, Iter last) {
- for (; first != last; ++first) {
- // value_type ctor needed because this might be called with std::pair's
- insert(value_type(*first));
- }
- }
-
- template <typename... Args>
- std::pair<iterator, bool> emplace(Args&&... args) {
- ROBIN_HOOD_TRACE(this)
- Node n{*this, std::forward<Args>(args)...};
- auto r = doInsert(std::move(n));
- if (!r.second) {
- // insertion not possible: destroy node
- // NOLINTNEXTLINE(bugprone-use-after-move)
- n.destroy(*this);
- }
- return r;
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
- return try_emplace_impl(key, std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
- return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const_iterator hint, const key_type& key,
- Args&&... args) {
- (void)hint;
- return try_emplace_impl(key, std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
- (void)hint;
- return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
- return insert_or_assign_impl(key, std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
- return insert_or_assign_impl(std::move(key), std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const_iterator hint, const key_type& key,
- Mapped&& obj) {
- (void)hint;
- return insert_or_assign_impl(key, std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) {
- (void)hint;
- return insert_or_assign_impl(std::move(key), std::forward<Mapped>(obj));
- }
-
- std::pair<iterator, bool> insert(const value_type& keyval) {
- ROBIN_HOOD_TRACE(this)
- return doInsert(keyval);
- }
-
- std::pair<iterator, bool> insert(value_type&& keyval) {
- return doInsert(std::move(keyval));
- }
-
- // Returns 1 if key is found, 0 otherwise.
- size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- return 1;
- }
- return 0;
- }
-
- template <typename OtherKey, typename Self_ = Self>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- return 1;
- }
- return 0;
- }
-
- bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- return 1U == count(key);
- }
-
- template <typename OtherKey, typename Self_ = Self>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
- return 1U == count(key);
- }
-
- // Returns a reference to the value found for key.
- // Throws std::out_of_range if element cannot be found
- template <typename Q = mapped_type>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- doThrow<std::out_of_range>("key not found");
- }
- return kv->getSecond();
- }
-
- // Returns a reference to the value found for key.
- // Throws std::out_of_range if element cannot be found
- template <typename Q = mapped_type>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- doThrow<std::out_of_range>("key not found");
- }
- return kv->getSecond();
- }
-
- const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey>
- const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey, typename Self_ = Self>
- typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard)
- const_iterator>::type // NOLINT(modernize-use-nodiscard)
- find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- iterator find(const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey>
- iterator find(const OtherKey& key, is_transparent_tag /*unused*/) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey, typename Self_ = Self>
- typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- iterator begin() {
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- return end();
- }
- return iterator(mKeyVals, mInfo, fast_forward_tag{});
- }
- const_iterator begin() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return cbegin();
- }
- const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- return cend();
- }
- return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
- }
-
- iterator end() {
- ROBIN_HOOD_TRACE(this)
- // no need to supply valid info pointer: end() must not be dereferenced, and only node
- // pointer is compared.
- return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
- }
- const_iterator end() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return cend();
- }
- const_iterator cend() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
- }
-
- iterator erase(const_iterator pos) {
- ROBIN_HOOD_TRACE(this)
- // its safe to perform const cast here
- // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
- return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
- }
-
- // Erases element at pos, returns iterator to the next element.
- iterator erase(iterator pos) {
- ROBIN_HOOD_TRACE(this)
- // we assume that pos always points to a valid entry, and not end().
- auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
-
- shiftDown(idx);
- --mNumElements;
-
- if (*pos.mInfo) {
- // we've backward shifted, return this again
- return pos;
- }
-
- // no backward shift, return next element
- return ++pos;
- }
-
- size_t erase(const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
-
- // check while info matches with the source idx
- do {
- if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
- shiftDown(idx);
- --mNumElements;
- return 1;
- }
- next(&info, &idx);
- } while (info <= mInfo[idx]);
-
- // nothing found to delete
- return 0;
- }
-
- // reserves space for the specified number of elements. Makes sure the old data fits.
- // exactly the same as reserve(c).
- void rehash(size_t c) {
- // forces a reserve
- reserve(c, true);
- }
-
- // reserves space for the specified number of elements. Makes sure the old data fits.
- // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
- void reserve(size_t c) {
- // reserve, but don't force rehash
- reserve(c, false);
- }
-
- // If possible reallocates the map to a smaller one. This frees the underlying table.
- // Does not do anything if load_factor is too large for decreasing the table's size.
- void compact() {
- ROBIN_HOOD_TRACE(this)
- auto newSize = InitialNumElements;
- while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
- newSize *= 2;
- }
- if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
- throwOverflowError();
- }
-
- ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
-
- // only actually do anything when the new size is bigger than the old one. This prevents to
- // continuously allocate for each reserve() call.
- if (newSize < mMask + 1) {
- rehashPowerOfTwo(newSize, true);
- }
- }
-
- size_type size() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return mNumElements;
- }
-
- size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return static_cast<size_type>(-1);
- }
-
- ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return 0 == mNumElements;
- }
-
- float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return MaxLoadFactor100 / 100.0F;
- }
-
- // Average number of elements per bucket. Since we allow only 1 per bucket
- float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return static_cast<float>(size()) / static_cast<float>(mMask + 1);
- }
-
- ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return mMask;
- }
-
-#ifndef IGNORE_STANDARD_METHODS
- ROBIN_HOOD(NODISCARD) size_t bucket_count() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return mMask;
- }
- void max_load_factor(float) {
- ROBIN_HOOD_TRACE(this)
- // warning, not used
- }
-#endif
-
- ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
- if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
- return maxElements * MaxLoadFactor100 / 100;
- }
-
- // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
- return (maxElements / 100) * MaxLoadFactor100;
- }
-
- ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
- // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
- // 64bit types.
- return numElements + sizeof(uint64_t);
- }
-
- ROBIN_HOOD(NODISCARD)
- size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
- auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
- return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
- }
-
- // calculation only allowed for 2^n values
- ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
-#if ROBIN_HOOD(BITNESS) == 64
- return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
-#else
- // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
- auto const ne = static_cast<uint64_t>(numElements);
- auto const s = static_cast<uint64_t>(sizeof(Node));
- auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
-
- auto const total64 = ne * s + infos;
- auto const total = static_cast<size_t>(total64);
-
- if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
- throwOverflowError();
- }
- return total;
-#endif
- }
-
-private:
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
- ROBIN_HOOD_TRACE(this)
- auto it = find(e.first);
- return it != end() && it->second == e.second;
- }
-
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
- ROBIN_HOOD_TRACE(this)
- return find(e) != end();
- }
-
- void reserve(size_t c, bool forceRehash) {
- ROBIN_HOOD_TRACE(this)
- auto const minElementsAllowed = (std::max)(c, mNumElements);
- auto newSize = InitialNumElements;
- while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
- newSize *= 2;
- }
- if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
- throwOverflowError();
- }
-
- ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
-
- // only actually do anything when the new size is bigger than the old one. This prevents to
- // continuously allocate for each reserve() call.
- if (forceRehash || newSize > mMask + 1) {
- rehashPowerOfTwo(newSize, false);
- }
- }
-
- // reserves space for at least the specified number of elements.
- // only works if numBuckets if power of two
- void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
- ROBIN_HOOD_TRACE(this)
-
- Node* const oldKeyVals = mKeyVals;
- uint8_t const* const oldInfo = mInfo;
-
- const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
-
- // resize operation: move stuff
- init_data(numBuckets);
- if (oldMaxElementsWithBuffer > 1) {
- for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
- if (oldInfo[i] != 0) {
- insert_move(std::move(oldKeyVals[i]));
- // destroy the node but DON'T destroy the data.
- oldKeyVals[i].~Node();
- }
- }
-
- // this check is not necessary as it's guarded by the previous if, but it helps silence
- // g++'s overeager "attempt to free a non-heap object 'map'
- // [-Werror=free-nonheap-object]" warning.
- if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
- // don't destroy old data: put it into the pool instead
- if (forceFree) {
- std::free(oldKeyVals);
- } else {
- DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
- }
- }
- }
- }
-
- ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
-#if ROBIN_HOOD(HAS_EXCEPTIONS)
- throw std::overflow_error("robin_hood::map overflow");
-#else
- abort();
-#endif
- }
-
- template <typename OtherKey, typename... Args>
- std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
- ROBIN_HOOD_TRACE(this)
- auto it = find(key);
- if (it == end()) {
- return emplace(std::piecewise_construct,
- std::forward_as_tuple(std::forward<OtherKey>(key)),
- std::forward_as_tuple(std::forward<Args>(args)...));
- }
- return {it, false};
- }
-
- template <typename OtherKey, typename Mapped>
- std::pair<iterator, bool> insert_or_assign_impl(OtherKey&& key, Mapped&& obj) {
- ROBIN_HOOD_TRACE(this)
- auto it = find(key);
- if (it == end()) {
- return emplace(std::forward<OtherKey>(key), std::forward<Mapped>(obj));
- }
- it->second = std::forward<Mapped>(obj);
- return {it, false};
- }
-
- void init_data(size_t max_elements) {
- mNumElements = 0;
- mMask = max_elements - 1;
- mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
-
- // calloc also zeroes everything
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
- ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = reinterpret_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::calloc(1, numBytesTotal)));
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
-
- // set sentinel
- mInfo[numElementsWithBuffer] = 1;
-
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- template <typename Arg, typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type doCreateByKey(Arg&& key) {
- while (true) {
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
- nextWhileLess(&info, &idx);
-
- // while we potentially have a match. Can't do a do-while here because when mInfo is
- // 0 we don't want to skip forward
- while (info == mInfo[idx]) {
- if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
- // key already exists, do not insert.
- return mKeyVals[idx].getSecond();
- }
- next(&info, &idx);
- }
-
- // unlikely that this evaluates to true
- if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
- increase_size();
- continue;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = info;
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- // put at empty spot. This forwards all arguments into the node where the object
- // is constructed exactly where it is needed.
- ::new (static_cast<void*>(&l))
- Node(*this, std::piecewise_construct,
- std::forward_as_tuple(std::forward<Arg>(key)), std::forward_as_tuple());
- } else {
- shiftUp(idx, insertion_idx);
- l = Node(*this, std::piecewise_construct,
- std::forward_as_tuple(std::forward<Arg>(key)), std::forward_as_tuple());
- }
-
- // mKeyVals[idx].getFirst() = std::move(key);
- mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
-
- ++mNumElements;
- return mKeyVals[insertion_idx].getSecond();
- }
- }
-
- // This is exactly the same code as operator[], except for the return values
- template <typename Arg>
- std::pair<iterator, bool> doInsert(Arg&& keyval) {
- while (true) {
- size_t idx{};
- InfoType info{};
- keyToIdx(getFirstConst(keyval), &idx, &info);
- nextWhileLess(&info, &idx);
-
- // while we potentially have a match
- while (info == mInfo[idx]) {
- if (WKeyEqual::operator()(getFirstConst(keyval), mKeyVals[idx].getFirst())) {
- // key already exists, do NOT insert.
- // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
- return std::make_pair<iterator, bool>(iterator(mKeyVals + idx, mInfo + idx),
- false);
- }
- next(&info, &idx);
- }
-
- // unlikely that this evaluates to true
- if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
- increase_size();
- continue;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = info;
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- ::new (static_cast<void*>(&l)) Node(*this, std::forward<Arg>(keyval));
- } else {
- shiftUp(idx, insertion_idx);
- l = Node(*this, std::forward<Arg>(keyval));
- }
-
- // put at empty spot
- mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
-
- ++mNumElements;
- return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true);
- }
- }
-
- bool try_increase_info() {
- ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
- << ", maxNumElementsAllowed="
- << calcMaxNumElementsAllowed(mMask + 1))
- if (mInfoInc <= 2) {
- // need to be > 2 so that shift works (otherwise undefined behavior!)
- return false;
- }
- // we got space left, try to make info smaller
- mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
-
- // remove one bit of the hash, leaving more space for the distance info.
- // This is extremely fast because we can operate on 8 bytes at once.
- ++mInfoHashShift;
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
-
- for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
- auto val = unaligned_load<uint64_t>(mInfo + i);
- val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
- std::memcpy(mInfo + i, &val, sizeof(val));
- }
- // update sentinel, which might have been cleared out!
- mInfo[numElementsWithBuffer] = 1;
-
- mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
- return true;
- }
-
- void increase_size() {
- // nothing allocated yet? just allocate InitialNumElements
- if (0 == mMask) {
- init_data(InitialNumElements);
- return;
- }
-
- auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
- if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
- return;
- }
-
- ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
- << maxNumElementsAllowed << ", load="
- << (static_cast<double>(mNumElements) * 100.0 /
- (static_cast<double>(mMask) + 1)))
- // it seems we have a really bad hash function! don't try to resize again
- if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
- throwOverflowError();
- }
-
- rehashPowerOfTwo((mMask + 1) * 2, false);
- }
-
- void destroy() {
- if (0 == mMask) {
- // don't deallocate!
- return;
- }
-
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
- .nodesDoNotDeallocate(*this);
-
- // This protection against not deleting mMask shouldn't be needed as it's sufficiently
- // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
- // reports a compile error: attempt to free a non-heap object 'fm'
- // [-Werror=free-nonheap-object]
- if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
- ROBIN_HOOD_LOG("std::free")
- std::free(mKeyVals);
- }
- }
-
- void init() noexcept {
- mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
- mInfo = reinterpret_cast<uint8_t*>(&mMask);
- mNumElements = 0;
- mMask = 0;
- mMaxNumElementsAllowed = 0;
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- // members are sorted so no padding occurs
- Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 8
- uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 16
- size_t mNumElements = 0; // 8 byte 24
- size_t mMask = 0; // 8 byte 32
- size_t mMaxNumElementsAllowed = 0; // 8 byte 40
- InfoType mInfoInc = InitialInfoInc; // 4 byte 44
- InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48
- // 16 byte 56 if NodeAllocator
-};
-
-} // namespace detail
-
-// map
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_flat_map = detail::Table<true, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_node_map = detail::Table<false, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_map =
- detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
- std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
- std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
- MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-// set
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_flat_set = detail::Table<true, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_node_set = detail::Table<false, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 &&
- std::is_nothrow_move_constructible<Key>::value &&
- std::is_nothrow_move_assignable<Key>::value,
- MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-} // namespace robin_hood
-
-#endif
\ No newline at end of file +// ______ _____ ______ _________ +// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ / +// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ / +// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ / +// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/ +// _/_____/ +// +// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20 +// https://github.com/martinus/robin-hood-hashing +// +// Licensed under the MIT License <http://opensource.org/licenses/MIT>. +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com> +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROBIN_HOOD_H_INCLUDED +#define ROBIN_HOOD_H_INCLUDED + +// see https://semver.org/ +#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes +#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_PATCH 3 // for backwards-compatible bug fixes + +#include <algorithm> +#include <cstdlib> +#include <cstring> +#include <functional> +#include <limits> +#include <memory> // only to support hash of smart pointers +#include <stdexcept> +#include <string> +#include <type_traits> +#include <utility> +#if __cplusplus >= 201703L +# include <string_view> +#endif + +// #define ROBIN_HOOD_LOG_ENABLED +#ifdef ROBIN_HOOD_LOG_ENABLED +# include <iostream> +# define ROBIN_HOOD_LOG(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#else +# define ROBIN_HOOD_LOG(x) +#endif + +// #define ROBIN_HOOD_TRACE_ENABLED +#ifdef ROBIN_HOOD_TRACE_ENABLED +# include <iostream> +# define ROBIN_HOOD_TRACE(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#else +# define ROBIN_HOOD_TRACE(x) +#endif + +// #define ROBIN_HOOD_COUNT_ENABLED +#ifdef ROBIN_HOOD_COUNT_ENABLED +# include <iostream> +# define ROBIN_HOOD_COUNT(x) ++counts().x; +namespace robin_hood { +struct Counts { + uint64_t shiftUp{}; + uint64_t shiftDown{}; +}; +inline std::ostream& operator<<(std::ostream& os, Counts const& c) { + return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; +} + +static Counts& counts() { + static Counts counts{}; + return counts; +} +} // namespace robin_hood +#else +# define ROBIN_HOOD_COUNT(x) +#endif + +// all non-argument macros should use this facility. See +// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/ +#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x() + +// mark unused members with this macro +#define ROBIN_HOOD_UNUSED(identifier) + +// bitness +#if SIZE_MAX == UINT32_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 +#elif SIZE_MAX == UINT64_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 +#else +# error Unsupported bitness +#endif + +// endianess +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#endif + +// inline +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) +#endif + +// exceptions +#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 +#endif + +// count leading/trailing bits +#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS) +# ifdef _MSC_VER +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +# endif +# include <intrin.h> +# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ + [](size_t mask) noexcept -> int { \ + unsigned long index; \ + return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \ + : ROBIN_HOOD(BITNESS); \ + }(x) +# else +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll +# endif +# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) +# endif +#endif + +// fallthrough +#ifndef __has_cpp_attribute // For backwards compatibility +# define __has_cpp_attribute(x) 0 +#endif +#if __has_cpp_attribute(clang::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] +#elif __has_cpp_attribute(gnu::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() +#endif + +// likely/unlikely +#ifdef _MSC_VER +# define ROBIN_HOOD_LIKELY(condition) condition +# define ROBIN_HOOD_UNLIKELY(condition) condition +#else +# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) +# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) +#endif + +// detect if native wchar_t type is availiable in MSVC +#ifdef _MSC_VER +# ifdef _NATIVE_WCHAR_T_DEFINED +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +#endif + +// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr +#ifdef _MSC_VER +# if _MSC_VER <= 1900 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +#endif + +// workaround missing "is_trivially_copyable" in g++ < 5.0 +// See https://stackoverflow.com/a/31798726/48181 +#if defined(__GNUC__) && __GNUC__ < 5 +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +#else +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value +#endif + +// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() +#endif + +namespace robin_hood { + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) +# define ROBIN_HOOD_STD std +#else + +// c++11 compatibility layer +namespace ROBIN_HOOD_STD { +template <class T> +struct alignment_of + : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {}; + +template <class T, T... Ints> +class integer_sequence { +public: + using value_type = T; + static_assert(std::is_integral<value_type>::value, "not integral type"); + static constexpr std::size_t size() noexcept { + return sizeof...(Ints); + } +}; +template <std::size_t... Inds> +using index_sequence = integer_sequence<std::size_t, Inds...>; + +namespace detail_ { +template <class T, T Begin, T End, bool> +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral<TValue>::value, "not integral type"); + static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); + + template <class, class> + struct IntSeqCombiner; + + template <TValue... Inds0, TValue... Inds1> + struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> { + using TResult = integer_sequence<TValue, Inds0..., Inds1...>; + }; + + using TResult = + typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2, + (End - Begin) / 2 == 1>::TResult, + typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End, + (End - Begin + 1) / 2 == 1>::TResult>::TResult; +}; + +template <class T, T Begin> +struct IntSeqImpl<T, Begin, Begin, false> { + using TValue = T; + static_assert(std::is_integral<TValue>::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence<TValue>; +}; + +template <class T, T Begin, T End> +struct IntSeqImpl<T, Begin, End, true> { + using TValue = T; + static_assert(std::is_integral<TValue>::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence<TValue, Begin>; +}; +} // namespace detail_ + +template <class T, T N> +using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult; + +template <std::size_t N> +using make_index_sequence = make_integer_sequence<std::size_t, N>; + +template <class... T> +using index_sequence_for = make_index_sequence<sizeof...(T)>; + +} // namespace ROBIN_HOOD_STD + +#endif + +namespace detail { + +// make sure we static_cast to the correct type for hash_int +#if ROBIN_HOOD(BITNESS) == 64 +using SizeT = uint64_t; +#else +using SizeT = uint32_t; +#endif + +template <typename T> +T rotr(T x, unsigned k) { + return (x >> k) | (x << (8U * sizeof(T) - k)); +} + +// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to +// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with +// care! +template <typename T> +inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept { + return reinterpret_cast<T>(ptr); +} + +template <typename T> +inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { + return reinterpret_cast<T>(ptr); +} + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +template <typename E, typename... Args> +[[noreturn]] ROBIN_HOOD(NOINLINE) +#if ROBIN_HOOD(HAS_EXCEPTIONS) + void doThrow(Args&&... args) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + throw E(std::forward<Args>(args)...); +} +#else + void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { + abort(); +} +#endif + +template <typename E, typename T, typename... Args> +T* assertNotNull(T* t, Args&&... args) { + if (ROBIN_HOOD_UNLIKELY(nullptr == t)) { + doThrow<E>(std::forward<Args>(args)...); + } + return t; +} + +template <typename T> +inline T unaligned_load(void const* ptr) noexcept { + // using memcpy so we don't get into unaligned load problems. + // compiler should optimize this very well anyways. + T t; + std::memcpy(&t, ptr, sizeof(T)); + return t; +} + +// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor, +// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a +// pointer. +template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256> +class BulkPoolAllocator { +public: + BulkPoolAllocator() noexcept = default; + + // does not copy anything, just creates a new allocator. + BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept + : mHead(nullptr) + , mListForFree(nullptr) {} + + BulkPoolAllocator(BulkPoolAllocator&& o) noexcept + : mHead(o.mHead) + , mListForFree(o.mListForFree) { + o.mListForFree = nullptr; + o.mHead = nullptr; + } + + BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept { + reset(); + mHead = o.mHead; + mListForFree = o.mListForFree; + o.mListForFree = nullptr; + o.mHead = nullptr; + return *this; + } + + BulkPoolAllocator& + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { + // does not do anything + return *this; + } + + ~BulkPoolAllocator() noexcept { + reset(); + } + + // Deallocates all allocated memory. + void reset() noexcept { + while (mListForFree) { + T* tmp = *mListForFree; + ROBIN_HOOD_LOG("std::free") + std::free(mListForFree); + mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp); + } + mHead = nullptr; + } + + // allocates, but does NOT initialize. Use in-place new constructor, e.g. + // T* obj = pool.allocate(); + // ::new (static_cast<void*>(obj)) T(); + T* allocate() { + T* tmp = mHead; + if (!tmp) { + tmp = performAllocation(); + } + + mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp); + return tmp; + } + + // does not actually deallocate but puts it in store. + // make sure you have already called the destructor! e.g. with + // obj->~T(); + // pool.deallocate(obj); + void deallocate(T* obj) noexcept { + *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead; + mHead = obj; + } + + // Adds an already allocated block of memory to the allocator. This allocator is from now on + // responsible for freeing the data (with free()). If the provided data is not large enough to + // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor. + void addOrFree(void* ptr, const size_t numBytes) noexcept { + // calculate number of available elements in ptr + if (numBytes < ALIGNMENT + ALIGNED_SIZE) { + // not enough data for at least one element. Free and return. + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } else { + ROBIN_HOOD_LOG("add to buffer") + add(ptr, numBytes); + } + } + + void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs>& other) noexcept { + using std::swap; + swap(mHead, other.mHead); + swap(mListForFree, other.mListForFree); + } + +private: + // iterates the list of allocated memory to calculate how many to alloc next. + // Recalculating this each time saves us a size_t member. + // This ignores the fact that memory blocks might have been added manually with addOrFree. In + // practice, this should not matter much. + ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept { + auto tmp = mListForFree; + size_t numAllocs = MinNumAllocs; + + while (numAllocs * 2 <= MaxNumAllocs && tmp) { + auto x = reinterpret_cast<T***>(tmp); + tmp = *x; + numAllocs *= 2; + } + + return numAllocs; + } + + // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). + void add(void* ptr, const size_t numBytes) noexcept { + const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; + + auto data = reinterpret_cast<T**>(ptr); + + // link free list + auto x = reinterpret_cast<T***>(data); + *x = mListForFree; + mListForFree = data; + + // create linked list for newly allocated data + auto* const headT = + reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT); + + auto* const head = reinterpret_cast<char*>(headT); + + // Visual Studio compiler automatically unrolls this loop, which is pretty cool + for (size_t i = 0; i < numElements; ++i) { + *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) = + head + (i + 1) * ALIGNED_SIZE; + } + + // last one points to 0 + *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) = + mHead; + mHead = headT; + } + + // Called when no memory is available (mHead == 0). + // Don't inline this slow path. + ROBIN_HOOD(NOINLINE) T* performAllocation() { + size_t const numElementsToAlloc = calcNumElementsToAlloc(); + + // alloc new memory: [prev |T, T, ... T] + size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; + ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE + << " * " << numElementsToAlloc) + add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes); + return mHead; + } + + // enforce byte alignment of the T's +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) + static constexpr size_t ALIGNMENT = + (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value); +#else + static const size_t ALIGNMENT = + (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value) + ? ROBIN_HOOD_STD::alignment_of<T>::value + : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround +#endif + + static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; + + static_assert(MinNumAllocs >= 1, "MinNumAllocs"); + static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); + static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE"); + static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod"); + static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT"); + + T* mHead{nullptr}; + T** mListForFree{nullptr}; +}; + +template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat> +struct NodeAllocator; + +// dummy allocator that does nothing +template <typename T, size_t MinSize, size_t MaxSize> +struct NodeAllocator<T, MinSize, MaxSize, true> { + + // we are not using the data, so just free it. + void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } +}; + +template <typename T, size_t MinSize, size_t MaxSize> +struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {}; + +// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making +// my own here. +namespace swappable { +#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) +using std::swap; +template <typename T> +struct nothrow { + static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>())); +}; +#else +template <typename T> +struct nothrow { + static const bool value = std::is_nothrow_swappable<T>::value; +}; +#endif +} // namespace swappable + +} // namespace detail + +struct is_transparent_tag {}; + +// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable, +// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is +// also tested. +template <typename T1, typename T2> +struct pair { + using first_type = T1; + using second_type = T2; + + template <typename U1 = T1, typename U2 = T2, + typename = typename std::enable_if<std::is_default_constructible<U1>::value && + std::is_default_constructible<U2>::value>::type> + constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) + : first() + , second() {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair<T1, T2> const& o) noexcept( + noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>()))) + : first(o.first) + , second(o.second) {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept( + T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>())))) + : first(std::move(o.first)) + , second(std::move(o.second)) {} + + constexpr pair(T1&& a, T2&& b) noexcept(noexcept( + T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>())))) + : first(std::move(a)) + , second(std::move(b)) {} + + template <typename U1, typename U2> + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>( + std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>())))) + : first(std::forward<U1>(a)) + , second(std::forward<U2>(b)) {} + + template <typename... U1, typename... U2> + // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members" + // if this constructor is constexpr +#if !ROBIN_HOOD(BROKEN_CONSTEXPR) + constexpr +#endif + pair(std::piecewise_construct_t /*unused*/, std::tuple<U1...> a, + std::tuple<U2...> + b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(), + std::declval<std::tuple<U2...>&>(), + ROBIN_HOOD_STD::index_sequence_for<U1...>(), + ROBIN_HOOD_STD::index_sequence_for<U2...>()))) + : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(), + ROBIN_HOOD_STD::index_sequence_for<U2...>()) { + } + + // constructor called from the std::piecewise_construct_t ctor + template <typename... U1, size_t... I1, typename... U2, size_t... I2> + pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept( + noexcept(T1(std::forward<U1>(std::get<I1>( + std::declval<std::tuple< + U1...>&>()))...)) && noexcept(T2(std:: + forward<U2>(std::get<I2>( + std::declval<std::tuple<U2...>&>()))...))) + : first(std::forward<U1>(std::get<I1>(a))...) + , second(std::forward<U2>(std::get<I2>(b))...) { + // make visual studio compiler happy about warning about unused a & b. + // Visual studio's pair implementation disables warning 4100. + (void)a; + (void)b; + } + + void swap(pair<T1, T2>& o) noexcept((detail::swappable::nothrow<T1>::value) && + (detail::swappable::nothrow<T2>::value)) { + using std::swap; + swap(first, o.first); + swap(second, o.second); + } + + T1 first; // NOLINT(misc-non-private-member-variables-in-classes) + T2 second; // NOLINT(misc-non-private-member-variables-in-classes) +}; + +template <typename A, typename B> +inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept( + noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) { + a.swap(b); +} + +template <typename A, typename B> +inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) { + return (x.first == y.first) && (x.second == y.second); +} +template <typename A, typename B> +inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) { + return !(x == y); +} +template <typename A, typename B> +inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept( + std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() < + std::declval<B const&>())) { + return x.first < y.first || (!(y.first < x.first) && x.second < y.second); +} +template <typename A, typename B> +inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) { + return y < x; +} +template <typename A, typename B> +inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) { + return !(x > y); +} +template <typename A, typename B> +inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) { + return !(x < y); +} + +inline size_t hash_bytes(void const* ptr, size_t len) noexcept { + static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); + static constexpr uint64_t seed = UINT64_C(0xe17a1465); + static constexpr unsigned int r = 47; + + auto const* const data64 = static_cast<uint64_t const*>(ptr); + uint64_t h = seed ^ (len * m); + + size_t const n_blocks = len / 8; + for (size_t i = 0; i < n_blocks; ++i) { + auto k = detail::unaligned_load<uint64_t>(data64 + i); + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks); + switch (len & 7U) { + case 7: + h ^= static_cast<uint64_t>(data8[6]) << 48U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 6: + h ^= static_cast<uint64_t>(data8[5]) << 40U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 5: + h ^= static_cast<uint64_t>(data8[4]) << 32U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 4: + h ^= static_cast<uint64_t>(data8[3]) << 24U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 3: + h ^= static_cast<uint64_t>(data8[2]) << 16U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 2: + h ^= static_cast<uint64_t>(data8[1]) << 8U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 1: + h ^= static_cast<uint64_t>(data8[0]); + h *= m; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + default: + break; + } + + h ^= h >> r; + + // not doing the final step here, because this will be done by keyToIdx anyways + // h *= m; + // h ^= h >> r; + return static_cast<size_t>(h); +} + +inline size_t hash_int(uint64_t x) noexcept { + // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested, + // and doesn't need any special 128bit operations. + x ^= x >> 33U; + x *= UINT64_C(0xff51afd7ed558ccd); + x ^= x >> 33U; + + // not doing the final step here, because this will be done by keyToIdx anyways + // x *= UINT64_C(0xc4ceb9fe1a85ec53); + // x ^= x >> 33U; + return static_cast<size_t>(x); +} + +// A thin wrapper around std::hash, performing an additional simple mixing step of the result. +template <typename T, typename Enable = void> +struct hash : public std::hash<T> { + size_t operator()(T const& obj) const + noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) { + // call base hash + auto result = std::hash<T>::operator()(obj); + // return mixed of that, to be save against identity has + return hash_int(static_cast<detail::SizeT>(result)); + } +}; + +template <typename CharT> +struct hash<std::basic_string<CharT>> { + size_t operator()(std::basic_string<CharT> const& str) const noexcept { + return hash_bytes(str.data(), sizeof(CharT) * str.size()); + } +}; + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +template <typename CharT> +struct hash<std::basic_string_view<CharT>> { + size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept { + return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); + } +}; +#endif + +template <class T> +struct hash<T*> { + size_t operator()(T* ptr) const noexcept { + return hash_int(reinterpret_cast<detail::SizeT>(ptr)); + } +}; + +template <class T> +struct hash<std::unique_ptr<T>> { + size_t operator()(std::unique_ptr<T> const& ptr) const noexcept { + return hash_int(reinterpret_cast<detail::SizeT>(ptr.get())); + } +}; + +template <class T> +struct hash<std::shared_ptr<T>> { + size_t operator()(std::shared_ptr<T> const& ptr) const noexcept { + return hash_int(reinterpret_cast<detail::SizeT>(ptr.get())); + } +}; + +template <typename Enum> +struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> { + size_t operator()(Enum e) const noexcept { + using Underlying = typename std::underlying_type<Enum>::type; + return hash<Underlying>{}(static_cast<Underlying>(e)); + } +}; + +#define ROBIN_HOOD_HASH_INT(T) \ + template <> \ + struct hash<T> { \ + size_t operator()(T const& obj) const noexcept { \ + return hash_int(static_cast<uint64_t>(obj)); \ + } \ + } + +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +// see https://en.cppreference.com/w/cpp/utility/hash +ROBIN_HOOD_HASH_INT(bool); +ROBIN_HOOD_HASH_INT(char); +ROBIN_HOOD_HASH_INT(signed char); +ROBIN_HOOD_HASH_INT(unsigned char); +ROBIN_HOOD_HASH_INT(char16_t); +ROBIN_HOOD_HASH_INT(char32_t); +#if ROBIN_HOOD(HAS_NATIVE_WCHART) +ROBIN_HOOD_HASH_INT(wchar_t); +#endif +ROBIN_HOOD_HASH_INT(short); +ROBIN_HOOD_HASH_INT(unsigned short); +ROBIN_HOOD_HASH_INT(int); +ROBIN_HOOD_HASH_INT(unsigned int); +ROBIN_HOOD_HASH_INT(long); +ROBIN_HOOD_HASH_INT(long long); +ROBIN_HOOD_HASH_INT(unsigned long); +ROBIN_HOOD_HASH_INT(unsigned long long); +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +#endif +namespace detail { + +template <typename T> +struct void_type { + using type = void; +}; + +template <typename T, typename = void> +struct has_is_transparent : public std::false_type {}; + +template <typename T> +struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type> + : public std::true_type {}; + +// using wrapper classes for hash and key_equal prevents the diamond problem when the same type +// is used. see https://stackoverflow.com/a/28771920/48181 +template <typename T> +struct WrapHash : public T { + WrapHash() = default; + explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>()))) + : T(o) {} +}; + +template <typename T> +struct WrapKeyEqual : public T { + WrapKeyEqual() = default; + explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>()))) + : T(o) {} +}; + +// A highly optimized hashmap implementation, using the Robin Hood algorithm. +// +// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but +// be about 2x faster in most cases and require much less allocations. +// +// This implementation uses the following memory layout: +// +// [Node, Node, ... Node | info, info, ... infoSentinel ] +// +// * Node: either a DataNode that directly has the std::pair<key, val> as member, +// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use +// depends on how fast the swap() operation is. Heuristically, this is automatically choosen +// based on sizeof(). there are always 2^n Nodes. +// +// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes. +// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the +// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it +// actually belongs to the previous position and was pushed out because that place is already +// taken. +// +// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the +// need for a idx variable. +// +// According to STL, order of templates has effect on throughput. That's why I've moved the +// boolean to the front. +// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ +template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash, + typename KeyEqual> +class Table + : public WrapHash<Hash>, + public WrapKeyEqual<KeyEqual>, + detail::NodeAllocator< + typename std::conditional< + std::is_void<T>::value, Key, + robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type, + 4, 16384, IsFlat> { +public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void<T>::value; + static constexpr bool is_set = !is_map; + static constexpr bool is_transparent = + has_is_transparent<Hash>::value && has_is_transparent<KeyEqual>::value; + + using key_type = Key; + using mapped_type = T; + using value_type = typename std::conditional< + is_set, Key, + robin_hood::pair<typename std::conditional<is_flat, Key, Key const>::type, T>>::type; + using size_type = size_t; + using hasher = Hash; + using key_equal = KeyEqual; + using Self = Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>; + +private: + static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, + "MaxLoadFactor100 needs to be >10 && < 100"); + + using WHash = WrapHash<Hash>; + using WKeyEqual = WrapKeyEqual<KeyEqual>; + + // configuration defaults + + // make sure we have 8 elements, needed to quickly rehash mInfo + static constexpr size_t InitialNumElements = sizeof(uint64_t); + static constexpr uint32_t InitialInfoNumBits = 5; + static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; + static constexpr size_t InfoMask = InitialInfoInc - 1U; + static constexpr uint8_t InitialInfoHashShift = 0; + using DataPool = detail::NodeAllocator<value_type, 4, 16384, IsFlat>; + + // type needs to be wider than uint8_t. + using InfoType = uint32_t; + + // DataNode //////////////////////////////////////////////////////// + + // Primary template for the data node. We have special implementations for small and big + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these + // on the heap so swap merely swaps a pointer. + template <typename M, bool> + class DataNode {}; + + // Small: just allocate on the stack. + template <typename M> + class DataNode<M, true> final { + public: + template <typename... Args> + explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept( + noexcept(value_type(std::forward<Args>(args)...))) + : mData(std::forward<Args>(args)...) {} + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept( + std::is_nothrow_move_constructible<value_type>::value) + : mData(std::move(n.mData)) {} + + // doesn't do anything + void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {} + void destroyDoNotDeallocate() noexcept {} + + value_type const* operator->() const noexcept { + return &mData; + } + value_type* operator->() noexcept { + return &mData; + } + + const value_type& operator*() const noexcept { + return mData; + } + + value_type& operator*() noexcept { + return mData; + } + + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept { + return mData.first; + } + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_set, VT&>::type getFirst() noexcept { + return mData; + } + + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, typename VT::first_type const&>::type + getFirst() const noexcept { + return mData.first; + } + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept { + return mData; + } + + template <typename MT = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, MT&>::type getSecond() noexcept { + return mData.second; + } + + template <typename MT = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept { + return mData.second; + } + + void swap(DataNode<M, true>& o) noexcept( + noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) { + mData.swap(o.mData); + } + + private: + value_type mData; + }; + + // big object: allocate on heap. + template <typename M> + class DataNode<M, false> { + public: + template <typename... Args> + explicit DataNode(M& map, Args&&... args) + : mData(map.allocate()) { + ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...); + } + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept + : mData(std::move(n.mData)) {} + + void destroy(M& map) noexcept { + // don't deallocate, just put it into list of datapool. + mData->~value_type(); + map.deallocate(mData); + } + + void destroyDoNotDeallocate() noexcept { + mData->~value_type(); + } + + value_type const* operator->() const noexcept { + return mData; + } + + value_type* operator->() noexcept { + return mData; + } + + const value_type& operator*() const { + return *mData; + } + + value_type& operator*() { + return *mData; + } + + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept { + return mData->first; + } + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_set, VT&>::type getFirst() noexcept { + return *mData; + } + + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, typename VT::first_type const&>::type + getFirst() const noexcept { + return mData->first; + } + template <typename VT = value_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept { + return *mData; + } + + template <typename MT = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, MT&>::type getSecond() noexcept { + return mData->second; + } + + template <typename MT = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept { + return mData->second; + } + + void swap(DataNode<M, false>& o) noexcept { + using std::swap; + swap(mData, o.mData); + } + + private: + value_type* mData; + }; + + using Node = DataNode<Self, IsFlat>; + + // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required) + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { + return n.getFirst(); + } + + // in case we have void mapped_type, we are not using a pair, thus we just route k through. + // No need to disable this because it's just not used if not applicable. + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept { + return k; + } + + // in case we have non-void mapped_type, we have a standard robin_hood::pair + template <typename Q = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type + getFirstConst(value_type const& vt) const noexcept { + return vt.first; + } + + // Cloner ////////////////////////////////////////////////////////// + + template <typename M, bool UseMemcpy> + struct Cloner; + + // fast path: Just copy data, without allocating anything. + template <typename M> + struct Cloner<M, true> { + void operator()(M const& source, M& target) const { + auto const* const src = reinterpret_cast<char const*>(source.mKeyVals); + auto* tgt = reinterpret_cast<char*>(target.mKeyVals); + auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); + std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); + } + }; + + template <typename M> + struct Cloner<M, false> { + void operator()(M const& s, M& t) const { + auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); + std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); + + for (size_t i = 0; i < numElementsWithBuffer; ++i) { + if (t.mInfo[i]) { + ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); + } + } + } + }; + + // Destroyer /////////////////////////////////////////////////////// + + template <typename M, bool IsFlatAndTrivial> + struct Destroyer {}; + + template <typename M> + struct Destroyer<M, true> { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + } + }; + + template <typename M> + struct Destroyer<M, false> { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroy(m); + n.~Node(); + } + } + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroyDoNotDeallocate(); + n.~Node(); + } + } + } + }; + + // Iter //////////////////////////////////////////////////////////// + + struct fast_forward_tag {}; + + // generic iterator for both const_iterator and iterator. + template <bool IsConst> + // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) + class Iter { + private: + using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type; + + public: + using difference_type = std::ptrdiff_t; + using value_type = typename Self::value_type; + using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type; + using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type; + using iterator_category = std::forward_iterator_tag; + + // default constructed iterator can be compared to itself, but WON'T return true when + // compared to end(). + Iter() = default; + + // Rule of zero: nothing specified. The conversion constructor is only enabled for + // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. + + // Conversion constructor from iterator to const_iterator. + template <bool OtherIsConst, + typename = typename std::enable_if<IsConst && !OtherIsConst>::type> + // NOLINTNEXTLINE(hicpp-explicit-conversions) + Iter(Iter<OtherIsConst> const& other) noexcept + : mKeyVals(other.mKeyVals) + , mInfo(other.mInfo) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr, + fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) { + fastForward(); + } + + template <bool OtherIsConst, + typename = typename std::enable_if<IsConst && !OtherIsConst>::type> + Iter& operator=(Iter<OtherIsConst> const& other) noexcept { + mKeyVals = other.mKeyVals; + mInfo = other.mInfo; + return *this; + } + + // prefix increment. Undefined behavior if we are at end()! + Iter& operator++() noexcept { + mInfo++; + mKeyVals++; + fastForward(); + return *this; + } + + Iter operator++(int) noexcept { + Iter tmp = *this; + ++(*this); + return tmp; + } + + reference operator*() const { + return **mKeyVals; + } + + pointer operator->() const { + return &**mKeyVals; + } + + template <bool O> + bool operator==(Iter<O> const& o) const noexcept { + return mKeyVals == o.mKeyVals; + } + + template <bool O> + bool operator!=(Iter<O> const& o) const noexcept { + return mKeyVals != o.mKeyVals; + } + + private: + // fast forward to the next non-free info byte + // I've tried a few variants that don't depend on intrinsics, but unfortunately they are + // quite a bit slower than this one. So I've reverted that change again. See map_benchmark. + void fastForward() noexcept { + size_t n = 0; + while (0U == (n = detail::unaligned_load<size_t>(mInfo))) { + mInfo += sizeof(size_t); + mKeyVals += sizeof(size_t); + } +#if defined(ROBIN_HOOD_DISABLE_INTRINSICS) + // we know for certain that within the next 8 bytes we'll find a non-zero one. + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) { + mInfo += 4; + mKeyVals += 4; + } + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) { + mInfo += 2; + mKeyVals += 2; + } + if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) { + mInfo += 1; + mKeyVals += 1; + } +#else +# if ROBIN_HOOD(LITTLE_ENDIAN) + auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; +# else + auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +# endif + mInfo += inc; + mKeyVals += inc; +#endif + } + + friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>; + NodePtr mKeyVals{nullptr}; + uint8_t const* mInfo{nullptr}; + }; + + //////////////////////////////////////////////////////////////////// + + // highly performance relevant code. + // Lower bits are used for indexing into the array (2^n size) + // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. + template <typename HashKey> + void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { + // In addition to whatever hash is used, add another mul & shift so we get better hashing. + // This serves as a bad hash prevention, if the given data is + // badly mixed. + auto h = static_cast<uint64_t>(WHash::operator()(key)); + + h *= mHashMultiplier; + h ^= h >> 33U; + + // the lower InitialInfoNumBits are reserved for info. + *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift); + *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask; + } + + // forwards the index by one, wrapping around at the end + void next(InfoType* info, size_t* idx) const noexcept { + *idx = *idx + 1; + *info += mInfoInc; + } + + void nextWhileLess(InfoType* info, size_t* idx) const noexcept { + // unrolling this by hand did not bring any speedups. + while (*info < mInfo[*idx]) { + next(info, idx); + } + } + + // Shift everything up by one element. Tries to move stuff around. + void + shiftUp(size_t startIdx, + size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) { + auto idx = startIdx; + ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); + while (--idx != insertion_idx) { + mKeyVals[idx] = std::move(mKeyVals[idx - 1]); + } + + idx = startIdx; + while (idx != insertion_idx) { + ROBIN_HOOD_COUNT(shiftUp) + mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc); + if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + --idx; + } + } + + void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) { + // until we find one that is either empty or has zero offset. + // TODO(martinus) we don't need to move everything, just the last one for the same + // bucket. + mKeyVals[idx].destroy(*this); + + // until we find one that is either empty or has zero offset. + while (mInfo[idx + 1] >= 2 * mInfoInc) { + ROBIN_HOOD_COUNT(shiftDown) + mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc); + mKeyVals[idx] = std::move(mKeyVals[idx + 1]); + ++idx; + } + + mInfo[idx] = 0; + // don't destroy, we've moved it + // mKeyVals[idx].destroy(*this); + mKeyVals[idx].~Node(); + } + + // copy of find(), except that it returns iterator instead of const_iterator. + template <typename Other> + ROBIN_HOOD(NODISCARD) + size_t findIdx(Other const& key) const { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + + do { + // unrolling this twice gives a bit of a speedup. More unrolling did not help. + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found! + return mMask == 0 ? 0 + : static_cast<size_t>(std::distance( + mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo))); + } + + void cloneData(const Table& o) { + Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this); + } + + // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. + // @return True on success, false if something went wrong + void insert_move(Node&& keyval) { + // we don't retry, fail if overflowing + // don't need to check max num elements + if (0 == mMaxNumElementsAllowed && !try_increase_info()) { + throwOverflowError(); + } + + size_t idx{}; + InfoType info{}; + keyToIdx(keyval.getFirst(), &idx, &info); + + // skip forward. Use <= because we are certain that the element is not there. + while (info <= mInfo[idx]) { + idx = idx + 1; + info += mInfoInc; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = static_cast<uint8_t>(info); + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) { + ::new (static_cast<void*>(&l)) Node(std::move(keyval)); + } else { + shiftUp(idx, insertion_idx); + l = std::move(keyval); + } + + // put at empty spot + mInfo[insertion_idx] = insertion_info; + + ++mNumElements; + } + +public: + using iterator = Iter<false>; + using const_iterator = Iter<true>; + + Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) + : WHash() + , WKeyEqual() { + ROBIN_HOOD_TRACE(this) + } + + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. + // This tremendously speeds up ctor & dtor of a map that never receives an element. The + // penalty is payed at the first insert, and not before. Lookup of this empty map works + // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the + // standard, but we can ignore it. + explicit Table( + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + } + + template <typename Iter> + Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, + const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + insert(first, last); + } + + Table(std::initializer_list<value_type> initlist, + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + insert(initlist.begin(), initlist.end()); + } + + Table(Table&& o) noexcept + : WHash(std::move(static_cast<WHash&>(o))) + , WKeyEqual(std::move(static_cast<WKeyEqual&>(o))) + , DataPool(std::move(static_cast<DataPool&>(o))) { + ROBIN_HOOD_TRACE(this) + if (o.mMask) { + mHashMultiplier = std::move(o.mHashMultiplier); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + // set other's mask to 0 so its destructor won't do anything + o.init(); + } + } + + Table& operator=(Table&& o) noexcept { + ROBIN_HOOD_TRACE(this) + if (&o != this) { + if (o.mMask) { + // only move stuff if the other map actually has some data + destroy(); + mHashMultiplier = std::move(o.mHashMultiplier); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + WHash::operator=(std::move(static_cast<WHash&>(o))); + WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o))); + DataPool::operator=(std::move(static_cast<DataPool&>(o))); + + o.init(); + + } else { + // nothing in the other map => just clear us. + clear(); + } + } + return *this; + } + + Table(const Table& o) + : WHash(static_cast<const WHash&>(o)) + , WKeyEqual(static_cast<const WKeyEqual&>(o)) + , DataPool(static_cast<const DataPool&>(o)) { + ROBIN_HOOD_TRACE(this) + if (!o.empty()) { + // not empty: create an exact copy. it is also possible to just iterate through all + // elements and insert them, but copying is probably faster. + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mHashMultiplier = o.mHashMultiplier; + mKeyVals = static_cast<Node*>( + detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal))); + // no need for calloc because clonData does memcpy + mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + } + } + + // Creates a copy of the given map. Copy constructor of each entry is used. + // Not sure why clang-tidy thinks this doesn't handle self assignment, it does + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + Table& operator=(Table const& o) { + ROBIN_HOOD_TRACE(this) + if (&o == this) { + // prevent assigning of itself + return *this; + } + + // we keep using the old allocator and not assign the new one, because we want to keep + // the memory available. when it is the same size. + if (o.empty()) { + if (0 == mMask) { + // nothing to do, we are empty too + return *this; + } + + // not empty: destroy what we have there + // clear also resets mInfo to 0, that's sometimes not necessary. + destroy(); + init(); + WHash::operator=(static_cast<const WHash&>(o)); + WKeyEqual::operator=(static_cast<const WKeyEqual&>(o)); + DataPool::operator=(static_cast<DataPool const&>(o)); + + return *this; + } + + // clean up old stuff + Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this); + + if (mMask != o.mMask) { + // no luck: we don't have the same array size allocated, so we need to realloc. + if (0 != mMask) { + // only deallocate if we actually have data! + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mKeyVals = static_cast<Node*>( + detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal))); + + // no need for calloc here because cloneData performs a memcpy. + mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer); + // sentinel is set in cloneData + } + WHash::operator=(static_cast<const WHash&>(o)); + WKeyEqual::operator=(static_cast<const WKeyEqual&>(o)); + DataPool::operator=(static_cast<DataPool const&>(o)); + mHashMultiplier = o.mHashMultiplier; + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + + return *this; + } + + // Swaps everything between the two maps. + void swap(Table& o) { + ROBIN_HOOD_TRACE(this) + using std::swap; + swap(o, *this); + } + + // Clears all data, without resizing. + void clear() { + ROBIN_HOOD_TRACE(this) + if (empty()) { + // don't do anything! also important because we don't want to write to + // DummyInfoByte::b, even though we would just write 0 to it. + return; + } + + Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + // clear everything, then set the sentinel again + uint8_t const z = 0; + std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // Destroys the map and all it's contents. + ~Table() { + ROBIN_HOOD_TRACE(this) + destroy(); + } + + // Checks if both tables contain the same entries. Order is irrelevant. + bool operator==(const Table& other) const { + ROBIN_HOOD_TRACE(this) + if (other.size() != size()) { + return false; + } + for (auto const& otherEntry : other) { + if (!has(otherEntry)) { + return false; + } + } + + return true; + } + + bool operator!=(const Table& other) const { + ROBIN_HOOD_TRACE(this) + return !operator==(other); + } + + template <typename Q = mapped_type> + typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); + } + + template <typename Q = mapped_type> + typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); + } + + template <typename Iter> + void insert(Iter first, Iter last) { + for (; first != last; ++first) { + // value_type ctor needed because this might be called with std::pair's + insert(value_type(*first)); + } + } + + void insert(std::initializer_list<value_type> ilist) { + for (auto&& vt : ilist) { + insert(std::move(vt)); + } + } + + template <typename... Args> + std::pair<iterator, bool> emplace(Args&&... args) { + ROBIN_HOOD_TRACE(this) + Node n{*this, std::forward<Args>(args)...}; + auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n)); + switch (idxAndState.second) { + case InsertionState::key_found: + n.destroy(*this); + break; + + case InsertionState::new_node: + ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = std::move(n); + break; + + case InsertionState::overflow_error: + n.destroy(*this); + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template <typename... Args> + std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) { + return try_emplace_impl(key, std::forward<Args>(args)...); + } + + template <typename... Args> + std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) { + return try_emplace_impl(std::move(key), std::forward<Args>(args)...); + } + + template <typename... Args> + std::pair<iterator, bool> try_emplace(const_iterator hint, const key_type& key, + Args&&... args) { + (void)hint; + return try_emplace_impl(key, std::forward<Args>(args)...); + } + + template <typename... Args> + std::pair<iterator, bool> try_emplace(const_iterator hint, key_type&& key, Args&&... args) { + (void)hint; + return try_emplace_impl(std::move(key), std::forward<Args>(args)...); + } + + template <typename Mapped> + std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) { + return insertOrAssignImpl(key, std::forward<Mapped>(obj)); + } + + template <typename Mapped> + std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) { + return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)); + } + + template <typename Mapped> + std::pair<iterator, bool> insert_or_assign(const_iterator hint, const key_type& key, + Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(key, std::forward<Mapped>(obj)); + } + + template <typename Mapped> + std::pair<iterator, bool> insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)); + } + + std::pair<iterator, bool> insert(const value_type& keyval) { + ROBIN_HOOD_TRACE(this) + return emplace(keyval); + } + + std::pair<iterator, bool> insert(value_type&& keyval) { + return emplace(std::move(keyval)); + } + + // Returns 1 if key is found, 0 otherwise. + size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) { + return 1; + } + return 0; + } + + template <typename OtherKey, typename Self_ = Self> + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + template <typename OtherKey, typename Self_ = Self> + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const { + return 1U == count(key); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template <typename Q = mapped_type> + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) { + doThrow<std::out_of_range>("key not found"); + } + return kv->getSecond(); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template <typename Q = mapped_type> + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) { + doThrow<std::out_of_range>("key not found"); + } + return kv->getSecond(); + } + + const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template <typename OtherKey> + const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template <typename OtherKey, typename Self_ = Self> + typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard) + const_iterator>::type // NOLINT(modernize-use-nodiscard) + find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator find(const key_type& key) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template <typename OtherKey> + iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template <typename OtherKey, typename Self_ = Self> + typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator begin() { + ROBIN_HOOD_TRACE(this) + if (empty()) { + return end(); + } + return iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + const_iterator begin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cbegin(); + } + const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + if (empty()) { + return cend(); + } + return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + + iterator end() { + ROBIN_HOOD_TRACE(this) + // no need to supply valid info pointer: end() must not be dereferenced, and only node + // pointer is compared. + return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr}; + } + const_iterator end() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cend(); + } + const_iterator cend() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr}; + } + + iterator erase(const_iterator pos) { + ROBIN_HOOD_TRACE(this) + // its safe to perform const cast here + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)}); + } + + // Erases element at pos, returns iterator to the next element. + iterator erase(iterator pos) { + ROBIN_HOOD_TRACE(this) + // we assume that pos always points to a valid entry, and not end(). + auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals); + + shiftDown(idx); + --mNumElements; + + if (*pos.mInfo) { + // we've backward shifted, return this again + return pos; + } + + // no backward shift, return next element + return ++pos; + } + + size_t erase(const key_type& key) { + ROBIN_HOOD_TRACE(this) + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + + // check while info matches with the source idx + do { + if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + shiftDown(idx); + --mNumElements; + return 1; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found to delete + return 0; + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // exactly the same as reserve(c). + void rehash(size_t c) { + // forces a reserve + reserve(c, true); + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. + void reserve(size_t c) { + // reserve, but don't force rehash + reserve(c, false); + } + + // If possible reallocates the map to a smaller one. This frees the underlying table. + // Does not do anything if load_factor is too large for decreasing the table's size. + void compact() { + ROBIN_HOOD_TRACE(this) + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (newSize < mMask + 1) { + rehashPowerOfTwo(newSize, true); + } + } + + size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return mNumElements; + } + + size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast<size_type>(-1); + } + + ROBIN_HOOD(NODISCARD) bool empty() const noexcept { + ROBIN_HOOD_TRACE(this) + return 0 == mNumElements; + } + + float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return MaxLoadFactor100 / 100.0F; + } + void max_load_factor(float) const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + } + + + // Average number of elements per bucket. Since we allow only 1 per bucket + float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast<float>(size()) / static_cast<float>(mMask + 1); + } + + ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { + ROBIN_HOOD_TRACE(this) + return mMask; + } + + ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept { + if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) { + return maxElements * MaxLoadFactor100 / 100; + } + + // we might be a bit inprecise, but since maxElements is quite large that doesn't matter + return (maxElements / 100) * MaxLoadFactor100; + } + + ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { + // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load + // 64bit types. + return numElements + sizeof(uint64_t); + } + + ROBIN_HOOD(NODISCARD) + size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { + auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); + return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF))); + } + + // calculation only allowed for 2^n values + ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { +#if ROBIN_HOOD(BITNESS) == 64 + return numElements * sizeof(Node) + calcNumBytesInfo(numElements); +#else + // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows. + auto const ne = static_cast<uint64_t>(numElements); + auto const s = static_cast<uint64_t>(sizeof(Node)); + auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements)); + + auto const total64 = ne * s + infos; + auto const total = static_cast<size_t>(total64); + + if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) { + throwOverflowError(); + } + return total; +#endif + } + +private: + template <typename Q = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template <typename Q = mapped_type> + ROBIN_HOOD(NODISCARD) + typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + return find(e) != end(); + } + + void reserve(size_t c, bool forceRehash) { + ROBIN_HOOD_TRACE(this) + auto const minElementsAllowed = (std::max)(c, mNumElements); + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (forceRehash || newSize > mMask + 1) { + rehashPowerOfTwo(newSize, false); + } + } + + // reserves space for at least the specified number of elements. + // only works if numBuckets if power of two + // True on success, false otherwise + void rehashPowerOfTwo(size_t numBuckets, bool forceFree) { + ROBIN_HOOD_TRACE(this) + + Node* const oldKeyVals = mKeyVals; + uint8_t const* const oldInfo = mInfo; + + const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + // resize operation: move stuff + initData(numBuckets); + if (oldMaxElementsWithBuffer > 1) { + for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { + if (oldInfo[i] != 0) { + // might throw an exception, which is really bad since we are in the middle of + // moving stuff. + insert_move(std::move(oldKeyVals[i])); + // destroy the node but DON'T destroy the data. + oldKeyVals[i].~Node(); + } + } + + // this check is not necessary as it's guarded by the previous if, but it helps + // silence g++'s overeager "attempt to free a non-heap object 'map' + // [-Werror=free-nonheap-object]" warning. + if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) { + // don't destroy old data: put it into the pool instead + if (forceFree) { + std::free(oldKeyVals); + } else { + DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); + } + } + } + } + + ROBIN_HOOD(NOINLINE) void throwOverflowError() const { +#if ROBIN_HOOD(HAS_EXCEPTIONS) + throw std::overflow_error("robin_hood::map overflow"); +#else + abort(); +#endif + } + + template <typename OtherKey, typename... Args> + std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)), + std::forward_as_tuple(std::forward<Args>(args)...)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward<OtherKey>(key)), + std::forward_as_tuple(std::forward<Args>(args)...)); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template <typename OtherKey, typename Mapped> + std::pair<iterator, bool> insertOrAssignImpl(OtherKey&& key, Mapped&& obj) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj); + break; + + case InsertionState::new_node: + ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)), + std::forward_as_tuple(std::forward<Mapped>(obj))); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward<OtherKey>(key)), + std::forward_as_tuple(std::forward<Mapped>(obj))); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + void initData(size_t max_elements) { + mNumElements = 0; + mMask = max_elements - 1; + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); + + // calloc also zeroes everything + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mKeyVals = reinterpret_cast<Node*>( + detail::assertNotNull<std::bad_alloc>(std::calloc(1, numBytesTotal))); + mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer); + + // set sentinel + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + enum class InsertionState { overflow_error, key_found, new_node, overwrite_node }; + + // Finds key, and if not already present prepares a spot where to pot the key & value. + // This potentially shifts nodes out of the way, updates mInfo and number of inserted + // elements, so the only operation left to do is create/assign a new node at that spot. + template <typename OtherKey> + std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey&& key) { + for (int i = 0; i < 256; ++i) { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match + while (info == mInfo[idx]) { + if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + // key already exists, do NOT insert. + // see http://en.cppreference.com/w/cpp/container/unordered_map/insert + return std::make_pair(idx, InsertionState::key_found); + } + next(&info, &idx); + } + + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { + if (!increase_size()) { + return std::make_pair(size_t(0), InsertionState::overflow_error); + } + continue; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + if (idx != insertion_idx) { + shiftUp(idx, insertion_idx); + } + // put at empty spot + mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info); + ++mNumElements; + return std::make_pair(insertion_idx, idx == insertion_idx + ? InsertionState::new_node + : InsertionState::overwrite_node); + } + + // enough attempts failed, so finally give up. + return std::make_pair(size_t(0), InsertionState::overflow_error); + } + + bool try_increase_info() { + ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements + << ", maxNumElementsAllowed=" + << calcMaxNumElementsAllowed(mMask + 1)) + if (mInfoInc <= 2) { + // need to be > 2 so that shift works (otherwise undefined behavior!) + return false; + } + // we got space left, try to make info smaller + mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U); + + // remove one bit of the hash, leaving more space for the distance info. + // This is extremely fast because we can operate on 8 bytes at once. + ++mInfoHashShift; + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + for (size_t i = 0; i < numElementsWithBuffer; i += 8) { + auto val = unaligned_load<uint64_t>(mInfo + i); + val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + std::memcpy(mInfo + i, &val, sizeof(val)); + } + // update sentinel, which might have been cleared out! + mInfo[numElementsWithBuffer] = 1; + + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + return true; + } + + // True if resize was possible, false otherwise + bool increase_size() { + // nothing allocated yet? just allocate InitialNumElements + if (0 == mMask) { + initData(InitialNumElements); + return true; + } + + auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + if (mNumElements < maxNumElementsAllowed && try_increase_info()) { + return true; + } + + ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" + << maxNumElementsAllowed << ", load=" + << (static_cast<double>(mNumElements) * 100.0 / + (static_cast<double>(mMask) + 1))) + + if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { + // we have to resize, even though there would still be plenty of space left! + // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case + // we have to rehash a few times + nextHashMultiplier(); + rehashPowerOfTwo(mMask + 1, true); + } else { + // we've reached the capacity of the map, so the hash seems to work nice. Keep using it. + rehashPowerOfTwo((mMask + 1) * 2, false); + } + return true; + } + + void nextHashMultiplier() { + // adding an *even* number, so that the multiplier will always stay odd. This is necessary + // so that the hash stays a mixing function (and thus doesn't have any information loss). + mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54); + } + + void destroy() { + if (0 == mMask) { + // don't deallocate! + return; + } + + Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{} + .nodesDoNotDeallocate(*this); + + // This protection against not deleting mMask shouldn't be needed as it's sufficiently + // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise + // reports a compile error: attempt to free a non-heap object 'fm' + // [-Werror=free-nonheap-object] + if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) { + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + } + + void init() noexcept { + mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); + mInfo = reinterpret_cast<uint8_t*>(&mMask); + mNumElements = 0; + mMask = 0; + mMaxNumElementsAllowed = 0; + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // members are sorted so no padding occurs + uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8 + Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 16 + uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 24 + size_t mNumElements = 0; // 8 byte 32 + size_t mMask = 0; // 8 byte 40 + size_t mMaxNumElementsAllowed = 0; // 8 byte 48 + InfoType mInfoInc = InitialInfoInc; // 4 byte 52 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56 + // 16 byte 56 if NodeAllocator +}; + +} // namespace detail + +// map + +template <typename Key, typename T, typename Hash = hash<Key>, + typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80> +using unordered_flat_map = detail::Table<true, MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +template <typename Key, typename T, typename Hash = hash<Key>, + typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80> +using unordered_node_map = detail::Table<false, MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +template <typename Key, typename T, typename Hash = hash<Key>, + typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80> +using unordered_map = + detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value && + std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value, + MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +// set + +template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, + size_t MaxLoadFactor100 = 80> +using unordered_flat_set = detail::Table<true, MaxLoadFactor100, Key, void, Hash, KeyEqual>; + +template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, + size_t MaxLoadFactor100 = 80> +using unordered_node_set = detail::Table<false, MaxLoadFactor100, Key, void, Hash, KeyEqual>; + +template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, + size_t MaxLoadFactor100 = 80> +using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible<Key>::value && + std::is_nothrow_move_assignable<Key>::value, + MaxLoadFactor100, Key, void, Hash, KeyEqual>; + +} // namespace robin_hood + +#endif diff --git a/benchmarks/others/sparsepp/spp.h b/benchmarks/others/sparsepp/spp.h deleted file mode 100644 index 26aa2001..00000000 --- a/benchmarks/others/sparsepp/spp.h +++ /dev/null @@ -1,4361 +0,0 @@ -#if !defined(sparsepp_h_guard_) -#define sparsepp_h_guard_ - - -// ---------------------------------------------------------------------- -// Copyright (c) 2016, Gregory Popovitch - [email protected] -// All rights reserved. -// -// This work is derived from Google's sparsehash library -// -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// ---------------------------------------------------------------------- - - -// some macros for portability -// --------------------------- -// includes -// -------- -#include <cassert> -#include <cstring> -#include <string> -#include <limits> // for numeric_limits -#include <algorithm> // For swap(), eg -#include <iterator> // for iterator tags -#include <functional> // for equal_to<>, select1st<>, std::unary_function, etc -#include <memory> // for alloc, uninitialized_copy, uninitialized_fill -#include <cstdlib> // for malloc/realloc/free -#include <cstddef> // for ptrdiff_t -#include <new> // for placement new -#include <stdexcept> // For length_error -#include <utility> // for pair<> -#include <cstdio> -#include <iosfwd> -#include <ios> - -#include "spp_stdint.h" // includes spp_config.h -#include "spp_traits.h" -#include "spp_utils.h" - -#ifdef SPP_INCLUDE_SPP_ALLOC - #include "spp_dlalloc.h" -#endif - -#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST) - #include <initializer_list> -#endif - -#if (SPP_GROUP_SIZE == 32) - #define SPP_SHIFT_ 5 - #define SPP_MASK_ 0x1F - typedef uint32_t group_bm_type; -#elif (SPP_GROUP_SIZE == 64) - #define SPP_SHIFT_ 6 - #define SPP_MASK_ 0x3F - typedef uint64_t group_bm_type; -#else - #error "SPP_GROUP_SIZE must be either 32 or 64" -#endif - -namespace spp_ { - -// ---------------------------------------------------------------------- -// U T I L F U N C T I O N S -// ---------------------------------------------------------------------- -template <class E> -inline void throw_exception(const E& exception) -{ -#if !defined(SPP_NO_EXCEPTIONS) - throw exception; -#else - assert(0); - abort(); -#endif -} - -// ---------------------------------------------------------------------- -// M U T A B L E P A I R H A C K -// turn std::pair<const K, V> into mutable std::pair<K, V> -// ---------------------------------------------------------------------- -template <class T> -struct cvt -{ - typedef T type; -}; - -template <class K, class V> -struct cvt<std::pair<const K, V> > -{ - typedef std::pair<K, V> type; -}; - -template <class K, class V> -struct cvt<const std::pair<const K, V> > -{ - typedef const std::pair<K, V> type; -}; - -// ---------------------------------------------------------------------- -// M O V E I T E R A T O R -// ---------------------------------------------------------------------- -#ifdef SPP_NO_CXX11_RVALUE_REFERENCES - #define MK_MOVE_IT(p) (p) -#else - #define MK_MOVE_IT(p) std::make_move_iterator(p) -#endif - - -// ---------------------------------------------------------------------- -// I N T E R N A L S T U F F -// ---------------------------------------------------------------------- -#ifdef SPP_NO_CXX11_STATIC_ASSERT - template <bool> struct SppCompileAssert { }; - #define SPP_COMPILE_ASSERT(expr, msg) \ - SPP_ATTRIBUTE_UNUSED typedef SppCompileAssert<(bool(expr))> spp_bogus_[bool(expr) ? 1 : -1] -#else - #define SPP_COMPILE_ASSERT static_assert -#endif - -namespace sparsehash_internal -{ - -// Adaptor methods for reading/writing data from an INPUT or OUPTUT -// variable passed to serialize() or unserialize(). For now we -// have implemented INPUT/OUTPUT for FILE*, istream*/ostream* (note -// they are pointers, unlike typical use), or else a pointer to -// something that supports a Read()/Write() method. -// -// For technical reasons, we implement read_data/write_data in two -// stages. The actual work is done in *_data_internal, which takes -// the stream argument twice: once as a template type, and once with -// normal type information. (We only use the second version.) We do -// this because of how C++ picks what function overload to use. If we -// implemented this the naive way: -// bool read_data(istream* is, const void* data, size_t length); -// template<typename T> read_data(T* fp, const void* data, size_t length); -// C++ would prefer the second version for every stream type except -// istream. However, we want C++ to prefer the first version for -// streams that are *subclasses* of istream, such as istringstream. -// This is not possible given the way template types are resolved. So -// we split the stream argument in two, one of which is templated and -// one of which is not. The specialized functions (like the istream -// version above) ignore the template arg and use the second, 'type' -// arg, getting subclass matching as normal. The 'catch-all' -// functions (the second version above) use the template arg to deduce -// the type, and use a second, void* arg to achieve the desired -// 'catch-all' semantics. - - // ----- low-level I/O for FILE* ---- - - template<typename Ignored> - inline bool read_data_internal(Ignored* /*unused*/, FILE* fp, - void* data, size_t length) - { - return fread(data, length, 1, fp) == 1; - } - - template<typename Ignored> - inline bool write_data_internal(Ignored* /*unused*/, FILE* fp, - const void* data, size_t length) - { - return fwrite(data, length, 1, fp) == 1; - } - - // ----- low-level I/O for iostream ---- - - // We want the caller to be responsible for #including <iostream>, not - // us, because iostream is a big header! According to the standard, - // it's only legal to delay the instantiation the way we want to if - // the istream/ostream is a template type. So we jump through hoops. - template<typename ISTREAM> - inline bool read_data_internal_for_istream(ISTREAM* fp, - void* data, size_t length) - { - return fp->read(reinterpret_cast<char*>(data), - static_cast<std::streamsize>(length)).good(); - } - template<typename Ignored> - inline bool read_data_internal(Ignored* /*unused*/, std::istream* fp, - void* data, size_t length) - { - return read_data_internal_for_istream(fp, data, length); - } - - template<typename OSTREAM> - inline bool write_data_internal_for_ostream(OSTREAM* fp, - const void* data, size_t length) - { - return fp->write(reinterpret_cast<const char*>(data), - static_cast<std::streamsize>(length)).good(); - } - template<typename Ignored> - inline bool write_data_internal(Ignored* /*unused*/, std::ostream* fp, - const void* data, size_t length) - { - return write_data_internal_for_ostream(fp, data, length); - } - - // ----- low-level I/O for custom streams ---- - - // The INPUT type needs to support a Read() method that takes a - // buffer and a length and returns the number of bytes read. - template <typename INPUT> - inline bool read_data_internal(INPUT* fp, void* /*unused*/, - void* data, size_t length) - { - return static_cast<size_t>(fp->Read(data, length)) == length; - } - - // The OUTPUT type needs to support a Write() operation that takes - // a buffer and a length and returns the number of bytes written. - template <typename OUTPUT> - inline bool write_data_internal(OUTPUT* fp, void* /*unused*/, - const void* data, size_t length) - { - return static_cast<size_t>(fp->Write(data, length)) == length; - } - - // ----- low-level I/O: the public API ---- - - template <typename INPUT> - inline bool read_data(INPUT* fp, void* data, size_t length) - { - return read_data_internal(fp, fp, data, length); - } - - template <typename OUTPUT> - inline bool write_data(OUTPUT* fp, const void* data, size_t length) - { - return write_data_internal(fp, fp, data, length); - } - - // Uses read_data() and write_data() to read/write an integer. - // length is the number of bytes to read/write (which may differ - // from sizeof(IntType), allowing us to save on a 32-bit system - // and load on a 64-bit system). Excess bytes are taken to be 0. - // INPUT and OUTPUT must match legal inputs to read/write_data (above). - // -------------------------------------------------------------------- - template <typename INPUT, typename IntType> - bool read_bigendian_number(INPUT* fp, IntType* value, size_t length) - { - *value = 0; - unsigned char byte; - // We require IntType to be unsigned or else the shifting gets all screwy. - SPP_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), "serializing_int_requires_an_unsigned_type"); - for (size_t i = 0; i < length; ++i) - { - if (!read_data(fp, &byte, sizeof(byte))) - return false; - *value |= static_cast<IntType>(byte) << ((length - 1 - i) * 8); - } - return true; - } - - template <typename OUTPUT, typename IntType> - bool write_bigendian_number(OUTPUT* fp, IntType value, size_t length) - { - unsigned char byte; - // We require IntType to be unsigned or else the shifting gets all screwy. - SPP_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), "serializing_int_requires_an_unsigned_type"); - for (size_t i = 0; i < length; ++i) - { - byte = (sizeof(value) <= length-1 - i) - ? static_cast<unsigned char>(0) : static_cast<unsigned char>((value >> ((length-1 - i) * 8)) & 255); - if (!write_data(fp, &byte, sizeof(byte))) return false; - } - return true; - } - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - // This is the type used for NopointerSerializer. - // --------------------------------------------------------------- - template <typename value_type> struct pod_serializer - { - template <typename INPUT> - bool operator()(INPUT* fp, value_type* value) const - { - return read_data(fp, value, sizeof(*value)); - } - - template <typename OUTPUT> - bool operator()(OUTPUT* fp, const value_type& value) const - { - return write_data(fp, &value, sizeof(value)); - } - }; - - - // Settings contains parameters for growing and shrinking the table. - // It also packages zero-size functor (ie. hasher). - // - // It does some munging of the hash value for the cases where - // the original hash function is not be very good. - // --------------------------------------------------------------- - template<typename Key, typename HashFunc, typename SizeType, int HT_MIN_BUCKETS> - class sh_hashtable_settings : public HashFunc - { - private: -#ifndef SPP_MIX_HASH - template <class T, int sz> struct Mixer - { - inline T operator()(T h) const { return h; } - }; -#else - template <class T, int sz> struct Mixer - { - inline T operator()(T h) const; - }; - - template <class T> struct Mixer<T, 4> - { - inline T operator()(T h) const - { - // from Thomas Wang - https://gist.github.com/badboy/6267743 - // --------------------------------------------------------- - h = (h ^ 61) ^ (h >> 16); - h = h + (h << 3); - h = h ^ (h >> 4); - h = h * 0x27d4eb2d; - h = h ^ (h >> 15); - return h; - } - }; - - template <class T> struct Mixer<T, 8> - { - inline T operator()(T h) const - { - // from Thomas Wang - https://gist.github.com/badboy/6267743 - // --------------------------------------------------------- - h = (~h) + (h << 21); // h = (h << 21) - h - 1; - h = h ^ (h >> 24); - h = (h + (h << 3)) + (h << 8); // h * 265 - h = h ^ (h >> 14); - h = (h + (h << 2)) + (h << 4); // h * 21 - h = h ^ (h >> 28); - h = h + (h << 31); - return h; - } - }; -#endif - - public: - typedef Key key_type; - typedef HashFunc hasher; - typedef SizeType size_type; - - public: - sh_hashtable_settings(const hasher& hf, - const float ht_occupancy_flt, - const float ht_empty_flt) - : hasher(hf), - enlarge_threshold_(0), - shrink_threshold_(0), - consider_shrink_(false), - num_ht_copies_(0) - { - set_enlarge_factor(ht_occupancy_flt); - set_shrink_factor(ht_empty_flt); - } - - size_t hash(const key_type& v) const - { - size_t h = hasher::operator()(v); - Mixer<size_t, sizeof(size_t)> mixer; - - return mixer(h); - } - - float enlarge_factor() const { return enlarge_factor_; } - void set_enlarge_factor(float f) { enlarge_factor_ = f; } - float shrink_factor() const { return shrink_factor_; } - void set_shrink_factor(float f) { shrink_factor_ = f; } - - size_type enlarge_threshold() const { return enlarge_threshold_; } - void set_enlarge_threshold(size_type t) { enlarge_threshold_ = t; } - size_type shrink_threshold() const { return shrink_threshold_; } - void set_shrink_threshold(size_type t) { shrink_threshold_ = t; } - - size_type enlarge_size(size_type x) const { return static_cast<size_type>(x * enlarge_factor_); } - size_type shrink_size(size_type x) const { return static_cast<size_type>(x * shrink_factor_); } - - bool consider_shrink() const { return consider_shrink_; } - void set_consider_shrink(bool t) { consider_shrink_ = t; } - - unsigned int num_ht_copies() const { return num_ht_copies_; } - void inc_num_ht_copies() { ++num_ht_copies_; } - - // Reset the enlarge and shrink thresholds - void reset_thresholds(size_type num_buckets) - { - set_enlarge_threshold(enlarge_size(num_buckets)); - set_shrink_threshold(shrink_size(num_buckets)); - // whatever caused us to reset already considered - set_consider_shrink(false); - } - - // Caller is resposible for calling reset_threshold right after - // set_resizing_parameters. - // ------------------------------------------------------------ - void set_resizing_parameters(float shrink, float grow) - { - assert(shrink >= 0); - assert(grow <= 1); - if (shrink > grow/2.0f) - shrink = grow / 2.0f; // otherwise we thrash hashtable size - set_shrink_factor(shrink); - set_enlarge_factor(grow); - } - - // This is the smallest size a hashtable can be without being too crowded - // If you like, you can give a min #buckets as well as a min #elts - // ---------------------------------------------------------------------- - size_type min_buckets(size_type num_elts, size_type min_buckets_wanted) - { - float enlarge = enlarge_factor(); - size_type sz = HT_MIN_BUCKETS; // min buckets allowed - while (sz < min_buckets_wanted || - num_elts >= static_cast<size_type>(sz * enlarge)) - { - // This just prevents overflowing size_type, since sz can exceed - // max_size() here. - // ------------------------------------------------------------- - if (static_cast<size_type>(sz * 2) < sz) - throw_exception(std::length_error("resize overflow")); // protect against overflow - sz *= 2; - } - return sz; - } - - private: - size_type enlarge_threshold_; // table.size() * enlarge_factor - size_type shrink_threshold_; // table.size() * shrink_factor - float enlarge_factor_; // how full before resize - float shrink_factor_; // how empty before resize - bool consider_shrink_; // if we should try to shrink before next insert - - unsigned int num_ht_copies_; // num_ht_copies is a counter incremented every Copy/Move - }; - -} // namespace sparsehash_internal - -#undef SPP_COMPILE_ASSERT - -// ---------------------------------------------------------------------- -// S P A R S E T A B L E -// ---------------------------------------------------------------------- -// -// A sparsetable is a random container that implements a sparse array, -// that is, an array that uses very little memory to store unassigned -// indices (in this case, between 1-2 bits per unassigned index). For -// instance, if you allocate an array of size 5 and assign a[2] = <big -// struct>, then a[2] will take up a lot of memory but a[0], a[1], -// a[3], and a[4] will not. Array elements that have a value are -// called "assigned". Array elements that have no value yet, or have -// had their value cleared using erase() or clear(), are called -// "unassigned". -// -// Unassigned values seem to have the default value of T (see below). -// Nevertheless, there is a difference between an unassigned index and -// one explicitly assigned the value of T(). The latter is considered -// assigned. -// -// Access to an array element is constant time, as is insertion and -// deletion. Insertion and deletion may be fairly slow, however: -// because of this container's memory economy, each insert and delete -// causes a memory reallocation. -// -// NOTE: You should not test(), get(), or set() any index that is -// greater than sparsetable.size(). If you need to do that, call -// resize() first. -// -// --- Template parameters -// PARAMETER DESCRIPTION DEFAULT -// T The value of the array: the type of -- -// object that is stored in the array. -// -// Alloc: Allocator to use to allocate memory. -// -// --- Model of -// Random Access Container -// -// --- Type requirements -// T must be Copy Constructible. It need not be Assignable. -// -// --- Public base classes -// None. -// -// --- Members -// -// [*] All iterators are const in a sparsetable (though nonempty_iterators -// may not be). Use get() and set() to assign values, not iterators. -// -// [+] iterators are random-access iterators. nonempty_iterators are -// bidirectional iterators. - -// [*] If you shrink a sparsetable using resize(), assigned elements -// past the end of the table are removed using erase(). If you grow -// a sparsetable, new unassigned indices are created. -// -// [+] Note that operator[] returns a const reference. You must use -// set() to change the value of a table element. -// -// [!] Unassignment also calls the destructor. -// -// Iterators are invalidated whenever an item is inserted or -// deleted (ie set() or erase() is used) or when the size of -// the table changes (ie resize() or clear() is used). - - - -// --------------------------------------------------------------------------- -// Our iterator as simple as iterators can be: basically it's just -// the index into our table. Dereference, the only complicated -// thing, we punt to the table class. This just goes to show how -// much machinery STL requires to do even the most trivial tasks. -// -// A NOTE ON ASSIGNING: -// A sparse table does not actually allocate memory for entries -// that are not filled. Because of this, it becomes complicated -// to have a non-const iterator: we don't know, if the iterator points -// to a not-filled bucket, whether you plan to fill it with something -// or whether you plan to read its value (in which case you'll get -// the default bucket value). Therefore, while we can define const -// operations in a pretty 'normal' way, for non-const operations, we -// define something that returns a helper object with operator= and -// operator& that allocate a bucket lazily. We use this for table[] -// and also for regular table iterators. - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -// Our iterator as simple as iterators can be: basically it's just -// the index into our table. Dereference, the only complicated -// thing, we punt to the table class. This just goes to show how -// much machinery STL requires to do even the most trivial tasks. -// -// By templatizing over tabletype, we have one iterator type which -// we can use for both sparsetables and sparsebins. In fact it -// works on any class that allows size() and operator[] (eg vector), -// as long as it does the standard STL typedefs too (eg value_type). - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -template <class tabletype> -class table_iterator -{ -public: - typedef table_iterator iterator; - - typedef std::random_access_iterator_tag iterator_category; - typedef typename tabletype::value_type value_type; - typedef typename tabletype::difference_type difference_type; - typedef typename tabletype::size_type size_type; - - explicit table_iterator(tabletype *tbl = 0, size_type p = 0) : - table(tbl), pos(p) - { } - - // Helper function to assert things are ok; eg pos is still in range - void check() const - { - assert(table); - assert(pos <= table->size()); - } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - iterator& operator+=(size_type t) { pos += t; check(); return *this; } - iterator& operator-=(size_type t) { pos -= t; check(); return *this; } - iterator& operator++() { ++pos; check(); return *this; } - iterator& operator--() { --pos; check(); return *this; } - iterator operator++(int) - { - iterator tmp(*this); // for x++ - ++pos; check(); return tmp; - } - - iterator operator--(int) - { - iterator tmp(*this); // for x-- - --pos; check(); return tmp; - } - - iterator operator+(difference_type i) const - { - iterator tmp(*this); - tmp += i; return tmp; - } - - iterator operator-(difference_type i) const - { - iterator tmp(*this); - tmp -= i; return tmp; - } - - difference_type operator-(iterator it) const - { - // for "x = it2 - it" - assert(table == it.table); - return pos - it.pos; - } - - // Comparisons. - bool operator==(const iterator& it) const - { - return table == it.table && pos == it.pos; - } - - bool operator<(const iterator& it) const - { - assert(table == it.table); // life is bad bad bad otherwise - return pos < it.pos; - } - - bool operator!=(const iterator& it) const { return !(*this == it); } - bool operator<=(const iterator& it) const { return !(it < *this); } - bool operator>(const iterator& it) const { return it < *this; } - bool operator>=(const iterator& it) const { return !(*this < it); } - - // Here's the info we actually need to be an iterator - tabletype *table; // so we can dereference and bounds-check - size_type pos; // index into the table -}; - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -template <class tabletype> -class const_table_iterator -{ -public: - typedef table_iterator<tabletype> iterator; - typedef const_table_iterator const_iterator; - - typedef std::random_access_iterator_tag iterator_category; - typedef typename tabletype::value_type value_type; - typedef typename tabletype::difference_type difference_type; - typedef typename tabletype::size_type size_type; - typedef typename tabletype::const_reference reference; // we're const-only - typedef typename tabletype::const_pointer pointer; - - // The "real" constructor - const_table_iterator(const tabletype *tbl, size_type p) - : table(tbl), pos(p) { } - - // The default constructor, used when I define vars of type table::iterator - const_table_iterator() : table(NULL), pos(0) { } - - // The copy constructor, for when I say table::iterator foo = tbl.begin() - // Also converts normal iterators to const iterators // not explicit on purpose - const_table_iterator(const iterator &from) - : table(from.table), pos(from.pos) { } - - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // The main thing our iterator does is dereference. If the table entry - // we point to is empty, we return the default value type. - reference operator*() const { return (*table)[pos]; } - pointer operator->() const { return &(operator*()); } - - // Helper function to assert things are ok; eg pos is still in range - void check() const - { - assert(table); - assert(pos <= table->size()); - } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - const_iterator& operator+=(size_type t) { pos += t; check(); return *this; } - const_iterator& operator-=(size_type t) { pos -= t; check(); return *this; } - const_iterator& operator++() { ++pos; check(); return *this; } - const_iterator& operator--() { --pos; check(); return *this; } - const_iterator operator++(int) - { - const_iterator tmp(*this); // for x++ - ++pos; check(); - return tmp; - } - const_iterator operator--(int) - { - const_iterator tmp(*this); // for x-- - --pos; check(); - return tmp; - } - const_iterator operator+(difference_type i) const - { - const_iterator tmp(*this); - tmp += i; - return tmp; - } - const_iterator operator-(difference_type i) const - { - const_iterator tmp(*this); - tmp -= i; - return tmp; - } - difference_type operator-(const_iterator it) const - { - // for "x = it2 - it" - assert(table == it.table); - return pos - it.pos; - } - reference operator[](difference_type n) const - { - return *(*this + n); // simple though not totally efficient - } - - // Comparisons. - bool operator==(const const_iterator& it) const - { - return table == it.table && pos == it.pos; - } - - bool operator<(const const_iterator& it) const - { - assert(table == it.table); // life is bad bad bad otherwise - return pos < it.pos; - } - bool operator!=(const const_iterator& it) const { return !(*this == it); } - bool operator<=(const const_iterator& it) const { return !(it < *this); } - bool operator>(const const_iterator& it) const { return it < *this; } - bool operator>=(const const_iterator& it) const { return !(*this < it); } - - // Here's the info we actually need to be an iterator - const tabletype *table; // so we can dereference and bounds-check - size_type pos; // index into the table -}; - -// --------------------------------------------------------------------------- -// This is a 2-D iterator. You specify a begin and end over a list -// of *containers*. We iterate over each container by iterating over -// it. It's actually simple: -// VECTOR.begin() VECTOR[0].begin() --------> VECTOR[0].end() ---, -// | ________________________________________________/ -// | \_> VECTOR[1].begin() --------> VECTOR[1].end() -, -// | ___________________________________________________/ -// v \_> ...... -// VECTOR.end() -// -// It's impossible to do random access on one of these things in constant -// time, so it's just a bidirectional iterator. -// -// Unfortunately, because we need to use this for a non-empty iterator, -// we use ne_begin() and ne_end() instead of begin() and end() -// (though only going across, not down). -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -template <class T, class row_it, class col_it, class iter_type> -class Two_d_iterator -{ -public: - typedef Two_d_iterator iterator; - typedef iter_type iterator_category; - typedef T value_type; - typedef std::ptrdiff_t difference_type; - typedef T* pointer; - typedef T& reference; - - explicit Two_d_iterator(row_it curr) : row_current(curr), col_current(0) - { - if (row_current && !row_current->is_marked()) - { - col_current = row_current->ne_begin(); - advance_past_end(); // in case cur->begin() == cur->end() - } - } - - explicit Two_d_iterator(row_it curr, col_it col) : row_current(curr), col_current(col) - { - assert(col); - } - - // The default constructor - Two_d_iterator() : row_current(0), col_current(0) { } - - // Need this explicitly so we can convert normal iterators <=> const iterators - // not explicit on purpose - // --------------------------------------------------------------------------- - template <class T2, class row_it2, class col_it2, class iter_type2> - Two_d_iterator(const Two_d_iterator<T2, row_it2, col_it2, iter_type2>& it) : - row_current (*(row_it *)&it.row_current), - col_current (*(col_it *)&it.col_current) - { } - - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - value_type& operator*() const { return *(col_current); } - value_type* operator->() const { return &(operator*()); } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - // NOTE: this is not amortized constant time! What do we do about it? - // ------------------------------------------------------------------ - void advance_past_end() - { - // used when col_current points to end() - while (col_current == row_current->ne_end()) - { - // end of current row - // ------------------ - ++row_current; // go to beginning of next - if (!row_current->is_marked()) // col is irrelevant at end - col_current = row_current->ne_begin(); - else - break; // don't go past row_end - } - } - - friend size_t operator-(iterator l, iterator f) - { - if (f.row_current->is_marked()) - return 0; - - size_t diff(0); - while (f != l) - { - ++diff; - ++f; - } - return diff; - } - - iterator& operator++() - { - // assert(!row_current->is_marked()); // how to ++ from there? - ++col_current; - advance_past_end(); // in case col_current is at end() - return *this; - } - - iterator& operator--() - { - while (row_current->is_marked() || - col_current == row_current->ne_begin()) - { - --row_current; - col_current = row_current->ne_end(); // this is 1 too far - } - --col_current; - return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } - - - // Comparisons. - bool operator==(const iterator& it) const - { - return (row_current == it.row_current && - (!row_current || row_current->is_marked() || col_current == it.col_current)); - } - - bool operator!=(const iterator& it) const { return !(*this == it); } - - // Here's the info we actually need to be an iterator - // These need to be public so we convert from iterator to const_iterator - // --------------------------------------------------------------------- - row_it row_current; - col_it col_current; -}; - - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -template <class T, class row_it, class col_it, class iter_type, class Alloc> -class Two_d_destructive_iterator : public Two_d_iterator<T, row_it, col_it, iter_type> -{ -public: - typedef Two_d_destructive_iterator iterator; - - Two_d_destructive_iterator(Alloc &alloc, row_it curr) : - _alloc(alloc) - { - this->row_current = curr; - this->col_current = 0; - if (this->row_current && !this->row_current->is_marked()) - { - this->col_current = this->row_current->ne_begin(); - advance_past_end(); // in case cur->begin() == cur->end() - } - } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - // NOTE: this is not amortized constant time! What do we do about it? - // ------------------------------------------------------------------ - void advance_past_end() - { - // used when col_current points to end() - while (this->col_current == this->row_current->ne_end()) - { - this->row_current->clear(_alloc, true); // This is what differs from non-destructive iterators above - - // end of current row - // ------------------ - ++this->row_current; // go to beginning of next - if (!this->row_current->is_marked()) // col is irrelevant at end - this->col_current = this->row_current->ne_begin(); - else - break; // don't go past row_end - } - } - - iterator& operator++() - { - // assert(!this->row_current->is_marked()); // how to ++ from there? - ++this->col_current; - advance_past_end(); // in case col_current is at end() - return *this; - } - -private: - Two_d_destructive_iterator& operator=(const Two_d_destructive_iterator &o); - - Alloc &_alloc; -}; - - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -#if defined(SPP_POPCNT_CHECK) -static inline bool spp_popcount_check() -{ - int cpuInfo[4] = { -1 }; - spp_cpuid(cpuInfo, 1); - if (cpuInfo[2] & (1 << 23)) - return true; // means SPP_POPCNT supported - return false; -} -#endif - -#if defined(SPP_POPCNT_CHECK) && defined(SPP_POPCNT) - -static inline uint32_t spp_popcount(uint32_t i) -{ - static const bool s_ok = spp_popcount_check(); - return s_ok ? SPP_POPCNT(i) : s_spp_popcount_default(i); -} - -#else - -static inline uint32_t spp_popcount(uint32_t i) -{ -#if defined(SPP_POPCNT) - return static_cast<uint32_t>(SPP_POPCNT(i)); -#else - return s_spp_popcount_default(i); -#endif -} - -#endif - -#if defined(SPP_POPCNT_CHECK) && defined(SPP_POPCNT64) - -static inline uint32_t spp_popcount(uint64_t i) -{ - static const bool s_ok = spp_popcount_check(); - return s_ok ? (uint32_t)SPP_POPCNT64(i) : s_spp_popcount_default(i); -} - -#else - -static inline uint32_t spp_popcount(uint64_t i) -{ -#if defined(SPP_POPCNT64) - return static_cast<uint32_t>(SPP_POPCNT64(i)); -#elif 1 - return s_spp_popcount_default(i); -#endif -} - -#endif - -// --------------------------------------------------------------------------- -// SPARSE-TABLE -// ------------ -// The idea is that a table with (logically) t buckets is divided -// into t/M *groups* of M buckets each. (M is a constant, typically -// 32) Each group is stored sparsely. -// Thus, inserting into the table causes some array to grow, which is -// slow but still constant time. Lookup involves doing a -// logical-position-to-sparse-position lookup, which is also slow but -// constant time. The larger M is, the slower these operations are -// but the less overhead (slightly). -// -// To store the sparse array, we store a bitmap B, where B[i] = 1 iff -// bucket i is non-empty. Then to look up bucket i we really look up -// array[# of 1s before i in B]. This is constant time for fixed M. -// -// Terminology: the position of an item in the overall table (from -// 1 .. t) is called its "location." The logical position in a group -// (from 1 .. M) is called its "position." The actual location in -// the array (from 1 .. # of non-empty buckets in the group) is -// called its "offset." -// --------------------------------------------------------------------------- - -template <class T, class Alloc> -class sparsegroup -{ -public: - // Basic types - typedef T value_type; - typedef Alloc allocator_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef value_type* pointer; - typedef const value_type* const_pointer; - - typedef uint8_t size_type; // max # of buckets - - // These are our special iterators, that go over non-empty buckets in a - // group. These aren't const-only because you can change non-empty bcks. - // --------------------------------------------------------------------- - typedef pointer ne_iterator; - typedef const_pointer const_ne_iterator; - typedef std::reverse_iterator<ne_iterator> reverse_ne_iterator; - typedef std::reverse_iterator<const_ne_iterator> const_reverse_ne_iterator; - - // We'll have versions for our special non-empty iterator too - // ---------------------------------------------------------- - ne_iterator ne_begin() { return reinterpret_cast<pointer>(_group); } - const_ne_iterator ne_begin() const { return reinterpret_cast<pointer>(_group); } - const_ne_iterator ne_cbegin() const { return reinterpret_cast<pointer>(_group); } - ne_iterator ne_end() { return reinterpret_cast<pointer>(_group + _num_items()); } - const_ne_iterator ne_end() const { return reinterpret_cast<pointer>(_group + _num_items()); } - const_ne_iterator ne_cend() const { return reinterpret_cast<pointer>(_group + _num_items()); } - reverse_ne_iterator ne_rbegin() { return reverse_ne_iterator(ne_end()); } - const_reverse_ne_iterator ne_rbegin() const { return const_reverse_ne_iterator(ne_cend()); } - const_reverse_ne_iterator ne_crbegin() const { return const_reverse_ne_iterator(ne_cend()); } - reverse_ne_iterator ne_rend() { return reverse_ne_iterator(ne_begin()); } - const_reverse_ne_iterator ne_rend() const { return const_reverse_ne_iterator(ne_cbegin()); } - const_reverse_ne_iterator ne_crend() const { return const_reverse_ne_iterator(ne_cbegin()); } - -private: - // T can be std::pair<const K, V>, but sometime we need to cast to a mutable type - // ------------------------------------------------------------------------------ - typedef typename spp_::cvt<T>::type mutable_value_type; - typedef mutable_value_type & mutable_reference; - typedef mutable_value_type * mutable_pointer; - typedef const mutable_value_type * const_mutable_pointer; - - bool _bmtest(size_type i) const { return !!(_bitmap & (static_cast<group_bm_type>(1) << i)); } - void _bmset(size_type i) { _bitmap |= static_cast<group_bm_type>(1) << i; } - void _bmclear(size_type i) { _bitmap &= ~(static_cast<group_bm_type>(1) << i); } - - bool _bme_test(size_type i) const { return !!(_bm_erased & (static_cast<group_bm_type>(1) << i)); } - void _bme_set(size_type i) { _bm_erased |= static_cast<group_bm_type>(1) << i; } - void _bme_clear(size_type i) { _bm_erased &= ~(static_cast<group_bm_type>(1) << i); } - - bool _bmtest_strict(size_type i) const - { return !!((_bitmap | _bm_erased) & (static_cast<group_bm_type>(1) << i)); } - - - static uint32_t _sizing(uint32_t n) - { -#if !defined(SPP_ALLOC_SZ) || (SPP_ALLOC_SZ == 0) - // aggressive allocation first, then decreasing as sparsegroups fill up - // -------------------------------------------------------------------- - struct alloc_batch_size - { - // 32 bit bitmap - // ........ .... .... .. .. .. .. . . . . . . . . - // 8 12 16 18 20 22 24 25 26 ... 32 - // ------------------------------------------------------ - SPP_CXX14_CONSTEXPR alloc_batch_size() - : data() - { - uint8_t group_sz = SPP_GROUP_SIZE / 4; - uint8_t group_start_alloc = SPP_GROUP_SIZE / 8; //4; - uint8_t alloc_sz = group_start_alloc; - for (int i=0; i<4; ++i) - { - for (int j=0; j<group_sz; ++j) - { - if (j && j % group_start_alloc == 0) - alloc_sz += group_start_alloc; - data[i * group_sz + j] = alloc_sz; - } - if (group_start_alloc > 2) - group_start_alloc /= 2; - alloc_sz += group_start_alloc; - } - } - uint8_t data[SPP_GROUP_SIZE]; - }; - - static alloc_batch_size s_alloc_batch_sz; - return n ? static_cast<uint32_t>(s_alloc_batch_sz.data[n-1]) : 0; // more aggressive alloc at the beginning - -#elif (SPP_ALLOC_SZ == 1) - // use as little memory as possible - slowest insert/delete in table - // ----------------------------------------------------------------- - return n; -#else - // decent compromise when SPP_ALLOC_SZ == 2 - // ---------------------------------------- - static size_type sz_minus_1 = SPP_ALLOC_SZ - 1; - return (n + sz_minus_1) & ~sz_minus_1; -#endif - } - - pointer _allocate_group(allocator_type &alloc, uint32_t n /* , bool tight = false */) - { - // ignore tight since we don't store num_alloc - // num_alloc = (uint8_t)(tight ? n : _sizing(n)); - - uint32_t num_alloc = (uint8_t)_sizing(n); - _set_num_alloc(num_alloc); - pointer retval = alloc.allocate(static_cast<size_type>(num_alloc)); - if (retval == NULL) - { - // the allocator is supposed to throw an exception if the allocation fails. - throw_exception(std::bad_alloc()); - } - return retval; - } - - void _free_group(allocator_type &alloc, uint32_t num_alloc) - { - if (_group) - { - uint32_t num_buckets = _num_items(); - if (num_buckets) - { - mutable_pointer end_it = (mutable_pointer)(_group + num_buckets); - for (mutable_pointer p = (mutable_pointer)_group; p != end_it; ++p) - p->~mutable_value_type(); - } - alloc.deallocate(_group, (typename allocator_type::size_type)num_alloc); - _group = NULL; - } - } - - // private because should not be called - no allocator! - sparsegroup &operator=(const sparsegroup& x); - - static size_type _pos_to_offset(group_bm_type bm, size_type pos) - { - //return (size_type)((uint32_t)~((int32_t(-1) + pos) >> 31) & spp_popcount(bm << (SPP_GROUP_SIZE - pos))); - //return (size_type)(pos ? spp_popcount(bm << (SPP_GROUP_SIZE - pos)) : 0); - return static_cast<size_type>(spp_popcount(bm & ((static_cast<group_bm_type>(1) << pos) - 1))); - } - -public: - - // get_iter() in sparsetable needs it - size_type pos_to_offset(size_type pos) const - { - return _pos_to_offset(_bitmap, pos); - } - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4146) -#endif - - // Returns the (logical) position in the bm[] array, i, such that - // bm[i] is the offset-th set bit in the array. It is the inverse - // of pos_to_offset. get_pos() uses this function to find the index - // of an ne_iterator in the table. Bit-twiddling from - // http://hackersdelight.org/basics.pdf - // ----------------------------------------------------------------- - static size_type offset_to_pos(group_bm_type bm, size_type offset) - { - for (; offset > 0; offset--) - bm &= (bm-1); // remove right-most set bit - - // Clear all bits to the left of the rightmost bit (the &), - // and then clear the rightmost bit but set all bits to the - // right of it (the -1). - // -------------------------------------------------------- - bm = (bm & -bm) - 1; - return static_cast<size_type>(spp_popcount(bm)); - } - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - size_type offset_to_pos(size_type offset) const - { - return offset_to_pos(_bitmap, offset); - } - -public: - // Constructors -- default and copy -- and destructor - explicit sparsegroup() : - _group(0), _bitmap(0), _bm_erased(0) - { - _set_num_items(0); - _set_num_alloc(0); - } - - sparsegroup(const sparsegroup& x) : - _group(0), _bitmap(x._bitmap), _bm_erased(x._bm_erased) - { - _set_num_items(0); - _set_num_alloc(0); - assert(_group == 0); - } - - sparsegroup(const sparsegroup& x, allocator_type& a) : - _group(0), _bitmap(x._bitmap), _bm_erased(x._bm_erased) - { - _set_num_items(0); - _set_num_alloc(0); - - uint32_t num_items = x._num_items(); - if (num_items) - { - _group = _allocate_group(a, num_items /* , true */); - _set_num_items(num_items); - std::uninitialized_copy(x._group, x._group + num_items, _group); - } - } - - ~sparsegroup() { assert(_group == 0); } - - void destruct(allocator_type& a) { _free_group(a, _num_alloc()); } - - // Many STL algorithms use swap instead of copy constructors - void swap(sparsegroup& x) - { - using std::swap; - - swap(_group, x._group); - swap(_bitmap, x._bitmap); - swap(_bm_erased, x._bm_erased); -#ifdef SPP_STORE_NUM_ITEMS - swap(_num_buckets, x._num_buckets); - swap(_num_allocated, x._num_allocated); -#endif - } - - // It's always nice to be able to clear a table without deallocating it - void clear(allocator_type &alloc, bool erased) - { - _free_group(alloc, _num_alloc()); - _bitmap = 0; - if (erased) - _bm_erased = 0; - _set_num_items(0); - _set_num_alloc(0); - } - - // Functions that tell you about size. Alas, these aren't so useful - // because our table is always fixed size. - size_type size() const { return static_cast<size_type>(SPP_GROUP_SIZE); } - size_type max_size() const { return static_cast<size_type>(SPP_GROUP_SIZE); } - - bool empty() const { return false; } - - // We also may want to know how many *used* buckets there are - size_type num_nonempty() const { return (size_type)_num_items(); } - - // TODO(csilvers): make protected + friend - // This is used by sparse_hashtable to get an element from the table - // when we know it exists. - reference unsafe_get(size_type i) const - { - // assert(_bmtest(i)); - return (reference)_group[pos_to_offset(i)]; - } - - typedef std::pair<pointer, bool> SetResult; - -private: - //typedef spp_::integral_constant<bool, spp_::is_relocatable<value_type>::value> check_relocatable; - typedef spp_::true_type realloc_ok_type; - typedef spp_::false_type realloc_not_ok_type; - - //typedef spp_::zero_type libc_reloc_type; - //typedef spp_::one_type spp_reloc_type; - //typedef spp_::two_type spp_not_reloc_type; - //typedef spp_::three_type generic_alloc_type; - -#if 1 - typedef typename if_<((spp_::is_same<allocator_type, libc_allocator<value_type> >::value || - spp_::is_same<allocator_type, spp_allocator<value_type> >::value) && - spp_::is_relocatable<value_type>::value), realloc_ok_type, realloc_not_ok_type>::type - check_alloc_type; -#else - typedef typename if_<spp_::is_same<allocator_type, spp_allocator<value_type> >::value, - typename if_<spp_::is_relocatable<value_type>::value, spp_reloc_type, spp_not_reloc_type>::type, - typename if_<(spp_::is_same<allocator_type, libc_allocator<value_type> >::value && - spp_::is_relocatable<value_type>::value), libc_reloc_type, generic_alloc_type>::type >::type - check_alloc_type; -#endif - - - //typedef if_<spp_::is_same<allocator_type, libc_allocator<value_type> >::value, - // libc_alloc_type, - // if_<spp_::is_same<allocator_type, spp_allocator<value_type> >::value, - // spp_alloc_type, user_alloc_type> > check_alloc_type; - - //typedef spp_::integral_constant<bool, - // (spp_::is_relocatable<value_type>::value && - // (spp_::is_same<allocator_type, spp_allocator<value_type> >::value || - // spp_::is_same<allocator_type, libc_allocator<value_type> >::value)) > - // realloc_and_memmove_ok; - - // ------------------------- memory at *p is uninitialized => need to construct - void _init_val(mutable_value_type *p, reference val) - { -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - ::new (p) value_type(std::move((mutable_reference)val)); -#else - ::new (p) value_type((mutable_reference)val); -#endif - } - - // ------------------------- memory at *p is uninitialized => need to construct - void _init_val(mutable_value_type *p, const_reference val) - { - ::new (p) value_type(val); - } - - // ------------------------------------------------ memory at *p is initialized - void _set_val(value_type *p, reference val) - { -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - *(mutable_pointer)p = std::move((mutable_reference)val); -#else - using std::swap; - swap(*(mutable_pointer)p, *(mutable_pointer)&val); -#endif - } - - // ------------------------------------------------ memory at *p is initialized - void _set_val(value_type *p, const_reference val) - { - *(mutable_pointer)p = *(const_mutable_pointer)&val; - } - - // Create space at _group[offset], assuming value_type is relocatable, and the - // allocator_type is the spp allocator. - // return true if the slot was constructed (i.e. contains a valid value_type - // --------------------------------------------------------------------------------- - template <class Val> - void _set_aux(allocator_type &alloc, size_type offset, Val &val, realloc_ok_type) - { - //static int x=0; if (++x < 10) printf("x\n"); // check we are getting here - - uint32_t num_items = _num_items(); - uint32_t num_alloc = _sizing(num_items); - - if (num_items == num_alloc) - { - num_alloc = _sizing(num_items + 1); - _group = alloc.reallocate(_group, num_alloc); - _set_num_alloc(num_alloc); - } - - for (uint32_t i = num_items; i > offset; --i) - memcpy(static_cast<void *>(_group + i), _group + i-1, sizeof(*_group)); - - _init_val((mutable_pointer)(_group + offset), val); - } - - // Create space at _group[offset], assuming value_type is *not* relocatable, and the - // allocator_type is the spp allocator. - // return true if the slot was constructed (i.e. contains a valid value_type - // --------------------------------------------------------------------------------- - template <class Val> - void _set_aux(allocator_type &alloc, size_type offset, Val &val, realloc_not_ok_type) - { - uint32_t num_items = _num_items(); - uint32_t num_alloc = _sizing(num_items); - - //assert(num_alloc == (uint32_t)_num_allocated); - if (num_items < num_alloc) - { - // create new object at end and rotate it to position - _init_val((mutable_pointer)&_group[num_items], val); - std::rotate((mutable_pointer)(_group + offset), - (mutable_pointer)(_group + num_items), - (mutable_pointer)(_group + num_items + 1)); - return; - } - - // This is valid because 0 <= offset <= num_items - pointer p = _allocate_group(alloc, _sizing(num_items + 1)); - if (offset) - std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)_group), - MK_MOVE_IT((mutable_pointer)(_group + offset)), - (mutable_pointer)p); - if (num_items > offset) - std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group + offset)), - MK_MOVE_IT((mutable_pointer)(_group + num_items)), - (mutable_pointer)(p + offset + 1)); - _init_val((mutable_pointer)(p + offset), val); - _free_group(alloc, num_alloc); - _group = p; - } - - // ---------------------------------------------------------------------------------- - template <class Val> - void _set(allocator_type &alloc, size_type i, size_type offset, Val &val) - { - if (!_bmtest(i)) - { - _set_aux(alloc, offset, val, check_alloc_type()); - _incr_num_items(); - _bmset(i); - } - else - _set_val(&_group[offset], val); - } - -public: - - // This returns the pointer to the inserted item - // --------------------------------------------- - template <class Val> - pointer set(allocator_type &alloc, size_type i, Val &val) - { - _bme_clear(i); // in case this was an "erased" location - - size_type offset = pos_to_offset(i); - _set(alloc, i, offset, val); // may change _group pointer - return (pointer)(_group + offset); - } - - // We let you see if a bucket is non-empty without retrieving it - // ------------------------------------------------------------- - bool test(size_type i) const - { - return _bmtest(i); - } - - // also tests for erased values - // ---------------------------- - bool test_strict(size_type i) const - { - return _bmtest_strict(i); - } - -private: - // Shrink the array, assuming value_type is relocatable, and the - // allocator_type is the libc allocator (supporting reallocate). - // ------------------------------------------------------------- - void _group_erase_aux(allocator_type &alloc, size_type offset, realloc_ok_type) - { - // static int x=0; if (++x < 10) printf("Y\n"); // check we are getting here - uint32_t num_items = _num_items(); - uint32_t num_alloc = _sizing(num_items); - - if (num_items == 1) - { - assert(offset == 0); - _free_group(alloc, num_alloc); - _set_num_alloc(0); - return; - } - - _group[offset].~value_type(); - - for (size_type i = offset; i < num_items - 1; ++i) - memcpy(static_cast<void *>(_group + i), _group + i + 1, sizeof(*_group)); - - if (_sizing(num_items - 1) != num_alloc) - { - num_alloc = _sizing(num_items - 1); - assert(num_alloc); // because we have at least 1 item left - _set_num_alloc(num_alloc); - _group = alloc.reallocate(_group, num_alloc); - } - } - - // Shrink the array, without any special assumptions about value_type and - // allocator_type. - // -------------------------------------------------------------------------- - void _group_erase_aux(allocator_type &alloc, size_type offset, realloc_not_ok_type) - { - uint32_t num_items = _num_items(); - uint32_t num_alloc = _sizing(num_items); - - if (_sizing(num_items - 1) != num_alloc) - { - pointer p = 0; - if (num_items > 1) - { - p = _allocate_group(alloc, num_items - 1); - if (offset) - std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group)), - MK_MOVE_IT((mutable_pointer)(_group + offset)), - (mutable_pointer)(p)); - if (static_cast<uint32_t>(offset + 1) < num_items) - std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group + offset + 1)), - MK_MOVE_IT((mutable_pointer)(_group + num_items)), - (mutable_pointer)(p + offset)); - } - else - { - assert(offset == 0); - _set_num_alloc(0); - } - _free_group(alloc, num_alloc); - _group = p; - } - else - { - std::rotate((mutable_pointer)(_group + offset), - (mutable_pointer)(_group + offset + 1), - (mutable_pointer)(_group + num_items)); - ((mutable_pointer)(_group + num_items - 1))->~mutable_value_type(); - } - } - - void _group_erase(allocator_type &alloc, size_type offset) - { - _group_erase_aux(alloc, offset, check_alloc_type()); - } - -public: - template <class twod_iter> - bool erase_ne(allocator_type &alloc, twod_iter &it) - { - assert(_group && it.col_current != ne_end()); - size_type offset = (size_type)(it.col_current - ne_begin()); - size_type pos = offset_to_pos(offset); - - if (_num_items() <= 1) - { - clear(alloc, false); - it.col_current = 0; - } - else - { - _group_erase(alloc, offset); - _decr_num_items(); - _bmclear(pos); - - // in case _group_erase reallocated the buffer - it.col_current = reinterpret_cast<pointer>(_group) + offset; - } - _bme_set(pos); // remember that this position has been erased - it.advance_past_end(); - return true; - } - - - // This takes the specified elements out of the group. This is - // "undefining", rather than "clearing". - // TODO(austern): Make this exception safe: handle exceptions from - // value_type's copy constructor. - // --------------------------------------------------------------- - void erase(allocator_type &alloc, size_type i) - { - if (_bmtest(i)) - { - // trivial to erase empty bucket - if (_num_items() == 1) - clear(alloc, false); - else - { - _group_erase(alloc, pos_to_offset(i)); - _decr_num_items(); - _bmclear(i); - } - _bme_set(i); // remember that this position has been erased - } - } - - // I/O - // We support reading and writing groups to disk. We don't store - // the actual array contents (which we don't know how to store), - // just the bitmap and size. Meant to be used with table I/O. - // -------------------------------------------------------------- - template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const - { - // warning: we write 4 or 8 bytes for the bitmap, instead of 6 in the - // original google sparsehash - // ------------------------------------------------------------------ - if (!sparsehash_internal::write_data(fp, &_bitmap, sizeof(_bitmap))) - return false; - - return true; - } - - // Reading destroys the old group contents! Returns true if all was ok. - template <typename INPUT> bool read_metadata(allocator_type &alloc, INPUT *fp) - { - clear(alloc, true); - - if (!sparsehash_internal::read_data(fp, &_bitmap, sizeof(_bitmap))) - return false; - - // We'll allocate the space, but we won't fill it: it will be - // left as uninitialized raw memory. - uint32_t num_items = spp_popcount(_bitmap); // yes, _num_buckets not set - _set_num_items(num_items); - _group = num_items ? _allocate_group(alloc, num_items/* , true */) : 0; - return true; - } - - // Again, only meaningful if value_type is a POD. - template <typename INPUT> bool read_nopointer_data(INPUT *fp) - { - for (ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!sparsehash_internal::read_data(fp, &(*it), sizeof(*it))) - return false; - return true; - } - - // If your keys and values are simple enough, we can write them - // to disk for you. "simple enough" means POD and no pointers. - // However, we don't try to normalize endianness. - // ------------------------------------------------------------ - template <typename OUTPUT> bool write_nopointer_data(OUTPUT *fp) const - { - for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!sparsehash_internal::write_data(fp, &(*it), sizeof(*it))) - return false; - return true; - } - - - // Comparisons. We only need to define == and < -- we get - // != > <= >= via relops.h (which we happily included above). - // Note the comparisons are pretty arbitrary: we compare - // values of the first index that isn't equal (using default - // value for empty buckets). - // --------------------------------------------------------- - bool operator==(const sparsegroup& x) const - { - return (_bitmap == x._bitmap && - _bm_erased == x._bm_erased && - std::equal(_group, _group + _num_items(), x._group)); - } - - bool operator<(const sparsegroup& x) const - { - // also from <algorithm> - return std::lexicographical_compare(_group, _group + _num_items(), - x._group, x._group + x._num_items()); - } - - bool operator!=(const sparsegroup& x) const { return !(*this == x); } - bool operator<=(const sparsegroup& x) const { return !(x < *this); } - bool operator> (const sparsegroup& x) const { return x < *this; } - bool operator>=(const sparsegroup& x) const { return !(*this < x); } - - void mark() { _group = (value_type *)static_cast<uintptr_t>(-1); } - bool is_marked() const { return _group == (value_type *)static_cast<uintptr_t>(-1); } - -private: - // --------------------------------------------------------------------------- - template <class A> - class alloc_impl : public A - { - public: - typedef typename A::pointer pointer; - typedef typename A::size_type size_type; - - // Convert a normal allocator to one that has realloc_or_die() - explicit alloc_impl(const A& a) : A(a) { } - - // realloc_or_die should only be used when using the default - // allocator (spp::spp_allocator). - pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) - { - throw_exception(std::runtime_error("realloc_or_die is only supported for spp::spp_allocator\n")); - return NULL; - } - }; - - // A template specialization of alloc_impl for - // spp::libc_allocator that can handle realloc_or_die. - // ----------------------------------------------------------- - template <class A> - class alloc_impl<spp_::libc_allocator<A> > : public spp_::libc_allocator<A> - { - public: - typedef typename spp_::libc_allocator<A>::pointer pointer; - typedef typename spp_::libc_allocator<A>::size_type size_type; - - explicit alloc_impl(const spp_::libc_allocator<A>& a) - : spp_::libc_allocator<A>(a) - { } - - pointer realloc_or_die(pointer ptr, size_type n) - { - pointer retval = this->reallocate(ptr, n); - if (retval == NULL) - { - // the allocator is supposed to throw an exception if the allocation fails. - throw_exception(std::bad_alloc()); - } - return retval; - } - }; - - // A template specialization of alloc_impl for - // spp::spp_allocator that can handle realloc_or_die. - // ----------------------------------------------------------- - template <class A> - class alloc_impl<spp_::spp_allocator<A> > : public spp_::spp_allocator<A> - { - public: - typedef typename spp_::spp_allocator<A>::pointer pointer; - typedef typename spp_::spp_allocator<A>::size_type size_type; - - explicit alloc_impl(const spp_::spp_allocator<A>& a) - : spp_::spp_allocator<A>(a) - { } - - pointer realloc_or_die(pointer ptr, size_type n) - { - pointer retval = this->reallocate(ptr, n); - if (retval == NULL) - { - // the allocator is supposed to throw an exception if the allocation fails. - throw_exception(std::bad_alloc()); - } - return retval; - } - }; - - -#ifdef SPP_STORE_NUM_ITEMS - uint32_t _num_items() const { return (uint32_t)_num_buckets; } - void _set_num_items(uint32_t val) { _num_buckets = static_cast<size_type>(val); } - void _incr_num_items() { ++_num_buckets; } - void _decr_num_items() { --_num_buckets; } - uint32_t _num_alloc() const { return (uint32_t)_num_allocated; } - void _set_num_alloc(uint32_t val) { _num_allocated = static_cast<size_type>(val); } -#else - uint32_t _num_items() const { return spp_popcount(_bitmap); } - void _set_num_items(uint32_t ) { } - void _incr_num_items() { } - void _decr_num_items() { } - uint32_t _num_alloc() const { return _sizing(_num_items()); } - void _set_num_alloc(uint32_t val) { } -#endif - - // The actual data - // --------------- - value_type * _group; // (small) array of T's - group_bm_type _bitmap; - group_bm_type _bm_erased; // ones where items have been erased - -#ifdef SPP_STORE_NUM_ITEMS - size_type _num_buckets; - size_type _num_allocated; -#endif -}; - -// --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -template <class T, class Alloc> -class sparsetable -{ -public: - typedef T value_type; - typedef Alloc allocator_type; - typedef sparsegroup<value_type, allocator_type> group_type; - -private: - typedef typename Alloc::template rebind<group_type>::other group_alloc_type; - typedef typename group_alloc_type::size_type group_size_type; - -public: - // Basic types - // ----------- - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef value_type* pointer; - typedef const value_type* const_pointer; - - typedef group_type& GroupsReference; - typedef const group_type& GroupsConstReference; - - typedef typename group_type::ne_iterator ColIterator; - typedef typename group_type::const_ne_iterator ColConstIterator; - - typedef table_iterator<sparsetable<T, allocator_type> > iterator; // defined with index - typedef const_table_iterator<sparsetable<T, allocator_type> > const_iterator; // defined with index - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; - - // These are our special iterators, that go over non-empty buckets in a - // table. These aren't const only because you can change non-empty bcks. - // ---------------------------------------------------------------------- - typedef Two_d_iterator<T, - group_type *, - ColIterator, - std::bidirectional_iterator_tag> ne_iterator; - - typedef Two_d_iterator<const T, - const group_type *, - ColConstIterator, - std::bidirectional_iterator_tag> const_ne_iterator; - - // Another special iterator: it frees memory as it iterates (used to resize). - // Obviously, you can only iterate over it once, which is why it's an input iterator - // --------------------------------------------------------------------------------- - typedef Two_d_destructive_iterator<T, - group_type *, - ColIterator, - std::input_iterator_tag, - allocator_type> destructive_iterator; - - typedef std::reverse_iterator<ne_iterator> reverse_ne_iterator; - typedef std::reverse_iterator<const_ne_iterator> const_reverse_ne_iterator; - - - // Iterator functions - // ------------------ - iterator begin() { return iterator(this, 0); } - const_iterator begin() const { return const_iterator(this, 0); } - const_iterator cbegin() const { return const_iterator(this, 0); } - iterator end() { return iterator(this, size()); } - const_iterator end() const { return const_iterator(this, size()); } - const_iterator cend() const { return const_iterator(this, size()); } - reverse_iterator rbegin() { return reverse_iterator(end()); } - const_reverse_iterator rbegin() const { return const_reverse_iterator(cend()); } - const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); } - reverse_iterator rend() { return reverse_iterator(begin()); } - const_reverse_iterator rend() const { return const_reverse_iterator(cbegin()); } - const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); } - - // Versions for our special non-empty iterator - // ------------------------------------------ - ne_iterator ne_begin() { return ne_iterator (_first_group); } - const_ne_iterator ne_begin() const { return const_ne_iterator(_first_group); } - const_ne_iterator ne_cbegin() const { return const_ne_iterator(_first_group); } - ne_iterator ne_end() { return ne_iterator (_last_group); } - const_ne_iterator ne_end() const { return const_ne_iterator(_last_group); } - const_ne_iterator ne_cend() const { return const_ne_iterator(_last_group); } - - reverse_ne_iterator ne_rbegin() { return reverse_ne_iterator(ne_end()); } - const_reverse_ne_iterator ne_rbegin() const { return const_reverse_ne_iterator(ne_end()); } - const_reverse_ne_iterator ne_crbegin() const { return const_reverse_ne_iterator(ne_end()); } - reverse_ne_iterator ne_rend() { return reverse_ne_iterator(ne_begin()); } - const_reverse_ne_iterator ne_rend() const { return const_reverse_ne_iterator(ne_begin()); } - const_reverse_ne_iterator ne_crend() const { return const_reverse_ne_iterator(ne_begin()); } - - destructive_iterator destructive_begin() - { - return destructive_iterator(_alloc, _first_group); - } - - destructive_iterator destructive_end() - { - return destructive_iterator(_alloc, _last_group); - } - - // How to deal with the proper group - static group_size_type num_groups(size_type num) - { - // how many to hold num buckets - return num == 0 ? (group_size_type)0 : - (group_size_type)(((num-1) / SPP_GROUP_SIZE) + 1); - } - - typename group_type::size_type pos_in_group(size_type i) const - { - return static_cast<typename group_type::size_type>(i & SPP_MASK_); - } - - size_type group_num(size_type i) const - { - return (size_type)(i >> SPP_SHIFT_); - } - - GroupsReference which_group(size_type i) - { - return _first_group[group_num(i)]; - } - - GroupsConstReference which_group(size_type i) const - { - return _first_group[group_num(i)]; - } - - void _alloc_group_array(group_size_type sz, group_type *&first, group_type *&last) - { - if (sz) - { - first = _group_alloc.allocate((size_type)(sz + 1)); // + 1 for end marker - first[sz].mark(); // for the ne_iterator - last = first + sz; - } - } - - void _free_group_array(group_type *&first, group_type *&last) - { - if (first) - { - _group_alloc.deallocate(first, (group_size_type)(last - first + 1)); // + 1 for end marker - first = last = 0; - } - } - - void _allocate_groups(size_type sz) - { - if (sz) - { - _alloc_group_array(sz, _first_group, _last_group); - std::uninitialized_fill(_first_group, _last_group, group_type()); - } - } - - void _free_groups() - { - if (_first_group) - { - for (group_type *g = _first_group; g != _last_group; ++g) - g->destruct(_alloc); - _free_group_array(_first_group, _last_group); - } - } - - void _cleanup() - { - _free_groups(); // sets _first_group = _last_group = 0 - _table_size = 0; - _num_buckets = 0; - } - - void _init() - { - _first_group = 0; - _last_group = 0; - _table_size = 0; - _num_buckets = 0; - } - - void _copy(const sparsetable &o) - { - _table_size = o._table_size; - _num_buckets = o._num_buckets; - _alloc = o._alloc; // todo - copy or move allocator according to... - _group_alloc = o._group_alloc; // http://en.cppreference.com/w/cpp/container/unordered_map/unordered_map - - group_size_type sz = (group_size_type)(o._last_group - o._first_group); - if (sz) - { - _alloc_group_array(sz, _first_group, _last_group); - for (group_size_type i=0; i<sz; ++i) - new (_first_group + i) group_type(o._first_group[i], _alloc); - } - } - -public: - // Constructors -- default, normal (when you specify size), and copy - explicit sparsetable(size_type sz = 0, const allocator_type &alloc = allocator_type()) : - _first_group(0), - _last_group(0), - _table_size(sz), - _num_buckets(0), - _group_alloc(alloc), - _alloc(alloc) - // todo - copy or move allocator according to - // http://en.cppreference.com/w/cpp/container/unordered_map/unordered_map - { - _allocate_groups(num_groups(sz)); - } - - ~sparsetable() - { - _free_groups(); - } - - sparsetable(const sparsetable &o) - { - _init(); - _copy(o); - } - - sparsetable& operator=(const sparsetable &o) - { - _cleanup(); - _copy(o); - return *this; - } - - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - sparsetable(sparsetable&& o) - { - _init(); - this->swap(o); - } - - sparsetable(sparsetable&& o, const allocator_type &alloc) - { - _init(); - this->swap(o); - _alloc = alloc; // [gp todo] is this correct? - } - - sparsetable& operator=(sparsetable&& o) - { - _cleanup(); - this->swap(o); - return *this; - } -#endif - - // Many STL algorithms use swap instead of copy constructors - void swap(sparsetable& o) - { - using std::swap; - - swap(_first_group, o._first_group); - swap(_last_group, o._last_group); - swap(_table_size, o._table_size); - swap(_num_buckets, o._num_buckets); - if (_alloc != o._alloc) - swap(_alloc, o._alloc); - if (_group_alloc != o._group_alloc) - swap(_group_alloc, o._group_alloc); - } - - // It's always nice to be able to clear a table without deallocating it - void clear() - { - _free_groups(); - _num_buckets = 0; - _table_size = 0; - } - - inline allocator_type get_allocator() const - { - return _alloc; - } - - - // Functions that tell you about size. - // NOTE: empty() is non-intuitive! It does not tell you the number - // of not-empty buckets (use num_nonempty() for that). Instead - // it says whether you've allocated any buckets or not. - // ---------------------------------------------------------------- - size_type size() const { return _table_size; } - size_type max_size() const { return _alloc.max_size(); } - bool empty() const { return _table_size == 0; } - size_type num_nonempty() const { return _num_buckets; } - - // OK, we'll let you resize one of these puppies - void resize(size_type new_size) - { - group_size_type sz = num_groups(new_size); - group_size_type old_sz = (group_size_type)(_last_group - _first_group); - - if (sz != old_sz) - { - // resize group array - // ------------------ - group_type *first = 0, *last = 0; - if (sz) - { - _alloc_group_array(sz, first, last); - if (old_sz) - memcpy(static_cast<void *>(first), _first_group, sizeof(*first) * (std::min)(sz, old_sz)); - } - - if (sz < old_sz) - { - for (group_type *g = _first_group + sz; g != _last_group; ++g) - g->destruct(_alloc); - } - else - std::uninitialized_fill(first + old_sz, last, group_type()); - - _free_group_array(_first_group, _last_group); - _first_group = first; - _last_group = last; - } -#if 0 - // used only in test program - // todo: fix if sparsetable to be used directly - // -------------------------------------------- - if (new_size < _table_size) - { - // lower num_buckets, clear last group - if (pos_in_group(new_size) > 0) // need to clear inside last group - groups.back().erase(_alloc, groups.back().begin() + pos_in_group(new_size), - groups.back().end()); - _num_buckets = 0; // refigure # of used buckets - for (const group_type *group = _first_group; group != _last_group; ++group) - _num_buckets += group->num_nonempty(); - } -#endif - _table_size = new_size; - } - - // We let you see if a bucket is non-empty without retrieving it - // ------------------------------------------------------------- - bool test(size_type i) const - { - // assert(i < _table_size); - return which_group(i).test(pos_in_group(i)); - } - - // also tests for erased values - // ---------------------------- - bool test_strict(size_type i) const - { - // assert(i < _table_size); - return which_group(i).test_strict(pos_in_group(i)); - } - - friend struct GrpPos; - - struct GrpPos - { - typedef typename sparsetable::ne_iterator ne_iter; - GrpPos(const sparsetable &table, size_type i) : - grp(table.which_group(i)), pos(table.pos_in_group(i)) {} - - bool test_strict() const { return grp.test_strict(pos); } - bool test() const { return grp.test(pos); } - typename sparsetable::reference unsafe_get() const { return grp.unsafe_get(pos); } - ne_iter get_iter(typename sparsetable::reference ref) - { - return ne_iter((group_type *)&grp, &ref); - } - - void erase(sparsetable &table) // item *must* be present - { - assert(table._num_buckets); - ((group_type &)grp).erase(table._alloc, pos); - --table._num_buckets; - } - - private: - GrpPos* operator=(const GrpPos&); - - const group_type &grp; - typename group_type::size_type pos; - }; - - bool test(iterator pos) const - { - return which_group(pos.pos).test(pos_in_group(pos.pos)); - } - - bool test(const_iterator pos) const - { - return which_group(pos.pos).test(pos_in_group(pos.pos)); - } - - // TODO(csilvers): make protected + friend - // This is used by sparse_hashtable to get an element from the table - // when we know it exists (because the caller has called test(i)). - // ----------------------------------------------------------------- - reference unsafe_get(size_type i) const - { - assert(i < _table_size); - // assert(test(i)); - return which_group(i).unsafe_get(pos_in_group(i)); - } - - // Needed for hashtables, gets as a ne_iterator. Crashes for empty bcks - const_ne_iterator get_iter(size_type i) const - { - //assert(test(i)); // how can a ne_iterator point to an empty bucket? - - size_type grp_idx = group_num(i); - - return const_ne_iterator(_first_group + grp_idx, - (_first_group[grp_idx].ne_begin() + - _first_group[grp_idx].pos_to_offset(pos_in_group(i)))); - } - - const_ne_iterator get_iter(size_type i, ColIterator col_it) const - { - return const_ne_iterator(_first_group + group_num(i), col_it); - } - - // For nonempty we can return a non-const version - ne_iterator get_iter(size_type i) - { - //assert(test(i)); // how can a nonempty_iterator point to an empty bucket? - - size_type grp_idx = group_num(i); - - return ne_iterator(_first_group + grp_idx, - (_first_group[grp_idx].ne_begin() + - _first_group[grp_idx].pos_to_offset(pos_in_group(i)))); - } - - ne_iterator get_iter(size_type i, ColIterator col_it) - { - return ne_iterator(_first_group + group_num(i), col_it); - } - - // And the reverse transformation. - size_type get_pos(const const_ne_iterator& it) const - { - difference_type current_row = it.row_current - _first_group; - difference_type current_col = (it.col_current - _first_group[current_row].ne_begin()); - return ((current_row * SPP_GROUP_SIZE) + - _first_group[current_row].offset_to_pos(current_col)); - } - - // Val can be reference or const_reference - // --------------------------------------- - template <class Val> - reference set(size_type i, Val &val) - { - assert(i < _table_size); - group_type &group = which_group(i); - typename group_type::size_type old_numbuckets = group.num_nonempty(); - pointer p(group.set(_alloc, pos_in_group(i), val)); - _num_buckets += group.num_nonempty() - old_numbuckets; - return *p; - } - - // used in _move_from (where we can move the old value instead of copying it - void move(size_type i, reference val) - { - assert(i < _table_size); - which_group(i).set(_alloc, pos_in_group(i), val); - ++_num_buckets; - } - - // This takes the specified elements out of the table. - // -------------------------------------------------- - void erase(size_type i) - { - assert(i < _table_size); - - GroupsReference grp(which_group(i)); - typename group_type::size_type old_numbuckets = grp.num_nonempty(); - grp.erase(_alloc, pos_in_group(i)); - _num_buckets += grp.num_nonempty() - old_numbuckets; - } - - void erase(iterator pos) - { - erase(pos.pos); - } - - void erase(iterator start_it, iterator end_it) - { - // This could be more efficient, but then we'd need to figure - // out if we spanned groups or not. Doesn't seem worth it. - for (; start_it != end_it; ++start_it) - erase(start_it); - } - - const_ne_iterator erase(const_ne_iterator it) - { - ne_iterator res(it); - if (res.row_current->erase_ne(_alloc, res)) - _num_buckets--; - return res; - } - - const_ne_iterator erase(const_ne_iterator f, const_ne_iterator l) - { - size_t diff = l - f; - while (diff--) - f = erase(f); - return f; - } - - // We support reading and writing tables to disk. We don't store - // the actual array contents (which we don't know how to store), - // just the groups and sizes. Returns true if all went ok. - -private: - // Every time the disk format changes, this should probably change too - typedef unsigned long MagicNumberType; - static const MagicNumberType MAGIC_NUMBER = 0x24687531; - - // Old versions of this code write all data in 32 bits. We need to - // support these files as well as having support for 64-bit systems. - // So we use the following encoding scheme: for values < 2^32-1, we - // store in 4 bytes in big-endian order. For values > 2^32, we - // store 0xFFFFFFF followed by 8 bytes in big-endian order. This - // causes us to mis-read old-version code that stores exactly - // 0xFFFFFFF, but I don't think that is likely to have happened for - // these particular values. - template <typename OUTPUT, typename IntType> - static bool write_32_or_64(OUTPUT* fp, IntType value) - { - if (value < 0xFFFFFFFFULL) // fits in 4 bytes - { - if (!sparsehash_internal::write_bigendian_number(fp, value, 4)) - return false; - } - else - { - if (!sparsehash_internal::write_bigendian_number(fp, 0xFFFFFFFFUL, 4)) - return false; - if (!sparsehash_internal::write_bigendian_number(fp, value, 8)) - return false; - } - return true; - } - - template <typename INPUT, typename IntType> - static bool read_32_or_64(INPUT* fp, IntType *value) - { - // reads into value - MagicNumberType first4 = 0; // a convenient 32-bit unsigned type - if (!sparsehash_internal::read_bigendian_number(fp, &first4, 4)) - return false; - - if (first4 < 0xFFFFFFFFULL) - { - *value = first4; - } - else - { - if (!sparsehash_internal::read_bigendian_number(fp, value, 8)) - return false; - } - return true; - } - -public: - // read/write_metadata() and read_write/nopointer_data() are DEPRECATED. - // Use serialize() and unserialize(), below, for new code. - - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) const - { - if (!write_32_or_64(fp, MAGIC_NUMBER)) return false; - if (!write_32_or_64(fp, _table_size)) return false; - if (!write_32_or_64(fp, _num_buckets)) return false; - - for (const group_type *group = _first_group; group != _last_group; ++group) - if (group->write_metadata(fp) == false) - return false; - return true; - } - - // Reading destroys the old table contents! Returns true if read ok. - template <typename INPUT> - bool read_metadata(INPUT *fp) - { - size_type magic_read = 0; - if (!read_32_or_64(fp, &magic_read)) return false; - if (magic_read != MAGIC_NUMBER) - { - clear(); // just to be consistent - return false; - } - - if (!read_32_or_64(fp, &_table_size)) return false; - if (!read_32_or_64(fp, &_num_buckets)) return false; - - resize(_table_size); // so the vector's sized ok - for (group_type *group = _first_group; group != _last_group; ++group) - if (group->read_metadata(_alloc, fp) == false) - return false; - return true; - } - - // This code is identical to that for SparseGroup - // If your keys and values are simple enough, we can write them - // to disk for you. "simple enough" means no pointers. - // However, we don't try to normalize endianness - bool write_nopointer_data(FILE *fp) const - { - for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!fwrite(&*it, sizeof(*it), 1, fp)) - return false; - return true; - } - - // When reading, we have to override the potential const-ness of *it - bool read_nopointer_data(FILE *fp) - { - for (ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!fread(reinterpret_cast<void*>(&(*it)), sizeof(*it), 1, fp)) - return false; - return true; - } - - // INPUT and OUTPUT must be either a FILE, *or* a C++ stream - // (istream, ostream, etc) *or* a class providing - // Read(void*, size_t) and Write(const void*, size_t) - // (respectively), which writes a buffer into a stream - // (which the INPUT/OUTPUT instance presumably owns). - - typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; - - // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT *fp) - { - if (!write_metadata(fp)) - return false; - for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!serializer(fp, *it)) - return false; - return true; - } - - // ValueSerializer: a functor. operator()(INPUT*, value_type*) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT *fp) - { - clear(); - if (!read_metadata(fp)) - return false; - for (ne_iterator it = ne_begin(); it != ne_end(); ++it) - if (!serializer(fp, &*it)) - return false; - return true; - } - - // Comparisons. Note the comparisons are pretty arbitrary: we - // compare values of the first index that isn't equal (using default - // value for empty buckets). - bool operator==(const sparsetable& x) const - { - return (_table_size == x._table_size && - _num_buckets == x._num_buckets && - _first_group == x._first_group); - } - - bool operator<(const sparsetable& x) const - { - return std::lexicographical_compare(begin(), end(), x.begin(), x.end()); - } - bool operator!=(const sparsetable& x) const { return !(*this == x); } - bool operator<=(const sparsetable& x) const { return !(x < *this); } - bool operator>(const sparsetable& x) const { return x < *this; } - bool operator>=(const sparsetable& x) const { return !(*this < x); } - - -private: - // The actual data - // --------------- - group_type * _first_group; - group_type * _last_group; - size_type _table_size; // how many buckets they want - size_type _num_buckets; // number of non-empty buckets - group_alloc_type _group_alloc; - allocator_type _alloc; -}; - -// ---------------------------------------------------------------------- -// S P A R S E _ H A S H T A B L E -// ---------------------------------------------------------------------- -// Hashtable class, used to implement the hashed associative containers -// hash_set and hash_map. -// -// Value: what is stored in the table (each bucket is a Value). -// Key: something in a 1-to-1 correspondence to a Value, that can be used -// to search for a Value in the table (find() takes a Key). -// HashFcn: Takes a Key and returns an integer, the more unique the better. -// ExtractKey: given a Value, returns the unique Key associated with it. -// Must inherit from unary_function, or at least have a -// result_type enum indicating the return type of operator(). -// EqualKey: Given two Keys, says whether they are the same (that is, -// if they are both associated with the same Value). -// Alloc: STL allocator to use to allocate memory. -// -// ---------------------------------------------------------------------- - -// The probing method -// ------------------ -// Linear probing -// #define JUMP_(key, num_probes) ( 1 ) -// Quadratic probing -#define JUMP_(key, num_probes) ( num_probes ) - - -// ------------------------------------------------------------------- -// ------------------------------------------------------------------- -template <class Value, class Key, class HashFcn, - class ExtractKey, class SetKey, class EqualKey, class Alloc> -class sparse_hashtable -{ -public: - typedef Key key_type; - typedef Value value_type; - typedef HashFcn hasher; // user provided or spp_hash<Key> - typedef EqualKey key_equal; - typedef Alloc allocator_type; - - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef value_type* pointer; - typedef const value_type* const_pointer; - - // Table is the main storage class. - typedef sparsetable<value_type, allocator_type> Table; - typedef typename Table::ne_iterator ne_it; - typedef typename Table::const_ne_iterator cne_it; - typedef typename Table::destructive_iterator dest_it; - typedef typename Table::ColIterator ColIterator; - - typedef ne_it iterator; - typedef cne_it const_iterator; - typedef dest_it destructive_iterator; - - // These come from tr1. For us they're the same as regular iterators. - // ------------------------------------------------------------------- - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; - - // How full we let the table get before we resize - // ---------------------------------------------- - static const int HT_OCCUPANCY_PCT; // = 80 (out of 100); - - // How empty we let the table get before we resize lower, by default. - // (0.0 means never resize lower.) - // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing - // ------------------------------------------------------------------ - static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT; - - // Minimum size we're willing to let hashtables be. - // Must be a power of two, and at least 4. - // Note, however, that for a given hashtable, the initial size is a - // function of the first constructor arg, and may be >HT_MIN_BUCKETS. - // ------------------------------------------------------------------ - static const size_type HT_MIN_BUCKETS = 4; - - // By default, if you don't specify a hashtable size at - // construction-time, we use this size. Must be a power of two, and - // at least HT_MIN_BUCKETS. - // ----------------------------------------------------------------- - static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; - - // iterators - // --------- - iterator begin() { return _mk_iterator(table.ne_begin()); } - iterator end() { return _mk_iterator(table.ne_end()); } - const_iterator begin() const { return _mk_const_iterator(table.ne_cbegin()); } - const_iterator end() const { return _mk_const_iterator(table.ne_cend()); } - const_iterator cbegin() const { return _mk_const_iterator(table.ne_cbegin()); } - const_iterator cend() const { return _mk_const_iterator(table.ne_cend()); } - - // These come from tr1 unordered_map. They iterate over 'bucket' n. - // For sparsehashtable, we could consider each 'group' to be a bucket, - // I guess, but I don't really see the point. We'll just consider - // bucket n to be the n-th element of the sparsetable, if it's occupied, - // or some empty element, otherwise. - // --------------------------------------------------------------------- - local_iterator begin(size_type i) - { - return _mk_iterator(table.test(i) ? table.get_iter(i) : table.ne_end()); - } - - local_iterator end(size_type i) - { - local_iterator it = begin(i); - if (table.test(i)) - ++it; - return _mk_iterator(it); - } - - const_local_iterator begin(size_type i) const - { - return _mk_const_iterator(table.test(i) ? table.get_iter(i) : table.ne_cend()); - } - - const_local_iterator end(size_type i) const - { - const_local_iterator it = begin(i); - if (table.test(i)) - ++it; - return _mk_const_iterator(it); - } - - const_local_iterator cbegin(size_type i) const { return begin(i); } - const_local_iterator cend(size_type i) const { return end(i); } - - // This is used when resizing - // -------------------------- - destructive_iterator destructive_begin() { return _mk_destructive_iterator(table.destructive_begin()); } - destructive_iterator destructive_end() { return _mk_destructive_iterator(table.destructive_end()); } - - - // accessor functions for the things we templatize on, basically - // ------------------------------------------------------------- - hasher hash_funct() const { return settings; } - key_equal key_eq() const { return key_info; } - allocator_type get_allocator() const { return table.get_allocator(); } - - // Accessor function for statistics gathering. - unsigned int num_table_copies() const { return settings.num_ht_copies(); } - -private: - // This is used as a tag for the copy constructor, saying to destroy its - // arg We have two ways of destructively copying: with potentially growing - // the hashtable as we copy, and without. To make sure the outside world - // can't do a destructive copy, we make the typename private. - // ----------------------------------------------------------------------- - enum MoveDontCopyT {MoveDontCopy, MoveDontGrow}; - - // creating iterators from sparsetable::ne_iterators - // ------------------------------------------------- - iterator _mk_iterator(ne_it it) const { return it; } - const_iterator _mk_const_iterator(cne_it it) const { return it; } - destructive_iterator _mk_destructive_iterator(dest_it it) const { return it; } - -public: - size_type size() const { return table.num_nonempty(); } - size_type max_size() const { return table.max_size(); } - bool empty() const { return size() == 0; } - size_type bucket_count() const { return table.size(); } - size_type max_bucket_count() const { return max_size(); } - // These are tr1 methods. Their idea of 'bucket' doesn't map well to - // what we do. We just say every bucket has 0 or 1 items in it. - size_type bucket_size(size_type i) const - { - return (size_type)(begin(i) == end(i) ? 0 : 1); - } - -private: - // Because of the above, size_type(-1) is never legal; use it for errors - // --------------------------------------------------------------------- - static const size_type ILLEGAL_BUCKET = size_type(-1); - - // Used after a string of deletes. Returns true if we actually shrunk. - // TODO(csilvers): take a delta so we can take into account inserts - // done after shrinking. Maybe make part of the Settings class? - // -------------------------------------------------------------------- - bool _maybe_shrink() - { - assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two - assert(bucket_count() >= HT_MIN_BUCKETS); - bool retval = false; - - // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, - // we'll never shrink until you get relatively big, and we'll never - // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something - // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will - // shrink us down to HT_MIN_BUCKETS buckets, which is too small. - // --------------------------------------------------------------- - const size_type num_remain = table.num_nonempty(); - const size_type shrink_threshold = settings.shrink_threshold(); - if (shrink_threshold > 0 && num_remain < shrink_threshold && - bucket_count() > HT_DEFAULT_STARTING_BUCKETS) - { - const float shrink_factor = settings.shrink_factor(); - size_type sz = (size_type)(bucket_count() / 2); // find how much we should shrink - while (sz > HT_DEFAULT_STARTING_BUCKETS && - num_remain < static_cast<size_type>(sz * shrink_factor)) - { - sz /= 2; // stay a power of 2 - } - sparse_hashtable tmp(MoveDontCopy, *this, sz); - swap(tmp); // now we are tmp - retval = true; - } - settings.set_consider_shrink(false); // because we just considered it - return retval; - } - - // We'll let you resize a hashtable -- though this makes us copy all! - // When you resize, you say, "make it big enough for this many more elements" - // Returns true if we actually resized, false if size was already ok. - // -------------------------------------------------------------------------- - bool _resize_delta(size_type delta) - { - bool did_resize = false; - if (settings.consider_shrink()) - { - // see if lots of deletes happened - if (_maybe_shrink()) - did_resize = true; - } - if (table.num_nonempty() >= - (std::numeric_limits<size_type>::max)() - delta) - { - throw_exception(std::length_error("resize overflow")); - } - - size_type num_occupied = (size_type)(table.num_nonempty() + num_deleted); - - if (bucket_count() >= HT_MIN_BUCKETS && - (num_occupied + delta) <= settings.enlarge_threshold()) - return did_resize; // we're ok as we are - - // Sometimes, we need to resize just to get rid of all the - // "deleted" buckets that are clogging up the hashtable. So when - // deciding whether to resize, count the deleted buckets (which - // are currently taking up room). - // ------------------------------------------------------------- - const size_type needed_size = - settings.min_buckets((size_type)(num_occupied + delta), (size_type)0); - - if (needed_size <= bucket_count()) // we have enough buckets - return did_resize; - - size_type resize_to = settings.min_buckets((size_type)(num_occupied + delta), bucket_count()); - - if (resize_to < needed_size && // may double resize_to - resize_to < (std::numeric_limits<size_type>::max)() / 2) - { - // This situation means that we have enough deleted elements, - // that once we purge them, we won't actually have needed to - // grow. But we may want to grow anyway: if we just purge one - // element, say, we'll have to grow anyway next time we - // insert. Might as well grow now, since we're already going - // through the trouble of copying (in order to purge the - // deleted elements). - const size_type target = - static_cast<size_type>(settings.shrink_size((size_type)(resize_to*2))); - if (table.num_nonempty() + delta >= target) - { - // Good, we won't be below the shrink threshhold even if we double. - resize_to *= 2; - } - } - - sparse_hashtable tmp(MoveDontCopy, *this, resize_to); - swap(tmp); // now we are tmp - return true; - } - - // Used to actually do the rehashing when we grow/shrink a hashtable - // ----------------------------------------------------------------- - void _copy_from(const sparse_hashtable &ht, size_type min_buckets_wanted) - { - clear(); // clear table, set num_deleted to 0 - - // If we need to change the size of our table, do it now - const size_type resize_to = settings.min_buckets(ht.size(), min_buckets_wanted); - - if (resize_to > bucket_count()) - { - // we don't have enough buckets - table.resize(resize_to); // sets the number of buckets - settings.reset_thresholds(bucket_count()); - } - - // We use a normal iterator to get bcks from ht - // We could use insert() here, but since we know there are - // no duplicates, we can be more efficient - assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two - for (const_iterator it = ht.begin(); it != ht.end(); ++it) - { - size_type num_probes = 0; // how many times we've probed - size_type bucknum; - const size_type bucket_count_minus_one = bucket_count() - 1; - for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; - table.test(bucknum); // table.test() OK since no erase() - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) - { - ++num_probes; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - table.set(bucknum, *it); // copies the value to here - } - settings.inc_num_ht_copies(); - } - - // Implementation is like _copy_from, but it destroys the table of the - // "from" guy by freeing sparsetable memory as we iterate. This is - // useful in resizing, since we're throwing away the "from" guy anyway. - // -------------------------------------------------------------------- - void _move_from(MoveDontCopyT mover, sparse_hashtable &ht, - size_type min_buckets_wanted) - { - clear(); - - // If we need to change the size of our table, do it now - size_type resize_to; - if (mover == MoveDontGrow) - resize_to = ht.bucket_count(); // keep same size as old ht - else // MoveDontCopy - resize_to = settings.min_buckets(ht.size(), min_buckets_wanted); - if (resize_to > bucket_count()) - { - // we don't have enough buckets - table.resize(resize_to); // sets the number of buckets - settings.reset_thresholds(bucket_count()); - } - - // We use a normal iterator to get bcks from ht - // We could use insert() here, but since we know there are - // no duplicates, we can be more efficient - assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two - const size_type bucket_count_minus_one = (const size_type)(bucket_count() - 1); - - // THIS IS THE MAJOR LINE THAT DIFFERS FROM COPY_FROM(): - for (destructive_iterator it = ht.destructive_begin(); - it != ht.destructive_end(); ++it) - { - size_type num_probes = 0; - size_type bucknum; - for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; - table.test(bucknum); // table.test() OK since no erase() - bucknum = (size_type)((bucknum + JUMP_(key, num_probes)) & (bucket_count()-1))) - { - ++num_probes; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - table.move(bucknum, *it); // moves the value to here - } - settings.inc_num_ht_copies(); - } - - - // Required by the spec for hashed associative container -public: - // Though the docs say this should be num_buckets, I think it's much - // more useful as num_elements. As a special feature, calling with - // req_elements==0 will cause us to shrink if we can, saving space. - // ----------------------------------------------------------------- - void resize(size_type req_elements) - { - // resize to this or larger - if (settings.consider_shrink() || req_elements == 0) - _maybe_shrink(); - if (req_elements > table.num_nonempty()) // we only grow - _resize_delta((size_type)(req_elements - table.num_nonempty())); - } - - // Get and change the value of shrink_factor and enlarge_factor. The - // description at the beginning of this file explains how to choose - // the values. Setting the shrink parameter to 0.0 ensures that the - // table never shrinks. - // ------------------------------------------------------------------ - void get_resizing_parameters(float* shrink, float* grow) const - { - *shrink = settings.shrink_factor(); - *grow = settings.enlarge_factor(); - } - - float get_shrink_factor() const { return settings.shrink_factor(); } - float get_enlarge_factor() const { return settings.enlarge_factor(); } - - void set_resizing_parameters(float shrink, float grow) - { - settings.set_resizing_parameters(shrink, grow); - settings.reset_thresholds(bucket_count()); - } - - void set_shrink_factor(float shrink) - { - set_resizing_parameters(shrink, get_enlarge_factor()); - } - - void set_enlarge_factor(float grow) - { - set_resizing_parameters(get_shrink_factor(), grow); - } - - // CONSTRUCTORS -- as required by the specs, we take a size, - // but also let you specify a hashfunction, key comparator, - // and key extractor. We also define a copy constructor and =. - // DESTRUCTOR -- the default is fine, surprisingly. - // ------------------------------------------------------------ - explicit sparse_hashtable(size_type expected_max_items_in_table = 0, - const HashFcn& hf = HashFcn(), - const EqualKey& eql = EqualKey(), - const ExtractKey& ext = ExtractKey(), - const SetKey& set = SetKey(), - const allocator_type& alloc = allocator_type()) - : settings(hf), - key_info(ext, set, eql), - num_deleted(0), - table((expected_max_items_in_table == 0 - ? HT_DEFAULT_STARTING_BUCKETS - : settings.min_buckets(expected_max_items_in_table, 0)), - alloc) - { - settings.reset_thresholds(bucket_count()); - } - - // As a convenience for resize(), we allow an optional second argument - // which lets you make this new hashtable a different size than ht. - // We also provide a mechanism of saying you want to "move" the ht argument - // into us instead of copying. - // ------------------------------------------------------------------------ - sparse_hashtable(const sparse_hashtable& ht, - size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) - : settings(ht.settings), - key_info(ht.key_info), - num_deleted(0), - table(0) - { - settings.reset_thresholds(bucket_count()); - _copy_from(ht, min_buckets_wanted); - } - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - - sparse_hashtable(sparse_hashtable&& o, const allocator_type& alloc = allocator_type()) : - settings(o.settings), - key_info(o.key_info), - num_deleted(0), - table(HT_DEFAULT_STARTING_BUCKETS, alloc) - { - settings.reset_thresholds(bucket_count()); - this->swap(o); - } - - sparse_hashtable& operator=(sparse_hashtable&& o) - { - this->swap(o); - return *this; - } -#endif - - sparse_hashtable(MoveDontCopyT mover, - sparse_hashtable& ht, - size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) - : settings(ht.settings), - key_info(ht.key_info), - num_deleted(0), - table(min_buckets_wanted, ht.table.get_allocator()) - //table(min_buckets_wanted) - { - settings.reset_thresholds(bucket_count()); - _move_from(mover, ht, min_buckets_wanted); - } - - sparse_hashtable& operator=(const sparse_hashtable& ht) - { - if (&ht == this) - return *this; // don't copy onto ourselves - settings = ht.settings; - key_info = ht.key_info; - num_deleted = ht.num_deleted; - - // _copy_from() calls clear and sets num_deleted to 0 too - _copy_from(ht, HT_MIN_BUCKETS); - - // we purposefully don't copy the allocator, which may not be copyable - return *this; - } - - // Many STL algorithms use swap instead of copy constructors - void swap(sparse_hashtable& ht) - { - using std::swap; - - swap(settings, ht.settings); - swap(key_info, ht.key_info); - swap(num_deleted, ht.num_deleted); - table.swap(ht.table); - settings.reset_thresholds(bucket_count()); // also resets consider_shrink - ht.settings.reset_thresholds(ht.bucket_count()); - // we purposefully don't swap the allocator, which may not be swap-able - } - - // It's always nice to be able to clear a table without deallocating it - void clear() - { - if (!empty() || num_deleted != 0) - { - table.clear(); - table = Table(HT_DEFAULT_STARTING_BUCKETS, table.get_allocator()); - } - settings.reset_thresholds(bucket_count()); - num_deleted = 0; - } - - // LOOKUP ROUTINES -private: - - enum pos_type { pt_empty = 0, pt_erased, pt_full }; - // ------------------------------------------------------------------- - class Position - { - public: - - Position() : _t(pt_empty) {} - Position(pos_type t, size_type idx) : _t(t), _idx(idx) {} - - pos_type _t; - size_type _idx; - }; - - // Returns a pair: - // - 'first' is a code, 2 if key already present, 0 or 1 otherwise. - // - 'second' is a position, where the key should go - // Note: because of deletions where-to-insert is not trivial: it's the - // first deleted bucket we see, as long as we don't find the key later - // ------------------------------------------------------------------- - Position _find_position(const key_type &key) const - { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = (const size_type)(bucket_count() - 1); - size_type bucknum = hash(key) & bucket_count_minus_one; - Position pos; - - while (1) - { - // probe until something happens - // ----------------------------- - typename Table::GrpPos grp_pos(table, bucknum); - - if (!grp_pos.test_strict()) - { - // bucket is empty => key not present - return pos._t ? pos : Position(pt_empty, bucknum); - } - else if (grp_pos.test()) - { - reference ref(grp_pos.unsafe_get()); - - if (equals(key, get_key(ref))) - return Position(pt_full, bucknum); - } - else if (pos._t == pt_empty) - { - // first erased position - pos._t = pt_erased; - pos._idx = bucknum; - } - - ++num_probes; // we're doing another probe - bucknum = (size_type)((bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one); - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - -public: - // I hate to duplicate find() like that, but it is - // significantly faster to not have the intermediate pair - // ------------------------------------------------------------------ - iterator find(const key_type& key) - { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - - while (1) // probe until something happens - { - typename Table::GrpPos grp_pos(table, bucknum); - - if (!grp_pos.test_strict()) - return end(); // bucket is empty - if (grp_pos.test()) - { - reference ref(grp_pos.unsafe_get()); - - if (equals(key, get_key(ref))) - return grp_pos.get_iter(ref); - } - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - // Wish I could avoid the duplicate find() const and non-const. - // ------------------------------------------------------------ - const_iterator find(const key_type& key) const - { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - - while (1) // probe until something happens - { - typename Table::GrpPos grp_pos(table, bucknum); - - if (!grp_pos.test_strict()) - return end(); // bucket is empty - else if (grp_pos.test()) - { - reference ref(grp_pos.unsafe_get()); - - if (equals(key, get_key(ref))) - return _mk_const_iterator(table.get_iter(bucknum, &ref)); - } - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - // This is a tr1 method: the bucket a given key is in, or what bucket - // it would be put in, if it were to be inserted. Shrug. - // ------------------------------------------------------------------ - size_type bucket(const key_type& key) const - { - Position pos = _find_position(key); - return pos._idx; - } - - // Counts how many elements have key key. For maps, it's either 0 or 1. - // --------------------------------------------------------------------- - size_type count(const key_type &key) const - { - Position pos = _find_position(key); - return (size_type)(pos._t == pt_full ? 1 : 0); - } - - // Likewise, equal_range doesn't really make sense for us. Oh well. - // ----------------------------------------------------------------- - std::pair<iterator,iterator> equal_range(const key_type& key) - { - iterator pos = find(key); // either an iterator or end - if (pos == end()) - return std::pair<iterator,iterator>(pos, pos); - else - { - const iterator startpos = pos++; - return std::pair<iterator,iterator>(startpos, pos); - } - } - - std::pair<const_iterator,const_iterator> equal_range(const key_type& key) const - { - const_iterator pos = find(key); // either an iterator or end - if (pos == end()) - return std::pair<const_iterator,const_iterator>(pos, pos); - else - { - const const_iterator startpos = pos++; - return std::pair<const_iterator,const_iterator>(startpos, pos); - } - } - - - // INSERTION ROUTINES -private: - // Private method used by insert_noresize and find_or_insert. - template <class T> - reference _insert_at(T& obj, size_type pos, bool erased) - { - if (size() >= max_size()) - { - throw_exception(std::length_error("insert overflow")); - } - if (erased) - { - assert(num_deleted); - --num_deleted; - } - return table.set(pos, obj); - } - - // If you know *this is big enough to hold obj, use this routine - template <class T> - std::pair<iterator, bool> _insert_noresize(T& obj) - { - Position pos = _find_position(get_key(obj)); - bool already_there = (pos._t == pt_full); - - if (!already_there) - { - reference ref(_insert_at(obj, pos._idx, pos._t == pt_erased)); - return std::pair<iterator, bool>(_mk_iterator(table.get_iter(pos._idx, &ref)), true); - } - return std::pair<iterator,bool>(_mk_iterator(table.get_iter(pos._idx)), false); - } - - // Specializations of insert(it, it) depending on the power of the iterator: - // (1) Iterator supports operator-, resize before inserting - template <class ForwardIterator> - void _insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag /*unused*/) - { - int64_t dist = std::distance(f, l); - if (dist < 0 || static_cast<size_t>(dist) >= (std::numeric_limits<size_type>::max)()) - throw_exception(std::length_error("insert-range overflow")); - - _resize_delta(static_cast<size_type>(dist)); - - for (; dist > 0; --dist, ++f) - _insert_noresize(*f); - } - - // (2) Arbitrary iterator, can't tell how much to resize - template <class InputIterator> - void _insert(InputIterator f, InputIterator l, std::input_iterator_tag /*unused*/) - { - for (; f != l; ++f) - _insert(*f); - } - -public: - -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class... Args> - std::pair<iterator, bool> emplace(Args&&... args) - { - _resize_delta(1); - value_type obj(std::forward<Args>(args)...); - return _insert_noresize(obj); - } -#endif - - // This is the normal insert routine, used by the outside world - std::pair<iterator, bool> insert(const_reference obj) - { - _resize_delta(1); // adding an object, grow if need be - return _insert_noresize(obj); - } - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - template< class P > - std::pair<iterator, bool> insert(P &&obj) - { - _resize_delta(1); // adding an object, grow if need be - value_type val(std::forward<P>(obj)); - return _insert_noresize(val); - } -#endif - - // When inserting a lot at a time, we specialize on the type of iterator - template <class InputIterator> - void insert(InputIterator f, InputIterator l) - { - // specializes on iterator type - _insert(f, l, - typename std::iterator_traits<InputIterator>::iterator_category()); - } - - // DefaultValue is a functor that takes a key and returns a value_type - // representing the default value to be inserted if none is found. -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class DefaultValue, class KT> - value_type& find_or_insert(KT&& key) -#else - template <class DefaultValue> - value_type& find_or_insert(const key_type& key) -#endif - { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - DefaultValue default_value; - size_type erased_pos = 0; - bool erased = false; - - while (1) // probe until something happens - { - typename Table::GrpPos grp_pos(table, bucknum); - - if (!grp_pos.test_strict()) - { - // not found -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - auto&& def(default_value(std::forward<KT>(key))); -#else - value_type def(default_value(key)); -#endif - if (_resize_delta(1)) - { - // needed to rehash to make room - // Since we resized, we can't use pos, so recalculate where to insert. - return *(_insert_noresize(def).first); - } - else - { - // no need to rehash, insert right here - return _insert_at(def, erased ? erased_pos : bucknum, erased); - } - } - if (grp_pos.test()) - { - reference ref(grp_pos.unsafe_get()); - - if (equals(key, get_key(ref))) - return ref; - } - else if (!erased) - { - // first erased position - erased_pos = bucknum; - erased = true; - } - - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - size_type erase(const key_type& key) - { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - - while (1) // probe until something happens - { - typename Table::GrpPos grp_pos(table, bucknum); - - if (!grp_pos.test_strict()) - return 0; // bucket is empty, we deleted nothing - if (grp_pos.test()) - { - reference ref(grp_pos.unsafe_get()); - - if (equals(key, get_key(ref))) - { - grp_pos.erase(table); - ++num_deleted; - settings.set_consider_shrink(true); // will think about shrink after next insert - return 1; // because we deleted one thing - } - } - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - const_iterator erase(const_iterator pos) - { - if (pos == cend()) - return cend(); // sanity check - - const_iterator nextpos = table.erase(pos); - ++num_deleted; - settings.set_consider_shrink(true); - return nextpos; - } - - const_iterator erase(const_iterator f, const_iterator l) - { - if (f == cend()) - return cend(); // sanity check - - size_type num_before = table.num_nonempty(); - const_iterator nextpos = table.erase(f, l); - num_deleted += num_before - table.num_nonempty(); - settings.set_consider_shrink(true); - return nextpos; - } - - // Deleted key routines - just to keep google test framework happy - // we don't actually use the deleted key - // --------------------------------------------------------------- - void set_deleted_key(const key_type&) - { - } - - void clear_deleted_key() - { - } - - bool operator==(const sparse_hashtable& ht) const - { - if (this == &ht) - return true; - - if (size() != ht.size()) - return false; - - for (const_iterator it = begin(); it != end(); ++it) - { - const_iterator it2 = ht.find(get_key(*it)); - if ((it2 == ht.end()) || (*it != *it2)) - return false; - } - - return true; - } - - bool operator!=(const sparse_hashtable& ht) const - { - return !(*this == ht); - } - - - // I/O - // We support reading and writing hashtables to disk. NOTE that - // this only stores the hashtable metadata, not the stuff you've - // actually put in the hashtable! Alas, since I don't know how to - // write a hasher or key_equal, you have to make sure everything - // but the table is the same. We compact before writing. - // - // The OUTPUT type needs to support a Write() operation. File and - // OutputBuffer are appropriate types to pass in. - // - // The INPUT type needs to support a Read() operation. File and - // InputBuffer are appropriate types to pass in. - // ------------------------------------------------------------- - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) - { - return table.write_metadata(fp); - } - - template <typename INPUT> - bool read_metadata(INPUT *fp) - { - num_deleted = 0; // since we got rid before writing - const bool result = table.read_metadata(fp); - settings.reset_thresholds(bucket_count()); - return result; - } - - // Only meaningful if value_type is a POD. - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) - { - return table.write_nopointer_data(fp); - } - - // Only meaningful if value_type is a POD. - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) - { - return table.read_nopointer_data(fp); - } - - // INPUT and OUTPUT must be either a FILE, *or* a C++ stream - // (istream, ostream, etc) *or* a class providing - // Read(void*, size_t) and Write(const void*, size_t) - // (respectively), which writes a buffer into a stream - // (which the INPUT/OUTPUT instance presumably owns). - - typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; - - // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT *fp) - { - return table.serialize(serializer, fp); - } - - // ValueSerializer: a functor. operator()(INPUT*, value_type*) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT *fp) - { - num_deleted = 0; // since we got rid before writing - const bool result = table.unserialize(serializer, fp); - settings.reset_thresholds(bucket_count()); - return result; - } - -private: - - // Package templated functors with the other types to eliminate memory - // needed for storing these zero-size operators. Since ExtractKey and - // hasher's operator() might have the same function signature, they - // must be packaged in different classes. - // ------------------------------------------------------------------------- - struct Settings : - sparsehash_internal::sh_hashtable_settings<key_type, hasher, - size_type, HT_MIN_BUCKETS> - { - explicit Settings(const hasher& hf) - : sparsehash_internal::sh_hashtable_settings<key_type, hasher, size_type, - HT_MIN_BUCKETS> - (hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} - }; - - // KeyInfo stores delete key and packages zero-size functors: - // ExtractKey and SetKey. - // --------------------------------------------------------- - class KeyInfo : public ExtractKey, public SetKey, public EqualKey - { - public: - KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq) - : ExtractKey(ek), SetKey(sk), EqualKey(eq) - { - } - - // We want to return the exact same type as ExtractKey: Key or const Key& - typename ExtractKey::result_type get_key(const_reference v) const - { - return ExtractKey::operator()(v); - } - - bool equals(const key_type& a, const key_type& b) const - { - return EqualKey::operator()(a, b); - } - }; - - // Utility functions to access the templated operators - size_t hash(const key_type& v) const - { - return settings.hash(v); - } - - bool equals(const key_type& a, const key_type& b) const - { - return key_info.equals(a, b); - } - - typename ExtractKey::result_type get_key(const_reference v) const - { - return key_info.get_key(v); - } - -private: - // Actual data - // ----------- - Settings settings; - KeyInfo key_info; - size_type num_deleted; - Table table; // holds num_buckets and num_elements too -}; - -#undef JUMP_ - -// ----------------------------------------------------------------------------- -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const typename sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type -sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET; - -// How full we let the table get before we resize. Knuth says .8 is -// good -- higher causes us to probe too much, though saves memory -// ----------------------------------------------------------------------------- -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 50; - -// How empty we let the table get before we resize lower. -// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing -// ----------------------------------------------------------------------------- -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT -= static_cast<int>(0.4 * - sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT); - - -// ---------------------------------------------------------------------- -// S P A R S E _ H A S H _ M A P -// ---------------------------------------------------------------------- -template <class Key, class T, - class HashFcn = spp_hash<Key>, - class EqualKey = std::equal_to<Key>, - class Alloc = SPP_DEFAULT_ALLOCATOR<std::pair<const Key, T> > > -class sparse_hash_map -{ -public: - typedef typename std::pair<const Key, T> value_type; - -private: - // Apparently select1st is not stl-standard, so we define our own - struct SelectKey - { - typedef const Key& result_type; - - inline const Key& operator()(const value_type& p) const - { - return p.first; - } - }; - - struct SetKey - { - inline void operator()(value_type* value, const Key& new_key) const - { - *const_cast<Key*>(&value->first) = new_key; - } - }; - - // For operator[]. - struct DefaultValue - { -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class KT> - inline value_type operator()(KT&& key) const - { - return { std::forward<KT>(key), T() }; - } -#else - inline value_type operator()(const Key& key) const - { - return std::make_pair(key, T()); - } -#endif - }; - - // The actual data - typedef sparse_hashtable<value_type, Key, HashFcn, SelectKey, - SetKey, EqualKey, Alloc> ht; - -public: - typedef typename ht::key_type key_type; - typedef T data_type; - typedef T mapped_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - // Iterator functions - iterator begin() { return rep.begin(); } - iterator end() { return rep.end(); } - const_iterator begin() const { return rep.cbegin(); } - const_iterator end() const { return rep.cend(); } - const_iterator cbegin() const { return rep.cbegin(); } - const_iterator cend() const { return rep.cend(); } - - // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) { return rep.begin(i); } - local_iterator end(size_type i) { return rep.end(i); } - const_local_iterator begin(size_type i) const { return rep.begin(i); } - const_local_iterator end(size_type i) const { return rep.end(i); } - const_local_iterator cbegin(size_type i) const { return rep.cbegin(i); } - const_local_iterator cend(size_type i) const { return rep.cend(i); } - - // Accessor functions - // ------------------ - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - // ------------ - explicit sparse_hash_map(size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(n, hf, eql, SelectKey(), SetKey(), alloc) - { - } - - explicit sparse_hash_map(const allocator_type& alloc) : - rep(0, hasher(), key_equal(), SelectKey(), SetKey(), alloc) - { - } - - sparse_hash_map(size_type n, const allocator_type& alloc) : - rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc) - { - } - - sparse_hash_map(size_type n, const hasher& hf, const allocator_type& alloc) : - rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc) - { - } - - template <class InputIterator> - sparse_hash_map(InputIterator f, InputIterator l, - size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(n, hf, eql, SelectKey(), SetKey(), alloc) - { - rep.insert(f, l); - } - - template <class InputIterator> - sparse_hash_map(InputIterator f, InputIterator l, - size_type n, const allocator_type& alloc) - : rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc) - { - rep.insert(f, l); - } - - template <class InputIterator> - sparse_hash_map(InputIterator f, InputIterator l, - size_type n, const hasher& hf, const allocator_type& alloc) - : rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc) - { - rep.insert(f, l); - } - - sparse_hash_map(const sparse_hash_map &o) : - rep(o.rep) - {} - - sparse_hash_map(const sparse_hash_map &o, - const allocator_type& alloc) : - rep(o.rep, alloc) - {} - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - sparse_hash_map(sparse_hash_map &&o) : - rep(std::move(o.rep)) - {} - - sparse_hash_map(sparse_hash_map &&o, - const allocator_type& alloc) : - rep(std::move(o.rep), alloc) - {} - - sparse_hash_map& operator=(sparse_hash_map &&o) - { - rep = std::move(o.rep); - } -#endif - -#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST) - sparse_hash_map(std::initializer_list<value_type> init, - size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(n, hf, eql, SelectKey(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_map(std::initializer_list<value_type> init, - size_type n, const allocator_type& alloc) : - rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_map(std::initializer_list<value_type> init, - size_type n, const hasher& hf, const allocator_type& alloc) : - rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_map& operator=(std::initializer_list<value_type> init) - { - rep.clear(); - rep.insert(init.begin(), init.end()); - return *this; - } - - void insert(std::initializer_list<value_type> init) - { - rep.insert(init.begin(), init.end()); - } -#endif - - sparse_hash_map& operator=(const sparse_hash_map &o) - { - rep = o.rep; - return *this; - } - - void clear() { rep.clear(); } - void swap(sparse_hash_map& hs) { rep.swap(hs.rep); } - - // Functions concerning size - // ------------------------- - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - float load_factor() const { return size() * 1.0f / bucket_count(); } - - float max_load_factor() const { return rep.get_enlarge_factor(); } - void max_load_factor(float grow) { rep.set_enlarge_factor(grow); } - - float min_load_factor() const { return rep.get_shrink_factor(); } - void min_load_factor(float shrink){ rep.set_shrink_factor(shrink); } - - void set_resizing_parameters(float shrink, float grow) - { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type cnt) { rep.resize(cnt); } - void rehash(size_type cnt) { resize(cnt); } // c++11 name - void reserve(size_type cnt) { resize(cnt); } // c++11 - - // Lookup - // ------ - iterator find(const key_type& key) { return rep.find(key); } - const_iterator find(const key_type& key) const { return rep.find(key); } - bool contains(const key_type& key) const { return rep.find(key) != rep.end(); } - -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class KT> - mapped_type& operator[](KT&& key) - { - return rep.template find_or_insert<DefaultValue>(std::forward<KT>(key)).second; - } -#else - mapped_type& operator[](const key_type& key) - { - return rep.template find_or_insert<DefaultValue>(key).second; - } -#endif - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> - equal_range(const key_type& key) { return rep.equal_range(key); } - - std::pair<const_iterator, const_iterator> - equal_range(const key_type& key) const { return rep.equal_range(key); } - - mapped_type& at(const key_type& key) - { - iterator it = rep.find(key); - if (it == rep.end()) - throw_exception(std::out_of_range("at: key not present")); - return it->second; - } - - const mapped_type& at(const key_type& key) const - { - const_iterator it = rep.find(key); - if (it == rep.cend()) - throw_exception(std::out_of_range("at: key not present")); - return it->second; - } - -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class... Args> - std::pair<iterator, bool> emplace(Args&&... args) - { - return rep.emplace(std::forward<Args>(args)...); - } - - template <class... Args> - iterator emplace_hint(const_iterator , Args&&... args) - { - return rep.emplace(std::forward<Args>(args)...).first; - } -#endif - - // Insert - // ------ - std::pair<iterator, bool> - insert(const value_type& obj) { return rep.insert(obj); } - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - template< class P > - std::pair<iterator, bool> insert(P&& obj) { return rep.insert(std::forward<P>(obj)); } -#endif - - template <class InputIterator> - void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } - - void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } - - iterator insert(iterator /*unused*/, const value_type& obj) { return insert(obj).first; } - iterator insert(const_iterator /*unused*/, const value_type& obj) { return insert(obj).first; } - - // Deleted key routines - just to keep google test framework happy - // we don't actually use the deleted key - // --------------------------------------------------------------- - void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // Erase - // ----- - size_type erase(const key_type& key) { return rep.erase(key); } - iterator erase(iterator it) { return rep.erase(it); } - iterator erase(iterator f, iterator l) { return rep.erase(f, l); } - iterator erase(const_iterator it) { return rep.erase(it); } - iterator erase(const_iterator f, const_iterator l){ return rep.erase(f, l); } - - // Comparison - // ---------- - bool operator==(const sparse_hash_map& hs) const { return rep == hs.rep; } - bool operator!=(const sparse_hash_map& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing metainformation to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - // --------------------------------------------------------------- - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - // --------------------------------------------------------------- - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) - { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer - // may need to do a const cast in order to fill in the key. - // NOTE: if Key or T are not POD types, the serializer MUST use - // placement-new to initialize their values, rather than a normal - // equals-assignment or similar. (The value_type* passed into the - // serializer points to garbage memory.) - // --------------------------------------------------------------- - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) - { - return rep.unserialize(serializer, fp); - } - - // The four methods below are DEPRECATED. - // Use serialize() and unserialize() for new code. - // ----------------------------------------------- - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } - - template <typename INPUT> - bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } - - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } - - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } - - -private: - // The actual data - // --------------- - ht rep; -}; - -// ---------------------------------------------------------------------- -// S P A R S E _ H A S H _ S E T -// ---------------------------------------------------------------------- - -template <class Value, - class HashFcn = spp_hash<Value>, - class EqualKey = std::equal_to<Value>, - class Alloc = SPP_DEFAULT_ALLOCATOR<Value> > -class sparse_hash_set -{ -private: - // Apparently identity is not stl-standard, so we define our own - struct Identity - { - typedef const Value& result_type; - inline const Value& operator()(const Value& v) const { return v; } - }; - - struct SetKey - { - inline void operator()(Value* value, const Value& new_key) const - { - *value = new_key; - } - }; - - typedef sparse_hashtable<Value, Value, HashFcn, Identity, SetKey, - EqualKey, Alloc> ht; - -public: - typedef typename ht::key_type key_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::const_pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::const_reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::const_iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::const_local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - - // Iterator functions -- recall all iterators are const - iterator begin() const { return rep.begin(); } - iterator end() const { return rep.end(); } - const_iterator cbegin() const { return rep.cbegin(); } - const_iterator cend() const { return rep.cend(); } - - // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) const { return rep.begin(i); } - local_iterator end(size_type i) const { return rep.end(i); } - local_iterator cbegin(size_type i) const { return rep.cbegin(i); } - local_iterator cend(size_type i) const { return rep.cend(i); } - - - // Accessor functions - // ------------------ - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } // tr1 name - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - // ------------ - explicit sparse_hash_set(size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) : - rep(n, hf, eql, Identity(), SetKey(), alloc) - { - } - - explicit sparse_hash_set(const allocator_type& alloc) : - rep(0, hasher(), key_equal(), Identity(), SetKey(), alloc) - { - } - - sparse_hash_set(size_type n, const allocator_type& alloc) : - rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc) - { - } - - sparse_hash_set(size_type n, const hasher& hf, - const allocator_type& alloc) : - rep(n, hf, key_equal(), Identity(), SetKey(), alloc) - { - } - - template <class InputIterator> - sparse_hash_set(InputIterator f, InputIterator l, - size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(n, hf, eql, Identity(), SetKey(), alloc) - { - rep.insert(f, l); - } - - template <class InputIterator> - sparse_hash_set(InputIterator f, InputIterator l, - size_type n, const allocator_type& alloc) - : rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc) - { - rep.insert(f, l); - } - - template <class InputIterator> - sparse_hash_set(InputIterator f, InputIterator l, - size_type n, const hasher& hf, const allocator_type& alloc) - : rep(n, hf, key_equal(), Identity(), SetKey(), alloc) - { - rep.insert(f, l); - } - - sparse_hash_set(const sparse_hash_set &o) : - rep(o.rep) - {} - - sparse_hash_set(const sparse_hash_set &o, - const allocator_type& alloc) : - rep(o.rep, alloc) - {} - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - sparse_hash_set(sparse_hash_set &&o) : - rep(std::move(o.rep)) - {} - - sparse_hash_set(sparse_hash_set &&o, - const allocator_type& alloc) : - rep(std::move(o.rep), alloc) - {} -#endif - -#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST) - sparse_hash_set(std::initializer_list<value_type> init, - size_type n = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) : - rep(n, hf, eql, Identity(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_set(std::initializer_list<value_type> init, - size_type n, const allocator_type& alloc) : - rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_set(std::initializer_list<value_type> init, - size_type n, const hasher& hf, - const allocator_type& alloc) : - rep(n, hf, key_equal(), Identity(), SetKey(), alloc) - { - rep.insert(init.begin(), init.end()); - } - - sparse_hash_set& operator=(std::initializer_list<value_type> init) - { - rep.clear(); - rep.insert(init.begin(), init.end()); - return *this; - } - - void insert(std::initializer_list<value_type> init) - { - rep.insert(init.begin(), init.end()); - } - -#endif - - sparse_hash_set& operator=(const sparse_hash_set &o) - { - rep = o.rep; - return *this; - } - - void clear() { rep.clear(); } - void swap(sparse_hash_set& hs) { rep.swap(hs.rep); } - - - // Functions concerning size - // ------------------------- - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - - float load_factor() const { return size() * 1.0f / bucket_count(); } - - float max_load_factor() const { return rep.get_enlarge_factor(); } - void max_load_factor(float grow) { rep.set_enlarge_factor(grow); } - - float min_load_factor() const { return rep.get_shrink_factor(); } - void min_load_factor(float shrink){ rep.set_shrink_factor(shrink); } - - void set_resizing_parameters(float shrink, float grow) - { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type cnt) { rep.resize(cnt); } - void rehash(size_type cnt) { resize(cnt); } // c++11 name - void reserve(size_type cnt) { resize(cnt); } // c++11 - - // Lookup - // ------ - iterator find(const key_type& key) const { return rep.find(key); } - bool contains(const key_type& key) const { return rep.find(key) != rep.end(); } - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> - equal_range(const key_type& key) const { return rep.equal_range(key); } - -#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES) - template <class... Args> - std::pair<iterator, bool> emplace(Args&&... args) - { - return rep.emplace(std::forward<Args>(args)...); - } - - template <class... Args> - iterator emplace_hint(const_iterator , Args&&... args) - { - return rep.emplace(std::forward<Args>(args)...).first; - } -#endif - - // Insert - // ------ - std::pair<iterator, bool> insert(const value_type& obj) - { - std::pair<typename ht::iterator, bool> p = rep.insert(obj); - return std::pair<iterator, bool>(p.first, p.second); // const to non-const - } - -#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES) - template<class P> - std::pair<iterator, bool> insert(P&& obj) { return rep.insert(std::forward<P>(obj)); } -#endif - - template <class InputIterator> - void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } - - void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } - - iterator insert(iterator /*unused*/, const value_type& obj) { return insert(obj).first; } - - // Deleted key - do nothing - just to keep google test framework happy - // ------------------------------------------------------------------- - void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // Erase - // ----- - size_type erase(const key_type& key) { return rep.erase(key); } - iterator erase(iterator it) { return rep.erase(it); } - iterator erase(iterator f, iterator l) { return rep.erase(f, l); } - - // Comparison - // ---------- - bool operator==(const sparse_hash_set& hs) const { return rep == hs.rep; } - bool operator!=(const sparse_hash_set& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing metainformation to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - // --------------------------------------------------------------- - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - // --------------------------------------------------------------- - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) - { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - // NOTE: Since value_type is const Key, ValueSerializer - // may need to do a const cast in order to fill in the key. - // NOTE: if Key is not a POD type, the serializer MUST use - // placement-new to initialize its value, rather than a normal - // equals-assignment or similar. (The value_type* passed into - // the serializer points to garbage memory.) - // --------------------------------------------------------------- - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) - { - return rep.unserialize(serializer, fp); - } - - // The four methods below are DEPRECATED. - // Use serialize() and unserialize() for new code. - // ----------------------------------------------- - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } - - template <typename INPUT> - bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } - - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } - - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } - -private: - // The actual data - // --------------- - ht rep; -}; - -} // spp_ namespace - - -// We need a global swap for all our classes as well -// ------------------------------------------------- - -template <class T, class Alloc> -inline void swap(spp_::sparsegroup<T,Alloc> &x, spp_::sparsegroup<T,Alloc> &y) -{ - x.swap(y); -} - -template <class T, class Alloc> -inline void swap(spp_::sparsetable<T,Alloc> &x, spp_::sparsetable<T,Alloc> &y) -{ - x.swap(y); -} - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -inline void swap(spp_::sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &x, - spp_::sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &y) -{ - x.swap(y); -} - -template <class Key, class T, class HashFcn, class EqualKey, class Alloc> -inline void swap(spp_::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1, - spp_::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2) -{ - hm1.swap(hm2); -} - -template <class Val, class HashFcn, class EqualKey, class Alloc> -inline void swap(spp_::sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1, - spp_::sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2) -{ - hs1.swap(hs2); -} - -#endif // sparsepp_h_guard_ diff --git a/benchmarks/others/sparsepp/spp_config.h b/benchmarks/others/sparsepp/spp_config.h deleted file mode 100644 index 46eeee5c..00000000 --- a/benchmarks/others/sparsepp/spp_config.h +++ /dev/null @@ -1,781 +0,0 @@ -#if !defined(spp_config_h_guard) -#define spp_config_h_guard - -// -------------------------------------------------- -// Sparsepp config macros -// some can be overriden on the command line -// -------------------------------------------------- -#ifndef SPP_NAMESPACE - #define SPP_NAMESPACE spp -#endif - -#ifndef spp_ - #define spp_ SPP_NAMESPACE -#endif - -#ifndef SPP_DEFAULT_ALLOCATOR - #if (defined(SPP_USE_SPP_ALLOC) && SPP_USE_SPP_ALLOC) && defined(_MSC_VER) - // ----------------------------------------------------------------------------- - // When building with the Microsoft compiler, we use a custom allocator because - // the default one fragments memory when reallocating. This is desirable only - // when creating large sparsepp hash maps. If you create lots of small hash_maps, - // define the following before including spp.h: - // #define SPP_DEFAULT_ALLOCATOR spp::libc_allocator - // ----------------------------------------------------------------------------- - #define SPP_DEFAULT_ALLOCATOR spp_::spp_allocator - #define SPP_INCLUDE_SPP_ALLOC - #else - #define SPP_DEFAULT_ALLOCATOR spp_::libc_allocator - #endif -#endif - -#ifndef SPP_GROUP_SIZE - // must be 32 or 64 - #define SPP_GROUP_SIZE 32 -#endif - -#ifndef SPP_ALLOC_SZ - // must be power of 2 (0 = agressive alloc, 1 = smallest memory usage, 2 = good compromise) - #define SPP_ALLOC_SZ 0 -#endif - -#ifndef SPP_STORE_NUM_ITEMS - // 1 uses a little bit more memory, but faster!! - #define SPP_STORE_NUM_ITEMS 1 -#endif - - -// --------------------------------------------------------------------------- -// Compiler detection code (SPP_ proprocessor macros) derived from Boost -// libraries. Therefore Boost software licence reproduced below. -// --------------------------------------------------------------------------- -// Boost Software License - Version 1.0 - August 17th, 2003 -// -// Permission is hereby granted, free of charge, to any person or organization -// obtaining a copy of the software and accompanying documentation covered by -// this license (the "Software") to use, reproduce, display, distribute, -// execute, and transmit the Software, and to prepare derivative works of the -// Software, and to permit third-parties to whom the Software is furnished to -// do so, all subject to the following: -// -// The copyright notices in the Software and this entire statement, including -// the above license grant, this restriction and the following disclaimer, -// must be included in all copies of the Software, in whole or in part, and -// all derivative works of the Software, unless such copies or derivative -// works are solely in the form of machine-executable object code generated by -// a source language processor. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// --------------------------------------------------------------------------- - -// Boost like configuration -// ------------------------ -#if defined __clang__ - - #if defined(i386) - #include <cpuid.h> - inline void spp_cpuid(int info[4], int InfoType) { - __cpuid_count(InfoType, 0, info[0], info[1], info[2], info[3]); - } - #endif - - #define SPP_POPCNT __builtin_popcount - #define SPP_POPCNT64 __builtin_popcountll - - #define SPP_HAS_CSTDINT - - #ifndef __has_extension - #define __has_extension __has_feature - #endif - - #if !__has_feature(cxx_exceptions) && !defined(SPP_NO_EXCEPTIONS) - #define SPP_NO_EXCEPTIONS - #endif - - #if !__has_feature(cxx_rtti) && !defined(SPP_NO_RTTI) - #define SPP_NO_RTTI - #endif - - #if !__has_feature(cxx_rtti) && !defined(SPP_NO_TYPEID) - #define SPP_NO_TYPEID - #endif - - #if defined(__int64) && !defined(__GNUC__) - #define SPP_HAS_MS_INT64 - #endif - - #define SPP_HAS_NRVO - - // Branch prediction hints - #if defined(__has_builtin) - #if __has_builtin(__builtin_expect) - #define SPP_LIKELY(x) __builtin_expect(x, 1) - #define SPP_UNLIKELY(x) __builtin_expect(x, 0) - #endif - #endif - - // Clang supports "long long" in all compilation modes. - #define SPP_HAS_LONG_LONG - - #if !__has_feature(cxx_constexpr) - #define SPP_NO_CXX11_CONSTEXPR - #endif - - #if !__has_feature(cxx_decltype) - #define SPP_NO_CXX11_DECLTYPE - #endif - - #if !__has_feature(cxx_decltype_incomplete_return_types) - #define SPP_NO_CXX11_DECLTYPE_N3276 - #endif - - #if !__has_feature(cxx_defaulted_functions) - #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS - #endif - - #if !__has_feature(cxx_deleted_functions) - #define SPP_NO_CXX11_DELETED_FUNCTIONS - #endif - - #if !__has_feature(cxx_explicit_conversions) - #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS - #endif - - #if !__has_feature(cxx_default_function_template_args) - #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS - #endif - - #if !__has_feature(cxx_generalized_initializers) - #define SPP_NO_CXX11_HDR_INITIALIZER_LIST - #endif - - #if !__has_feature(cxx_lambdas) - #define SPP_NO_CXX11_LAMBDAS - #endif - - #if !__has_feature(cxx_local_type_template_args) - #define SPP_NO_CXX11_LOCAL_CLASS_TEMPLATE_PARAMETERS - #endif - - #if !__has_feature(cxx_raw_string_literals) - #define SPP_NO_CXX11_RAW_LITERALS - #endif - - #if !__has_feature(cxx_reference_qualified_functions) - #define SPP_NO_CXX11_REF_QUALIFIERS - #endif - - #if !__has_feature(cxx_generalized_initializers) - #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX - #endif - - #if !__has_feature(cxx_rvalue_references) - #define SPP_NO_CXX11_RVALUE_REFERENCES - #endif - - #if !__has_feature(cxx_static_assert) - #define SPP_NO_CXX11_STATIC_ASSERT - #endif - - #if !__has_feature(cxx_alias_templates) - #define SPP_NO_CXX11_TEMPLATE_ALIASES - #endif - - #if !__has_feature(cxx_variadic_templates) - #define SPP_NO_CXX11_VARIADIC_TEMPLATES - #endif - - #if !__has_feature(cxx_user_literals) - #define SPP_NO_CXX11_USER_DEFINED_LITERALS - #endif - - #if !__has_feature(cxx_alignas) - #define SPP_NO_CXX11_ALIGNAS - #endif - - #if !__has_feature(cxx_trailing_return) - #define SPP_NO_CXX11_TRAILING_RESULT_TYPES - #endif - - #if !__has_feature(cxx_inline_namespaces) - #define SPP_NO_CXX11_INLINE_NAMESPACES - #endif - - #if !__has_feature(cxx_override_control) - #define SPP_NO_CXX11_FINAL - #endif - - #if !(__has_feature(__cxx_binary_literals__) || __has_extension(__cxx_binary_literals__)) - #define SPP_NO_CXX14_BINARY_LITERALS - #endif - - #if !__has_feature(__cxx_decltype_auto__) - #define SPP_NO_CXX14_DECLTYPE_AUTO - #endif - - #if !__has_feature(__cxx_init_captures__) - #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES - #endif - - #if !__has_feature(__cxx_generic_lambdas__) - #define SPP_NO_CXX14_GENERIC_LAMBDAS - #endif - - - #if !__has_feature(__cxx_generic_lambdas__) || !__has_feature(__cxx_relaxed_constexpr__) - #define SPP_NO_CXX14_CONSTEXPR - #endif - - #if !__has_feature(__cxx_return_type_deduction__) - #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION - #endif - - #if !__has_feature(__cxx_variable_templates__) - #define SPP_NO_CXX14_VARIABLE_TEMPLATES - #endif - - #if __cplusplus < 201400 - #define SPP_NO_CXX14_DIGIT_SEPARATORS - #endif - - #if defined(__has_builtin) && __has_builtin(__builtin_unreachable) - #define SPP_UNREACHABLE_RETURN(x) __builtin_unreachable(); - #endif - - #define SPP_ATTRIBUTE_UNUSED __attribute__((__unused__)) - - #ifndef SPP_COMPILER - #define SPP_COMPILER "Clang version " __clang_version__ - #endif - - #define SPP_CLANG 1 - - -#elif defined __GNUC__ - - #define SPP_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - - // definition to expand macro then apply to pragma message - // #define VALUE_TO_STRING(x) #x - // #define VALUE(x) VALUE_TO_STRING(x) - // #define VAR_NAME_VALUE(var) #var "=" VALUE(var) - // #pragma message(VAR_NAME_VALUE(SPP_GCC_VERSION)) - - #if defined(i386) - #include <cpuid.h> - inline void spp_cpuid(int info[4], int InfoType) { - __cpuid_count(InfoType, 0, info[0], info[1], info[2], info[3]); - } - #endif - - // __POPCNT__ defined when the compiled with popcount support - // (-mpopcnt compiler option is given for example) - #ifdef __POPCNT__ - // slower unless compiled iwith -mpopcnt - #define SPP_POPCNT __builtin_popcount - #define SPP_POPCNT64 __builtin_popcountll - #endif - - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || (__cplusplus >= 201103L) - #define SPP_GCC_CXX11 - #endif - - #if __GNUC__ == 3 - #if defined (__PATHSCALE__) - #define SPP_NO_TWO_PHASE_NAME_LOOKUP - #define SPP_NO_IS_ABSTRACT - #endif - - #if __GNUC_MINOR__ < 4 - #define SPP_NO_IS_ABSTRACT - #endif - - #define SPP_NO_CXX11_EXTERN_TEMPLATE - #endif - - #if __GNUC__ < 4 - // - // All problems to gcc-3.x and earlier here: - // - #define SPP_NO_TWO_PHASE_NAME_LOOKUP - #ifdef __OPEN64__ - #define SPP_NO_IS_ABSTRACT - #endif - #endif - - // GCC prior to 3.4 had #pragma once too but it didn't work well with filesystem links - #if SPP_GCC_VERSION >= 30400 - #define SPP_HAS_PRAGMA_ONCE - #endif - - #if SPP_GCC_VERSION < 40400 - // Previous versions of GCC did not completely implement value-initialization: - // GCC Bug 30111, "Value-initialization of POD base class doesn't initialize - // members", reported by Jonathan Wakely in 2006, - // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=30111 (fixed for GCC 4.4) - // GCC Bug 33916, "Default constructor fails to initialize array members", - // reported by Michael Elizabeth Chastain in 2007, - // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=33916 (fixed for GCC 4.2.4) - // See also: http://www.boost.org/libs/utility/value_init.htm #compiler_issues - #define SPP_NO_COMPLETE_VALUE_INITIALIZATION - #endif - - #if !defined(__EXCEPTIONS) && !defined(SPP_NO_EXCEPTIONS) - #define SPP_NO_EXCEPTIONS - #endif - - // - // Threading support: Turn this on unconditionally here (except for - // those platforms where we can know for sure). It will get turned off again - // later if no threading API is detected. - // - #if !defined(__MINGW32__) && !defined(linux) && !defined(__linux) && !defined(__linux__) - #define SPP_HAS_THREADS - #endif - - // - // gcc has "long long" - // Except on Darwin with standard compliance enabled (-pedantic) - // Apple gcc helpfully defines this macro we can query - // - #if !defined(__DARWIN_NO_LONG_LONG) - #define SPP_HAS_LONG_LONG - #endif - - // - // gcc implements the named return value optimization since version 3.1 - // - #define SPP_HAS_NRVO - - // Branch prediction hints - #define SPP_LIKELY(x) __builtin_expect(x, 1) - #define SPP_UNLIKELY(x) __builtin_expect(x, 0) - - // - // Dynamic shared object (DSO) and dynamic-link library (DLL) support - // - #if __GNUC__ >= 4 - #if (defined(_WIN32) || defined(__WIN32__) || defined(WIN32)) && !defined(__CYGWIN__) - // All Win32 development environments, including 64-bit Windows and MinGW, define - // _WIN32 or one of its variant spellings. Note that Cygwin is a POSIX environment, - // so does not define _WIN32 or its variants. - #define SPP_HAS_DECLSPEC - #define SPP_SYMBOL_EXPORT __attribute__((__dllexport__)) - #define SPP_SYMBOL_IMPORT __attribute__((__dllimport__)) - #else - #define SPP_SYMBOL_EXPORT __attribute__((__visibility__("default"))) - #define SPP_SYMBOL_IMPORT - #endif - - #define SPP_SYMBOL_VISIBLE __attribute__((__visibility__("default"))) - #else - // config/platform/win32.hpp will define SPP_SYMBOL_EXPORT, etc., unless already defined - #define SPP_SYMBOL_EXPORT - #endif - - // - // RTTI and typeinfo detection is possible post gcc-4.3: - // - #if SPP_GCC_VERSION > 40300 - #ifndef __GXX_RTTI - #ifndef SPP_NO_TYPEID - #define SPP_NO_TYPEID - #endif - #ifndef SPP_NO_RTTI - #define SPP_NO_RTTI - #endif - #endif - #endif - - // - // Recent GCC versions have __int128 when in 64-bit mode. - // - // We disable this if the compiler is really nvcc with C++03 as it - // doesn't actually support __int128 as of CUDA_VERSION=7500 - // even though it defines __SIZEOF_INT128__. - // See https://svn.boost.org/trac/boost/ticket/8048 - // https://svn.boost.org/trac/boost/ticket/11852 - // Only re-enable this for nvcc if you're absolutely sure - // of the circumstances under which it's supported: - // - #if defined(__CUDACC__) - #if defined(SPP_GCC_CXX11) - #define SPP_NVCC_CXX11 - #else - #define SPP_NVCC_CXX03 - #endif - #endif - - #if defined(__SIZEOF_INT128__) && !defined(SPP_NVCC_CXX03) - #define SPP_HAS_INT128 - #endif - // - // Recent GCC versions have a __float128 native type, we need to - // include a std lib header to detect this - not ideal, but we'll - // be including <cstddef> later anyway when we select the std lib. - // - // Nevertheless, as of CUDA 7.5, using __float128 with the host - // compiler in pre-C++11 mode is still not supported. - // See https://svn.boost.org/trac/boost/ticket/11852 - // - #ifdef __cplusplus - #include <cstddef> - #else - #include <stddef.h> - #endif - - #if defined(_GLIBCXX_USE_FLOAT128) && !defined(__STRICT_ANSI__) && !defined(SPP_NVCC_CXX03) - #define SPP_HAS_FLOAT128 - #endif - - // C++0x features in 4.3.n and later - // - #if (SPP_GCC_VERSION >= 40300) && defined(SPP_GCC_CXX11) - // C++0x features are only enabled when -std=c++0x or -std=gnu++0x are - // passed on the command line, which in turn defines - // __GXX_EXPERIMENTAL_CXX0X__. - #define SPP_HAS_DECLTYPE - #define SPP_HAS_RVALUE_REFS - #define SPP_HAS_STATIC_ASSERT - #define SPP_HAS_VARIADIC_TMPL - #define SPP_HAS_CSTDINT - #else - #define SPP_NO_CXX11_DECLTYPE - #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS - #define SPP_NO_CXX11_RVALUE_REFERENCES - #define SPP_NO_CXX11_STATIC_ASSERT - #endif - - // C++0x features in 4.4.n and later - // - #if (SPP_GCC_VERSION < 40400) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_AUTO_DECLARATIONS - #define SPP_NO_CXX11_AUTO_MULTIDECLARATIONS - #define SPP_NO_CXX11_CHAR16_T - #define SPP_NO_CXX11_CHAR32_T - #define SPP_NO_CXX11_HDR_INITIALIZER_LIST - #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS - #define SPP_NO_CXX11_DELETED_FUNCTIONS - #define SPP_NO_CXX11_TRAILING_RESULT_TYPES - #define SPP_NO_CXX11_INLINE_NAMESPACES - #define SPP_NO_CXX11_VARIADIC_TEMPLATES - #endif - - #if SPP_GCC_VERSION < 40500 - #define SPP_NO_SFINAE_EXPR - #endif - - // GCC 4.5 forbids declaration of defaulted functions in private or protected sections - #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ == 5) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_NON_PUBLIC_DEFAULTED_FUNCTIONS - #endif - - // C++0x features in 4.5.0 and later - // - #if (SPP_GCC_VERSION < 40500) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS - #define SPP_NO_CXX11_LAMBDAS - #define SPP_NO_CXX11_LOCAL_CLASS_TEMPLATE_PARAMETERS - #define SPP_NO_CXX11_RAW_LITERALS - #endif - - // C++0x features in 4.6.n and later - // - #if (SPP_GCC_VERSION < 40600) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_CONSTEXPR - #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX - #endif - - // C++0x features in 4.7.n and later - // - #if (SPP_GCC_VERSION < 40700) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_FINAL - #define SPP_NO_CXX11_TEMPLATE_ALIASES - #define SPP_NO_CXX11_USER_DEFINED_LITERALS - #define SPP_NO_CXX11_FIXED_LENGTH_VARIADIC_TEMPLATE_EXPANSION_PACKS - #endif - - // C++0x features in 4.8.n and later - // - #if (SPP_GCC_VERSION < 40800) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_ALIGNAS - #endif - - // C++0x features in 4.8.1 and later - // - #if (SPP_GCC_VERSION < 40801) || !defined(SPP_GCC_CXX11) - #define SPP_NO_CXX11_DECLTYPE_N3276 - #define SPP_NO_CXX11_REF_QUALIFIERS - #define SPP_NO_CXX14_BINARY_LITERALS - #endif - - // C++14 features in 4.9.0 and later - // - #if (SPP_GCC_VERSION < 40900) || (__cplusplus < 201300) - #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION - #define SPP_NO_CXX14_GENERIC_LAMBDAS - #define SPP_NO_CXX14_DIGIT_SEPARATORS - #define SPP_NO_CXX14_DECLTYPE_AUTO - #if !((SPP_GCC_VERSION >= 40801) && (SPP_GCC_VERSION < 40900) && defined(SPP_GCC_CXX11)) - #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES - #endif - #endif - - - // C++ 14: - #if !defined(__cpp_constexpr) || (__cpp_constexpr < 201304) - #define SPP_NO_CXX14_CONSTEXPR - #endif - #if !defined(__cpp_variable_templates) || (__cpp_variable_templates < 201304) - #define SPP_NO_CXX14_VARIABLE_TEMPLATES - #endif - - // - // Unused attribute: - #if __GNUC__ >= 4 - #define SPP_ATTRIBUTE_UNUSED __attribute__((__unused__)) - #endif - // - // __builtin_unreachable: - #if SPP_GCC_VERSION >= 40800 - #define SPP_UNREACHABLE_RETURN(x) __builtin_unreachable(); - #endif - - #ifndef SPP_COMPILER - #define SPP_COMPILER "GNU C++ version " __VERSION__ - #endif - - // ConceptGCC compiler: - // http://www.generic-programming.org/software/ConceptGCC/ - #ifdef __GXX_CONCEPTS__ - #define SPP_HAS_CONCEPTS - #define SPP_COMPILER "ConceptGCC version " __VERSION__ - #endif - -#elif defined _MSC_VER - - #include <intrin.h> // for __popcnt() - - #define SPP_POPCNT_CHECK // slower when defined, but we have to check! - #define spp_cpuid(info, x) __cpuid(info, x) - - #define SPP_POPCNT __popcnt - #if (SPP_GROUP_SIZE == 64 && INTPTR_MAX == INT64_MAX) - #define SPP_POPCNT64 __popcnt64 - #endif - - // Attempt to suppress VC6 warnings about the length of decorated names (obsolete): - #pragma warning( disable : 4503 ) // warning: decorated name length exceeded - - #define SPP_HAS_PRAGMA_ONCE - #define SPP_HAS_CSTDINT - - // - // versions check: - // we don't support Visual C++ prior to version 7.1: - #if _MSC_VER < 1310 - #error "Antique compiler not supported" - #endif - - #if _MSC_FULL_VER < 180020827 - #define SPP_NO_FENV_H - #endif - - #if _MSC_VER < 1400 - // although a conforming signature for swprint exists in VC7.1 - // it appears not to actually work: - #define SPP_NO_SWPRINTF - - // Our extern template tests also fail for this compiler: - #define SPP_NO_CXX11_EXTERN_TEMPLATE - - // Variadic macros do not exist for VC7.1 and lower - #define SPP_NO_CXX11_VARIADIC_MACROS - #endif - - #if _MSC_VER < 1500 // 140X == VC++ 8.0 - #undef SPP_HAS_CSTDINT - #define SPP_NO_MEMBER_TEMPLATE_FRIENDS - #endif - - #if _MSC_VER < 1600 // 150X == VC++ 9.0 - // A bug in VC9: - #define SPP_NO_ADL_BARRIER - #endif - - - // MSVC (including the latest checked version) has not yet completely - // implemented value-initialization, as is reported: - // "VC++ does not value-initialize members of derived classes without - // user-declared constructor", reported in 2009 by Sylvester Hesp: - // https: //connect.microsoft.com/VisualStudio/feedback/details/484295 - // "Presence of copy constructor breaks member class initialization", - // reported in 2009 by Alex Vakulenko: - // https: //connect.microsoft.com/VisualStudio/feedback/details/499606 - // "Value-initialization in new-expression", reported in 2005 by - // Pavel Kuznetsov (MetaCommunications Engineering): - // https: //connect.microsoft.com/VisualStudio/feedback/details/100744 - // See also: http: //www.boost.org/libs/utility/value_init.htm #compiler_issues - // (Niels Dekker, LKEB, May 2010) - #define SPP_NO_COMPLETE_VALUE_INITIALIZATION - - #ifndef _NATIVE_WCHAR_T_DEFINED - #define SPP_NO_INTRINSIC_WCHAR_T - #endif - - // - // check for exception handling support: - #if !defined(_CPPUNWIND) && !defined(SPP_NO_EXCEPTIONS) - #define SPP_NO_EXCEPTIONS - #endif - - // - // __int64 support: - // - #define SPP_HAS_MS_INT64 - #if defined(_MSC_EXTENSIONS) || (_MSC_VER >= 1400) - #define SPP_HAS_LONG_LONG - #else - #define SPP_NO_LONG_LONG - #endif - - #if (_MSC_VER >= 1400) && !defined(_DEBUG) - #define SPP_HAS_NRVO - #endif - - #if _MSC_VER >= 1500 // 150X == VC++ 9.0 - #define SPP_HAS_PRAGMA_DETECT_MISMATCH - #endif - - // - // disable Win32 API's if compiler extensions are - // turned off: - // - #if !defined(_MSC_EXTENSIONS) && !defined(SPP_DISABLE_WIN32) - #define SPP_DISABLE_WIN32 - #endif - - #if !defined(_CPPRTTI) && !defined(SPP_NO_RTTI) - #define SPP_NO_RTTI - #endif - - // - // TR1 features: - // - #if _MSC_VER >= 1700 - // #define SPP_HAS_TR1_HASH // don't know if this is true yet. - // #define SPP_HAS_TR1_TYPE_TRAITS // don't know if this is true yet. - #define SPP_HAS_TR1_UNORDERED_MAP - #define SPP_HAS_TR1_UNORDERED_SET - #endif - - // - // C++0x features - // - // See above for SPP_NO_LONG_LONG - - // C++ features supported by VC++ 10 (aka 2010) - // - #if _MSC_VER < 1600 - #define SPP_NO_CXX11_AUTO_DECLARATIONS - #define SPP_NO_CXX11_AUTO_MULTIDECLARATIONS - #define SPP_NO_CXX11_LAMBDAS - #define SPP_NO_CXX11_RVALUE_REFERENCES - #define SPP_NO_CXX11_STATIC_ASSERT - #define SPP_NO_CXX11_DECLTYPE - #endif // _MSC_VER < 1600 - - #if _MSC_VER >= 1600 - #define SPP_HAS_STDINT_H - #endif - - // C++11 features supported by VC++ 11 (aka 2012) - // - #if _MSC_VER < 1700 - #define SPP_NO_CXX11_FINAL - #endif // _MSC_VER < 1700 - - // C++11 features supported by VC++ 12 (aka 2013). - // - #if _MSC_FULL_VER < 180020827 - #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS - #define SPP_NO_CXX11_DELETED_FUNCTIONS - #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS - #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS - #define SPP_NO_CXX11_RAW_LITERALS - #define SPP_NO_CXX11_TEMPLATE_ALIASES - #define SPP_NO_CXX11_TRAILING_RESULT_TYPES - #define SPP_NO_CXX11_VARIADIC_TEMPLATES - #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX - #define SPP_NO_CXX11_DECLTYPE_N3276 - #endif - - // C++11 features supported by VC++ 14 (aka 2014) CTP1 - #if (_MSC_FULL_VER < 190021730) - #define SPP_NO_CXX11_REF_QUALIFIERS - #define SPP_NO_CXX11_USER_DEFINED_LITERALS - #define SPP_NO_CXX11_ALIGNAS - #define SPP_NO_CXX11_INLINE_NAMESPACES - #define SPP_NO_CXX14_DECLTYPE_AUTO - #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES - #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION - #define SPP_NO_CXX11_HDR_INITIALIZER_LIST - #endif - - // C++11 features not supported by any versions - #define SPP_NO_CXX11_CHAR16_T - #define SPP_NO_CXX11_CHAR32_T - #define SPP_NO_CXX11_CONSTEXPR - #define SPP_NO_SFINAE_EXPR - #define SPP_NO_TWO_PHASE_NAME_LOOKUP - - // C++ 14: - #if !defined(__cpp_binary_literals) || (__cpp_binary_literals < 201304) - #define SPP_NO_CXX14_BINARY_LITERALS - #endif - - #if !defined(__cpp_constexpr) || (__cpp_constexpr < 201304) - #define SPP_NO_CXX14_CONSTEXPR - #endif - - #if (__cplusplus < 201304) // There's no SD6 check for this.... - #define SPP_NO_CXX14_DIGIT_SEPARATORS - #endif - - #if !defined(__cpp_generic_lambdas) || (__cpp_generic_lambdas < 201304) - #define SPP_NO_CXX14_GENERIC_LAMBDAS - #endif - - #if !defined(__cpp_variable_templates) || (__cpp_variable_templates < 201304) - #define SPP_NO_CXX14_VARIABLE_TEMPLATES - #endif - -#endif - -// from boost/config/suffix.hpp -// ---------------------------- -#ifndef SPP_ATTRIBUTE_UNUSED - #define SPP_ATTRIBUTE_UNUSED -#endif - -/* - Try to persuade compilers to inline. -*/ -#ifndef SPP_FORCEINLINE - #if defined(__GNUC__) - #define SPP_FORCEINLINE __inline __attribute__ ((always_inline)) - #elif defined(_MSC_VER) - #define SPP_FORCEINLINE __forceinline - #else - #define SPP_FORCEINLINE inline - #endif -#endif - - -#endif // spp_config_h_guard diff --git a/benchmarks/others/sparsepp/spp_dlalloc.h b/benchmarks/others/sparsepp/spp_dlalloc.h deleted file mode 100644 index f88aab7c..00000000 --- a/benchmarks/others/sparsepp/spp_dlalloc.h +++ /dev/null @@ -1,4044 +0,0 @@ -#ifndef spp_dlalloc__h_ -#define spp_dlalloc__h_ - -/* This is a C++ allocator created from Doug Lea's dlmalloc - (Version 2.8.6 Wed Aug 29 06:57:58 2012) - see: http://g.oswego.edu/dl/html/malloc.html -*/ - -#include "spp_utils.h" -#include "spp_smartptr.h" - - -#ifndef SPP_FORCEINLINE - #if defined(__GNUC__) - #define SPP_FORCEINLINE __inline __attribute__ ((always_inline)) - #elif defined(_MSC_VER) - #define SPP_FORCEINLINE __forceinline - #else - #define SPP_FORCEINLINE inline - #endif -#endif - - -#ifndef SPP_IMPL - #define SPP_IMPL SPP_FORCEINLINE -#endif - -#ifndef SPP_API - #define SPP_API static -#endif - - -namespace spp -{ - // ---------------------- allocator internal API ----------------------- - typedef void* mspace; - - /* - create_mspace creates and returns a new independent space with the - given initial capacity, or, if 0, the default granularity size. It - returns null if there is no system memory available to create the - space. If argument locked is non-zero, the space uses a separate - lock to control access. The capacity of the space will grow - dynamically as needed to service mspace_malloc requests. You can - control the sizes of incremental increases of this space by - compiling with a different SPP_DEFAULT_GRANULARITY or dynamically - setting with mallopt(M_GRANULARITY, value). - */ - SPP_API mspace create_mspace(size_t capacity, int locked); - SPP_API size_t destroy_mspace(mspace msp); - SPP_API void* mspace_malloc(mspace msp, size_t bytes); - SPP_API void mspace_free(mspace msp, void* mem); - SPP_API void* mspace_realloc(mspace msp, void* mem, size_t newsize); - -#if 0 - SPP_API mspace create_mspace_with_base(void* base, size_t capacity, int locked); - SPP_API int mspace_track_large_chunks(mspace msp, int enable); - SPP_API void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size); - SPP_API void* mspace_memalign(mspace msp, size_t alignment, size_t bytes); - SPP_API void** mspace_independent_calloc(mspace msp, size_t n_elements, - size_t elem_size, void* chunks[]); - SPP_API void** mspace_independent_comalloc(mspace msp, size_t n_elements, - size_t sizes[], void* chunks[]); - SPP_API size_t mspace_footprint(mspace msp); - SPP_API size_t mspace_max_footprint(mspace msp); - SPP_API size_t mspace_usable_size(const void* mem); - SPP_API int mspace_trim(mspace msp, size_t pad); - SPP_API int mspace_mallopt(int, int); -#endif - - // ----------------------------------------------------------- - // ----------------------------------------------------------- - struct MSpace : public spp_rc - { - MSpace() : - _sp(create_mspace(0, 0)) - {} - - ~MSpace() - { - destroy_mspace(_sp); - } - - mspace _sp; - }; - - // ----------------------------------------------------------- - // ----------------------------------------------------------- - template<class T> - class spp_allocator - { - public: - typedef T value_type; - typedef T* pointer; - typedef ptrdiff_t difference_type; - typedef const T* const_pointer; - typedef size_t size_type; - - MSpace *getSpace() const { return _space.get(); } - - spp_allocator() : _space(new MSpace) {} - - template<class U> - spp_allocator(const spp_allocator<U> &o) : _space(o.getSpace()) {} - - template<class U> - spp_allocator& operator=(const spp_allocator<U> &o) - { - if (&o != this) - _space = o.getSpace(); - return *this; - } - - void swap(spp_allocator &o) - { - std::swap(_space, o._space); - } - - pointer allocate(size_t n, const_pointer /* unused */ = 0) - { - pointer res = static_cast<pointer>(mspace_malloc(_space->_sp, n * sizeof(T))); - if (!res) - throw std::bad_alloc(); - return res; - } - - void deallocate(pointer p, size_t /* unused */) - { - mspace_free(_space->_sp, p); - } - - pointer reallocate(pointer p, size_t new_size) - { - pointer res = static_cast<pointer>(mspace_realloc(_space->_sp, p, new_size * sizeof(T))); - if (!res) - throw std::bad_alloc(); - return res; - } - - pointer reallocate(pointer p, size_type /* old_size */, size_t new_size) - { - return reallocate(p, new_size); - } - - size_type max_size() const - { - return static_cast<size_type>(-1) / sizeof(value_type); - } - - void construct(pointer p, const value_type& val) - { - new (p) value_type(val); - } - - void destroy(pointer p) { p->~value_type(); } - - template<class U> - struct rebind - { - // rebind to libc_allocator because we want to use malloc_inspect_all in destructive_iterator - // to reduce peak memory usage (we don't want <group_items> mixed with value_type when - // we traverse the allocated memory). - typedef spp::spp_allocator<U> other; - }; - - mspace space() const { return _space->_sp; } - - // check if we can clear the whole allocator memory at once => works only if the allocator - // is not be shared. If can_clear() returns true, we expect that the next allocator call - // will be clear() - not allocate() or deallocate() - bool can_clear() - { - assert(!_space_to_clear); - _space_to_clear.reset(); - _space_to_clear.swap(_space); - if (_space_to_clear->count() == 1) - return true; - else - _space_to_clear.swap(_space); - return false; - } - - void clear() - { - assert(!_space && !!_space_to_clear); - _space_to_clear.reset(); - _space = new MSpace; - } - - private: - spp_sptr<MSpace> _space; - spp_sptr<MSpace> _space_to_clear; - }; -} - - -// allocators are "equal" whenever memory allocated with one can be deallocated with the other -template<class T> -inline bool operator==(const spp_::spp_allocator<T> &a, const spp_::spp_allocator<T> &b) -{ - return a.space() == b.space(); -} - -template<class T> -inline bool operator!=(const spp_::spp_allocator<T> &a, const spp_::spp_allocator<T> &b) -{ - return !(a == b); -} - -namespace std -{ - template <class T> - inline void swap(spp_::spp_allocator<T> &a, spp_::spp_allocator<T> &b) - { - a.swap(b); - } -} - -#if !defined(SPP_EXCLUDE_IMPLEMENTATION) - -#ifndef WIN32 - #ifdef _WIN32 - #define WIN32 1 - #endif - #ifdef _WIN32_WCE - #define SPP_LACKS_FCNTL_H - #define WIN32 1 - #endif -#endif - -#ifdef WIN32 - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <tchar.h> - #define SPP_HAVE_MMAP 1 - #define SPP_LACKS_UNISTD_H - #define SPP_LACKS_SYS_PARAM_H - #define SPP_LACKS_SYS_MMAN_H - #define SPP_LACKS_STRING_H - #define SPP_LACKS_STRINGS_H - #define SPP_LACKS_SYS_TYPES_H - #define SPP_LACKS_ERRNO_H - #define SPP_LACKS_SCHED_H - #ifndef SPP_MALLOC_FAILURE_ACTION - #define SPP_MALLOC_FAILURE_ACTION - #endif - #ifndef SPP_MMAP_CLEARS - #ifdef _WIN32_WCE /* WINCE reportedly does not clear */ - #define SPP_MMAP_CLEARS 0 - #else - #define SPP_MMAP_CLEARS 1 - #endif - #endif -#endif - -#if defined(DARWIN) || defined(_DARWIN) - #define SPP_HAVE_MMAP 1 - /* OSX allocators provide 16 byte alignment */ - #ifndef SPP_MALLOC_ALIGNMENT - #define SPP_MALLOC_ALIGNMENT ((size_t)16U) - #endif -#endif - -#ifndef SPP_LACKS_SYS_TYPES_H - #include <sys/types.h> /* For size_t */ -#endif - -#ifndef SPP_MALLOC_ALIGNMENT - #define SPP_MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *))) -#endif - -/* ------------------- size_t and alignment properties -------------------- */ -static const size_t spp_max_size_t = ~(size_t)0; -static const size_t spp_size_t_bitsize = sizeof(size_t) << 3; -static const size_t spp_half_max_size_t = spp_max_size_t / 2U; -static const size_t spp_chunk_align_mask = SPP_MALLOC_ALIGNMENT - 1; - -#if defined(SPP_DEBUG) || !defined(NDEBUG) -static bool spp_is_aligned(void *p) { return ((size_t)p & spp_chunk_align_mask) == 0; } -#endif - -// the number of bytes to offset an address to align it -static size_t align_offset(void *p) -{ - return (((size_t)p & spp_chunk_align_mask) == 0) ? 0 : - ((SPP_MALLOC_ALIGNMENT - ((size_t)p & spp_chunk_align_mask)) & spp_chunk_align_mask); -} - - -#ifndef SPP_FOOTERS - #define SPP_FOOTERS 0 -#endif - -#ifndef SPP_ABORT - #define SPP_ABORT abort() -#endif - -#ifndef SPP_ABORT_ON_ASSERT_FAILURE - #define SPP_ABORT_ON_ASSERT_FAILURE 1 -#endif - -#ifndef SPP_PROCEED_ON_ERROR - #define SPP_PROCEED_ON_ERROR 0 -#endif - -#ifndef SPP_INSECURE - #define SPP_INSECURE 0 -#endif - -#ifndef SPP_MALLOC_INSPECT_ALL - #define SPP_MALLOC_INSPECT_ALL 0 -#endif - -#ifndef SPP_HAVE_MMAP - #define SPP_HAVE_MMAP 1 -#endif - -#ifndef SPP_MMAP_CLEARS - #define SPP_MMAP_CLEARS 1 -#endif - -#ifndef SPP_HAVE_MREMAP - #ifdef linux - #define SPP_HAVE_MREMAP 1 - #ifndef _GNU_SOURCE - #define _GNU_SOURCE /* Turns on mremap() definition */ - #endif - #else - #define SPP_HAVE_MREMAP 0 - #endif -#endif - -#ifndef SPP_MALLOC_FAILURE_ACTION - // ENOMEM = 12 - #define SPP_MALLOC_FAILURE_ACTION errno = 12 -#endif - - -#ifndef SPP_DEFAULT_GRANULARITY - #if defined(WIN32) - #define SPP_DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */ - #else - #define SPP_DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U) - #endif -#endif - -#ifndef SPP_DEFAULT_TRIM_THRESHOLD - #define SPP_DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U) -#endif - -#ifndef SPP_DEFAULT_MMAP_THRESHOLD - #if SPP_HAVE_MMAP - #define SPP_DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U) - #else - #define SPP_DEFAULT_MMAP_THRESHOLD spp_max_size_t - #endif -#endif - -#ifndef SPP_MAX_RELEASE_CHECK_RATE - #if SPP_HAVE_MMAP - #define SPP_MAX_RELEASE_CHECK_RATE 4095 - #else - #define SPP_MAX_RELEASE_CHECK_RATE spp_max_size_t - #endif -#endif - -#ifndef SPP_USE_BUILTIN_FFS - #define SPP_USE_BUILTIN_FFS 0 -#endif - -#ifndef SPP_USE_DEV_RANDOM - #define SPP_USE_DEV_RANDOM 0 -#endif - -#ifndef SPP_NO_SEGMENT_TRAVERSAL - #define SPP_NO_SEGMENT_TRAVERSAL 0 -#endif - - - -/*------------------------------ internal #includes ---------------------- */ - -#ifdef _MSC_VER - #pragma warning( disable : 4146 ) /* no "unsigned" warnings */ -#endif -#ifndef SPP_LACKS_ERRNO_H - #include <errno.h> /* for SPP_MALLOC_FAILURE_ACTION */ -#endif - -#ifdef SPP_DEBUG - #if SPP_ABORT_ON_ASSERT_FAILURE - #undef assert - #define assert(x) if(!(x)) SPP_ABORT - #else - #include <assert.h> - #endif -#else - #ifndef assert - #define assert(x) - #endif - #define SPP_DEBUG 0 -#endif - -#if !defined(WIN32) && !defined(SPP_LACKS_TIME_H) - #include <time.h> /* for magic initialization */ -#endif - -#ifndef SPP_LACKS_STDLIB_H - #include <stdlib.h> /* for abort() */ -#endif - -#ifndef SPP_LACKS_STRING_H - #include <string.h> /* for memset etc */ -#endif - -#if SPP_USE_BUILTIN_FFS - #ifndef SPP_LACKS_STRINGS_H - #include <strings.h> /* for ffs */ - #endif -#endif - -#if SPP_HAVE_MMAP - #ifndef SPP_LACKS_SYS_MMAN_H - /* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */ - #if (defined(linux) && !defined(__USE_GNU)) - #define __USE_GNU 1 - #include <sys/mman.h> /* for mmap */ - #undef __USE_GNU - #else - #include <sys/mman.h> /* for mmap */ - #endif - #endif - #ifndef SPP_LACKS_FCNTL_H - #include <fcntl.h> - #endif -#endif - -#ifndef SPP_LACKS_UNISTD_H - #include <unistd.h> /* for sbrk, sysconf */ -#else - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) - extern void* sbrk(ptrdiff_t); - #endif -#endif - -#include <new> - -namespace spp -{ - -/* Declarations for bit scanning on win32 */ -#if defined(_MSC_VER) && _MSC_VER>=1300 - #ifndef BitScanForward /* Try to avoid pulling in WinNT.h */ - extern "C" { - unsigned char _BitScanForward(unsigned long *index, unsigned long mask); - unsigned char _BitScanReverse(unsigned long *index, unsigned long mask); - } - - #define BitScanForward _BitScanForward - #define BitScanReverse _BitScanReverse - #pragma intrinsic(_BitScanForward) - #pragma intrinsic(_BitScanReverse) - #endif /* BitScanForward */ -#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */ - -#ifndef WIN32 - #ifndef malloc_getpagesize - #ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */ - #ifndef _SC_PAGE_SIZE - #define _SC_PAGE_SIZE _SC_PAGESIZE - #endif - #endif - #ifdef _SC_PAGE_SIZE - #define malloc_getpagesize sysconf(_SC_PAGE_SIZE) - #else - #if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE) - extern size_t getpagesize(); - #define malloc_getpagesize getpagesize() - #else - #ifdef WIN32 /* use supplied emulation of getpagesize */ - #define malloc_getpagesize getpagesize() - #else - #ifndef SPP_LACKS_SYS_PARAM_H - #include <sys/param.h> - #endif - #ifdef EXEC_PAGESIZE - #define malloc_getpagesize EXEC_PAGESIZE - #else - #ifdef NBPG - #ifndef CLSIZE - #define malloc_getpagesize NBPG - #else - #define malloc_getpagesize (NBPG * CLSIZE) - #endif - #else - #ifdef NBPC - #define malloc_getpagesize NBPC - #else - #ifdef PAGESIZE - #define malloc_getpagesize PAGESIZE - #else /* just guess */ - #define malloc_getpagesize ((size_t)4096U) - #endif - #endif - #endif - #endif - #endif - #endif - #endif - #endif -#endif - -/* -------------------------- MMAP preliminaries ------------------------- */ - -/* - If SPP_HAVE_MORECORE or SPP_HAVE_MMAP are false, we just define calls and - checks to fail so compiler optimizer can delete code rather than - using so many "#if"s. -*/ - - -/* MMAP must return mfail on failure */ -static void *mfail = (void*)spp_max_size_t; -static char *cmfail = (char*)mfail; - -#if SPP_HAVE_MMAP - -#ifndef WIN32 - #define SPP_MUNMAP_DEFAULT(a, s) munmap((a), (s)) - #define SPP_MMAP_PROT (PROT_READ | PROT_WRITE) - #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) - #define MAP_ANONYMOUS MAP_ANON - #endif - - #ifdef MAP_ANONYMOUS - #define SPP_MMAP_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS) - #define SPP_MMAP_DEFAULT(s) mmap(0, (s), SPP_MMAP_PROT, SPP_MMAP_FLAGS, -1, 0) - #else /* MAP_ANONYMOUS */ - /* - Nearly all versions of mmap support MAP_ANONYMOUS, so the following - is unlikely to be needed, but is supplied just in case. - */ - #define SPP_MMAP_FLAGS (MAP_PRIVATE) - static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */ - void SPP_MMAP_DEFAULT(size_t s) - { - if (dev_zero_fd < 0) - dev_zero_fd = open("/dev/zero", O_RDWR); - mmap(0, s, SPP_MMAP_PROT, SPP_MMAP_FLAGS, dev_zero_fd, 0); - } - #endif /* MAP_ANONYMOUS */ - - #define SPP_DIRECT_MMAP_DEFAULT(s) SPP_MMAP_DEFAULT(s) - -#else /* WIN32 */ - - /* Win32 MMAP via VirtualAlloc */ - static SPP_FORCEINLINE void* win32mmap(size_t size) - { - void* ptr = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - return (ptr != 0) ? ptr : mfail; - } - - /* For direct MMAP, use MEM_TOP_DOWN to minimize interference */ - static SPP_FORCEINLINE void* win32direct_mmap(size_t size) - { - void* ptr = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, - PAGE_READWRITE); - return (ptr != 0) ? ptr : mfail; - } - - /* This function supports releasing coalesed segments */ - static SPP_FORCEINLINE int win32munmap(void* ptr, size_t size) - { - MEMORY_BASIC_INFORMATION minfo; - char* cptr = (char*)ptr; - while (size) - { - if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0) - return -1; - if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr || - minfo.State != MEM_COMMIT || minfo.RegionSize > size) - return -1; - if (VirtualFree(cptr, 0, MEM_RELEASE) == 0) - return -1; - cptr += minfo.RegionSize; - size -= minfo.RegionSize; - } - return 0; - } - - #define SPP_MMAP_DEFAULT(s) win32mmap(s) - #define SPP_MUNMAP_DEFAULT(a, s) win32munmap((a), (s)) - #define SPP_DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s) -#endif /* WIN32 */ -#endif /* SPP_HAVE_MMAP */ - -#if SPP_HAVE_MREMAP - #ifndef WIN32 - #define SPP_MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv)) - #endif -#endif - -/** - * Define SPP_CALL_MMAP/SPP_CALL_MUNMAP/SPP_CALL_DIRECT_MMAP - */ -#if SPP_HAVE_MMAP - #define USE_MMAP_BIT 1 - - #ifdef SPP_MMAP - #define SPP_CALL_MMAP(s) SPP_MMAP(s) - #else - #define SPP_CALL_MMAP(s) SPP_MMAP_DEFAULT(s) - #endif - - #ifdef SPP_MUNMAP - #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP((a), (s)) - #else - #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP_DEFAULT((a), (s)) - #endif - - #ifdef SPP_DIRECT_MMAP - #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP(s) - #else - #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP_DEFAULT(s) - #endif - -#else /* SPP_HAVE_MMAP */ - #define USE_MMAP_BIT 0 - - #define SPP_MMAP(s) mfail - #define SPP_MUNMAP(a, s) (-1) - #define SPP_DIRECT_MMAP(s) mfail - #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP(s) - #define SPP_CALL_MMAP(s) SPP_MMAP(s) - #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP((a), (s)) -#endif - -/** - * Define SPP_CALL_MREMAP - */ -#if SPP_HAVE_MMAP && SPP_HAVE_MREMAP - #ifdef MREMAP - #define SPP_CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv)) - #else - #define SPP_CALL_MREMAP(addr, osz, nsz, mv) SPP_MREMAP_DEFAULT((addr), (osz), (nsz), (mv)) - #endif -#else - #define SPP_CALL_MREMAP(addr, osz, nsz, mv) mfail -#endif - -/* mstate bit set if continguous morecore disabled or failed */ -static const unsigned USE_NONCONTIGUOUS_BIT = 4U; - -/* segment bit set in create_mspace_with_base */ -static const unsigned EXTERN_BIT = 8U; - - -/* --------------------------- flags ------------------------ */ - -static const unsigned PINUSE_BIT = 1; -static const unsigned CINUSE_BIT = 2; -static const unsigned FLAG4_BIT = 4; -static const unsigned INUSE_BITS = (PINUSE_BIT | CINUSE_BIT); -static const unsigned FLAG_BITS = (PINUSE_BIT | CINUSE_BIT | FLAG4_BIT); - -/* ------------------- Chunks sizes and alignments ----------------------- */ - -#if SPP_FOOTERS - static const unsigned CHUNK_OVERHEAD = 2 * sizeof(size_t); -#else - static const unsigned CHUNK_OVERHEAD = sizeof(size_t); -#endif - -/* MMapped chunks need a second word of overhead ... */ -static const unsigned SPP_MMAP_CHUNK_OVERHEAD = 2 * sizeof(size_t); - -/* ... and additional padding for fake next-chunk at foot */ -static const unsigned SPP_MMAP_FOOT_PAD = 4 * sizeof(size_t); - -// =============================================================================== -struct malloc_chunk_header -{ - void set_size_and_pinuse_of_free_chunk(size_t s) - { - _head = s | PINUSE_BIT; - set_foot(s); - } - - void set_foot(size_t s) - { - ((malloc_chunk_header *)((char*)this + s))->_prev_foot = s; - } - - // extraction of fields from head words - bool cinuse() const { return !!(_head & CINUSE_BIT); } - bool pinuse() const { return !!(_head & PINUSE_BIT); } - bool flag4inuse() const { return !!(_head & FLAG4_BIT); } - bool is_inuse() const { return (_head & INUSE_BITS) != PINUSE_BIT; } - bool is_mmapped() const { return (_head & INUSE_BITS) == 0; } - - size_t chunksize() const { return _head & ~(FLAG_BITS); } - - void clear_pinuse() { _head &= ~PINUSE_BIT; } - void set_flag4() { _head |= FLAG4_BIT; } - void clear_flag4() { _head &= ~FLAG4_BIT; } - - // Treat space at ptr +/- offset as a chunk - malloc_chunk_header * chunk_plus_offset(size_t s) - { - return (malloc_chunk_header *)((char*)this + s); - } - malloc_chunk_header * chunk_minus_offset(size_t s) - { - return (malloc_chunk_header *)((char*)this - s); - } - - // Ptr to next or previous physical malloc_chunk. - malloc_chunk_header * next_chunk() - { - return (malloc_chunk_header *)((char*)this + (_head & ~FLAG_BITS)); - } - malloc_chunk_header * prev_chunk() - { - return (malloc_chunk_header *)((char*)this - (_prev_foot)); - } - - // extract next chunk's pinuse bit - size_t next_pinuse() { return next_chunk()->_head & PINUSE_BIT; } - - size_t _prev_foot; // Size of previous chunk (if free). - size_t _head; // Size and inuse bits. -}; - -// =============================================================================== -struct malloc_chunk : public malloc_chunk_header -{ - // Set size, pinuse bit, foot, and clear next pinuse - void set_free_with_pinuse(size_t s, malloc_chunk* n) - { - n->clear_pinuse(); - set_size_and_pinuse_of_free_chunk(s); - } - - // Get the internal overhead associated with chunk p - size_t overhead_for() { return is_mmapped() ? SPP_MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD; } - - // Return true if malloced space is not necessarily cleared - bool calloc_must_clear() - { -#if SPP_MMAP_CLEARS - return !is_mmapped(); -#else - return true; -#endif - } - - struct malloc_chunk* _fd; // double links -- used only if free. - struct malloc_chunk* _bk; -}; - -static const unsigned MCHUNK_SIZE = sizeof(malloc_chunk); - -/* The smallest size we can malloc is an aligned minimal chunk */ -static const unsigned MIN_CHUNK_SIZE = (MCHUNK_SIZE + spp_chunk_align_mask) & ~spp_chunk_align_mask; - -typedef malloc_chunk mchunk; -typedef malloc_chunk* mchunkptr; -typedef malloc_chunk_header *hchunkptr; -typedef malloc_chunk* sbinptr; // The type of bins of chunks -typedef unsigned int bindex_t; // Described below -typedef unsigned int binmap_t; // Described below -typedef unsigned int flag_t; // The type of various bit flag sets - -// conversion from malloc headers to user pointers, and back -static SPP_FORCEINLINE void *chunk2mem(const void *p) { return (void *)((char *)p + 2 * sizeof(size_t)); } -static SPP_FORCEINLINE mchunkptr mem2chunk(const void *mem) { return (mchunkptr)((char *)mem - 2 * sizeof(size_t)); } - -// chunk associated with aligned address A -static SPP_FORCEINLINE mchunkptr align_as_chunk(char *A) { return (mchunkptr)(A + align_offset(chunk2mem(A))); } - -// Bounds on request (not chunk) sizes. -static const unsigned MAX_REQUEST = (-MIN_CHUNK_SIZE) << 2; -static const unsigned MIN_REQUEST = MIN_CHUNK_SIZE - CHUNK_OVERHEAD - 1; - -// pad request bytes into a usable size -static SPP_FORCEINLINE size_t pad_request(size_t req) -{ - return (req + CHUNK_OVERHEAD + spp_chunk_align_mask) & ~spp_chunk_align_mask; -} - -// pad request, checking for minimum (but not maximum) -static SPP_FORCEINLINE size_t request2size(size_t req) -{ - return req < MIN_REQUEST ? MIN_CHUNK_SIZE : pad_request(req); -} - - -/* ------------------ Operations on head and foot fields ----------------- */ - -/* - The head field of a chunk is or'ed with PINUSE_BIT when previous - adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in - use, unless mmapped, in which case both bits are cleared. - - FLAG4_BIT is not used by this malloc, but might be useful in extensions. -*/ - -// Head value for fenceposts -static const unsigned FENCEPOST_HEAD = INUSE_BITS | sizeof(size_t); - - -/* ---------------------- Overlaid data structures ----------------------- */ - -/* - When chunks are not in use, they are treated as nodes of either - lists or trees. - - "Small" chunks are stored in circular doubly-linked lists, and look - like this: - - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of previous chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `head:' | Size of chunk, in bytes |P| - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Forward pointer to next chunk in list | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Back pointer to previous chunk in list | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Unused space (may be 0 bytes long) . - . . - . | -nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `foot:' | Size of chunk, in bytes | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - Larger chunks are kept in a form of bitwise digital trees (aka - tries) keyed on chunksizes. Because malloc_tree_chunks are only for - free chunks greater than 256 bytes, their size doesn't impose any - constraints on user chunk sizes. Each node looks like: - - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of previous chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `head:' | Size of chunk, in bytes |P| - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Forward pointer to next chunk of same size | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Back pointer to previous chunk of same size | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to left child (child[0]) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to right child (child[1]) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to parent | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | bin index of this chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Unused space . - . | -nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `foot:' | Size of chunk, in bytes | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - Each tree holding treenodes is a tree of unique chunk sizes. Chunks - of the same size are arranged in a circularly-linked list, with only - the oldest chunk (the next to be used, in our FIFO ordering) - actually in the tree. (Tree members are distinguished by a non-null - parent pointer.) If a chunk with the same size an an existing node - is inserted, it is linked off the existing node using pointers that - work in the same way as fd/bk pointers of small chunks. - - Each tree contains a power of 2 sized range of chunk sizes (the - smallest is 0x100 <= x < 0x180), which is is divided in half at each - tree level, with the chunks in the smaller half of the range (0x100 - <= x < 0x140 for the top nose) in the left subtree and the larger - half (0x140 <= x < 0x180) in the right subtree. This is, of course, - done by inspecting individual bits. - - Using these rules, each node's left subtree contains all smaller - sizes than its right subtree. However, the node at the root of each - subtree has no particular ordering relationship to either. (The - dividing line between the subtree sizes is based on trie relation.) - If we remove the last chunk of a given size from the interior of the - tree, we need to replace it with a leaf node. The tree ordering - rules permit a node to be replaced by any leaf below it. - - The smallest chunk in a tree (a common operation in a best-fit - allocator) can be found by walking a path to the leftmost leaf in - the tree. Unlike a usual binary tree, where we follow left child - pointers until we reach a null, here we follow the right child - pointer any time the left one is null, until we reach a leaf with - both child pointers null. The smallest chunk in the tree will be - somewhere along that path. - - The worst case number of steps to add, find, or remove a node is - bounded by the number of bits differentiating chunks within - bins. Under current bin calculations, this ranges from 6 up to 21 - (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case - is of course much better. -*/ - -// =============================================================================== -struct malloc_tree_chunk : public malloc_chunk_header -{ - malloc_tree_chunk *leftmost_child() - { - return _child[0] ? _child[0] : _child[1]; - } - - - malloc_tree_chunk* _fd; - malloc_tree_chunk* _bk; - - malloc_tree_chunk* _child[2]; - malloc_tree_chunk* _parent; - bindex_t _index; -}; - -typedef malloc_tree_chunk tchunk; -typedef malloc_tree_chunk* tchunkptr; -typedef malloc_tree_chunk* tbinptr; // The type of bins of trees - -/* ----------------------------- Segments -------------------------------- */ - -/* - Each malloc space may include non-contiguous segments, held in a - list headed by an embedded malloc_segment record representing the - top-most space. Segments also include flags holding properties of - the space. Large chunks that are directly allocated by mmap are not - included in this list. They are instead independently created and - destroyed without otherwise keeping track of them. - - Segment management mainly comes into play for spaces allocated by - MMAP. Any call to MMAP might or might not return memory that is - adjacent to an existing segment. MORECORE normally contiguously - extends the current space, so this space is almost always adjacent, - which is simpler and faster to deal with. (This is why MORECORE is - used preferentially to MMAP when both are available -- see - sys_alloc.) When allocating using MMAP, we don't use any of the - hinting mechanisms (inconsistently) supported in various - implementations of unix mmap, or distinguish reserving from - committing memory. Instead, we just ask for space, and exploit - contiguity when we get it. It is probably possible to do - better than this on some systems, but no general scheme seems - to be significantly better. - - Management entails a simpler variant of the consolidation scheme - used for chunks to reduce fragmentation -- new adjacent memory is - normally prepended or appended to an existing segment. However, - there are limitations compared to chunk consolidation that mostly - reflect the fact that segment processing is relatively infrequent - (occurring only when getting memory from system) and that we - don't expect to have huge numbers of segments: - - * Segments are not indexed, so traversal requires linear scans. (It - would be possible to index these, but is not worth the extra - overhead and complexity for most programs on most platforms.) - * New segments are only appended to old ones when holding top-most - memory; if they cannot be prepended to others, they are held in - different segments. - - Except for the top-most segment of an mstate, each segment record - is kept at the tail of its segment. Segments are added by pushing - segment records onto the list headed by &mstate.seg for the - containing mstate. - - Segment flags control allocation/merge/deallocation policies: - * If EXTERN_BIT set, then we did not allocate this segment, - and so should not try to deallocate or merge with others. - (This currently holds only for the initial segment passed - into create_mspace_with_base.) - * If USE_MMAP_BIT set, the segment may be merged with - other surrounding mmapped segments and trimmed/de-allocated - using munmap. - * If neither bit is set, then the segment was obtained using - MORECORE so can be merged with surrounding MORECORE'd segments - and deallocated/trimmed using MORECORE with negative arguments. -*/ - -// =============================================================================== -struct malloc_segment -{ - bool is_mmapped_segment() { return !!(_sflags & USE_MMAP_BIT); } - bool is_extern_segment() { return !!(_sflags & EXTERN_BIT); } - - char* _base; // base address - size_t _size; // allocated size - malloc_segment* _next; // ptr to next segment - flag_t _sflags; // mmap and extern flag -}; - -typedef malloc_segment msegment; -typedef malloc_segment* msegmentptr; - -/* ------------- Malloc_params ------------------- */ - -/* - malloc_params holds global properties, including those that can be - dynamically set using mallopt. There is a single instance, mparams, - initialized in init_mparams. Note that the non-zeroness of "magic" - also serves as an initialization flag. -*/ - -// =============================================================================== -struct malloc_params -{ - malloc_params() : _magic(0) {} - - void ensure_initialization() - { - if (!_magic) - _init(); - } - - SPP_IMPL int change(int param_number, int value); - - size_t page_align(size_t sz) - { - return (sz + (_page_size - 1)) & ~(_page_size - 1); - } - - size_t granularity_align(size_t sz) - { - return (sz + (_granularity - 1)) & ~(_granularity - 1); - } - - bool is_page_aligned(char *S) - { - return ((size_t)S & (_page_size - 1)) == 0; - } - - SPP_IMPL int _init(); - - size_t _magic; - size_t _page_size; - size_t _granularity; - size_t _mmap_threshold; - size_t _trim_threshold; - flag_t _default_mflags; -}; - -static malloc_params mparams; - -/* ---------------------------- malloc_state ----------------------------- */ - -/* - A malloc_state holds all of the bookkeeping for a space. - The main fields are: - - Top - The topmost chunk of the currently active segment. Its size is - cached in topsize. The actual size of topmost space is - topsize+TOP_FOOT_SIZE, which includes space reserved for adding - fenceposts and segment records if necessary when getting more - space from the system. The size at which to autotrim top is - cached from mparams in trim_check, except that it is disabled if - an autotrim fails. - - Designated victim (dv) - This is the preferred chunk for servicing small requests that - don't have exact fits. It is normally the chunk split off most - recently to service another small request. Its size is cached in - dvsize. The link fields of this chunk are not maintained since it - is not kept in a bin. - - SmallBins - An array of bin headers for free chunks. These bins hold chunks - with sizes less than MIN_LARGE_SIZE bytes. Each bin contains - chunks of all the same size, spaced 8 bytes apart. To simplify - use in double-linked lists, each bin header acts as a malloc_chunk - pointing to the real first node, if it exists (else pointing to - itself). This avoids special-casing for headers. But to avoid - waste, we allocate only the fd/bk pointers of bins, and then use - repositioning tricks to treat these as the fields of a chunk. - - TreeBins - Treebins are pointers to the roots of trees holding a range of - sizes. There are 2 equally spaced treebins for each power of two - from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything - larger. - - Bin maps - There is one bit map for small bins ("smallmap") and one for - treebins ("treemap). Each bin sets its bit when non-empty, and - clears the bit when empty. Bit operations are then used to avoid - bin-by-bin searching -- nearly all "search" is done without ever - looking at bins that won't be selected. The bit maps - conservatively use 32 bits per map word, even if on 64bit system. - For a good description of some of the bit-based techniques used - here, see Henry S. Warren Jr's book "Hacker's Delight" (and - supplement at http://hackersdelight.org/). Many of these are - intended to reduce the branchiness of paths through malloc etc, as - well as to reduce the number of memory locations read or written. - - Segments - A list of segments headed by an embedded malloc_segment record - representing the initial space. - - Address check support - The least_addr field is the least address ever obtained from - MORECORE or MMAP. Attempted frees and reallocs of any address less - than this are trapped (unless SPP_INSECURE is defined). - - Magic tag - A cross-check field that should always hold same value as mparams._magic. - - Max allowed footprint - The maximum allowed bytes to allocate from system (zero means no limit) - - Flags - Bits recording whether to use MMAP, locks, or contiguous MORECORE - - Statistics - Each space keeps track of current and maximum system memory - obtained via MORECORE or MMAP. - - Trim support - Fields holding the amount of unused topmost memory that should trigger - trimming, and a counter to force periodic scanning to release unused - non-topmost segments. - - Extension support - A void* pointer and a size_t field that can be used to help implement - extensions to this malloc. -*/ - - -// ================================================================================ -class malloc_state -{ -public: - /* ----------------------- _malloc, _free, etc... --- */ - SPP_FORCEINLINE void* _malloc(size_t bytes); - SPP_FORCEINLINE void _free(mchunkptr p); - - - /* ------------------------ Relays to internal calls to malloc/free from realloc, memalign etc */ - void *internal_malloc(size_t b) { return mspace_malloc(this, b); } - void internal_free(void *mem) { mspace_free(this, mem); } - - /* ------------------------ ----------------------- */ - - SPP_IMPL void init_top(mchunkptr p, size_t psize); - SPP_IMPL void init_bins(); - SPP_IMPL void init(char* tbase, size_t tsize); - - /* ------------------------ System alloc/dealloc -------------------------- */ - SPP_IMPL void* sys_alloc(size_t nb); - SPP_IMPL size_t release_unused_segments(); - SPP_IMPL int sys_trim(size_t pad); - SPP_IMPL void dispose_chunk(mchunkptr p, size_t psize); - - /* ----------------------- Internal support for realloc, memalign, etc --- */ - SPP_IMPL mchunkptr try_realloc_chunk(mchunkptr p, size_t nb, int can_move); - SPP_IMPL void* internal_memalign(size_t alignment, size_t bytes); - SPP_IMPL void** ialloc(size_t n_elements, size_t* sizes, int opts, void* chunks[]); - SPP_IMPL size_t internal_bulk_free(void* array[], size_t nelem); - SPP_IMPL void internal_inspect_all(void(*handler)(void *start, void *end, - size_t used_bytes, void* callback_arg), - void* arg); - - /* -------------------------- system alloc setup (Operations on mflags) ----- */ - bool use_lock() const { return false; } - void enable_lock() {} - void set_lock(int) {} - void disable_lock() {} - - bool use_mmap() const { return !!(_mflags & USE_MMAP_BIT); } - void enable_mmap() { _mflags |= USE_MMAP_BIT; } - -#if SPP_HAVE_MMAP - void disable_mmap() { _mflags &= ~USE_MMAP_BIT; } -#else - void disable_mmap() {} -#endif - - /* ----------------------- Runtime Check Support ------------------------- */ - - /* - For security, the main invariant is that malloc/free/etc never - writes to a static address other than malloc_state, unless static - malloc_state itself has been corrupted, which cannot occur via - malloc (because of these checks). In essence this means that we - believe all pointers, sizes, maps etc held in malloc_state, but - check all of those linked or offsetted from other embedded data - structures. These checks are interspersed with main code in a way - that tends to minimize their run-time cost. - - When SPP_FOOTERS is defined, in addition to range checking, we also - verify footer fields of inuse chunks, which can be used guarantee - that the mstate controlling malloc/free is intact. This is a - streamlined version of the approach described by William Robertson - et al in "Run-time Detection of Heap-based Overflows" LISA'03 - http://www.usenix.org/events/lisa03/tech/robertson.html The footer - of an inuse chunk holds the xor of its mstate and a random seed, - that is checked upon calls to free() and realloc(). This is - (probabalistically) unguessable from outside the program, but can be - computed by any code successfully malloc'ing any chunk, so does not - itself provide protection against code that has already broken - security through some other means. Unlike Robertson et al, we - always dynamically check addresses of all offset chunks (previous, - next, etc). This turns out to be cheaper than relying on hashes. - */ - - -#if !SPP_INSECURE - // Check if address a is at least as high as any from MORECORE or MMAP - bool ok_address(void *a) const { return (char *)a >= _least_addr; } - - // Check if address of next chunk n is higher than base chunk p - static bool ok_next(void *p, void *n) { return p < n; } - - // Check if p has inuse status - static bool ok_inuse(mchunkptr p) { return p->is_inuse(); } - - // Check if p has its pinuse bit on - static bool ok_pinuse(mchunkptr p) { return p->pinuse(); } - - // Check if (alleged) mstate m has expected magic field - bool ok_magic() const { return _magic == mparams._magic; } - - // In gcc, use __builtin_expect to minimize impact of checks - #if defined(__GNUC__) && __GNUC__ >= 3 - static bool rtcheck(bool e) { return __builtin_expect(e, 1); } - #else - static bool rtcheck(bool e) { return e; } - #endif -#else - static bool ok_address(void *) { return true; } - static bool ok_next(void *, void *) { return true; } - static bool ok_inuse(mchunkptr) { return true; } - static bool ok_pinuse(mchunkptr) { return true; } - static bool ok_magic() { return true; } - static bool rtcheck(bool) { return true; } -#endif - - bool is_initialized() const { return _top != 0; } - - bool use_noncontiguous() const { return !!(_mflags & USE_NONCONTIGUOUS_BIT); } - void disable_contiguous() { _mflags |= USE_NONCONTIGUOUS_BIT; } - - // Return segment holding given address - msegmentptr segment_holding(char* addr) const - { - msegmentptr sp = (msegmentptr)&_seg; - for (;;) - { - if (addr >= sp->_base && addr < sp->_base + sp->_size) - return sp; - if ((sp = sp->_next) == 0) - return 0; - } - } - - // Return true if segment contains a segment link - int has_segment_link(msegmentptr ss) const - { - msegmentptr sp = (msegmentptr)&_seg; - for (;;) - { - if ((char*)sp >= ss->_base && (char*)sp < ss->_base + ss->_size) - return 1; - if ((sp = sp->_next) == 0) - return 0; - } - } - - bool should_trim(size_t s) const { return s > _trim_check; } - - /* -------------------------- Debugging setup ---------------------------- */ - -#if ! SPP_DEBUG - void check_free_chunk(mchunkptr) {} - void check_inuse_chunk(mchunkptr) {} - void check_malloced_chunk(void*, size_t) {} - void check_mmapped_chunk(mchunkptr) {} - void check_malloc_state() {} - void check_top_chunk(mchunkptr) {} -#else /* SPP_DEBUG */ - void check_free_chunk(mchunkptr p) { do_check_free_chunk(p); } - void check_inuse_chunk(mchunkptr p) { do_check_inuse_chunk(p); } - void check_malloced_chunk(void* p, size_t s) { do_check_malloced_chunk(p, s); } - void check_mmapped_chunk(mchunkptr p) { do_check_mmapped_chunk(p); } - void check_malloc_state() { do_check_malloc_state(); } - void check_top_chunk(mchunkptr p) { do_check_top_chunk(p); } - - void do_check_any_chunk(mchunkptr p) const; - void do_check_top_chunk(mchunkptr p) const; - void do_check_mmapped_chunk(mchunkptr p) const; - void do_check_inuse_chunk(mchunkptr p) const; - void do_check_free_chunk(mchunkptr p) const; - void do_check_malloced_chunk(void* mem, size_t s) const; - void do_check_tree(tchunkptr t); - void do_check_treebin(bindex_t i); - void do_check_smallbin(bindex_t i); - void do_check_malloc_state(); - int bin_find(mchunkptr x); - size_t traverse_and_check(); -#endif - -private: - - /* ---------------------------- Indexing Bins ---------------------------- */ - - static bool is_small(size_t s) { return (s >> SMALLBIN_SHIFT) < NSMALLBINS; } - static bindex_t small_index(size_t s) { return (bindex_t)(s >> SMALLBIN_SHIFT); } - static size_t small_index2size(size_t i) { return i << SMALLBIN_SHIFT; } - static bindex_t MIN_SMALL_INDEX() { return small_index(MIN_CHUNK_SIZE); } - - // assign tree index for size S to variable I. Use x86 asm if possible -#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S) - { - unsigned int X = S >> TREEBIN_SHIFT; - if (X == 0) - return 0; - else if (X > 0xFFFF) - return NTREEBINS - 1; - - unsigned int K = (unsigned) sizeof(X) * __CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X); - return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1))); - } - -#elif defined (__INTEL_COMPILER) - SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S) - { - size_t X = S >> TREEBIN_SHIFT; - if (X == 0) - return 0; - else if (X > 0xFFFF) - return NTREEBINS - 1; - - unsigned int K = _bit_scan_reverse(X); - return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1))); - } - -#elif defined(_MSC_VER) && _MSC_VER>=1300 - SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S) - { - size_t X = S >> TREEBIN_SHIFT; - if (X == 0) - return 0; - else if (X > 0xFFFF) - return NTREEBINS - 1; - - unsigned int K; - _BitScanReverse((DWORD *) &K, (DWORD) X); - return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1))); - } - -#else // GNUC - SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S) - { - size_t X = S >> TREEBIN_SHIFT; - if (X == 0) - return 0; - else if (X > 0xFFFF) - return NTREEBINS - 1; - - unsigned int Y = (unsigned int)X; - unsigned int N = ((Y - 0x100) >> 16) & 8; - unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4; - N += K; - N += K = (((Y <<= K) - 0x4000) >> 16) & 2; - K = 14 - N + ((Y <<= K) >> 15); - return (K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1)); - } -#endif - - // Shift placing maximum resolved bit in a treebin at i as sign bit - static bindex_t leftshift_for_tree_index(bindex_t i) - { - return (i == NTREEBINS - 1) ? 0 : - ((spp_size_t_bitsize - 1) - ((i >> 1) + TREEBIN_SHIFT - 2)); - } - - // The size of the smallest chunk held in bin with index i - static bindex_t minsize_for_tree_index(bindex_t i) - { - return ((size_t)1 << ((i >> 1) + TREEBIN_SHIFT)) | - (((size_t)(i & 1)) << ((i >> 1) + TREEBIN_SHIFT - 1)); - } - - - // ----------- isolate the least set bit of a bitmap - static binmap_t least_bit(binmap_t x) { return x & -x; } - - // ----------- mask with all bits to left of least bit of x on - static binmap_t left_bits(binmap_t x) { return (x << 1) | -(x << 1); } - - // index corresponding to given bit. Use x86 asm if possible -#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - static bindex_t compute_bit2idx(binmap_t X) - { - unsigned int J; - J = __builtin_ctz(X); - return (bindex_t)J; - } - -#elif defined (__INTEL_COMPILER) - static bindex_t compute_bit2idx(binmap_t X) - { - unsigned int J; - J = _bit_scan_forward(X); - return (bindex_t)J; - } - -#elif defined(_MSC_VER) && _MSC_VER>=1300 - static bindex_t compute_bit2idx(binmap_t X) - { - unsigned int J; - _BitScanForward((DWORD *) &J, X); - return (bindex_t)J; - } - -#elif SPP_USE_BUILTIN_FFS - static bindex_t compute_bit2idx(binmap_t X) { return ffs(X) - 1; } - -#else - static bindex_t compute_bit2idx(binmap_t X) - { - unsigned int Y = X - 1; - unsigned int K = Y >> (16 - 4) & 16; - unsigned int N = K; Y >>= K; - N += K = Y >> (8 - 3) & 8; Y >>= K; - N += K = Y >> (4 - 2) & 4; Y >>= K; - N += K = Y >> (2 - 1) & 2; Y >>= K; - N += K = Y >> (1 - 0) & 1; Y >>= K; - return (bindex_t)(N + Y); - } -#endif - - /* ------------------------ Set up inuse chunks with or without footers ---*/ -#if !SPP_FOOTERS - void mark_inuse_foot(malloc_chunk_header *, size_t) {} -#else - //Set foot of inuse chunk to be xor of mstate and seed - void mark_inuse_foot(malloc_chunk_header *p, size_t s) - { - (((mchunkptr)((char*)p + s))->prev_foot = (size_t)this ^ mparams._magic); - } -#endif - - void set_inuse(malloc_chunk_header *p, size_t s) - { - p->_head = (p->_head & PINUSE_BIT) | s | CINUSE_BIT; - ((mchunkptr)(((char*)p) + s))->_head |= PINUSE_BIT; - mark_inuse_foot(p, s); - } - - void set_inuse_and_pinuse(malloc_chunk_header *p, size_t s) - { - p->_head = s | PINUSE_BIT | CINUSE_BIT; - ((mchunkptr)(((char*)p) + s))->_head |= PINUSE_BIT; - mark_inuse_foot(p, s); - } - - void set_size_and_pinuse_of_inuse_chunk(malloc_chunk_header *p, size_t s) - { - p->_head = s | PINUSE_BIT | CINUSE_BIT; - mark_inuse_foot(p, s); - } - - /* ------------------------ Addressing by index. See about smallbin repositioning --- */ - sbinptr smallbin_at(bindex_t i) const { return (sbinptr)((char*)&_smallbins[i << 1]); } - tbinptr* treebin_at(bindex_t i) { return &_treebins[i]; } - - /* ----------------------- bit corresponding to given index ---------*/ - static binmap_t idx2bit(bindex_t i) { return ((binmap_t)1 << i); } - - // --------------- Mark/Clear bits with given index - void mark_smallmap(bindex_t i) { _smallmap |= idx2bit(i); } - void clear_smallmap(bindex_t i) { _smallmap &= ~idx2bit(i); } - binmap_t smallmap_is_marked(bindex_t i) const { return _smallmap & idx2bit(i); } - - void mark_treemap(bindex_t i) { _treemap |= idx2bit(i); } - void clear_treemap(bindex_t i) { _treemap &= ~idx2bit(i); } - binmap_t treemap_is_marked(bindex_t i) const { return _treemap & idx2bit(i); } - - /* ------------------------ ----------------------- */ - SPP_FORCEINLINE void insert_small_chunk(mchunkptr P, size_t S); - SPP_FORCEINLINE void unlink_small_chunk(mchunkptr P, size_t S); - SPP_FORCEINLINE void unlink_first_small_chunk(mchunkptr B, mchunkptr P, bindex_t I); - SPP_FORCEINLINE void replace_dv(mchunkptr P, size_t S); - - /* ------------------------- Operations on trees ------------------------- */ - SPP_FORCEINLINE void insert_large_chunk(tchunkptr X, size_t S); - SPP_FORCEINLINE void unlink_large_chunk(tchunkptr X); - - /* ------------------------ Relays to large vs small bin operations */ - SPP_FORCEINLINE void insert_chunk(mchunkptr P, size_t S); - SPP_FORCEINLINE void unlink_chunk(mchunkptr P, size_t S); - - /* ----------------------- Direct-mmapping chunks ----------------------- */ - SPP_IMPL void* mmap_alloc(size_t nb); - SPP_IMPL mchunkptr mmap_resize(mchunkptr oldp, size_t nb, int flags); - - SPP_IMPL void reset_on_error(); - SPP_IMPL void* prepend_alloc(char* newbase, char* oldbase, size_t nb); - SPP_IMPL void add_segment(char* tbase, size_t tsize, flag_t mmapped); - - /* ------------------------ malloc --------------------------- */ - SPP_IMPL void* tmalloc_large(size_t nb); - SPP_IMPL void* tmalloc_small(size_t nb); - - /* ------------------------Bin types, widths and sizes -------- */ - static const size_t NSMALLBINS = 32; - static const size_t NTREEBINS = 32; - static const size_t SMALLBIN_SHIFT = 3; - static const size_t SMALLBIN_WIDTH = 1 << SMALLBIN_SHIFT; - static const size_t TREEBIN_SHIFT = 8; - static const size_t MIN_LARGE_SIZE = 1 << TREEBIN_SHIFT; - static const size_t MAX_SMALL_SIZE = (MIN_LARGE_SIZE - 1); - static const size_t MAX_SMALL_REQUEST = (MAX_SMALL_SIZE - spp_chunk_align_mask - CHUNK_OVERHEAD); - - /* ------------------------ data members --------------------------- */ - binmap_t _smallmap; - binmap_t _treemap; - size_t _dvsize; - size_t _topsize; - char* _least_addr; - mchunkptr _dv; - mchunkptr _top; - size_t _trim_check; - size_t _release_checks; - size_t _magic; - mchunkptr _smallbins[(NSMALLBINS + 1) * 2]; - tbinptr _treebins[NTREEBINS]; -public: - size_t _footprint; - size_t _max_footprint; - size_t _footprint_limit; // zero means no limit - flag_t _mflags; - - msegment _seg; - -private: - void* _extp; // Unused but available for extensions - size_t _exts; -}; - -typedef malloc_state* mstate; - -/* ------------- end malloc_state ------------------- */ - -#if SPP_FOOTERS -static malloc_state* get_mstate_for(malloc_chunk_header *p) -{ - return (malloc_state*)(((mchunkptr)((char*)(p) + - (p->chunksize())))->prev_foot ^ mparams._magic); -} -#endif - -/* -------------------------- system alloc setup ------------------------- */ - - - -// For mmap, use granularity alignment on windows, else page-align -#ifdef WIN32 - #define mmap_align(S) mparams.granularity_align(S) -#else - #define mmap_align(S) mparams.page_align(S) -#endif - -// True if segment S holds address A -static bool segment_holds(msegmentptr S, mchunkptr A) -{ - return (char*)A >= S->_base && (char*)A < S->_base + S->_size; -} - -/* - top_foot_size is padding at the end of a segment, including space - that may be needed to place segment records and fenceposts when new - noncontiguous segments are added. -*/ -static SPP_FORCEINLINE size_t top_foot_size() -{ - return align_offset(chunk2mem((void *)0)) + - pad_request(sizeof(struct malloc_segment)) + - MIN_CHUNK_SIZE; -} - - -// For sys_alloc, enough padding to ensure can malloc request on success -static SPP_FORCEINLINE size_t sys_alloc_padding() -{ - return top_foot_size() + SPP_MALLOC_ALIGNMENT; -} - - -#define SPP_USAGE_ERROR_ACTION(m,p) SPP_ABORT - -/* ---------------------------- setting mparams -------------------------- */ - -// Initialize mparams -int malloc_params::_init() -{ -#ifdef NEED_GLOBAL_LOCK_INIT - if (malloc_global_mutex_status <= 0) - init_malloc_global_mutex(); -#endif - - if (_magic == 0) - { - size_t magic; - size_t psize; - size_t gsize; - -#ifndef WIN32 - psize = malloc_getpagesize; - gsize = ((SPP_DEFAULT_GRANULARITY != 0) ? SPP_DEFAULT_GRANULARITY : psize); -#else - { - SYSTEM_INFO system_info; - GetSystemInfo(&system_info); - psize = system_info.dwPageSize; - gsize = ((SPP_DEFAULT_GRANULARITY != 0) ? - SPP_DEFAULT_GRANULARITY : system_info.dwAllocationGranularity); - } -#endif - - /* Sanity-check configuration: - size_t must be unsigned and as wide as pointer type. - ints must be at least 4 bytes. - alignment must be at least 8. - Alignment, min chunk size, and page size must all be powers of 2. - */ - if ((sizeof(size_t) != sizeof(char*)) || - (spp_max_size_t < MIN_CHUNK_SIZE) || - (sizeof(int) < 4) || - (SPP_MALLOC_ALIGNMENT < (size_t)8U) || - ((SPP_MALLOC_ALIGNMENT & (SPP_MALLOC_ALIGNMENT - 1)) != 0) || - ((MCHUNK_SIZE & (MCHUNK_SIZE - 1)) != 0) || - ((gsize & (gsize - 1)) != 0) || - ((psize & (psize - 1)) != 0)) - SPP_ABORT; - _granularity = gsize; - _page_size = psize; - _mmap_threshold = SPP_DEFAULT_MMAP_THRESHOLD; - _trim_threshold = SPP_DEFAULT_TRIM_THRESHOLD; - _default_mflags = USE_MMAP_BIT | USE_NONCONTIGUOUS_BIT; - - { -#if SPP_USE_DEV_RANDOM - int fd; - unsigned char buf[sizeof(size_t)]; - // Try to use /dev/urandom, else fall back on using time - if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 && - read(fd, buf, sizeof(buf)) == sizeof(buf)) - { - magic = *((size_t *) buf); - close(fd); - } - else -#endif - { -#ifdef WIN32 - magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U); -#elif defined(SPP_LACKS_TIME_H) - magic = (size_t)&magic ^ (size_t)0x55555555U; -#else - magic = (size_t)(time(0) ^ (size_t)0x55555555U); -#endif - } - magic |= (size_t)8U; // ensure nonzero - magic &= ~(size_t)7U; // improve chances of fault for bad values - // Until memory modes commonly available, use volatile-write - (*(volatile size_t *)(&(_magic))) = magic; - } - } - - return 1; -} - -/* - mallopt tuning options. SVID/XPG defines four standard parameter - numbers for mallopt, normally defined in malloc.h. None of these - are used in this malloc, so setting them has no effect. But this - malloc does support the following options. -*/ -static const int m_trim_threshold = -1; -static const int m_granularity = -2; -static const int m_mmap_threshold = -3; - -// support for mallopt -int malloc_params::change(int param_number, int value) -{ - size_t val; - ensure_initialization(); - val = (value == -1) ? spp_max_size_t : (size_t)value; - - switch (param_number) - { - case m_trim_threshold: - _trim_threshold = val; - return 1; - - case m_granularity: - if (val >= _page_size && ((val & (val - 1)) == 0)) - { - _granularity = val; - return 1; - } - else - return 0; - - case m_mmap_threshold: - _mmap_threshold = val; - return 1; - - default: - return 0; - } -} - -#if SPP_DEBUG -/* ------------------------- Debugging Support --------------------------- */ - -// Check properties of any chunk, whether free, inuse, mmapped etc -void malloc_state::do_check_any_chunk(mchunkptr p) const -{ - assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD)); - assert(ok_address(p)); -} - -// Check properties of top chunk -void malloc_state::do_check_top_chunk(mchunkptr p) const -{ - msegmentptr sp = segment_holding((char*)p); - size_t sz = p->_head & ~INUSE_BITS; // third-lowest bit can be set! - assert(sp != 0); - assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD)); - assert(ok_address(p)); - assert(sz == _topsize); - assert(sz > 0); - assert(sz == ((sp->_base + sp->_size) - (char*)p) - top_foot_size()); - assert(p->pinuse()); - assert(!p->chunk_plus_offset(sz)->pinuse()); -} - -// Check properties of (inuse) mmapped chunks -void malloc_state::do_check_mmapped_chunk(mchunkptr p) const -{ - size_t sz = p->chunksize(); - size_t len = (sz + (p->_prev_foot) + SPP_MMAP_FOOT_PAD); - assert(p->is_mmapped()); - assert(use_mmap()); - assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD)); - assert(ok_address(p)); - assert(!is_small(sz)); - assert((len & (mparams._page_size - 1)) == 0); - assert(p->chunk_plus_offset(sz)->_head == FENCEPOST_HEAD); - assert(p->chunk_plus_offset(sz + sizeof(size_t))->_head == 0); -} - -// Check properties of inuse chunks -void malloc_state::do_check_inuse_chunk(mchunkptr p) const -{ - do_check_any_chunk(p); - assert(p->is_inuse()); - assert(p->next_pinuse()); - // If not pinuse and not mmapped, previous chunk has OK offset - assert(p->is_mmapped() || p->pinuse() || (mchunkptr)p->prev_chunk()->next_chunk() == p); - if (p->is_mmapped()) - do_check_mmapped_chunk(p); -} - -// Check properties of free chunks -void malloc_state::do_check_free_chunk(mchunkptr p) const -{ - size_t sz = p->chunksize(); - mchunkptr next = (mchunkptr)p->chunk_plus_offset(sz); - do_check_any_chunk(p); - assert(!p->is_inuse()); - assert(!p->next_pinuse()); - assert(!p->is_mmapped()); - if (p != _dv && p != _top) - { - if (sz >= MIN_CHUNK_SIZE) - { - assert((sz & spp_chunk_align_mask) == 0); - assert(spp_is_aligned(chunk2mem(p))); - assert(next->_prev_foot == sz); - assert(p->pinuse()); - assert(next == _top || next->is_inuse()); - assert(p->_fd->_bk == p); - assert(p->_bk->_fd == p); - } - else // markers are always of size sizeof(size_t) - assert(sz == sizeof(size_t)); - } -} - -// Check properties of malloced chunks at the point they are malloced -void malloc_state::do_check_malloced_chunk(void* mem, size_t s) const -{ - if (mem != 0) - { - mchunkptr p = mem2chunk(mem); - size_t sz = p->_head & ~INUSE_BITS; - do_check_inuse_chunk(p); - assert((sz & spp_chunk_align_mask) == 0); - assert(sz >= MIN_CHUNK_SIZE); - assert(sz >= s); - // unless mmapped, size is less than MIN_CHUNK_SIZE more than request - assert(p->is_mmapped() || sz < (s + MIN_CHUNK_SIZE)); - } -} - -// Check a tree and its subtrees. -void malloc_state::do_check_tree(tchunkptr t) -{ - tchunkptr head = 0; - tchunkptr u = t; - bindex_t tindex = t->_index; - size_t tsize = t->chunksize(); - bindex_t idx = compute_tree_index(tsize); - assert(tindex == idx); - assert(tsize >= MIN_LARGE_SIZE); - assert(tsize >= minsize_for_tree_index(idx)); - assert((idx == NTREEBINS - 1) || (tsize < minsize_for_tree_index((idx + 1)))); - - do - { - // traverse through chain of same-sized nodes - do_check_any_chunk((mchunkptr)u); - assert(u->_index == tindex); - assert(u->chunksize() == tsize); - assert(!u->is_inuse()); - assert(!u->next_pinuse()); - assert(u->_fd->_bk == u); - assert(u->_bk->_fd == u); - if (u->_parent == 0) - { - assert(u->_child[0] == 0); - assert(u->_child[1] == 0); - } - else - { - assert(head == 0); // only one node on chain has parent - head = u; - assert(u->_parent != u); - assert(u->_parent->_child[0] == u || - u->_parent->_child[1] == u || - *((tbinptr*)(u->_parent)) == u); - if (u->_child[0] != 0) - { - assert(u->_child[0]->_parent == u); - assert(u->_child[0] != u); - do_check_tree(u->_child[0]); - } - if (u->_child[1] != 0) - { - assert(u->_child[1]->_parent == u); - assert(u->_child[1] != u); - do_check_tree(u->_child[1]); - } - if (u->_child[0] != 0 && u->_child[1] != 0) - assert(u->_child[0]->chunksize() < u->_child[1]->chunksize()); - } - u = u->_fd; - } - while (u != t); - assert(head != 0); -} - -// Check all the chunks in a treebin. -void malloc_state::do_check_treebin(bindex_t i) -{ - tbinptr* tb = (tbinptr*)treebin_at(i); - tchunkptr t = *tb; - int empty = (_treemap & (1U << i)) == 0; - if (t == 0) - assert(empty); - if (!empty) - do_check_tree(t); -} - -// Check all the chunks in a smallbin. -void malloc_state::do_check_smallbin(bindex_t i) -{ - sbinptr b = smallbin_at(i); - mchunkptr p = b->_bk; - unsigned int empty = (_smallmap & (1U << i)) == 0; - if (p == b) - assert(empty); - if (!empty) - { - for (; p != b; p = p->_bk) - { - size_t size = p->chunksize(); - mchunkptr q; - // each chunk claims to be free - do_check_free_chunk(p); - // chunk belongs in bin - assert(small_index(size) == i); - assert(p->_bk == b || p->_bk->chunksize() == p->chunksize()); - // chunk is followed by an inuse chunk - q = (mchunkptr)p->next_chunk(); - if (q->_head != FENCEPOST_HEAD) - do_check_inuse_chunk(q); - } - } -} - -// Find x in a bin. Used in other check functions. -int malloc_state::bin_find(mchunkptr x) -{ - size_t size = x->chunksize(); - if (is_small(size)) - { - bindex_t sidx = small_index(size); - sbinptr b = smallbin_at(sidx); - if (smallmap_is_marked(sidx)) - { - mchunkptr p = b; - do - { - if (p == x) - return 1; - } - while ((p = p->_fd) != b); - } - } - else - { - bindex_t tidx = compute_tree_index(size); - if (treemap_is_marked(tidx)) - { - tchunkptr t = *treebin_at(tidx); - size_t sizebits = size << leftshift_for_tree_index(tidx); - while (t != 0 && t->chunksize() != size) - { - t = t->_child[(sizebits >> (spp_size_t_bitsize - 1)) & 1]; - sizebits <<= 1; - } - if (t != 0) - { - tchunkptr u = t; - do - { - if (u == (tchunkptr)x) - return 1; - } - while ((u = u->_fd) != t); - } - } - } - return 0; -} - -// Traverse each chunk and check it; return total -size_t malloc_state::traverse_and_check() -{ - size_t sum = 0; - if (is_initialized()) - { - msegmentptr s = (msegmentptr)&_seg; - sum += _topsize + top_foot_size(); - while (s != 0) - { - mchunkptr q = align_as_chunk(s->_base); - mchunkptr lastq = 0; - assert(q->pinuse()); - while (segment_holds(s, q) && - q != _top && q->_head != FENCEPOST_HEAD) - { - sum += q->chunksize(); - if (q->is_inuse()) - { - assert(!bin_find(q)); - do_check_inuse_chunk(q); - } - else - { - assert(q == _dv || bin_find(q)); - assert(lastq == 0 || lastq->is_inuse()); // Not 2 consecutive free - do_check_free_chunk(q); - } - lastq = q; - q = (mchunkptr)q->next_chunk(); - } - s = s->_next; - } - } - return sum; -} - - -// Check all properties of malloc_state. -void malloc_state::do_check_malloc_state() -{ - bindex_t i; - size_t total; - // check bins - for (i = 0; i < NSMALLBINS; ++i) - do_check_smallbin(i); - for (i = 0; i < NTREEBINS; ++i) - do_check_treebin(i); - - if (_dvsize != 0) - { - // check dv chunk - do_check_any_chunk(_dv); - assert(_dvsize == _dv->chunksize()); - assert(_dvsize >= MIN_CHUNK_SIZE); - assert(bin_find(_dv) == 0); - } - - if (_top != 0) - { - // check top chunk - do_check_top_chunk(_top); - //assert(topsize == top->chunksize()); redundant - assert(_topsize > 0); - assert(bin_find(_top) == 0); - } - - total = traverse_and_check(); - assert(total <= _footprint); - assert(_footprint <= _max_footprint); -} -#endif // SPP_DEBUG - -/* ----------------------- Operations on smallbins ----------------------- */ - -/* - Various forms of linking and unlinking are defined as macros. Even - the ones for trees, which are very long but have very short typical - paths. This is ugly but reduces reliance on inlining support of - compilers. -*/ - -// Link a free chunk into a smallbin -void malloc_state::insert_small_chunk(mchunkptr p, size_t s) -{ - bindex_t I = small_index(s); - mchunkptr B = smallbin_at(I); - mchunkptr F = B; - assert(s >= MIN_CHUNK_SIZE); - if (!smallmap_is_marked(I)) - mark_smallmap(I); - else if (rtcheck(ok_address(B->_fd))) - F = B->_fd; - else - SPP_ABORT; - B->_fd = p; - F->_bk = p; - p->_fd = F; - p->_bk = B; -} - -// Unlink a chunk from a smallbin -void malloc_state::unlink_small_chunk(mchunkptr p, size_t s) -{ - mchunkptr F = p->_fd; - mchunkptr B = p->_bk; - bindex_t I = small_index(s); - assert(p != B); - assert(p != F); - assert(p->chunksize() == small_index2size(I)); - if (rtcheck(F == smallbin_at(I) || (ok_address(F) && F->_bk == p))) - { - if (B == F) - clear_smallmap(I); - else if (rtcheck(B == smallbin_at(I) || - (ok_address(B) && B->_fd == p))) - { - F->_bk = B; - B->_fd = F; - } - else - SPP_ABORT; - } - else - SPP_ABORT; -} - -// Unlink the first chunk from a smallbin -void malloc_state::unlink_first_small_chunk(mchunkptr B, mchunkptr p, bindex_t I) -{ - mchunkptr F = p->_fd; - assert(p != B); - assert(p != F); - assert(p->chunksize() == small_index2size(I)); - if (B == F) - clear_smallmap(I); - else if (rtcheck(ok_address(F) && F->_bk == p)) - { - F->_bk = B; - B->_fd = F; - } - else - SPP_ABORT; -} - -// Replace dv node, binning the old one -// Used only when dvsize known to be small -void malloc_state::replace_dv(mchunkptr p, size_t s) -{ - size_t DVS = _dvsize; - assert(is_small(DVS)); - if (DVS != 0) - { - mchunkptr DV = _dv; - insert_small_chunk(DV, DVS); - } - _dvsize = s; - _dv = p; -} - -/* ------------------------- Operations on trees ------------------------- */ - -// Insert chunk into tree -void malloc_state::insert_large_chunk(tchunkptr X, size_t s) -{ - tbinptr* H; - bindex_t I = compute_tree_index(s); - H = treebin_at(I); - X->_index = I; - X->_child[0] = X->_child[1] = 0; - if (!treemap_is_marked(I)) - { - mark_treemap(I); - *H = X; - X->_parent = (tchunkptr)H; - X->_fd = X->_bk = X; - } - else - { - tchunkptr T = *H; - size_t K = s << leftshift_for_tree_index(I); - for (;;) - { - if (T->chunksize() != s) - { - tchunkptr* C = &(T->_child[(K >> (spp_size_t_bitsize - 1)) & 1]); - K <<= 1; - if (*C != 0) - T = *C; - else if (rtcheck(ok_address(C))) - { - *C = X; - X->_parent = T; - X->_fd = X->_bk = X; - break; - } - else - { - SPP_ABORT; - break; - } - } - else - { - tchunkptr F = T->_fd; - if (rtcheck(ok_address(T) && ok_address(F))) - { - T->_fd = F->_bk = X; - X->_fd = F; - X->_bk = T; - X->_parent = 0; - break; - } - else - { - SPP_ABORT; - break; - } - } - } - } -} - -/* - Unlink steps: - - 1. If x is a chained node, unlink it from its same-sized fd/bk links - and choose its bk node as its replacement. - 2. If x was the last node of its size, but not a leaf node, it must - be replaced with a leaf node (not merely one with an open left or - right), to make sure that lefts and rights of descendents - correspond properly to bit masks. We use the rightmost descendent - of x. We could use any other leaf, but this is easy to locate and - tends to counteract removal of leftmosts elsewhere, and so keeps - paths shorter than minimally guaranteed. This doesn't loop much - because on average a node in a tree is near the bottom. - 3. If x is the base of a chain (i.e., has parent links) relink - x's parent and children to x's replacement (or null if none). -*/ - -void malloc_state::unlink_large_chunk(tchunkptr X) -{ - tchunkptr XP = X->_parent; - tchunkptr R; - if (X->_bk != X) - { - tchunkptr F = X->_fd; - R = X->_bk; - if (rtcheck(ok_address(F) && F->_bk == X && R->_fd == X)) - { - F->_bk = R; - R->_fd = F; - } - else - SPP_ABORT; - } - else - { - tchunkptr* RP; - if (((R = *(RP = &(X->_child[1]))) != 0) || - ((R = *(RP = &(X->_child[0]))) != 0)) - { - tchunkptr* CP; - while ((*(CP = &(R->_child[1])) != 0) || - (*(CP = &(R->_child[0])) != 0)) - R = *(RP = CP); - if (rtcheck(ok_address(RP))) - *RP = 0; - else - SPP_ABORT; - } - } - if (XP != 0) - { - tbinptr* H = treebin_at(X->_index); - if (X == *H) - { - if ((*H = R) == 0) - clear_treemap(X->_index); - } - else if (rtcheck(ok_address(XP))) - { - if (XP->_child[0] == X) - XP->_child[0] = R; - else - XP->_child[1] = R; - } - else - SPP_ABORT; - if (R != 0) - { - if (rtcheck(ok_address(R))) - { - tchunkptr C0, C1; - R->_parent = XP; - if ((C0 = X->_child[0]) != 0) - { - if (rtcheck(ok_address(C0))) - { - R->_child[0] = C0; - C0->_parent = R; - } - else - SPP_ABORT; - } - if ((C1 = X->_child[1]) != 0) - { - if (rtcheck(ok_address(C1))) - { - R->_child[1] = C1; - C1->_parent = R; - } - else - SPP_ABORT; - } - } - else - SPP_ABORT; - } - } -} - -// Relays to large vs small bin operations - -void malloc_state::insert_chunk(mchunkptr p, size_t s) -{ - if (is_small(s)) - insert_small_chunk(p, s); - else - { - tchunkptr tp = (tchunkptr)(p); - insert_large_chunk(tp, s); - } -} - -void malloc_state::unlink_chunk(mchunkptr p, size_t s) -{ - if (is_small(s)) - unlink_small_chunk(p, s); - else - { - tchunkptr tp = (tchunkptr)(p); - unlink_large_chunk(tp); - } -} - - -/* ----------------------- Direct-mmapping chunks ----------------------- */ - -/* - Directly mmapped chunks are set up with an offset to the start of - the mmapped region stored in the prev_foot field of the chunk. This - allows reconstruction of the required argument to MUNMAP when freed, - and also allows adjustment of the returned chunk to meet alignment - requirements (especially in memalign). -*/ - -// Malloc using mmap -void* malloc_state::mmap_alloc(size_t nb) -{ - size_t mmsize = mmap_align(nb + 6 * sizeof(size_t) + spp_chunk_align_mask); - if (_footprint_limit != 0) - { - size_t fp = _footprint + mmsize; - if (fp <= _footprint || fp > _footprint_limit) - return 0; - } - if (mmsize > nb) - { - // Check for wrap around 0 - char* mm = (char*)(SPP_CALL_DIRECT_MMAP(mmsize)); - if (mm != cmfail) - { - size_t offset = align_offset(chunk2mem(mm)); - size_t psize = mmsize - offset - SPP_MMAP_FOOT_PAD; - mchunkptr p = (mchunkptr)(mm + offset); - p->_prev_foot = offset; - p->_head = psize; - mark_inuse_foot(p, psize); - p->chunk_plus_offset(psize)->_head = FENCEPOST_HEAD; - p->chunk_plus_offset(psize + sizeof(size_t))->_head = 0; - - if (_least_addr == 0 || mm < _least_addr) - _least_addr = mm; - if ((_footprint += mmsize) > _max_footprint) - _max_footprint = _footprint; - assert(spp_is_aligned(chunk2mem(p))); - check_mmapped_chunk(p); - return chunk2mem(p); - } - } - return 0; -} - -// Realloc using mmap -mchunkptr malloc_state::mmap_resize(mchunkptr oldp, size_t nb, int flags) -{ - size_t oldsize = oldp->chunksize(); - (void)flags; // placate people compiling -Wunused - if (is_small(nb)) // Can't shrink mmap regions below small size - return 0; - - // Keep old chunk if big enough but not too big - if (oldsize >= nb + sizeof(size_t) && - (oldsize - nb) <= (mparams._granularity << 1)) - return oldp; - else - { - size_t offset = oldp->_prev_foot; - size_t oldmmsize = oldsize + offset + SPP_MMAP_FOOT_PAD; - size_t newmmsize = mmap_align(nb + 6 * sizeof(size_t) + spp_chunk_align_mask); - char* cp = (char*)SPP_CALL_MREMAP((char*)oldp - offset, - oldmmsize, newmmsize, flags); - if (cp != cmfail) - { - mchunkptr newp = (mchunkptr)(cp + offset); - size_t psize = newmmsize - offset - SPP_MMAP_FOOT_PAD; - newp->_head = psize; - mark_inuse_foot(newp, psize); - newp->chunk_plus_offset(psize)->_head = FENCEPOST_HEAD; - newp->chunk_plus_offset(psize + sizeof(size_t))->_head = 0; - - if (cp < _least_addr) - _least_addr = cp; - if ((_footprint += newmmsize - oldmmsize) > _max_footprint) - _max_footprint = _footprint; - check_mmapped_chunk(newp); - return newp; - } - } - return 0; -} - - -/* -------------------------- mspace management -------------------------- */ - -// Initialize top chunk and its size -void malloc_state::init_top(mchunkptr p, size_t psize) -{ - // Ensure alignment - size_t offset = align_offset(chunk2mem(p)); - p = (mchunkptr)((char*)p + offset); - psize -= offset; - - _top = p; - _topsize = psize; - p->_head = psize | PINUSE_BIT; - // set size of fake trailing chunk holding overhead space only once - p->chunk_plus_offset(psize)->_head = top_foot_size(); - _trim_check = mparams._trim_threshold; // reset on each update -} - -// Initialize bins for a new mstate that is otherwise zeroed out -void malloc_state::init_bins() -{ - // Establish circular links for smallbins - bindex_t i; - for (i = 0; i < NSMALLBINS; ++i) - { - sbinptr bin = smallbin_at(i); - bin->_fd = bin->_bk = bin; - } -} - -#if SPP_PROCEED_ON_ERROR - -// default corruption action -void malloc_state::reset_on_error() -{ - int i; - ++malloc_corruption_error_count; - // Reinitialize fields to forget about all memory - _smallmap = _treemap = 0; - _dvsize = _topsize = 0; - _seg._base = 0; - _seg._size = 0; - _seg._next = 0; - _top = _dv = 0; - for (i = 0; i < NTREEBINS; ++i) - *treebin_at(i) = 0; - init_bins(); -} -#endif - -/* Allocate chunk and prepend remainder with chunk in successor base. */ -void* malloc_state::prepend_alloc(char* newbase, char* oldbase, size_t nb) -{ - mchunkptr p = align_as_chunk(newbase); - mchunkptr oldfirst = align_as_chunk(oldbase); - size_t psize = (char*)oldfirst - (char*)p; - mchunkptr q = (mchunkptr)p->chunk_plus_offset(nb); - size_t qsize = psize - nb; - set_size_and_pinuse_of_inuse_chunk(p, nb); - - assert((char*)oldfirst > (char*)q); - assert(oldfirst->pinuse()); - assert(qsize >= MIN_CHUNK_SIZE); - - // consolidate remainder with first chunk of old base - if (oldfirst == _top) - { - size_t tsize = _topsize += qsize; - _top = q; - q->_head = tsize | PINUSE_BIT; - check_top_chunk(q); - } - else if (oldfirst == _dv) - { - size_t dsize = _dvsize += qsize; - _dv = q; - q->set_size_and_pinuse_of_free_chunk(dsize); - } - else - { - if (!oldfirst->is_inuse()) - { - size_t nsize = oldfirst->chunksize(); - unlink_chunk(oldfirst, nsize); - oldfirst = (mchunkptr)oldfirst->chunk_plus_offset(nsize); - qsize += nsize; - } - q->set_free_with_pinuse(qsize, oldfirst); - insert_chunk(q, qsize); - check_free_chunk(q); - } - - check_malloced_chunk(chunk2mem(p), nb); - return chunk2mem(p); -} - -// Add a segment to hold a new noncontiguous region -void malloc_state::add_segment(char* tbase, size_t tsize, flag_t mmapped) -{ - // Determine locations and sizes of segment, fenceposts, old top - char* old_top = (char*)_top; - msegmentptr oldsp = segment_holding(old_top); - char* old_end = oldsp->_base + oldsp->_size; - size_t ssize = pad_request(sizeof(struct malloc_segment)); - char* rawsp = old_end - (ssize + 4 * sizeof(size_t) + spp_chunk_align_mask); - size_t offset = align_offset(chunk2mem(rawsp)); - char* asp = rawsp + offset; - char* csp = (asp < (old_top + MIN_CHUNK_SIZE)) ? old_top : asp; - mchunkptr sp = (mchunkptr)csp; - msegmentptr ss = (msegmentptr)(chunk2mem(sp)); - mchunkptr tnext = (mchunkptr)sp->chunk_plus_offset(ssize); - mchunkptr p = tnext; - int nfences = 0; - - // reset top to new space - init_top((mchunkptr)tbase, tsize - top_foot_size()); - - // Set up segment record - assert(spp_is_aligned(ss)); - set_size_and_pinuse_of_inuse_chunk(sp, ssize); - *ss = _seg; // Push current record - _seg._base = tbase; - _seg._size = tsize; - _seg._sflags = mmapped; - _seg._next = ss; - - // Insert trailing fenceposts - for (;;) - { - mchunkptr nextp = (mchunkptr)p->chunk_plus_offset(sizeof(size_t)); - p->_head = FENCEPOST_HEAD; - ++nfences; - if ((char*)(&(nextp->_head)) < old_end) - p = nextp; - else - break; - } - assert(nfences >= 2); - - // Insert the rest of old top into a bin as an ordinary free chunk - if (csp != old_top) - { - mchunkptr q = (mchunkptr)old_top; - size_t psize = csp - old_top; - mchunkptr tn = (mchunkptr)q->chunk_plus_offset(psize); - q->set_free_with_pinuse(psize, tn); - insert_chunk(q, psize); - } - - check_top_chunk(_top); -} - -/* -------------------------- System allocation -------------------------- */ - -// Get memory from system using MMAP -void* malloc_state::sys_alloc(size_t nb) -{ - char* tbase = cmfail; - size_t tsize = 0; - flag_t mmap_flag = 0; - size_t asize; // allocation size - - mparams.ensure_initialization(); - - // Directly map large chunks, but only if already initialized - if (use_mmap() && nb >= mparams._mmap_threshold && _topsize != 0) - { - void* mem = mmap_alloc(nb); - if (mem != 0) - return mem; - } - - asize = mparams.granularity_align(nb + sys_alloc_padding()); - if (asize <= nb) - return 0; // wraparound - if (_footprint_limit != 0) - { - size_t fp = _footprint + asize; - if (fp <= _footprint || fp > _footprint_limit) - return 0; - } - - /* - Try getting memory with a call to MMAP new space (disabled if not SPP_HAVE_MMAP). - We need to request enough bytes from system to ensure - we can malloc nb bytes upon success, so pad with enough space for - top_foot, plus alignment-pad to make sure we don't lose bytes if - not on boundary, and round this up to a granularity unit. - */ - - if (SPP_HAVE_MMAP && tbase == cmfail) - { - // Try MMAP - char* mp = (char*)(SPP_CALL_MMAP(asize)); - if (mp != cmfail) - { - tbase = mp; - tsize = asize; - mmap_flag = USE_MMAP_BIT; - } - } - - if (tbase != cmfail) - { - - if ((_footprint += tsize) > _max_footprint) - _max_footprint = _footprint; - - if (!is_initialized()) - { - // first-time initialization - if (_least_addr == 0 || tbase < _least_addr) - _least_addr = tbase; - _seg._base = tbase; - _seg._size = tsize; - _seg._sflags = mmap_flag; - _magic = mparams._magic; - _release_checks = SPP_MAX_RELEASE_CHECK_RATE; - init_bins(); - - // Offset top by embedded malloc_state - mchunkptr mn = (mchunkptr)mem2chunk(this)->next_chunk(); - init_top(mn, (size_t)((tbase + tsize) - (char*)mn) - top_foot_size()); - } - - else - { - // Try to merge with an existing segment - msegmentptr sp = &_seg; - // Only consider most recent segment if traversal suppressed - while (sp != 0 && tbase != sp->_base + sp->_size) - sp = (SPP_NO_SEGMENT_TRAVERSAL) ? 0 : sp->_next; - if (sp != 0 && - !sp->is_extern_segment() && - (sp->_sflags & USE_MMAP_BIT) == mmap_flag && - segment_holds(sp, _top)) - { - // append - sp->_size += tsize; - init_top(_top, _topsize + tsize); - } - else - { - if (tbase < _least_addr) - _least_addr = tbase; - sp = &_seg; - while (sp != 0 && sp->_base != tbase + tsize) - sp = (SPP_NO_SEGMENT_TRAVERSAL) ? 0 : sp->_next; - if (sp != 0 && - !sp->is_extern_segment() && - (sp->_sflags & USE_MMAP_BIT) == mmap_flag) - { - char* oldbase = sp->_base; - sp->_base = tbase; - sp->_size += tsize; - return prepend_alloc(tbase, oldbase, nb); - } - else - add_segment(tbase, tsize, mmap_flag); - } - } - - if (nb < _topsize) - { - // Allocate from new or extended top space - size_t rsize = _topsize -= nb; - mchunkptr p = _top; - mchunkptr r = _top = (mchunkptr)p->chunk_plus_offset(nb); - r->_head = rsize | PINUSE_BIT; - set_size_and_pinuse_of_inuse_chunk(p, nb); - check_top_chunk(_top); - check_malloced_chunk(chunk2mem(p), nb); - return chunk2mem(p); - } - } - - SPP_MALLOC_FAILURE_ACTION; - return 0; -} - -/* ----------------------- system deallocation -------------------------- */ - -// Unmap and unlink any mmapped segments that don't contain used chunks -size_t malloc_state::release_unused_segments() -{ - size_t released = 0; - int nsegs = 0; - msegmentptr pred = &_seg; - msegmentptr sp = pred->_next; - while (sp != 0) - { - char* base = sp->_base; - size_t size = sp->_size; - msegmentptr next = sp->_next; - ++nsegs; - if (sp->is_mmapped_segment() && !sp->is_extern_segment()) - { - mchunkptr p = align_as_chunk(base); - size_t psize = p->chunksize(); - // Can unmap if first chunk holds entire segment and not pinned - if (!p->is_inuse() && (char*)p + psize >= base + size - top_foot_size()) - { - tchunkptr tp = (tchunkptr)p; - assert(segment_holds(sp, p)); - if (p == _dv) - { - _dv = 0; - _dvsize = 0; - } - else - unlink_large_chunk(tp); - if (SPP_CALL_MUNMAP(base, size) == 0) - { - released += size; - _footprint -= size; - // unlink obsoleted record - sp = pred; - sp->_next = next; - } - else - { - // back out if cannot unmap - insert_large_chunk(tp, psize); - } - } - } - if (SPP_NO_SEGMENT_TRAVERSAL) // scan only first segment - break; - pred = sp; - sp = next; - } - // Reset check counter - _release_checks = (((size_t) nsegs > (size_t) SPP_MAX_RELEASE_CHECK_RATE) ? - (size_t) nsegs : (size_t) SPP_MAX_RELEASE_CHECK_RATE); - return released; -} - -int malloc_state::sys_trim(size_t pad) -{ - size_t released = 0; - mparams.ensure_initialization(); - if (pad < MAX_REQUEST && is_initialized()) - { - pad += top_foot_size(); // ensure enough room for segment overhead - - if (_topsize > pad) - { - // Shrink top space in _granularity - size units, keeping at least one - size_t unit = mparams._granularity; - size_t extra = ((_topsize - pad + (unit - 1)) / unit - - 1) * unit; - msegmentptr sp = segment_holding((char*)_top); - - if (!sp->is_extern_segment()) - { - if (sp->is_mmapped_segment()) - { - if (SPP_HAVE_MMAP && - sp->_size >= extra && - !has_segment_link(sp)) - { - // can't shrink if pinned - size_t newsize = sp->_size - extra; - (void)newsize; // placate people compiling -Wunused-variable - // Prefer mremap, fall back to munmap - if ((SPP_CALL_MREMAP(sp->_base, sp->_size, newsize, 0) != mfail) || - (SPP_CALL_MUNMAP(sp->_base + newsize, extra) == 0)) - released = extra; - } - } - } - - if (released != 0) - { - sp->_size -= released; - _footprint -= released; - init_top(_top, _topsize - released); - check_top_chunk(_top); - } - } - - // Unmap any unused mmapped segments - if (SPP_HAVE_MMAP) - released += release_unused_segments(); - - // On failure, disable autotrim to avoid repeated failed future calls - if (released == 0 && _topsize > _trim_check) - _trim_check = spp_max_size_t; - } - - return (released != 0) ? 1 : 0; -} - -/* Consolidate and bin a chunk. Differs from exported versions - of free mainly in that the chunk need not be marked as inuse. -*/ -void malloc_state::dispose_chunk(mchunkptr p, size_t psize) -{ - mchunkptr next = (mchunkptr)p->chunk_plus_offset(psize); - if (!p->pinuse()) - { - mchunkptr prev; - size_t prevsize = p->_prev_foot; - if (p->is_mmapped()) - { - psize += prevsize + SPP_MMAP_FOOT_PAD; - if (SPP_CALL_MUNMAP((char*)p - prevsize, psize) == 0) - _footprint -= psize; - return; - } - prev = (mchunkptr)p->chunk_minus_offset(prevsize); - psize += prevsize; - p = prev; - if (rtcheck(ok_address(prev))) - { - // consolidate backward - if (p != _dv) - unlink_chunk(p, prevsize); - else if ((next->_head & INUSE_BITS) == INUSE_BITS) - { - _dvsize = psize; - p->set_free_with_pinuse(psize, next); - return; - } - } - else - { - SPP_ABORT; - return; - } - } - if (rtcheck(ok_address(next))) - { - if (!next->cinuse()) - { - // consolidate forward - if (next == _top) - { - size_t tsize = _topsize += psize; - _top = p; - p->_head = tsize | PINUSE_BIT; - if (p == _dv) - { - _dv = 0; - _dvsize = 0; - } - return; - } - else if (next == _dv) - { - size_t dsize = _dvsize += psize; - _dv = p; - p->set_size_and_pinuse_of_free_chunk(dsize); - return; - } - else - { - size_t nsize = next->chunksize(); - psize += nsize; - unlink_chunk(next, nsize); - p->set_size_and_pinuse_of_free_chunk(psize); - if (p == _dv) - { - _dvsize = psize; - return; - } - } - } - else - p->set_free_with_pinuse(psize, next); - insert_chunk(p, psize); - } - else - SPP_ABORT; -} - -/* ---------------------------- malloc --------------------------- */ - -// allocate a large request from the best fitting chunk in a treebin -void* malloc_state::tmalloc_large(size_t nb) -{ - tchunkptr v = 0; - size_t rsize = -nb; // Unsigned negation - tchunkptr t; - bindex_t idx = compute_tree_index(nb); - if ((t = *treebin_at(idx)) != 0) - { - // Traverse tree for this bin looking for node with size == nb - size_t sizebits = nb << leftshift_for_tree_index(idx); - tchunkptr rst = 0; // The deepest untaken right subtree - for (;;) - { - tchunkptr rt; - size_t trem = t->chunksize() - nb; - if (trem < rsize) - { - v = t; - if ((rsize = trem) == 0) - break; - } - rt = t->_child[1]; - t = t->_child[(sizebits >> (spp_size_t_bitsize - 1)) & 1]; - if (rt != 0 && rt != t) - rst = rt; - if (t == 0) - { - t = rst; // set t to least subtree holding sizes > nb - break; - } - sizebits <<= 1; - } - } - if (t == 0 && v == 0) - { - // set t to root of next non-empty treebin - binmap_t leftbits = left_bits(idx2bit(idx)) & _treemap; - if (leftbits != 0) - { - binmap_t leastbit = least_bit(leftbits); - bindex_t i = compute_bit2idx(leastbit); - t = *treebin_at(i); - } - } - - while (t != 0) - { - // find smallest of tree or subtree - size_t trem = t->chunksize() - nb; - if (trem < rsize) - { - rsize = trem; - v = t; - } - t = t->leftmost_child(); - } - - // If dv is a better fit, return 0 so malloc will use it - if (v != 0 && rsize < (size_t)(_dvsize - nb)) - { - if (rtcheck(ok_address(v))) - { - // split - mchunkptr r = (mchunkptr)v->chunk_plus_offset(nb); - assert(v->chunksize() == rsize + nb); - if (rtcheck(ok_next(v, r))) - { - unlink_large_chunk(v); - if (rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(v, (rsize + nb)); - else - { - set_size_and_pinuse_of_inuse_chunk(v, nb); - r->set_size_and_pinuse_of_free_chunk(rsize); - insert_chunk(r, rsize); - } - return chunk2mem(v); - } - } - SPP_ABORT; - } - return 0; -} - -// allocate a small request from the best fitting chunk in a treebin -void* malloc_state::tmalloc_small(size_t nb) -{ - tchunkptr t, v; - size_t rsize; - binmap_t leastbit = least_bit(_treemap); - bindex_t i = compute_bit2idx(leastbit); - v = t = *treebin_at(i); - rsize = t->chunksize() - nb; - - while ((t = t->leftmost_child()) != 0) - { - size_t trem = t->chunksize() - nb; - if (trem < rsize) - { - rsize = trem; - v = t; - } - } - - if (rtcheck(ok_address(v))) - { - mchunkptr r = (mchunkptr)v->chunk_plus_offset(nb); - assert(v->chunksize() == rsize + nb); - if (rtcheck(ok_next(v, r))) - { - unlink_large_chunk(v); - if (rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(v, (rsize + nb)); - else - { - set_size_and_pinuse_of_inuse_chunk(v, nb); - r->set_size_and_pinuse_of_free_chunk(rsize); - replace_dv(r, rsize); - } - return chunk2mem(v); - } - } - - SPP_ABORT; - return 0; -} - -/* ---------------------------- malloc --------------------------- */ - -void* malloc_state::_malloc(size_t bytes) -{ - if (1) - { - void* mem; - size_t nb; - if (bytes <= MAX_SMALL_REQUEST) - { - bindex_t idx; - binmap_t smallbits; - nb = (bytes < MIN_REQUEST) ? MIN_CHUNK_SIZE : pad_request(bytes); - idx = small_index(nb); - smallbits = _smallmap >> idx; - - if ((smallbits & 0x3U) != 0) - { - // Remainderless fit to a smallbin. - mchunkptr b, p; - idx += ~smallbits & 1; // Uses next bin if idx empty - b = smallbin_at(idx); - p = b->_fd; - assert(p->chunksize() == small_index2size(idx)); - unlink_first_small_chunk(b, p, idx); - set_inuse_and_pinuse(p, small_index2size(idx)); - mem = chunk2mem(p); - check_malloced_chunk(mem, nb); - goto postaction; - } - - else if (nb > _dvsize) - { - if (smallbits != 0) - { - // Use chunk in next nonempty smallbin - mchunkptr b, p, r; - size_t rsize; - binmap_t leftbits = (smallbits << idx) & left_bits(malloc_state::idx2bit(idx)); - binmap_t leastbit = least_bit(leftbits); - bindex_t i = compute_bit2idx(leastbit); - b = smallbin_at(i); - p = b->_fd; - assert(p->chunksize() == small_index2size(i)); - unlink_first_small_chunk(b, p, i); - rsize = small_index2size(i) - nb; - // Fit here cannot be remainderless if 4byte sizes - if (sizeof(size_t) != 4 && rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(p, small_index2size(i)); - else - { - set_size_and_pinuse_of_inuse_chunk(p, nb); - r = (mchunkptr)p->chunk_plus_offset(nb); - r->set_size_and_pinuse_of_free_chunk(rsize); - replace_dv(r, rsize); - } - mem = chunk2mem(p); - check_malloced_chunk(mem, nb); - goto postaction; - } - - else if (_treemap != 0 && (mem = tmalloc_small(nb)) != 0) - { - check_malloced_chunk(mem, nb); - goto postaction; - } - } - } - else if (bytes >= MAX_REQUEST) - nb = spp_max_size_t; // Too big to allocate. Force failure (in sys alloc) - else - { - nb = pad_request(bytes); - if (_treemap != 0 && (mem = tmalloc_large(nb)) != 0) - { - check_malloced_chunk(mem, nb); - goto postaction; - } - } - - if (nb <= _dvsize) - { - size_t rsize = _dvsize - nb; - mchunkptr p = _dv; - if (rsize >= MIN_CHUNK_SIZE) - { - // split dv - mchunkptr r = _dv = (mchunkptr)p->chunk_plus_offset(nb); - _dvsize = rsize; - r->set_size_and_pinuse_of_free_chunk(rsize); - set_size_and_pinuse_of_inuse_chunk(p, nb); - } - else // exhaust dv - { - size_t dvs = _dvsize; - _dvsize = 0; - _dv = 0; - set_inuse_and_pinuse(p, dvs); - } - mem = chunk2mem(p); - check_malloced_chunk(mem, nb); - goto postaction; - } - - else if (nb < _topsize) - { - // Split top - size_t rsize = _topsize -= nb; - mchunkptr p = _top; - mchunkptr r = _top = (mchunkptr)p->chunk_plus_offset(nb); - r->_head = rsize | PINUSE_BIT; - set_size_and_pinuse_of_inuse_chunk(p, nb); - mem = chunk2mem(p); - check_top_chunk(_top); - check_malloced_chunk(mem, nb); - goto postaction; - } - - mem = sys_alloc(nb); - -postaction: - return mem; - } - - return 0; -} - -/* ---------------------------- free --------------------------- */ - -void malloc_state::_free(mchunkptr p) -{ - if (1) - { - check_inuse_chunk(p); - if (rtcheck(ok_address(p) && ok_inuse(p))) - { - size_t psize = p->chunksize(); - mchunkptr next = (mchunkptr)p->chunk_plus_offset(psize); - if (!p->pinuse()) - { - size_t prevsize = p->_prev_foot; - if (p->is_mmapped()) - { - psize += prevsize + SPP_MMAP_FOOT_PAD; - if (SPP_CALL_MUNMAP((char*)p - prevsize, psize) == 0) - _footprint -= psize; - goto postaction; - } - else - { - mchunkptr prev = (mchunkptr)p->chunk_minus_offset(prevsize); - psize += prevsize; - p = prev; - if (rtcheck(ok_address(prev))) - { - // consolidate backward - if (p != _dv) - unlink_chunk(p, prevsize); - else if ((next->_head & INUSE_BITS) == INUSE_BITS) - { - _dvsize = psize; - p->set_free_with_pinuse(psize, next); - goto postaction; - } - } - else - goto erroraction; - } - } - - if (rtcheck(ok_next(p, next) && ok_pinuse(next))) - { - if (!next->cinuse()) - { - // consolidate forward - if (next == _top) - { - size_t tsize = _topsize += psize; - _top = p; - p->_head = tsize | PINUSE_BIT; - if (p == _dv) - { - _dv = 0; - _dvsize = 0; - } - if (should_trim(tsize)) - sys_trim(0); - goto postaction; - } - else if (next == _dv) - { - size_t dsize = _dvsize += psize; - _dv = p; - p->set_size_and_pinuse_of_free_chunk(dsize); - goto postaction; - } - else - { - size_t nsize = next->chunksize(); - psize += nsize; - unlink_chunk(next, nsize); - p->set_size_and_pinuse_of_free_chunk(psize); - if (p == _dv) - { - _dvsize = psize; - goto postaction; - } - } - } - else - p->set_free_with_pinuse(psize, next); - - if (is_small(psize)) - { - insert_small_chunk(p, psize); - check_free_chunk(p); - } - else - { - tchunkptr tp = (tchunkptr)p; - insert_large_chunk(tp, psize); - check_free_chunk(p); - if (--_release_checks == 0) - release_unused_segments(); - } - goto postaction; - } - } -erroraction: - SPP_USAGE_ERROR_ACTION(this, p); -postaction: - ; - } -} - -/* ------------ Internal support for realloc, memalign, etc -------------- */ - -// Try to realloc; only in-place unless can_move true -mchunkptr malloc_state::try_realloc_chunk(mchunkptr p, size_t nb, int can_move) -{ - mchunkptr newp = 0; - size_t oldsize = p->chunksize(); - mchunkptr next = (mchunkptr)p->chunk_plus_offset(oldsize); - if (rtcheck(ok_address(p) && ok_inuse(p) && - ok_next(p, next) && ok_pinuse(next))) - { - if (p->is_mmapped()) - newp = mmap_resize(p, nb, can_move); - else if (oldsize >= nb) - { - // already big enough - size_t rsize = oldsize - nb; - if (rsize >= MIN_CHUNK_SIZE) - { - // split off remainder - mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb); - set_inuse(p, nb); - set_inuse(r, rsize); - dispose_chunk(r, rsize); - } - newp = p; - } - else if (next == _top) - { - // extend into top - if (oldsize + _topsize > nb) - { - size_t newsize = oldsize + _topsize; - size_t newtopsize = newsize - nb; - mchunkptr newtop = (mchunkptr)p->chunk_plus_offset(nb); - set_inuse(p, nb); - newtop->_head = newtopsize | PINUSE_BIT; - _top = newtop; - _topsize = newtopsize; - newp = p; - } - } - else if (next == _dv) - { - // extend into dv - size_t dvs = _dvsize; - if (oldsize + dvs >= nb) - { - size_t dsize = oldsize + dvs - nb; - if (dsize >= MIN_CHUNK_SIZE) - { - mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb); - mchunkptr n = (mchunkptr)r->chunk_plus_offset(dsize); - set_inuse(p, nb); - r->set_size_and_pinuse_of_free_chunk(dsize); - n->clear_pinuse(); - _dvsize = dsize; - _dv = r; - } - else - { - // exhaust dv - size_t newsize = oldsize + dvs; - set_inuse(p, newsize); - _dvsize = 0; - _dv = 0; - } - newp = p; - } - } - else if (!next->cinuse()) - { - // extend into next free chunk - size_t nextsize = next->chunksize(); - if (oldsize + nextsize >= nb) - { - size_t rsize = oldsize + nextsize - nb; - unlink_chunk(next, nextsize); - if (rsize < MIN_CHUNK_SIZE) - { - size_t newsize = oldsize + nextsize; - set_inuse(p, newsize); - } - else - { - mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb); - set_inuse(p, nb); - set_inuse(r, rsize); - dispose_chunk(r, rsize); - } - newp = p; - } - } - } - else - SPP_USAGE_ERROR_ACTION(m, chunk2mem(p)); - return newp; -} - -void* malloc_state::internal_memalign(size_t alignment, size_t bytes) -{ - void* mem = 0; - if (alignment < MIN_CHUNK_SIZE) // must be at least a minimum chunk size - alignment = MIN_CHUNK_SIZE; - if ((alignment & (alignment - 1)) != 0) - { - // Ensure a power of 2 - size_t a = SPP_MALLOC_ALIGNMENT << 1; - while (a < alignment) - a <<= 1; - alignment = a; - } - if (bytes >= MAX_REQUEST - alignment) - SPP_MALLOC_FAILURE_ACTION; - else - { - size_t nb = request2size(bytes); - size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD; - mem = internal_malloc(req); - if (mem != 0) - { - mchunkptr p = mem2chunk(mem); - if ((((size_t)(mem)) & (alignment - 1)) != 0) - { - // misaligned - /* - Find an aligned spot inside chunk. Since we need to give - back leading space in a chunk of at least MIN_CHUNK_SIZE, if - the first calculation places us at a spot with less than - MIN_CHUNK_SIZE leader, we can move to the next aligned spot. - We've allocated enough total room so that this is always - possible. - */ - char* br = (char*)mem2chunk((void *)(((size_t)((char*)mem + alignment - 1)) & - -alignment)); - char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE) ? - br : br + alignment; - mchunkptr newp = (mchunkptr)pos; - size_t leadsize = pos - (char*)(p); - size_t newsize = p->chunksize() - leadsize; - - if (p->is_mmapped()) - { - // For mmapped chunks, just adjust offset - newp->_prev_foot = p->_prev_foot + leadsize; - newp->_head = newsize; - } - else - { - // Otherwise, give back leader, use the rest - set_inuse(newp, newsize); - set_inuse(p, leadsize); - dispose_chunk(p, leadsize); - } - p = newp; - } - - // Give back spare room at the end - if (!p->is_mmapped()) - { - size_t size = p->chunksize(); - if (size > nb + MIN_CHUNK_SIZE) - { - size_t remainder_size = size - nb; - mchunkptr remainder = (mchunkptr)p->chunk_plus_offset(nb); - set_inuse(p, nb); - set_inuse(remainder, remainder_size); - dispose_chunk(remainder, remainder_size); - } - } - - mem = chunk2mem(p); - assert(p->chunksize() >= nb); - assert(((size_t)mem & (alignment - 1)) == 0); - check_inuse_chunk(p); - } - } - return mem; -} - -/* - Common support for independent_X routines, handling - all of the combinations that can result. - The opts arg has: - bit 0 set if all elements are same size (using sizes[0]) - bit 1 set if elements should be zeroed -*/ -void** malloc_state::ialloc(size_t n_elements, size_t* sizes, int opts, - void* chunks[]) -{ - - size_t element_size; // chunksize of each element, if all same - size_t contents_size; // total size of elements - size_t array_size; // request size of pointer array - void* mem; // malloced aggregate space - mchunkptr p; // corresponding chunk - size_t remainder_size; // remaining bytes while splitting - void** marray; // either "chunks" or malloced ptr array - mchunkptr array_chunk; // chunk for malloced ptr array - flag_t was_enabled; // to disable mmap - size_t size; - size_t i; - - mparams.ensure_initialization(); - // compute array length, if needed - if (chunks != 0) - { - if (n_elements == 0) - return chunks; // nothing to do - marray = chunks; - array_size = 0; - } - else - { - // if empty req, must still return chunk representing empty array - if (n_elements == 0) - return (void**)internal_malloc(0); - marray = 0; - array_size = request2size(n_elements * (sizeof(void*))); - } - - // compute total element size - if (opts & 0x1) - { - // all-same-size - element_size = request2size(*sizes); - contents_size = n_elements * element_size; - } - else - { - // add up all the sizes - element_size = 0; - contents_size = 0; - for (i = 0; i != n_elements; ++i) - contents_size += request2size(sizes[i]); - } - - size = contents_size + array_size; - - /* - Allocate the aggregate chunk. First disable direct-mmapping so - malloc won't use it, since we would not be able to later - free/realloc space internal to a segregated mmap region. - */ - was_enabled = use_mmap(); - disable_mmap(); - mem = internal_malloc(size - CHUNK_OVERHEAD); - if (was_enabled) - enable_mmap(); - if (mem == 0) - return 0; - - p = mem2chunk(mem); - remainder_size = p->chunksize(); - - assert(!p->is_mmapped()); - - if (opts & 0x2) - { - // optionally clear the elements - memset((size_t*)mem, 0, remainder_size - sizeof(size_t) - array_size); - } - - // If not provided, allocate the pointer array as final part of chunk - if (marray == 0) - { - size_t array_chunk_size; - array_chunk = (mchunkptr)p->chunk_plus_offset(contents_size); - array_chunk_size = remainder_size - contents_size; - marray = (void**)(chunk2mem(array_chunk)); - set_size_and_pinuse_of_inuse_chunk(array_chunk, array_chunk_size); - remainder_size = contents_size; - } - - // split out elements - for (i = 0; ; ++i) - { - marray[i] = chunk2mem(p); - if (i != n_elements - 1) - { - if (element_size != 0) - size = element_size; - else - size = request2size(sizes[i]); - remainder_size -= size; - set_size_and_pinuse_of_inuse_chunk(p, size); - p = (mchunkptr)p->chunk_plus_offset(size); - } - else - { - // the final element absorbs any overallocation slop - set_size_and_pinuse_of_inuse_chunk(p, remainder_size); - break; - } - } - -#if SPP_DEBUG - if (marray != chunks) - { - // final element must have exactly exhausted chunk - if (element_size != 0) - assert(remainder_size == element_size); - else - assert(remainder_size == request2size(sizes[i])); - check_inuse_chunk(mem2chunk(marray)); - } - for (i = 0; i != n_elements; ++i) - check_inuse_chunk(mem2chunk(marray[i])); - -#endif - - return marray; -} - -/* Try to free all pointers in the given array. - Note: this could be made faster, by delaying consolidation, - at the price of disabling some user integrity checks, We - still optimize some consolidations by combining adjacent - chunks before freeing, which will occur often if allocated - with ialloc or the array is sorted. -*/ -size_t malloc_state::internal_bulk_free(void* array[], size_t nelem) -{ - size_t unfreed = 0; - if (1) - { - void** a; - void** fence = &(array[nelem]); - for (a = array; a != fence; ++a) - { - void* mem = *a; - if (mem != 0) - { - mchunkptr p = mem2chunk(mem); - size_t psize = p->chunksize(); -#if SPP_FOOTERS - if (get_mstate_for(p) != m) - { - ++unfreed; - continue; - } -#endif - check_inuse_chunk(p); - *a = 0; - if (rtcheck(ok_address(p) && ok_inuse(p))) - { - void ** b = a + 1; // try to merge with next chunk - mchunkptr next = (mchunkptr)p->next_chunk(); - if (b != fence && *b == chunk2mem(next)) - { - size_t newsize = next->chunksize() + psize; - set_inuse(p, newsize); - *b = chunk2mem(p); - } - else - dispose_chunk(p, psize); - } - else - { - SPP_ABORT; - break; - } - } - } - if (should_trim(_topsize)) - sys_trim(0); - } - return unfreed; -} - -void malloc_state::init(char* tbase, size_t tsize) -{ - _seg._base = _least_addr = tbase; - _seg._size = _footprint = _max_footprint = tsize; - _magic = mparams._magic; - _release_checks = SPP_MAX_RELEASE_CHECK_RATE; - _mflags = mparams._default_mflags; - _extp = 0; - _exts = 0; - disable_contiguous(); - init_bins(); - mchunkptr mn = (mchunkptr)mem2chunk(this)->next_chunk(); - init_top(mn, (size_t)((tbase + tsize) - (char*)mn) - top_foot_size()); - check_top_chunk(_top); -} - -/* Traversal */ -#if SPP_MALLOC_INSPECT_ALL -void malloc_state::internal_inspect_all(void(*handler)(void *start, void *end, - size_t used_bytes, - void* callback_arg), - void* arg) -{ - if (is_initialized()) - { - mchunkptr top = top; - msegmentptr s; - for (s = &seg; s != 0; s = s->next) - { - mchunkptr q = align_as_chunk(s->base); - while (segment_holds(s, q) && q->head != FENCEPOST_HEAD) - { - mchunkptr next = (mchunkptr)q->next_chunk(); - size_t sz = q->chunksize(); - size_t used; - void* start; - if (q->is_inuse()) - { - used = sz - CHUNK_OVERHEAD; // must not be mmapped - start = chunk2mem(q); - } - else - { - used = 0; - if (is_small(sz)) - { - // offset by possible bookkeeping - start = (void*)((char*)q + sizeof(struct malloc_chunk)); - } - else - start = (void*)((char*)q + sizeof(struct malloc_tree_chunk)); - } - if (start < (void*)next) // skip if all space is bookkeeping - handler(start, next, used, arg); - if (q == top) - break; - q = next; - } - } - } -} -#endif // SPP_MALLOC_INSPECT_ALL - - - -/* ----------------------------- user mspaces ---------------------------- */ - -static mstate init_user_mstate(char* tbase, size_t tsize) -{ - size_t msize = pad_request(sizeof(malloc_state)); - mchunkptr msp = align_as_chunk(tbase); - mstate m = (mstate)(chunk2mem(msp)); - memset(m, 0, msize); - msp->_head = (msize | INUSE_BITS); - m->init(tbase, tsize); - return m; -} - -SPP_API mspace create_mspace(size_t capacity, int locked) -{ - mstate m = 0; - size_t msize; - mparams.ensure_initialization(); - msize = pad_request(sizeof(malloc_state)); - if (capacity < (size_t) - (msize + top_foot_size() + mparams._page_size)) - { - size_t rs = ((capacity == 0) ? mparams._granularity : - (capacity + top_foot_size() + msize)); - size_t tsize = mparams.granularity_align(rs); - char* tbase = (char*)(SPP_CALL_MMAP(tsize)); - if (tbase != cmfail) - { - m = init_user_mstate(tbase, tsize); - m->_seg._sflags = USE_MMAP_BIT; - m->set_lock(locked); - } - } - return (mspace)m; -} - -SPP_API size_t destroy_mspace(mspace msp) -{ - size_t freed = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - { - msegmentptr sp = &ms->_seg; - while (sp != 0) - { - char* base = sp->_base; - size_t size = sp->_size; - flag_t flag = sp->_sflags; - (void)base; // placate people compiling -Wunused-variable - sp = sp->_next; - if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) && - SPP_CALL_MUNMAP(base, size) == 0) - freed += size; - } - } - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return freed; -} - -/* ---------------------------- mspace versions of malloc/calloc/free routines -------------------- */ -SPP_API void* mspace_malloc(mspace msp, size_t bytes) -{ - mstate ms = (mstate)msp; - if (!ms->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(ms, ms); - return 0; - } - return ms->_malloc(bytes); -} - -SPP_API void mspace_free(mspace msp, void* mem) -{ - if (mem != 0) - { - mchunkptr p = mem2chunk(mem); -#if SPP_FOOTERS - mstate fm = get_mstate_for(p); - (void)msp; // placate people compiling -Wunused -#else - mstate fm = (mstate)msp; -#endif - if (!fm->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(fm, p); - return; - } - fm->_free(p); - } -} - -SPP_API inline void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) -{ - void* mem; - size_t req = 0; - mstate ms = (mstate)msp; - if (!ms->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(ms, ms); - return 0; - } - if (n_elements != 0) - { - req = n_elements * elem_size; - if (((n_elements | elem_size) & ~(size_t)0xffff) && - (req / n_elements != elem_size)) - req = spp_max_size_t; // force downstream failure on overflow - } - mem = ms->internal_malloc(req); - if (mem != 0 && mem2chunk(mem)->calloc_must_clear()) - memset(mem, 0, req); - return mem; -} - -SPP_API inline void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) -{ - void* mem = 0; - if (oldmem == 0) - mem = mspace_malloc(msp, bytes); - else if (bytes >= MAX_REQUEST) - SPP_MALLOC_FAILURE_ACTION; -#ifdef REALLOC_ZERO_BYTES_FREES - else if (bytes == 0) - mspace_free(msp, oldmem); -#endif - else - { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! SPP_FOOTERS - mstate m = (mstate)msp; -#else - mstate m = get_mstate_for(oldp); - if (!m->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif - if (1) - { - mchunkptr newp = m->try_realloc_chunk(oldp, nb, 1); - if (newp != 0) - { - m->check_inuse_chunk(newp); - mem = chunk2mem(newp); - } - else - { - mem = mspace_malloc(m, bytes); - if (mem != 0) - { - size_t oc = oldp->chunksize() - oldp->overhead_for(); - memcpy(mem, oldmem, (oc < bytes) ? oc : bytes); - mspace_free(m, oldmem); - } - } - } - } - return mem; -} - -#if 0 - -SPP_API mspace create_mspace_with_base(void* base, size_t capacity, int locked) -{ - mstate m = 0; - size_t msize; - mparams.ensure_initialization(); - msize = pad_request(sizeof(malloc_state)); - if (capacity > msize + top_foot_size() && - capacity < (size_t) - (msize + top_foot_size() + mparams._page_size)) - { - m = init_user_mstate((char*)base, capacity); - m->_seg._sflags = EXTERN_BIT; - m->set_lock(locked); - } - return (mspace)m; -} - -SPP_API int mspace_track_large_chunks(mspace msp, int enable) -{ - int ret = 0; - mstate ms = (mstate)msp; - if (1) - { - if (!ms->use_mmap()) - ret = 1; - if (!enable) - ms->enable_mmap(); - else - ms->disable_mmap(); - } - return ret; -} - -SPP_API void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) -{ - void* mem = 0; - if (oldmem != 0) - { - if (bytes >= MAX_REQUEST) - SPP_MALLOC_FAILURE_ACTION; - else - { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! SPP_FOOTERS - mstate m = (mstate)msp; -#else - mstate m = get_mstate_for(oldp); - (void)msp; // placate people compiling -Wunused - if (!m->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif - if (1) - { - mchunkptr newp = m->try_realloc_chunk(oldp, nb, 0); - if (newp == oldp) - { - m->check_inuse_chunk(newp); - mem = oldmem; - } - } - } - } - return mem; -} - -SPP_API void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) -{ - mstate ms = (mstate)msp; - if (!ms->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(ms, ms); - return 0; - } - if (alignment <= SPP_MALLOC_ALIGNMENT) - return mspace_malloc(msp, bytes); - return ms->internal_memalign(alignment, bytes); -} - -SPP_API void** mspace_independent_calloc(mspace msp, size_t n_elements, - size_t elem_size, void* chunks[]) -{ - size_t sz = elem_size; // serves as 1-element array - mstate ms = (mstate)msp; - if (!ms->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(ms, ms); - return 0; - } - return ms->ialloc(n_elements, &sz, 3, chunks); -} - -SPP_API void** mspace_independent_comalloc(mspace msp, size_t n_elements, - size_t sizes[], void* chunks[]) -{ - mstate ms = (mstate)msp; - if (!ms->ok_magic()) - { - SPP_USAGE_ERROR_ACTION(ms, ms); - return 0; - } - return ms->ialloc(n_elements, sizes, 0, chunks); -} - -#endif - -SPP_API inline size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem) -{ - return ((mstate)msp)->internal_bulk_free(array, nelem); -} - -#if SPP_MALLOC_INSPECT_ALL -SPP_API void mspace_inspect_all(mspace msp, - void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg) -{ - mstate ms = (mstate)msp; - if (ms->ok_magic()) - internal_inspect_all(ms, handler, arg); - else - SPP_USAGE_ERROR_ACTION(ms, ms); -} -#endif - -SPP_API inline int mspace_trim(mspace msp, size_t pad) -{ - int result = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - result = ms->sys_trim(pad); - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return result; -} - -SPP_API inline size_t mspace_footprint(mspace msp) -{ - size_t result = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - result = ms->_footprint; - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return result; -} - -SPP_API inline size_t mspace_max_footprint(mspace msp) -{ - size_t result = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - result = ms->_max_footprint; - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return result; -} - -SPP_API inline size_t mspace_footprint_limit(mspace msp) -{ - size_t result = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - { - size_t maf = ms->_footprint_limit; - result = (maf == 0) ? spp_max_size_t : maf; - } - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return result; -} - -SPP_API inline size_t mspace_set_footprint_limit(mspace msp, size_t bytes) -{ - size_t result = 0; - mstate ms = (mstate)msp; - if (ms->ok_magic()) - { - if (bytes == 0) - result = mparams.granularity_align(1); // Use minimal size - if (bytes == spp_max_size_t) - result = 0; // disable - else - result = mparams.granularity_align(bytes); - ms->_footprint_limit = result; - } - else - SPP_USAGE_ERROR_ACTION(ms, ms); - return result; -} - -SPP_API inline size_t mspace_usable_size(const void* mem) -{ - if (mem != 0) - { - mchunkptr p = mem2chunk(mem); - if (p->is_inuse()) - return p->chunksize() - p->overhead_for(); - } - return 0; -} - -SPP_API inline int mspace_mallopt(int param_number, int value) -{ - return mparams.change(param_number, value); -} - -} // spp_ namespace - - -#endif // SPP_EXCLUDE_IMPLEMENTATION - -#endif // spp_dlalloc__h_ diff --git a/benchmarks/others/sparsepp/spp_smartptr.h b/benchmarks/others/sparsepp/spp_smartptr.h deleted file mode 100644 index fba3acfb..00000000 --- a/benchmarks/others/sparsepp/spp_smartptr.h +++ /dev/null @@ -1,71 +0,0 @@ -#if !defined(spp_smartptr_h_guard) -#define spp_smartptr_h_guard - - -/* ----------------------------------------------------------------------------------------------- - * quick version of intrusive_ptr - * ----------------------------------------------------------------------------------------------- - */ - -#include <cassert> -#include "spp_config.h" - -// ------------------------------------------------------------------------ -class spp_rc -{ -public: - spp_rc() : _cnt(0) {} - spp_rc(const spp_rc &) : _cnt(0) {} - void increment() const { ++_cnt; } - void decrement() const { assert(_cnt); if (--_cnt == 0) delete this; } - unsigned count() const { return _cnt; } - -protected: - virtual ~spp_rc() {} - -private: - mutable unsigned _cnt; -}; - -// ------------------------------------------------------------------------ -template <class T> -class spp_sptr -{ -public: - spp_sptr() : _p(0) {} - spp_sptr(T *p) : _p(p) { if (_p) _p->increment(); } - spp_sptr(const spp_sptr &o) : _p(o._p) { if (_p) _p->increment(); } -#ifndef SPP_NO_CXX11_RVALUE_REFERENCES - spp_sptr(spp_sptr &&o) : _p(o._p) { o._p = (T *)0; } - spp_sptr& operator=(spp_sptr &&o) { this->swap(o); return *this; } -#endif - ~spp_sptr() { if (_p) _p->decrement(); } - spp_sptr& operator=(const spp_sptr &o) { reset(o._p); return *this; } - T* get() const { return _p; } - void swap(spp_sptr &o) { T *tmp = _p; _p = o._p; o._p = tmp; } - void reset(const T *p = 0) - { - if (p == _p) - return; - if (_p) _p->decrement(); - _p = (T *)p; - if (_p) _p->increment(); - } - T* operator->() const { return const_cast<T *>(_p); } - bool operator!() const { return _p == 0; } - -private: - T *_p; -}; - -// ------------------------------------------------------------------------ -namespace std -{ - template <class T> - inline void swap(spp_sptr<T> &a, spp_sptr<T> &b) - { - a.swap(b); - } -} - -#endif // spp_smartptr_h_guard diff --git a/benchmarks/others/sparsepp/spp_stdint.h b/benchmarks/others/sparsepp/spp_stdint.h deleted file mode 100644 index 3adced9c..00000000 --- a/benchmarks/others/sparsepp/spp_stdint.h +++ /dev/null @@ -1,16 +0,0 @@ -#if !defined(spp_stdint_h_guard) -#define spp_stdint_h_guard - -#include "spp_config.h" - -#if defined(SPP_HAS_CSTDINT) && (__cplusplus >= 201103) - #include <cstdint> -#else - #if defined(__FreeBSD__) || defined(__IBMCPP__) || defined(_AIX) - #include <inttypes.h> - #else - #include <stdint.h> - #endif -#endif - -#endif // spp_stdint_h_guard diff --git a/benchmarks/others/sparsepp/spp_timer.h b/benchmarks/others/sparsepp/spp_timer.h deleted file mode 100644 index 48180f4d..00000000 --- a/benchmarks/others/sparsepp/spp_timer.h +++ /dev/null @@ -1,58 +0,0 @@ -/** - Copyright (c) 2016 Mariano Gonzalez - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#ifndef spp_timer_h_guard -#define spp_timer_h_guard - -#include <chrono> - -namespace spp -{ - template<typename time_unit = std::milli> - class Timer - { - public: - Timer() { reset(); } - void reset() { _start = _snap = clock::now(); } - void snap() { _snap = clock::now(); } - - float get_total() const { return get_diff<float>(_start, clock::now()); } - float get_delta() const { return get_diff<float>(_snap, clock::now()); } - - private: - using clock = std::chrono::high_resolution_clock; - using point = std::chrono::time_point<clock>; - - template<typename T> - static T get_diff(const point& start, const point& end) - { - using duration_t = std::chrono::duration<T, time_unit>; - - return std::chrono::duration_cast<duration_t>(end - start).count(); - } - - point _start; - point _snap; - }; -} - -#endif // spp_timer_h_guard diff --git a/benchmarks/others/sparsepp/spp_traits.h b/benchmarks/others/sparsepp/spp_traits.h deleted file mode 100644 index 792f52f2..00000000 --- a/benchmarks/others/sparsepp/spp_traits.h +++ /dev/null @@ -1,125 +0,0 @@ -#if !defined(spp_traits_h_guard) -#define spp_traits_h_guard - -#include "spp_config.h" - -template<int S, int H> class HashObject; // for Google's benchmark, not in spp namespace! - -namespace spp_ -{ - -// --------------------------------------------------------------------------- -// type_traits we need -// --------------------------------------------------------------------------- -template<class T, T v> -struct integral_constant { static const T value = v; }; - -template <class T, T v> const T integral_constant<T, v>::value; - -typedef integral_constant<bool, true> true_type; -typedef integral_constant<bool, false> false_type; - -typedef integral_constant<int, 0> zero_type; -typedef integral_constant<int, 1> one_type; -typedef integral_constant<int, 2> two_type; -typedef integral_constant<int, 3> three_type; - -template<typename T, typename U> struct is_same : public false_type { }; -template<typename T> struct is_same<T, T> : public true_type { }; - -template<typename T> struct remove_const { typedef T type; }; -template<typename T> struct remove_const<T const> { typedef T type; }; - -template<typename T> struct remove_volatile { typedef T type; }; -template<typename T> struct remove_volatile<T volatile> { typedef T type; }; - -template<typename T> struct remove_cv -{ - typedef typename remove_const<typename remove_volatile<T>::type>::type type; -}; - -// ---------------- is_integral ---------------------------------------- -template <class T> struct is_integral; -template <class T> struct is_integral : false_type { }; -template<> struct is_integral<bool> : true_type { }; -template<> struct is_integral<char> : true_type { }; -template<> struct is_integral<unsigned char> : true_type { }; -template<> struct is_integral<signed char> : true_type { }; -template<> struct is_integral<short> : true_type { }; -template<> struct is_integral<unsigned short> : true_type { }; -template<> struct is_integral<int> : true_type { }; -template<> struct is_integral<unsigned int> : true_type { }; -template<> struct is_integral<long> : true_type { }; -template<> struct is_integral<unsigned long> : true_type { }; -#ifdef SPP_HAS_LONG_LONG - template<> struct is_integral<long long> : true_type { }; - template<> struct is_integral<unsigned long long> : true_type { }; -#endif -template <class T> struct is_integral<const T> : is_integral<T> { }; -template <class T> struct is_integral<volatile T> : is_integral<T> { }; -template <class T> struct is_integral<const volatile T> : is_integral<T> { }; - -// ---------------- is_floating_point ---------------------------------------- -template <class T> struct is_floating_point; -template <class T> struct is_floating_point : false_type { }; -template<> struct is_floating_point<float> : true_type { }; -template<> struct is_floating_point<double> : true_type { }; -template<> struct is_floating_point<long double> : true_type { }; -template <class T> struct is_floating_point<const T> : is_floating_point<T> { }; -template <class T> struct is_floating_point<volatile T> : is_floating_point<T> { }; -template <class T> struct is_floating_point<const volatile T> : is_floating_point<T> { }; - -// ---------------- is_pointer ---------------------------------------- -template <class T> struct is_pointer; -template <class T> struct is_pointer : false_type { }; -template <class T> struct is_pointer<T*> : true_type { }; -template <class T> struct is_pointer<const T> : is_pointer<T> { }; -template <class T> struct is_pointer<volatile T> : is_pointer<T> { }; -template <class T> struct is_pointer<const volatile T> : is_pointer<T> { }; - -// ---------------- is_reference ---------------------------------------- -template <class T> struct is_reference; -template<typename T> struct is_reference : false_type {}; -template<typename T> struct is_reference<T&> : true_type {}; - -// ---------------- is_relocatable ---------------------------------------- -// relocatable values can be moved around in memory using memcpy and remain -// correct. Most types are relocatable, an example of a type who is not would -// be a struct which contains a pointer to a buffer inside itself - this is the -// case for std::string in gcc 5. -// ------------------------------------------------------------------------ -template <class T> struct is_relocatable; -template <class T> struct is_relocatable : - integral_constant<bool, (is_integral<T>::value || - is_floating_point<T>::value || - is_pointer<T>::value - )> -{ }; - -template<int S, int H> struct is_relocatable<HashObject<S, H> > : true_type { }; - -template <class T> struct is_relocatable<const T> : is_relocatable<T> { }; -template <class T> struct is_relocatable<volatile T> : is_relocatable<T> { }; -template <class T> struct is_relocatable<const volatile T> : is_relocatable<T> { }; -template <class A, int N> struct is_relocatable<A[N]> : is_relocatable<A> { }; -template <class T, class U> struct is_relocatable<std::pair<T, U> > : - integral_constant<bool, (is_relocatable<T>::value && is_relocatable<U>::value)> -{ }; - -// A template helper used to select A or B based on a condition. -// ------------------------------------------------------------ -template<bool cond, typename A, typename B> -struct if_ -{ - typedef A type; -}; - -template<typename A, typename B> -struct if_<false, A, B> -{ - typedef B type; -}; - -} // spp_ namespace - -#endif // spp_traits_h_guard diff --git a/benchmarks/others/sparsepp/spp_utils.h b/benchmarks/others/sparsepp/spp_utils.h deleted file mode 100644 index 4f2e9257..00000000 --- a/benchmarks/others/sparsepp/spp_utils.h +++ /dev/null @@ -1,477 +0,0 @@ -// ---------------------------------------------------------------------- -// Copyright (c) 2016, Steven Gregory Popovitch - [email protected] -// All rights reserved. -// -// Code derived derived from Boost libraries. -// Boost software licence reproduced below. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * The name of Steven Gregory Popovitch may not be used to -// endorse or promote products derived from this software without -// specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// ---------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Boost Software License - Version 1.0 - August 17th, 2003 -// -// Permission is hereby granted, free of charge, to any person or organization -// obtaining a copy of the software and accompanying documentation covered by -// this license (the "Software") to use, reproduce, display, distribute, -// execute, and transmit the Software, and to prepare derivative works of the -// Software, and to permit third-parties to whom the Software is furnished to -// do so, all subject to the following: -// -// The copyright notices in the Software and this entire statement, including -// the above license grant, this restriction and the following disclaimer, -// must be included in all copies of the Software, in whole or in part, and -// all derivative works of the Software, unless such copies or derivative -// works are solely in the form of machine-executable object code generated by -// a source language processor. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// --------------------------------------------------------------------------- - -// ---------------------------------------------------------------------- -// H A S H F U N C T I O N S -// ---------------------------- -// -// Implements spp::spp_hash() and spp::hash_combine() -// ---------------------------------------------------------------------- - -#if !defined(spp_utils_h_guard_) -#define spp_utils_h_guard_ - -#if defined(_MSC_VER) - #if (_MSC_VER >= 1600 ) // vs2010 (1900 is vs2015) - #include <functional> - #define SPP_HASH_CLASS std::hash - #else - #include <hash_map> - #define SPP_HASH_CLASS stdext::hash_compare - #endif - #if (_MSC_FULL_VER < 190021730) - #define SPP_NO_CXX11_NOEXCEPT - #endif -#elif defined __clang__ - #if __has_feature(cxx_noexcept) || defined(SPP_CXX11) // define SPP_CXX11 if your compiler has <functional> - #include <functional> - #define SPP_HASH_CLASS std::hash - #else - #include <tr1/functional> - #define SPP_HASH_CLASS std::tr1::hash - #endif - - #if !__has_feature(cxx_noexcept) - #define SPP_NO_CXX11_NOEXCEPT - #endif -#elif defined(__GNUC__) - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || (__cplusplus >= 201103L) - #include <functional> - #define SPP_HASH_CLASS std::hash - - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100) < 40600 - #define SPP_NO_CXX11_NOEXCEPT - #endif - #else - #include <tr1/unordered_map> - #define SPP_HASH_CLASS std::tr1::hash - #define SPP_NO_CXX11_NOEXCEPT - #endif -#else - #include <functional> - #define SPP_HASH_CLASS std::hash -#endif - -#ifdef SPP_NO_CXX11_NOEXCEPT - #define SPP_NOEXCEPT -#else - #define SPP_NOEXCEPT noexcept -#endif - -#ifdef SPP_NO_CXX11_CONSTEXPR - #define SPP_CONSTEXPR -#else - #define SPP_CONSTEXPR constexpr -#endif - -#ifdef SPP_NO_CXX14_CONSTEXPR - #define SPP_CXX14_CONSTEXPR -#else - #define SPP_CXX14_CONSTEXPR constexpr -#endif - -#define SPP_INLINE - -#ifndef spp_ - #define spp_ spp -#endif - -namespace spp_ -{ - -template <class T> T spp_min(T a, T b) { return a < b ? a : b; } -template <class T> T spp_max(T a, T b) { return a >= b ? a : b; } - -template <class T> -struct spp_hash -{ - SPP_INLINE size_t operator()(const T &__v) const SPP_NOEXCEPT - { - SPP_HASH_CLASS<T> hasher; - return hasher(__v); - } -}; - -template <class T> -struct spp_hash<T *> -{ - static size_t spp_log2 (size_t val) SPP_NOEXCEPT - { - size_t res = 0; - while (val > 1) - { - val >>= 1; - res++; - } - return res; - } - - SPP_INLINE size_t operator()(const T *__v) const SPP_NOEXCEPT - { - static const size_t shift = 3; // spp_log2(1 + sizeof(T)); // T might be incomplete! - const uintptr_t i = (const uintptr_t)__v; - return static_cast<size_t>(i >> shift); - } -}; - -// from http://burtleburtle.net/bob/hash/integer.html -// fast and efficient for power of two table sizes where we always -// consider the last bits. -// --------------------------------------------------------------- -inline size_t spp_mix_32(uint32_t a) -{ - a = a ^ (a >> 4); - a = (a ^ 0xdeadbeef) + (a << 5); - a = a ^ (a >> 11); - return static_cast<size_t>(a); -} - -// More thorough scrambling as described in -// https://gist.github.com/badboy/6267743 -// ---------------------------------------- -inline size_t spp_mix_64(uint64_t a) -{ - a = (~a) + (a << 21); // a = (a << 21) - a - 1; - a = a ^ (a >> 24); - a = (a + (a << 3)) + (a << 8); // a * 265 - a = a ^ (a >> 14); - a = (a + (a << 2)) + (a << 4); // a * 21 - a = a ^ (a >> 28); - a = a + (a << 31); - return static_cast<size_t>(a); -} - -template<class ArgumentType, class ResultType> -struct spp_unary_function -{ - typedef ArgumentType argument_type; - typedef ResultType result_type; -}; - -template <> -struct spp_hash<bool> : public spp_unary_function<bool, size_t> -{ - SPP_INLINE size_t operator()(bool __v) const SPP_NOEXCEPT - { return static_cast<size_t>(__v); } -}; - -template <> -struct spp_hash<char> : public spp_unary_function<char, size_t> -{ - SPP_INLINE size_t operator()(char __v) const SPP_NOEXCEPT - { return static_cast<size_t>(__v); } -}; - -template <> -struct spp_hash<signed char> : public spp_unary_function<signed char, size_t> -{ - SPP_INLINE size_t operator()(signed char __v) const SPP_NOEXCEPT - { return static_cast<size_t>(__v); } -}; - -template <> -struct spp_hash<unsigned char> : public spp_unary_function<unsigned char, size_t> -{ - SPP_INLINE size_t operator()(unsigned char __v) const SPP_NOEXCEPT - { return static_cast<size_t>(__v); } -}; - -template <> -struct spp_hash<wchar_t> : public spp_unary_function<wchar_t, size_t> -{ - SPP_INLINE size_t operator()(wchar_t __v) const SPP_NOEXCEPT - { return static_cast<size_t>(__v); } -}; - -template <> -struct spp_hash<int16_t> : public spp_unary_function<int16_t, size_t> -{ - SPP_INLINE size_t operator()(int16_t __v) const SPP_NOEXCEPT - { return spp_mix_32(static_cast<uint32_t>(__v)); } -}; - -template <> -struct spp_hash<uint16_t> : public spp_unary_function<uint16_t, size_t> -{ - SPP_INLINE size_t operator()(uint16_t __v) const SPP_NOEXCEPT - { return spp_mix_32(static_cast<uint32_t>(__v)); } -}; - -template <> -struct spp_hash<int32_t> : public spp_unary_function<int32_t, size_t> -{ - SPP_INLINE size_t operator()(int32_t __v) const SPP_NOEXCEPT - { return spp_mix_32(static_cast<uint32_t>(__v)); } -}; - -template <> -struct spp_hash<uint32_t> : public spp_unary_function<uint32_t, size_t> -{ - SPP_INLINE size_t operator()(uint32_t __v) const SPP_NOEXCEPT - { return spp_mix_32(static_cast<uint32_t>(__v)); } -}; - -template <> -struct spp_hash<int64_t> : public spp_unary_function<int64_t, size_t> -{ - SPP_INLINE size_t operator()(int64_t __v) const SPP_NOEXCEPT - { return spp_mix_64(static_cast<uint64_t>(__v)); } -}; - -template <> -struct spp_hash<uint64_t> : public spp_unary_function<uint64_t, size_t> -{ - SPP_INLINE size_t operator()(uint64_t __v) const SPP_NOEXCEPT - { return spp_mix_64(static_cast<uint64_t>(__v)); } -}; - -template <> -struct spp_hash<float> : public spp_unary_function<float, size_t> -{ - SPP_INLINE size_t operator()(float __v) const SPP_NOEXCEPT - { - // -0.0 and 0.0 should return same hash - uint32_t *as_int = reinterpret_cast<uint32_t *>(&__v); - return (__v == 0) ? static_cast<size_t>(0) : spp_mix_32(*as_int); - } -}; - -template <> -struct spp_hash<double> : public spp_unary_function<double, size_t> -{ - SPP_INLINE size_t operator()(double __v) const SPP_NOEXCEPT - { - // -0.0 and 0.0 should return same hash - uint64_t *as_int = reinterpret_cast<uint64_t *>(&__v); - return (__v == 0) ? static_cast<size_t>(0) : spp_mix_64(*as_int); - } -}; - -template <class T, int sz> struct Combiner -{ - inline void operator()(T& seed, T value); -}; - -template <class T> struct Combiner<T, 4> -{ - inline void operator()(T& seed, T value) - { - seed ^= value + 0x9e3779b9 + (seed << 6) + (seed >> 2); - } -}; - -template <class T> struct Combiner<T, 8> -{ - inline void operator()(T& seed, T value) - { - seed ^= value + T(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2); - } -}; - -template <class T> -inline void hash_combine(std::size_t& seed, T const& v) -{ - spp_::spp_hash<T> hasher; - Combiner<std::size_t, sizeof(std::size_t)> combiner; - - combiner(seed, hasher(v)); -} - -static inline uint32_t s_spp_popcount_default(uint32_t i) SPP_NOEXCEPT -{ - i = i - ((i >> 1) & 0x55555555); - i = (i & 0x33333333) + ((i >> 2) & 0x33333333); - return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24; -} - -static inline uint32_t s_spp_popcount_default(uint64_t x) SPP_NOEXCEPT -{ - const uint64_t m1 = uint64_t(0x5555555555555555); // binary: 0101... - const uint64_t m2 = uint64_t(0x3333333333333333); // binary: 00110011.. - const uint64_t m4 = uint64_t(0x0f0f0f0f0f0f0f0f); // binary: 4 zeros, 4 ones ... - const uint64_t h01 = uint64_t(0x0101010101010101); // the sum of 256 to the power of 0,1,2,3... - - x -= (x >> 1) & m1; // put count of each 2 bits into those 2 bits - x = (x & m2) + ((x >> 2) & m2); // put count of each 4 bits into those 4 bits - x = (x + (x >> 4)) & m4; // put count of each 8 bits into those 8 bits - return (x * h01)>>56; // returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24)+... -} - -#ifdef __APPLE__ - static inline uint32_t count_trailing_zeroes(size_t v) SPP_NOEXCEPT - { - size_t x = (v & -v) - 1; - // sadly sizeof() required to build on macos - return sizeof(size_t) == 8 ? s_spp_popcount_default((uint64_t)x) : s_spp_popcount_default((uint32_t)x); - } - - static inline uint32_t s_popcount(size_t v) SPP_NOEXCEPT - { - // sadly sizeof() required to build on macos - return sizeof(size_t) == 8 ? s_spp_popcount_default((uint64_t)v) : s_spp_popcount_default((uint32_t)v); - } -#else - static inline uint32_t count_trailing_zeroes(size_t v) SPP_NOEXCEPT - { - return s_spp_popcount_default((v & -(intptr_t)v) - 1); - } - - static inline uint32_t s_popcount(size_t v) SPP_NOEXCEPT - { - return s_spp_popcount_default(v); - } -#endif - -// ----------------------------------------------------------- -// ----------------------------------------------------------- -template<class T> -class libc_allocator -{ -public: - typedef T value_type; - typedef T* pointer; - typedef ptrdiff_t difference_type; - typedef const T* const_pointer; - typedef size_t size_type; - - libc_allocator() {} - libc_allocator(const libc_allocator&) {} - - template<class U> - libc_allocator(const libc_allocator<U> &) {} - - libc_allocator& operator=(const libc_allocator &) { return *this; } - - template<class U> - libc_allocator& operator=(const libc_allocator<U> &) { return *this; } - -#ifndef SPP_NO_CXX11_RVALUE_REFERENCES - libc_allocator(libc_allocator &&) {} - libc_allocator& operator=(libc_allocator &&) { return *this; } -#endif - - pointer allocate(size_t n, const_pointer /* unused */= 0) - { - pointer res = static_cast<pointer>(malloc(n * sizeof(T))); - if (!res) - throw std::bad_alloc(); - return res; - } - - void deallocate(pointer p, size_t /* unused */) - { - free(p); - } - - pointer reallocate(pointer p, size_t new_size) - { - pointer res = static_cast<pointer>(realloc(p, new_size * sizeof(T))); - if (!res) - throw std::bad_alloc(); - return res; - } - - // extra API to match spp_allocator interface - pointer reallocate(pointer p, size_t /* old_size */, size_t new_size) - { - return static_cast<pointer>(realloc(p, new_size * sizeof(T))); - } - - size_type max_size() const - { - return static_cast<size_type>(-1) / sizeof(value_type); - } - - void construct(pointer p, const value_type& val) - { - new(p) value_type(val); - } - - void destroy(pointer p) { p->~value_type(); } - - template<class U> - struct rebind - { - typedef spp_::libc_allocator<U> other; - }; - -}; - -// forward declaration -// ------------------- -template<class T> -class spp_allocator; - -} - -template<class T> -inline bool operator==(const spp_::libc_allocator<T> &, const spp_::libc_allocator<T> &) -{ - return true; -} - -template<class T> -inline bool operator!=(const spp_::libc_allocator<T> &, const spp_::libc_allocator<T> &) -{ - return false; -} - -#endif // spp_utils_h_guard_ - diff --git a/benchmarks/others/update.sh b/benchmarks/others/update.sh index 6af246d7..1ac8ad91 100644 --- a/benchmarks/others/update.sh +++ b/benchmarks/others/update.sh @@ -1,31 +1,30 @@ -tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl/"
-tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl/"
-greg="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp/"
-martinus="https://raw.github.com/martinus/robin-hood-hashing/master/src/include/"
-skarupke="https://raw.github.com/skarupke/flat_hash_map/master/"
-
-mkdir -p tsl sparsepp skarupke
-
-wget $martinus"robin_hood.h" -O "robin_hood.h"
-
-wget $skarupke"bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp"
-wget $skarupke"flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp"
-
-exit
-wget $greg"spp.h" -O "sparsepp/spp.h"
-wget $greg"spp_config.h" -O "sparsepp/spp_config.h"
-wget $greg"spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h"
-wget $greg"spp_memory.h" -O "sparsepp/spp_memory.h"
-wget $greg"spp_smartptr.h" -O "sparsepp/spp_smartptr.h"
-wget $greg"spp_stdint.h" -O "sparsepp/spp_stdint.h"
-wget $greg"spp_timer.h" -O "sparsepp/spp_timer.h"
-wget $greg"spp_traits.h" -O "sparsepp/spp_traits.h"
-wget $greg"spp_utils.h" -O "sparsepp/spp_utils.h"
-
-wget $tsl_h"hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h"
-wget $tsl_h"hopscotch_hash.h" -O "tsl/hopscotch_hash.h"
-wget $tsl_h"hopscotch_map.h" -O "tsl/hopscotch_map.h"
-
-wget $tsl_r"robin_growth_policy.h" -O "tsl/robin_growth_policy.h"
-wget $tsl_r"robin_hash.h" -O "tsl/robin_hash.h"
-wget $tsl_r"robin_map.h" -O "tsl/robin_map.h"
+tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl/" +tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl/" +greg="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp/" +martinus="https://raw.github.com/martinus/robin-hood-hashing/master/src/include/" +skarupke="https://raw.github.com/skarupke/flat_hash_map/master/" + +mkdir -p tsl sparsepp skarupke + +wget $martinus"robin_hood.h" -O "robin_hood.h" + +wget $skarupke"bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp" +wget $skarupke"flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp" + +wget $greg"spp.h" -O "sparsepp/spp.h" +wget $greg"spp_config.h" -O "sparsepp/spp_config.h" +wget $greg"spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h" +wget $greg"spp_memory.h" -O "sparsepp/spp_memory.h" +wget $greg"spp_smartptr.h" -O "sparsepp/spp_smartptr.h" +wget $greg"spp_stdint.h" -O "sparsepp/spp_stdint.h" +wget $greg"spp_timer.h" -O "sparsepp/spp_timer.h" +wget $greg"spp_traits.h" -O "sparsepp/spp_traits.h" +wget $greg"spp_utils.h" -O "sparsepp/spp_utils.h" + +wget $tsl_h"hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h" +wget $tsl_h"hopscotch_hash.h" -O "tsl/hopscotch_hash.h" +wget $tsl_h"hopscotch_map.h" -O "tsl/hopscotch_map.h" + +wget $tsl_r"robin_growth_policy.h" -O "tsl/robin_growth_policy.h" +wget $tsl_r"robin_hash.h" -O "tsl/robin_hash.h" +wget $tsl_r"robin_map.h" -O "tsl/robin_map.h" diff --git a/benchmarks/shootout1_cmap.cpp b/benchmarks/shootout1_cmap.cpp index fab25096..87790844 100644 --- a/benchmarks/shootout1_cmap.cpp +++ b/benchmarks/shootout1_cmap.cpp @@ -7,7 +7,7 @@ #include "others/robin_hood.hpp"
#include "others/skarupke/bytell_hash_map.hpp"
#include "others/tsl/hopscotch_map.h"
-#include "others/sparsepp/spp.h"
+#include "others/parallel_hashmap/phmap.h"
#define PICOBENCH_IMPLEMENT_WITH_MAIN
#include "picobench.hpp"
@@ -19,15 +19,15 @@ template <class K, class V> using umap = std::unordered_map<K, V>; template <class K, class V> using bmap = ska::bytell_hash_map<K, V>;
template <class K, class V> using fmap = ska::flat_hash_map<K, V>;
template <class K, class V> using hmap = tsl::hopscotch_map<K, V>;
-template <class K, class V> using smap = spp::sparse_hash_map<K, V>;
-template <class K, class V> using rmap = robin_hood::unordered_flat_map<K, V, robin_hood::hash<K>,
- std::equal_to<K>, MaxLoadFactor100>;
+template <class K, class V> using pmap = phmap::flat_hash_map<K, V>;
+template <class K, class V> using rmap = robin_hood::unordered_flat_map<K, V,
+ robin_hood::hash<K>, std::equal_to<K>, MaxLoadFactor100>;
#define DEFMAP(map, ...) \
using u##map = umap __VA_ARGS__; \
using b##map = bmap __VA_ARGS__; \
using f##map = fmap __VA_ARGS__; \
using h##map = hmap __VA_ARGS__; \
- using s##map = smap __VA_ARGS__; \
+ using p##map = pmap __VA_ARGS__; \
using r##map = rmap __VA_ARGS__
@@ -119,7 +119,7 @@ PICOBENCH(ins_and_erase_i<umap_x>).P; PICOBENCH(ins_and_erase_i<bmap_x>).P;
PICOBENCH(ins_and_erase_i<fmap_x>).P;
PICOBENCH(ins_and_erase_i<hmap_x>).P;
-PICOBENCH(ins_and_erase_i<smap_x>).P;
+PICOBENCH(ins_and_erase_i<pmap_x>).P;
PICOBENCH(ins_and_erase_i<rmap_x>).P;
PICOBENCH(ins_and_erase_cmap_x).P;
#undef P
@@ -161,7 +161,7 @@ PICOBENCH(ins_and_access_i<umap_i>).P; PICOBENCH(ins_and_access_i<bmap_i>).P;
PICOBENCH(ins_and_access_i<fmap_i>).P;
PICOBENCH(ins_and_access_i<hmap_i>).P;
-PICOBENCH(ins_and_access_i<smap_i>).P;
+PICOBENCH(ins_and_access_i<pmap_i>).P;
PICOBENCH(ins_and_access_i<rmap_i>).P;
PICOBENCH(ins_and_access_cmap_i).P;
#undef P
@@ -169,9 +169,11 @@ PICOBENCH(ins_and_access_cmap_i).P; PICOBENCH_SUITE("Map3");
static void randomize(char* str, size_t len) {
- union {uint64_t i; char c[8];} r = {.i = stc64_random()};
- for (int i = len - 7, j = 0; i < len; ++j, ++i)
- str[i] = (r.c[j] & 63) + 48;
+ for (int k=0; k < len; ++k) {
+ union {uint64_t i; char c[8];} r = {.i = stc64_random()};
+ for (int i=0; i<8 && k<len; ++k, ++i)
+ str[k] = (r.c[i] & 63) + 48;
+ }
}
template <class MapStr>
@@ -204,8 +206,9 @@ static void ins_and_access_cmap_s(picobench::state& s) picobench::scope scope(s);
c_forrange (s.iterations()) {
randomize(str.str, cstr_size(str));
+ //if (cstr_size(str) > 30) { printf("%s\n", str.str); exit(0); }
cmap_str_emplace(&map, str.str, str.str);
- randomize(str.str, cstr_size(str));
+ //randomize(str.str, cstr_size(str));
result += cmap_str_erase(&map, str.str);
}
s.set_result(result + cmap_str_size(map));
@@ -218,7 +221,7 @@ PICOBENCH(ins_and_access_s<umap_s>).P; PICOBENCH(ins_and_access_s<bmap_s>).P;
PICOBENCH(ins_and_access_s<fmap_s>).P;
PICOBENCH(ins_and_access_s<hmap_s>).P;
-PICOBENCH(ins_and_access_s<smap_s>).P;
+PICOBENCH(ins_and_access_s<pmap_s>).P;
PICOBENCH(ins_and_access_s<rmap_s>).P;
PICOBENCH(ins_and_access_cmap_s).P;
#undef P
@@ -291,7 +294,7 @@ PICOBENCH(iterate_x<umap_x>).P; PICOBENCH(iterate_x<bmap_x>).P;
PICOBENCH(iterate_x<fmap_x>).P;
PICOBENCH(iterate_x<hmap_x>).P;
-PICOBENCH(iterate_x<smap_x>).P;
+PICOBENCH(iterate_x<pmap_x>).P;
PICOBENCH(iterate_x<rmap_x>).P;
PICOBENCH(iterate_cmap_x).P;
#undef P
diff --git a/benchmarks/shootout2_cmap.cpp b/benchmarks/shootout2_cmap.cpp index 6ad6b293..1704079d 100644 --- a/benchmarks/shootout2_cmap.cpp +++ b/benchmarks/shootout2_cmap.cpp @@ -10,11 +10,10 @@ #include "others/robin_hood.hpp"
#include "others/skarupke/bytell_hash_map.hpp"
#include "others/tsl/hopscotch_map.h"
-#include "others/sparsepp/spp.h"
+#include "others/parallel_hashmap/phmap.h"
template<typename C> inline void destroy_me(C& c) { C().swap(c); }
#endif
-// Visual Studio: compile with -TP to force C++: cl -TP -EHsc -O2 benchmark.c
// cmap and khash template expansion
#define i_key int64_t
@@ -117,29 +116,45 @@ stc64_t rng; #define RMAP_CLEAR(X) UMAP_CLEAR(X)
#define RMAP_DTOR(X) UMAP_DTOR(X)
-#define SMAP_SETUP(X, Key, Value) spp::sparse_hash_map<Key, Value> map; map.max_load_factor(max_load_factor)
-#define SMAP_PUT(X, key, val) UMAP_PUT(X, key, val)
-#define SMAP_EMPLACE(X, key, val) UMAP_EMPLACE(X, key, val)
-#define SMAP_FIND(X, key) UMAP_FIND(X, key)
-#define SMAP_ERASE(X, key) UMAP_ERASE(X, key)
-#define SMAP_FOR(X, i) UMAP_FOR(X, i)
-#define SMAP_ITEM(X, i) UMAP_ITEM(X, i)
-#define SMAP_SIZE(X) UMAP_SIZE(X)
-#define SMAP_BUCKETS(X) UMAP_BUCKETS(X)
-#define SMAP_CLEAR(X) UMAP_CLEAR(X)
-#define SMAP_DTOR(X) UMAP_DTOR(X)
+#define PMAP_SETUP(X, Key, Value) phmap::flat_hash_map<Key, Value> map; map.max_load_factor(max_load_factor)
+#define PMAP_PUT(X, key, val) UMAP_PUT(X, key, val)
+#define PMAP_EMPLACE(X, key, val) UMAP_EMPLACE(X, key, val)
+#define PMAP_FIND(X, key) UMAP_FIND(X, key)
+#define PMAP_ERASE(X, key) UMAP_ERASE(X, key)
+#define PMAP_FOR(X, i) UMAP_FOR(X, i)
+#define PMAP_ITEM(X, i) UMAP_ITEM(X, i)
+#define PMAP_SIZE(X) UMAP_SIZE(X)
+#define PMAP_BUCKETS(X) UMAP_BUCKETS(X)
+#define PMAP_CLEAR(X) UMAP_CLEAR(X)
+#define PMAP_DTOR(X) UMAP_DTOR(X)
enum {
+ RR = 27,
FAC = 3,
+ N0 = 10000000 * FAC,
N1 = 10000000 * FAC,
N2 = 10000000 * FAC,
N3 = 10000000 * FAC,
N4 = 10000000 * FAC,
- RR = 24
};
int rr = RR;
+#define MAP_TEST0(M, X) \
+{ \
+ M##_SETUP(X, int64_t, int64_t); \
+ uint64_t checksum = 0; \
+ SEED(seed); \
+ clock_t difference, before = clock(); \
+ for (size_t i = 0; i < N0; ++i) { \
+ checksum += ++ M##_EMPLACE(X, RAND(rr), i); \
+ } \
+ difference = clock() - before; \
+ printf(#M ": time: %5.02f, sum: %zu, size: %zu, buckets: %8zu\n", \
+ (float) difference / CLOCKS_PER_SEC, checksum, (size_t) M##_SIZE(X), (size_t) M##_BUCKETS(X)); \
+ M##_DTOR(X); \
+}
+
#define MAP_TEST1(M, X) \
{ \
M##_SETUP(X, int64_t, int64_t); \
@@ -204,26 +219,11 @@ int rr = RR; M##_DTOR(X); \
}
-#define MAP_TEST5(M, X) \
-{ \
- M##_SETUP(X, int64_t, int64_t); \
- uint64_t checksum = 0; \
- SEED(seed); \
- clock_t difference, before = clock(); \
- for (size_t i = 0; i < N1; ++i) { \
- checksum += ++ M##_EMPLACE(X, RAND(rr), i); \
- } \
- difference = clock() - before; \
- printf(#M ": time: %5.02f, sum: %zu, size: %zu, buckets: %8zu\n", \
- (float) difference / CLOCKS_PER_SEC, checksum, (size_t) M##_SIZE(X), (size_t) M##_BUCKETS(X)); \
- M##_DTOR(X); \
-}
-
#ifdef __cplusplus
-#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii) MAP_TEST##n(UMAP, ii) /*MAP_TEST##n(SMAP, ii)*/ \
+#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii) MAP_TEST##n(UMAP, ii) MAP_TEST##n(PMAP, ii) \
+ MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) MAP_TEST##n(RMAP, ii) /*MAP_TEST##n(HMAP, ii)*/
+#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii) /*MAP_TEST##n(KMAP, ii)*/ MAP_TEST##n(UMAP, ii) MAP_TEST##n(PMAP, ii) \
MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) MAP_TEST##n(RMAP, ii) /*MAP_TEST##n(HMAP, ii)*/
-#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii) /*MAP_TEST##n(KMAP, ii)*/ MAP_TEST##n(UMAP, ii) MAP_TEST##n(SMAP, ii) \
- MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) /*MAP_TEST##n(RMAP, ii)*/ /*MAP_TEST##n(HMAP, ii)*/
#else
#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii)
#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii)
@@ -235,14 +235,21 @@ int main(int argc, char* argv[]) rr = argc == 2 ? atoi(argv[1]) : RR;
seed = time(NULL);
- printf("\nUnordered maps: Insert %d random (2^%d) keys:\n", N1, rr);
- RUN_TEST(5)
+ printf("\nRandom keys are in range [0, 2^%d), seed = %zu:\n", rr, seed);
+ printf("CMAP = STC cmap\n"
+ "KMAP = Klib khash\n"
+ "UMAP = std::unordered_map\n"
+ "PMAP = phmap::flat_hash_map\n"
+ "BMAP = ska::bytell_hash_map\n"
+ "FMAP = ska::flat_hash_map\n"
+ "RMAP = robin_hood::unordered_map\n");
+ printf("\nUnordered maps: Insert %d random keys:\n", N0);
+ RUN_TEST(0)
- printf("\nRandom keys are in range [0, 2^%d), seed = %zu:\n", rr, seed);
- printf("\nUnordered maps: %d repeats of Insert random key + try to remove a random key:\n", N1);
+ printf("\nUnordered maps: %d insert random key + try to remove another random key:\n", N1);
RUN_TEST(1)
- printf("\nUnordered maps: Insert %d index keys, then remove them in same order:\n", N2);
+ printf("\nUnordered maps: Insert %d sequential keys, then remove them in same order:\n", N2);
RUN_TEST(2)
printf("\nUnordered maps: Insert %d random keys, then remove them in same order:\n", N3);
|
