summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorTyge Løvset <[email protected]>2021-10-22 22:50:34 +0200
committerTyge Løvset <[email protected]>2021-10-22 22:50:34 +0200
commit63a09f6492b022eccd1583fd9b69c450ae3a8935 (patch)
tree126d3bb60d730535505f233b70adb91eb934a2b6
parent4b5c7250b59401a8888b68f4b12f3ddd69f83741 (diff)
downloadSTC-modified-63a09f6492b022eccd1583fd9b69c450ae3a8935.tar.gz
STC-modified-63a09f6492b022eccd1583fd9b69c450ae3a8935.zip
Updated shootout2_cmap.cpp and competing hashmaps. Replaced Gregs sparsepp with his parallel_hashmap as recommended by him.
-rw-r--r--benchmarks/others/parallel_hashmap/btree.h4050
-rw-r--r--benchmarks/others/parallel_hashmap/conanfile.py36
-rw-r--r--benchmarks/others/parallel_hashmap/meminfo.h (renamed from benchmarks/others/sparsepp/spp_memory.h)5
-rw-r--r--benchmarks/others/parallel_hashmap/phmap.h4788
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_base.h5171
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_bits.h663
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_config.h771
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_dump.h227
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_fwd_decl.h154
-rw-r--r--benchmarks/others/parallel_hashmap/phmap_utils.h378
-rw-r--r--benchmarks/others/robin_hood.hpp4998
-rw-r--r--benchmarks/others/sparsepp/spp.h4361
-rw-r--r--benchmarks/others/sparsepp/spp_config.h781
-rw-r--r--benchmarks/others/sparsepp/spp_dlalloc.h4044
-rw-r--r--benchmarks/others/sparsepp/spp_smartptr.h71
-rw-r--r--benchmarks/others/sparsepp/spp_stdint.h16
-rw-r--r--benchmarks/others/sparsepp/spp_timer.h58
-rw-r--r--benchmarks/others/sparsepp/spp_traits.h125
-rw-r--r--benchmarks/others/sparsepp/spp_utils.h477
-rw-r--r--benchmarks/others/update.sh61
-rw-r--r--benchmarks/shootout1_cmap.cpp29
-rw-r--r--benchmarks/shootout2_cmap.cpp81
22 files changed, 18866 insertions, 12479 deletions
diff --git a/benchmarks/others/parallel_hashmap/btree.h b/benchmarks/others/parallel_hashmap/btree.h
new file mode 100644
index 00000000..ae1aec5b
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/btree.h
@@ -0,0 +1,4050 @@
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
+// with modifications.
+//
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_
+#define PHMAP_BTREE_BTREE_CONTAINER_H_
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+
+ #pragma warning(disable : 4127) // conditional expression is constant
+ #pragma warning(disable : 4324) // structure was padded due to alignment specifier
+ #pragma warning(disable : 4355) // 'this': used in base member initializer list
+ #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+ #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted
+ #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted
+ #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted
+ #pragma warning(disable : 4710) // function not inlined
+ #pragma warning(disable : 4711) // selected for automatic inline expansion
+ #pragma warning(disable : 4820) // '6' bytes padding added after data member
+ #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list
+ #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted
+ #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted
+ #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified
+#endif
+
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <new>
+
+#include "phmap_fwd_decl.h"
+#include "phmap_base.h"
+
+#if PHMAP_HAVE_STD_STRING_VIEW
+ #include <string_view>
+#endif
+
+// MSVC constructibility traits do not detect destructor properties and so our
+// implementations should not use them as a source-of-truth.
+#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__)
+ #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1
+#endif
+
+namespace phmap {
+
+ // Defined and documented later on in this file.
+ template <typename T>
+ struct is_trivially_destructible;
+
+ // Defined and documented later on in this file.
+ template <typename T>
+ struct is_trivially_move_assignable;
+
+ namespace type_traits_internal {
+
+ // Silence MSVC warnings about the destructor being defined as deleted.
+#if defined(_MSC_VER) && !defined(__GNUC__)
+ #pragma warning(push)
+ #pragma warning(disable : 4624)
+#endif // defined(_MSC_VER) && !defined(__GNUC__)
+
+ template <class T>
+ union SingleMemberUnion {
+ T t;
+ };
+
+ // Restore the state of the destructor warning that was silenced above.
+#if defined(_MSC_VER) && !defined(__GNUC__)
+ #pragma warning(pop)
+#endif // defined(_MSC_VER) && !defined(__GNUC__)
+
+ template <class T>
+ struct IsTriviallyMoveConstructibleObject
+ : std::integral_constant<
+ bool, std::is_move_constructible<
+ type_traits_internal::SingleMemberUnion<T>>::value &&
+ phmap::is_trivially_destructible<T>::value> {};
+
+ template <class T>
+ struct IsTriviallyCopyConstructibleObject
+ : std::integral_constant<
+ bool, std::is_copy_constructible<
+ type_traits_internal::SingleMemberUnion<T>>::value &&
+ phmap::is_trivially_destructible<T>::value> {};
+
+ template <class T>
+ struct IsTriviallyMoveAssignableReference : std::false_type {};
+
+ template <class T>
+ struct IsTriviallyMoveAssignableReference<T&>
+ : phmap::is_trivially_move_assignable<T>::type {};
+
+ template <class T>
+ struct IsTriviallyMoveAssignableReference<T&&>
+ : phmap::is_trivially_move_assignable<T>::type {};
+
+ } // namespace type_traits_internal
+
+
+ template <typename... Ts>
+ using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type;
+
+
+ template <typename T>
+ struct is_function
+ : std::integral_constant<
+ bool, !(std::is_reference<T>::value ||
+ std::is_const<typename std::add_const<T>::type>::value)> {};
+
+
+ namespace type_traits_internal {
+
+ template <typename T>
+ class is_trivially_copyable_impl {
+ using ExtentsRemoved = typename std::remove_all_extents<T>::type;
+ static constexpr bool kIsCopyOrMoveConstructible =
+ std::is_copy_constructible<ExtentsRemoved>::value ||
+ std::is_move_constructible<ExtentsRemoved>::value;
+ static constexpr bool kIsCopyOrMoveAssignable =
+ phmap::is_copy_assignable<ExtentsRemoved>::value ||
+ phmap::is_move_assignable<ExtentsRemoved>::value;
+
+ public:
+ static constexpr bool kValue =
+ (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) &&
+ (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) &&
+ (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) &&
+ is_trivially_destructible<ExtentsRemoved>::value &&
+ // We need to check for this explicitly because otherwise we'll say
+ // references are trivial copyable when compiled by MSVC.
+ !std::is_reference<ExtentsRemoved>::value;
+ };
+
+ template <typename T>
+ struct is_trivially_copyable
+ : std::integral_constant<
+ bool, type_traits_internal::is_trivially_copyable_impl<T>::kValue> {};
+ } // namespace type_traits_internal
+
+ namespace swap_internal {
+
+ // Necessary for the traits.
+ using std::swap;
+
+ // This declaration prevents global `swap` and `phmap::swap` overloads from being
+ // considered unless ADL picks them up.
+ void swap();
+
+ template <class T>
+ using IsSwappableImpl = decltype(swap(std::declval<T&>(), std::declval<T&>()));
+
+ // NOTE: This dance with the default template parameter is for MSVC.
+ template <class T,
+ class IsNoexcept = std::integral_constant<
+ bool, noexcept(swap(std::declval<T&>(), std::declval<T&>()))>>
+ using IsNothrowSwappableImpl = typename std::enable_if<IsNoexcept::value>::type;
+
+ template <class T>
+ struct IsSwappable
+ : phmap::type_traits_internal::is_detected<IsSwappableImpl, T> {};
+
+ template <class T>
+ struct IsNothrowSwappable
+ : phmap::type_traits_internal::is_detected<IsNothrowSwappableImpl, T> {};
+
+ template <class T, phmap::enable_if_t<IsSwappable<T>::value, int> = 0>
+ void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable<T>::value) {
+ swap(lhs, rhs);
+ }
+
+ using StdSwapIsUnconstrained = IsSwappable<void()>;
+
+ } // namespace swap_internal
+
+ namespace type_traits_internal {
+
+ // Make the swap-related traits/function accessible from this namespace.
+ using swap_internal::IsNothrowSwappable;
+ using swap_internal::IsSwappable;
+ using swap_internal::Swap;
+ using swap_internal::StdSwapIsUnconstrained;
+
+ } // namespace type_traits_internal
+
+ namespace compare_internal {
+
+ using value_type = int8_t;
+
+ template <typename T>
+ struct Fail {
+ static_assert(sizeof(T) < 0, "Only literal `0` is allowed.");
+ };
+
+ template <typename NullPtrT = std::nullptr_t>
+ struct OnlyLiteralZero {
+ constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT
+
+ template <
+ typename T,
+ typename = typename std::enable_if<
+ std::is_same<T, std::nullptr_t>::value ||
+ (std::is_integral<T>::value && !std::is_same<T, int>::value)>::type,
+ typename = typename Fail<T>::type>
+ OnlyLiteralZero(T); // NOLINT
+ };
+
+ enum class eq : value_type {
+ equal = 0,
+ equivalent = equal,
+ nonequal = 1,
+ nonequivalent = nonequal,
+ };
+
+ enum class ord : value_type { less = -1, greater = 1 };
+
+ enum class ncmp : value_type { unordered = -127 };
+
+#if defined(__cpp_inline_variables) && !defined(_MSC_VER)
+
+#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name)
+
+#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \
+ static const type name;
+
+#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \
+ inline constexpr type type::name(init)
+
+#else // __cpp_inline_variables
+
+#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \
+ static const T name;
+
+#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name)
+
+#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \
+ template <typename T> \
+ const T compare_internal::type##_base<T>::name(init)
+
+#endif // __cpp_inline_variables
+
+ // These template base classes allow for defining the values of the constants
+ // in the header file (for performance) without using inline variables (which
+ // aren't available in C++11).
+ template <typename T>
+ struct weak_equality_base {
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent)
+ };
+
+ template <typename T>
+ struct strong_equality_base {
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent)
+ };
+
+ template <typename T>
+ struct partial_ordering_base {
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(less)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered)
+ };
+
+ template <typename T>
+ struct weak_ordering_base {
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(less)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater)
+ };
+
+ template <typename T>
+ struct strong_ordering_base {
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(less)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent)
+ PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater)
+ };
+
+ } // namespace compare_internal
+
+ class weak_equality
+ : public compare_internal::weak_equality_base<weak_equality> {
+ explicit constexpr weak_equality(compare_internal::eq v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ friend struct compare_internal::weak_equality_base<weak_equality>;
+
+ public:
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent)
+
+ // Comparisons
+ friend constexpr bool operator==(
+ weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ == 0;
+ }
+ friend constexpr bool operator!=(
+ weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ != 0;
+ }
+ friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>,
+ weak_equality v) noexcept {
+ return 0 == v.value_;
+ }
+ friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>,
+ weak_equality v) noexcept {
+ return 0 != v.value_;
+ }
+
+ private:
+ compare_internal::value_type value_;
+ };
+ PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent,
+ compare_internal::eq::equivalent);
+ PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent,
+ compare_internal::eq::nonequivalent);
+
+ class strong_equality
+ : public compare_internal::strong_equality_base<strong_equality> {
+ explicit constexpr strong_equality(compare_internal::eq v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ friend struct compare_internal::strong_equality_base<strong_equality>;
+
+ public:
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent)
+
+ // Conversion
+ constexpr operator weak_equality() const noexcept { // NOLINT
+ return value_ == 0 ? weak_equality::equivalent
+ : weak_equality::nonequivalent;
+ }
+ // Comparisons
+ friend constexpr bool operator==(
+ strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ == 0;
+ }
+ friend constexpr bool operator!=(
+ strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ != 0;
+ }
+ friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>,
+ strong_equality v) noexcept {
+ return 0 == v.value_;
+ }
+ friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>,
+ strong_equality v) noexcept {
+ return 0 != v.value_;
+ }
+
+ private:
+ compare_internal::value_type value_;
+ };
+
+ PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal);
+ PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal,
+ compare_internal::eq::nonequal);
+ PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent,
+ compare_internal::eq::equivalent);
+ PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent,
+ compare_internal::eq::nonequivalent);
+
+ class partial_ordering
+ : public compare_internal::partial_ordering_base<partial_ordering> {
+ explicit constexpr partial_ordering(compare_internal::eq v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ explicit constexpr partial_ordering(compare_internal::ord v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ friend struct compare_internal::partial_ordering_base<partial_ordering>;
+
+ constexpr bool is_ordered() const noexcept {
+ return value_ !=
+ compare_internal::value_type(compare_internal::ncmp::unordered);
+ }
+
+ public:
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered)
+
+ // Conversion
+ constexpr operator weak_equality() const noexcept { // NOLINT
+ return value_ == 0 ? weak_equality::equivalent
+ : weak_equality::nonequivalent;
+ }
+ // Comparisons
+ friend constexpr bool operator==(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.is_ordered() && v.value_ == 0;
+ }
+ friend constexpr bool operator!=(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return !v.is_ordered() || v.value_ != 0;
+ }
+ friend constexpr bool operator<(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.is_ordered() && v.value_ < 0;
+ }
+ friend constexpr bool operator<=(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.is_ordered() && v.value_ <= 0;
+ }
+ friend constexpr bool operator>(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.is_ordered() && v.value_ > 0;
+ }
+ friend constexpr bool operator>=(
+ partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.is_ordered() && v.value_ >= 0;
+ }
+ friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return v.is_ordered() && 0 == v.value_;
+ }
+ friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return !v.is_ordered() || 0 != v.value_;
+ }
+ friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return v.is_ordered() && 0 < v.value_;
+ }
+ friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return v.is_ordered() && 0 <= v.value_;
+ }
+ friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return v.is_ordered() && 0 > v.value_;
+ }
+ friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>,
+ partial_ordering v) noexcept {
+ return v.is_ordered() && 0 >= v.value_;
+ }
+
+ private:
+ compare_internal::value_type value_;
+ };
+
+ PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less);
+ PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent,
+ compare_internal::eq::equivalent);
+ PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater,
+ compare_internal::ord::greater);
+ PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered,
+ compare_internal::ncmp::unordered);
+
+ class weak_ordering
+ : public compare_internal::weak_ordering_base<weak_ordering> {
+ explicit constexpr weak_ordering(compare_internal::eq v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ explicit constexpr weak_ordering(compare_internal::ord v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ friend struct compare_internal::weak_ordering_base<weak_ordering>;
+
+ public:
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater)
+
+ // Conversions
+ constexpr operator weak_equality() const noexcept { // NOLINT
+ return value_ == 0 ? weak_equality::equivalent
+ : weak_equality::nonequivalent;
+ }
+ constexpr operator partial_ordering() const noexcept { // NOLINT
+ return value_ == 0 ? partial_ordering::equivalent
+ : (value_ < 0 ? partial_ordering::less
+ : partial_ordering::greater);
+ }
+ // Comparisons
+ friend constexpr bool operator==(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ == 0;
+ }
+ friend constexpr bool operator!=(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ != 0;
+ }
+ friend constexpr bool operator<(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ < 0;
+ }
+ friend constexpr bool operator<=(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ <= 0;
+ }
+ friend constexpr bool operator>(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ > 0;
+ }
+ friend constexpr bool operator>=(
+ weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ >= 0;
+ }
+ friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 == v.value_;
+ }
+ friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 != v.value_;
+ }
+ friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 < v.value_;
+ }
+ friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 <= v.value_;
+ }
+ friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 > v.value_;
+ }
+ friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>,
+ weak_ordering v) noexcept {
+ return 0 >= v.value_;
+ }
+
+ private:
+ compare_internal::value_type value_;
+ };
+
+ PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less);
+ PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent,
+ compare_internal::eq::equivalent);
+ PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater,
+ compare_internal::ord::greater);
+
+ class strong_ordering
+ : public compare_internal::strong_ordering_base<strong_ordering> {
+ explicit constexpr strong_ordering(compare_internal::eq v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ explicit constexpr strong_ordering(compare_internal::ord v) noexcept
+ : value_(static_cast<compare_internal::value_type>(v)) {}
+ friend struct compare_internal::strong_ordering_base<strong_ordering>;
+
+ public:
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent)
+ PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater)
+
+ // Conversions
+ constexpr operator weak_equality() const noexcept { // NOLINT
+ return value_ == 0 ? weak_equality::equivalent
+ : weak_equality::nonequivalent;
+ }
+ constexpr operator strong_equality() const noexcept { // NOLINT
+ return value_ == 0 ? strong_equality::equal : strong_equality::nonequal;
+ }
+ constexpr operator partial_ordering() const noexcept { // NOLINT
+ return value_ == 0 ? partial_ordering::equivalent
+ : (value_ < 0 ? partial_ordering::less
+ : partial_ordering::greater);
+ }
+ constexpr operator weak_ordering() const noexcept { // NOLINT
+ return value_ == 0
+ ? weak_ordering::equivalent
+ : (value_ < 0 ? weak_ordering::less : weak_ordering::greater);
+ }
+ // Comparisons
+ friend constexpr bool operator==(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ == 0;
+ }
+ friend constexpr bool operator!=(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ != 0;
+ }
+ friend constexpr bool operator<(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ < 0;
+ }
+ friend constexpr bool operator<=(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ <= 0;
+ }
+ friend constexpr bool operator>(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ > 0;
+ }
+ friend constexpr bool operator>=(
+ strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept {
+ return v.value_ >= 0;
+ }
+ friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 == v.value_;
+ }
+ friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 != v.value_;
+ }
+ friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 < v.value_;
+ }
+ friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 <= v.value_;
+ }
+ friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 > v.value_;
+ }
+ friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>,
+ strong_ordering v) noexcept {
+ return 0 >= v.value_;
+ }
+
+ private:
+ compare_internal::value_type value_;
+ };
+ PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less);
+ PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal);
+ PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent,
+ compare_internal::eq::equivalent);
+ PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater,
+ compare_internal::ord::greater);
+
+#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL
+#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL
+#undef PHMAP_COMPARE_INLINE_INIT
+
+ namespace compare_internal {
+ // We also provide these comparator adapter functions for internal phmap use.
+
+ // Helper functions to do a boolean comparison of two keys given a boolean
+ // or three-way comparator.
+ // SFINAE prevents implicit conversions to bool (such as from int).
+ template <typename BoolType,
+ phmap::enable_if_t<std::is_same<bool, BoolType>::value, int> = 0>
+ constexpr bool compare_result_as_less_than(const BoolType r) { return r; }
+ constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) {
+ return r < 0;
+ }
+
+ template <typename Compare, typename K, typename LK>
+ constexpr bool do_less_than_comparison(const Compare &compare, const K &x,
+ const LK &y) {
+ return compare_result_as_less_than(compare(x, y));
+ }
+
+ // Helper functions to do a three-way comparison of two keys given a boolean or
+ // three-way comparator.
+ // SFINAE prevents implicit conversions to int (such as from bool).
+ template <typename Int,
+ phmap::enable_if_t<std::is_same<int, Int>::value, int> = 0>
+ constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) {
+ return c < 0 ? phmap::weak_ordering::less
+ : c == 0 ? phmap::weak_ordering::equivalent
+ : phmap::weak_ordering::greater;
+ }
+ constexpr phmap::weak_ordering compare_result_as_ordering(
+ const phmap::weak_ordering c) {
+ return c;
+ }
+
+ template <
+ typename Compare, typename K, typename LK,
+ phmap::enable_if_t<!std::is_same<bool, phmap::invoke_result_t<
+ Compare, const K &, const LK &>>::value,
+ int> = 0>
+ constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare,
+ const K &x, const LK &y) {
+ return compare_result_as_ordering(compare(x, y));
+ }
+ template <
+ typename Compare, typename K, typename LK,
+ phmap::enable_if_t<std::is_same<bool, phmap::invoke_result_t<Compare,
+ const K &, const LK &>>::value,
+ int> = 0>
+ constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare,
+ const K &x, const LK &y) {
+ return compare(x, y) ? phmap::weak_ordering::less
+ : compare(y, x) ? phmap::weak_ordering::greater
+ : phmap::weak_ordering::equivalent;
+ }
+
+ } // namespace compare_internal
+}
+
+
+namespace phmap {
+
+namespace priv {
+
+ // A helper class that indicates if the Compare parameter is a key-compare-to
+ // comparator.
+ template <typename Compare, typename T>
+ using btree_is_key_compare_to =
+ std::is_convertible<phmap::invoke_result_t<Compare, const T &, const T &>,
+ phmap::weak_ordering>;
+
+ struct StringBtreeDefaultLess {
+ using is_transparent = void;
+
+ StringBtreeDefaultLess() = default;
+
+ // Compatibility constructor.
+ StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT
+#if PHMAP_HAVE_STD_STRING_VIEW
+ StringBtreeDefaultLess(std::less<std::string_view>) {} // NOLINT
+ StringBtreeDefaultLess(phmap::Less<std::string_view>) {} // NOLINT
+
+ phmap::weak_ordering operator()(std::string_view lhs,
+ std::string_view rhs) const {
+ return compare_internal::compare_result_as_ordering(lhs.compare(rhs));
+ }
+#else
+ phmap::weak_ordering operator()(std::string lhs,
+ std::string rhs) const {
+ return compare_internal::compare_result_as_ordering(lhs.compare(rhs));
+ }
+#endif
+ };
+
+ struct StringBtreeDefaultGreater {
+ using is_transparent = void;
+
+ StringBtreeDefaultGreater() = default;
+
+ StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT
+#if PHMAP_HAVE_STD_STRING_VIEW
+ StringBtreeDefaultGreater(std::greater<std::string_view>) {} // NOLINT
+
+ phmap::weak_ordering operator()(std::string_view lhs,
+ std::string_view rhs) const {
+ return compare_internal::compare_result_as_ordering(rhs.compare(lhs));
+ }
+#else
+ phmap::weak_ordering operator()(std::string lhs,
+ std::string rhs) const {
+ return compare_internal::compare_result_as_ordering(rhs.compare(lhs));
+ }
+#endif
+ };
+
+ // A helper class to convert a boolean comparison into a three-way "compare-to"
+ // comparison that returns a negative value to indicate less-than, zero to
+ // indicate equality and a positive value to indicate greater-than. This helper
+ // class is specialized for less<std::string>, greater<std::string>,
+ // less<std::string_view>, and greater<std::string_view>.
+ //
+ // key_compare_to_adapter is provided so that btree users
+ // automatically get the more efficient compare-to code when using common
+ // google string types with common comparison functors.
+ // These string-like specializations also turn on heterogeneous lookup by
+ // default.
+ template <typename Compare>
+ struct key_compare_to_adapter {
+ using type = Compare;
+ };
+
+ template <>
+ struct key_compare_to_adapter<std::less<std::string>> {
+ using type = StringBtreeDefaultLess;
+ };
+
+ template <>
+ struct key_compare_to_adapter<phmap::Less<std::string>> {
+ using type = StringBtreeDefaultLess;
+ };
+
+ template <>
+ struct key_compare_to_adapter<std::greater<std::string>> {
+ using type = StringBtreeDefaultGreater;
+ };
+
+#if PHMAP_HAVE_STD_STRING_VIEW
+ template <>
+ struct key_compare_to_adapter<std::less<std::string_view>> {
+ using type = StringBtreeDefaultLess;
+ };
+
+ template <>
+ struct key_compare_to_adapter<phmap::Less<std::string_view>> {
+ using type = StringBtreeDefaultLess;
+ };
+
+ template <>
+ struct key_compare_to_adapter<std::greater<std::string_view>> {
+ using type = StringBtreeDefaultGreater;
+ };
+#endif
+
+ template <typename Key, typename Compare, typename Alloc, int TargetNodeSize,
+ bool Multi, typename SlotPolicy>
+ struct common_params {
+ // If Compare is a common comparator for a std::string-like type, then we adapt it
+ // to use heterogeneous lookup and to be a key-compare-to comparator.
+ using key_compare = typename key_compare_to_adapter<Compare>::type;
+ // A type which indicates if we have a key-compare-to functor or a plain old
+ // key-compare functor.
+ using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>;
+
+ using allocator_type = Alloc;
+ using key_type = Key;
+ using size_type = std::size_t ;
+ using difference_type = ptrdiff_t;
+
+ // True if this is a multiset or multimap.
+ using is_multi_container = std::integral_constant<bool, Multi>;
+
+ using slot_policy = SlotPolicy;
+ using slot_type = typename slot_policy::slot_type;
+ using value_type = typename slot_policy::value_type;
+ using init_type = typename slot_policy::mutable_value_type;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+
+ enum {
+ kTargetNodeSize = TargetNodeSize,
+
+ // Upper bound for the available space for values. This is largest for leaf
+ // nodes, which have overhead of at least a pointer + 4 bytes (for storing
+ // 3 field_types and an enum).
+ kNodeValueSpace =
+ TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4),
+ };
+
+ // This is an integral type large enough to hold as many
+ // ValueSize-values as will fit a node of TargetNodeSize bytes.
+ using node_count_type =
+ phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) >
+ (std::numeric_limits<uint8_t>::max)()),
+ uint16_t, uint8_t>; // NOLINT
+
+ // The following methods are necessary for passing this struct as PolicyTraits
+ // for node_handle and/or are used within btree.
+ static value_type &element(slot_type *slot) {
+ return slot_policy::element(slot);
+ }
+ static const value_type &element(const slot_type *slot) {
+ return slot_policy::element(slot);
+ }
+ template <class... Args>
+ static void construct(Alloc *alloc, slot_type *slot, Args &&... args) {
+ slot_policy::construct(alloc, slot, std::forward<Args>(args)...);
+ }
+ static void construct(Alloc *alloc, slot_type *slot, slot_type *other) {
+ slot_policy::construct(alloc, slot, other);
+ }
+ static void destroy(Alloc *alloc, slot_type *slot) {
+ slot_policy::destroy(alloc, slot);
+ }
+ static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) {
+ construct(alloc, new_slot, old_slot);
+ destroy(alloc, old_slot);
+ }
+ static void swap(Alloc *alloc, slot_type *a, slot_type *b) {
+ slot_policy::swap(alloc, a, b);
+ }
+ static void move(Alloc *alloc, slot_type *src, slot_type *dest) {
+ slot_policy::move(alloc, src, dest);
+ }
+ static void move(Alloc *alloc, slot_type *first, slot_type *last,
+ slot_type *result) {
+ slot_policy::move(alloc, first, last, result);
+ }
+ };
+
+ // A parameters structure for holding the type parameters for a btree_map.
+ // Compare and Alloc should be nothrow copy-constructible.
+ template <typename Key, typename Data, typename Compare, typename Alloc,
+ int TargetNodeSize, bool Multi>
+ struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi,
+ phmap::priv::map_slot_policy<Key, Data>> {
+ using super_type = typename map_params::common_params;
+ using mapped_type = Data;
+ // This type allows us to move keys when it is safe to do so. It is safe
+ // for maps in which value_type and mutable_value_type are layout compatible.
+ using slot_policy = typename super_type::slot_policy;
+ using slot_type = typename super_type::slot_type;
+ using value_type = typename super_type::value_type;
+ using init_type = typename super_type::init_type;
+
+ using key_compare = typename super_type::key_compare;
+ // Inherit from key_compare for empty base class optimization.
+ struct value_compare : private key_compare {
+ value_compare() = default;
+ explicit value_compare(const key_compare &cmp) : key_compare(cmp) {}
+
+ template <typename T, typename U>
+ auto operator()(const T &left, const U &right) const
+ -> decltype(std::declval<key_compare>()(left.first, right.first)) {
+ return key_compare::operator()(left.first, right.first);
+ }
+ };
+ using is_map_container = std::true_type;
+
+ static const Key &key(const value_type &x) { return x.first; }
+ static const Key &key(const init_type &x) { return x.first; }
+ static const Key &key(const slot_type *x) { return slot_policy::key(x); }
+ static mapped_type &value(value_type *value) { return value->second; }
+ };
+
+ // This type implements the necessary functions from the
+ // btree::priv::slot_type interface.
+ template <typename Key>
+ struct set_slot_policy {
+ using slot_type = Key;
+ using value_type = Key;
+ using mutable_value_type = Key;
+
+ static value_type &element(slot_type *slot) { return *slot; }
+ static const value_type &element(const slot_type *slot) { return *slot; }
+
+ template <typename Alloc, class... Args>
+ static void construct(Alloc *alloc, slot_type *slot, Args &&... args) {
+ phmap::allocator_traits<Alloc>::construct(*alloc, slot,
+ std::forward<Args>(args)...);
+ }
+
+ template <typename Alloc>
+ static void construct(Alloc *alloc, slot_type *slot, slot_type *other) {
+ phmap::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other));
+ }
+
+ template <typename Alloc>
+ static void destroy(Alloc *alloc, slot_type *slot) {
+ phmap::allocator_traits<Alloc>::destroy(*alloc, slot);
+ }
+
+ template <typename Alloc>
+ static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) {
+ using std::swap;
+ swap(*a, *b);
+ }
+
+ template <typename Alloc>
+ static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) {
+ *dest = std::move(*src);
+ }
+
+ template <typename Alloc>
+ static void move(Alloc *alloc, slot_type *first, slot_type *last,
+ slot_type *result) {
+ for (slot_type *src = first, *dest = result; src != last; ++src, ++dest)
+ move(alloc, src, dest);
+ }
+ };
+
+ // A parameters structure for holding the type parameters for a btree_set.
+ // Compare and Alloc should be nothrow copy-constructible.
+ template <typename Key, typename Compare, typename Alloc, int TargetNodeSize,
+ bool Multi>
+ struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi,
+ set_slot_policy<Key>> {
+ using value_type = Key;
+ using slot_type = typename set_params::common_params::slot_type;
+ using value_compare = typename set_params::common_params::key_compare;
+ using is_map_container = std::false_type;
+
+ static const Key &key(const value_type &x) { return x; }
+ static const Key &key(const slot_type *x) { return *x; }
+ };
+
+ // An adapter class that converts a lower-bound compare into an upper-bound
+ // compare. Note: there is no need to make a version of this adapter specialized
+ // for key-compare-to functors because the upper-bound (the first value greater
+ // than the input) is never an exact match.
+ template <typename Compare>
+ struct upper_bound_adapter {
+ explicit upper_bound_adapter(const Compare &c) : comp(c) {}
+ template <typename K, typename LK>
+ bool operator()(const K &a, const LK &b) const {
+ // Returns true when a is not greater than b.
+ return !phmap::compare_internal::compare_result_as_less_than(comp(b, a));
+ }
+
+ private:
+ Compare comp;
+ };
+
+ enum class MatchKind : uint8_t { kEq, kNe };
+
+ template <typename V, bool IsCompareTo>
+ struct SearchResult {
+ V value;
+ MatchKind match;
+
+ static constexpr bool HasMatch() { return true; }
+ bool IsEq() const { return match == MatchKind::kEq; }
+ };
+
+ // When we don't use CompareTo, `match` is not present.
+ // This ensures that callers can't use it accidentally when it provides no
+ // useful information.
+ template <typename V>
+ struct SearchResult<V, false> {
+ V value;
+
+ static constexpr bool HasMatch() { return false; }
+ static constexpr bool IsEq() { return false; }
+ };
+
+ // A node in the btree holding. The same node type is used for both internal
+ // and leaf nodes in the btree, though the nodes are allocated in such a way
+ // that the children array is only valid in internal nodes.
+ template <typename Params>
+ class btree_node {
+ using is_key_compare_to = typename Params::is_key_compare_to;
+ using is_multi_container = typename Params::is_multi_container;
+ using field_type = typename Params::node_count_type;
+ using allocator_type = typename Params::allocator_type;
+ using slot_type = typename Params::slot_type;
+
+ public:
+ using params_type = Params;
+ using key_type = typename Params::key_type;
+ using value_type = typename Params::value_type;
+ using pointer = typename Params::pointer;
+ using const_pointer = typename Params::const_pointer;
+ using reference = typename Params::reference;
+ using const_reference = typename Params::const_reference;
+ using key_compare = typename Params::key_compare;
+ using size_type = typename Params::size_type;
+ using difference_type = typename Params::difference_type;
+
+ // Btree decides whether to use linear node search as follows:
+ // - If the key is arithmetic and the comparator is std::less or
+ // std::greater, choose linear.
+ // - Otherwise, choose binary.
+ // TODO(ezb): Might make sense to add condition(s) based on node-size.
+ using use_linear_search = std::integral_constant<
+ bool,
+ std::is_arithmetic<key_type>::value &&
+ (std::is_same<phmap::Less<key_type>, key_compare>::value ||
+ std::is_same<std::less<key_type>, key_compare>::value ||
+ std::is_same<std::greater<key_type>, key_compare>::value)>;
+
+
+ ~btree_node() = default;
+ btree_node(btree_node const &) = delete;
+ btree_node &operator=(btree_node const &) = delete;
+
+ // Public for EmptyNodeType.
+ constexpr static size_type Alignment() {
+ static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(),
+ "Alignment of all nodes must be equal.");
+ return (size_type)InternalLayout().Alignment();
+ }
+
+ protected:
+ btree_node() = default;
+
+ private:
+ using layout_type = phmap::priv::Layout<btree_node *, field_type,
+ slot_type, btree_node *>;
+ constexpr static size_type SizeWithNValues(size_type n) {
+ return (size_type)layout_type(/*parent*/ 1,
+ /*position, start, count, max_count*/ 4,
+ /*values*/ (size_t)n,
+ /*children*/ 0)
+ .AllocSize();
+ }
+ // A lower bound for the overhead of fields other than values in a leaf node.
+ constexpr static size_type MinimumOverhead() {
+ return (size_type)(SizeWithNValues(1) - sizeof(value_type));
+ }
+
+ // Compute how many values we can fit onto a leaf node taking into account
+ // padding.
+ constexpr static size_type NodeTargetValues(const int begin, const int end) {
+ return begin == end ? begin
+ : SizeWithNValues((begin + end) / 2 + 1) >
+ params_type::kTargetNodeSize
+ ? NodeTargetValues(begin, (begin + end) / 2)
+ : NodeTargetValues((begin + end) / 2 + 1, end);
+ }
+
+ enum {
+ kTargetNodeSize = params_type::kTargetNodeSize,
+ kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize),
+
+ // We need a minimum of 3 values per internal node in order to perform
+ // splitting (1 value for the two nodes involved in the split and 1 value
+ // propagated to the parent as the delimiter for the split).
+ kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3,
+
+ // The node is internal (i.e. is not a leaf node) if and only if `max_count`
+ // has this value.
+ kInternalNodeMaxCount = 0,
+ };
+
+ // Leaves can have less than kNodeValues values.
+ constexpr static layout_type LeafLayout(const int max_values = kNodeValues) {
+ return layout_type(/*parent*/ 1,
+ /*position, start, count, max_count*/ 4,
+ /*values*/ (size_t)max_values,
+ /*children*/ 0);
+ }
+ constexpr static layout_type InternalLayout() {
+ return layout_type(/*parent*/ 1,
+ /*position, start, count, max_count*/ 4,
+ /*values*/ kNodeValues,
+ /*children*/ kNodeValues + 1);
+ }
+ constexpr static size_type LeafSize(const int max_values = kNodeValues) {
+ return (size_type)LeafLayout(max_values).AllocSize();
+ }
+ constexpr static size_type InternalSize() {
+ return (size_type)InternalLayout().AllocSize();
+ }
+
+ // N is the index of the type in the Layout definition.
+ // ElementType<N> is the Nth type in the Layout definition.
+ template <size_type N>
+ inline typename layout_type::template ElementType<N> *GetField() {
+ // We assert that we don't read from values that aren't there.
+ assert(N < 3 || !leaf());
+ return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this));
+ }
+
+ template <size_type N>
+ inline const typename layout_type::template ElementType<N> *GetField() const {
+ assert(N < 3 || !leaf());
+ return InternalLayout().template Pointer<N>(
+ reinterpret_cast<const char *>(this));
+ }
+
+ void set_parent(btree_node *p) { *GetField<0>() = p; }
+ field_type &mutable_count() { return GetField<1>()[2]; }
+ slot_type *slot(size_type i) { return &GetField<2>()[i]; }
+ const slot_type *slot(size_type i) const { return &GetField<2>()[i]; }
+ void set_position(field_type v) { GetField<1>()[0] = v; }
+ void set_start(field_type v) { GetField<1>()[1] = v; }
+ void set_count(field_type v) { GetField<1>()[2] = v; }
+ void set_max_count(field_type v) { GetField<1>()[3] = v; }
+
+ public:
+ // Whether this is a leaf node or not. This value doesn't change after the
+ // node is created.
+ bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; }
+
+ // Getter for the position of this node in its parent.
+ field_type position() const { return GetField<1>()[0]; }
+
+ // Getter for the offset of the first value in the `values` array.
+ field_type start() const { return GetField<1>()[1]; }
+
+ // Getters for the number of values stored in this node.
+ field_type count() const { return GetField<1>()[2]; }
+ field_type max_count() const {
+ // Internal nodes have max_count==kInternalNodeMaxCount.
+ // Leaf nodes have max_count in [1, kNodeValues].
+ const field_type max_cnt = GetField<1>()[3];
+ return max_cnt == field_type{kInternalNodeMaxCount}
+ ? field_type{kNodeValues}
+ : max_cnt;
+ }
+
+ // Getter for the parent of this node.
+ btree_node *parent() const { return *GetField<0>(); }
+ // Getter for whether the node is the root of the tree. The parent of the
+ // root of the tree is the leftmost node in the tree which is guaranteed to
+ // be a leaf.
+ bool is_root() const { return parent()->leaf(); }
+ void make_root() {
+ assert(parent()->is_root());
+ set_parent(parent()->parent());
+ }
+
+ // Getters for the key/value at position i in the node.
+ const key_type &key(size_type i) const { return params_type::key(slot(i)); }
+ reference value(size_type i) { return params_type::element(slot(i)); }
+ const_reference value(size_type i) const { return params_type::element(slot(i)); }
+
+ // Getters/setter for the child at position i in the node.
+ btree_node *child(size_type i) const { return GetField<3>()[i]; }
+ btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; }
+ void clear_child(size_type i) {
+ phmap::priv::SanitizerPoisonObject(&mutable_child(i));
+ }
+ void set_child(size_type i, btree_node *c) {
+ phmap::priv::SanitizerUnpoisonObject(&mutable_child(i));
+ mutable_child(i) = c;
+ c->set_position((field_type)i);
+ }
+ void init_child(int i, btree_node *c) {
+ set_child(i, c);
+ c->set_parent(this);
+ }
+
+ // Returns the position of the first value whose key is not less than k.
+ template <typename K>
+ SearchResult<int, is_key_compare_to::value> lower_bound(
+ const K &k, const key_compare &comp) const {
+ return use_linear_search::value ? linear_search(k, comp)
+ : binary_search(k, comp);
+ }
+ // Returns the position of the first value whose key is greater than k.
+ template <typename K>
+ int upper_bound(const K &k, const key_compare &comp) const {
+ auto upper_compare = upper_bound_adapter<key_compare>(comp);
+ return use_linear_search::value ? linear_search(k, upper_compare).value
+ : binary_search(k, upper_compare).value;
+ }
+
+ template <typename K, typename Compare>
+ SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value>
+ linear_search(const K &k, const Compare &comp) const {
+ return linear_search_impl(k, 0, count(), comp,
+ btree_is_key_compare_to<Compare, key_type>());
+ }
+
+ template <typename K, typename Compare>
+ SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value>
+ binary_search(const K &k, const Compare &comp) const {
+ return binary_search_impl(k, 0, count(), comp,
+ btree_is_key_compare_to<Compare, key_type>());
+ }
+
+ // Returns the position of the first value whose key is not less than k using
+ // linear search performed using plain compare.
+ template <typename K, typename Compare>
+ SearchResult<int, false> linear_search_impl(
+ const K &k, int s, const int e, const Compare &comp,
+ std::false_type /* IsCompareTo */) const {
+ while (s < e) {
+ if (!comp(key(s), k)) {
+ break;
+ }
+ ++s;
+ }
+ return {s};
+ }
+
+ // Returns the position of the first value whose key is not less than k using
+ // linear search performed using compare-to.
+ template <typename K, typename Compare>
+ SearchResult<int, true> linear_search_impl(
+ const K &k, int s, const int e, const Compare &comp,
+ std::true_type /* IsCompareTo */) const {
+ while (s < e) {
+ const phmap::weak_ordering c = comp(key(s), k);
+ if (c == 0) {
+ return {s, MatchKind::kEq};
+ } else if (c > 0) {
+ break;
+ }
+ ++s;
+ }
+ return {s, MatchKind::kNe};
+ }
+
+ // Returns the position of the first value whose key is not less than k using
+ // binary search performed using plain compare.
+ template <typename K, typename Compare>
+ SearchResult<int, false> binary_search_impl(
+ const K &k, int s, int e, const Compare &comp,
+ std::false_type /* IsCompareTo */) const {
+ while (s != e) {
+ const int mid = (s + e) >> 1;
+ if (comp(key(mid), k)) {
+ s = mid + 1;
+ } else {
+ e = mid;
+ }
+ }
+ return {s};
+ }
+
+ // Returns the position of the first value whose key is not less than k using
+ // binary search performed using compare-to.
+ template <typename K, typename CompareTo>
+ SearchResult<int, true> binary_search_impl(
+ const K &k, int s, int e, const CompareTo &comp,
+ std::true_type /* IsCompareTo */) const {
+ if (is_multi_container::value) {
+ MatchKind exact_match = MatchKind::kNe;
+ while (s != e) {
+ const int mid = (s + e) >> 1;
+ const phmap::weak_ordering c = comp(key(mid), k);
+ if (c < 0) {
+ s = mid + 1;
+ } else {
+ e = mid;
+ if (c == 0) {
+ // Need to return the first value whose key is not less than k,
+ // which requires continuing the binary search if this is a
+ // multi-container.
+ exact_match = MatchKind::kEq;
+ }
+ }
+ }
+ return {s, exact_match};
+ } else { // Not a multi-container.
+ while (s != e) {
+ const int mid = (s + e) >> 1;
+ const phmap::weak_ordering c = comp(key(mid), k);
+ if (c < 0) {
+ s = mid + 1;
+ } else if (c > 0) {
+ e = mid;
+ } else {
+ return {mid, MatchKind::kEq};
+ }
+ }
+ return {s, MatchKind::kNe};
+ }
+ }
+
+ // Emplaces a value at position i, shifting all existing values and
+ // children at positions >= i to the right by 1.
+ template <typename... Args>
+ void emplace_value(size_type i, allocator_type *alloc, Args &&... args);
+
+ // Removes the value at position i, shifting all existing values and children
+ // at positions > i to the left by 1.
+ void remove_value(int i, allocator_type *alloc);
+
+ // Removes the values at positions [i, i + to_erase), shifting all values
+ // after that range to the left by to_erase. Does not change children at all.
+ void remove_values_ignore_children(int i, size_type to_erase,
+ allocator_type *alloc);
+
+ // Rebalances a node with its right sibling.
+ void rebalance_right_to_left(int to_move, btree_node *right,
+ allocator_type *alloc);
+ void rebalance_left_to_right(int to_move, btree_node *right,
+ allocator_type *alloc);
+
+ // Splits a node, moving a portion of the node's values to its right sibling.
+ void split(int insert_position, btree_node *dest, allocator_type *alloc);
+
+ // Merges a node with its right sibling, moving all of the values and the
+ // delimiting key in the parent node onto itself.
+ void merge(btree_node *sibling, allocator_type *alloc);
+
+ // Swap the contents of "this" and "src".
+ void swap(btree_node *src, allocator_type *alloc);
+
+ // Node allocation/deletion routines.
+ static btree_node *init_leaf(btree_node *n, btree_node *parent,
+ int max_cnt) {
+ n->set_parent(parent);
+ n->set_position(0);
+ n->set_start(0);
+ n->set_count(0);
+ n->set_max_count((field_type)max_cnt);
+ phmap::priv::SanitizerPoisonMemoryRegion(
+ n->slot(0), max_cnt * sizeof(slot_type));
+ return n;
+ }
+ static btree_node *init_internal(btree_node *n, btree_node *parent) {
+ init_leaf(n, parent, kNodeValues);
+ // Set `max_count` to a sentinel value to indicate that this node is
+ // internal.
+ n->set_max_count(kInternalNodeMaxCount);
+ phmap::priv::SanitizerPoisonMemoryRegion(
+ &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *));
+ return n;
+ }
+ void destroy(allocator_type *alloc) {
+ for (int i = 0; i < count(); ++i) {
+ value_destroy(i, alloc);
+ }
+ }
+
+ public:
+ // Exposed only for tests.
+ static bool testonly_uses_linear_node_search() {
+ return use_linear_search::value;
+ }
+
+ private:
+ template <typename... Args>
+ void value_init(const size_type i, allocator_type *alloc, Args &&... args) {
+ phmap::priv::SanitizerUnpoisonObject(slot(i));
+ params_type::construct(alloc, slot(i), std::forward<Args>(args)...);
+ }
+ void value_destroy(const size_type i, allocator_type *alloc) {
+ params_type::destroy(alloc, slot(i));
+ phmap::priv::SanitizerPoisonObject(slot(i));
+ }
+
+ // Move n values starting at value i in this node into the values starting at
+ // value j in node x.
+ void uninitialized_move_n(const size_type n, const size_type i,
+ const size_type j, btree_node *x,
+ allocator_type *alloc) {
+ phmap::priv::SanitizerUnpoisonMemoryRegion(
+ x->slot(j), n * sizeof(slot_type));
+ for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j);
+ src != end; ++src, ++dest) {
+ params_type::construct(alloc, dest, src);
+ }
+ }
+
+ // Destroys a range of n values, starting at index i.
+ void value_destroy_n(const size_type i, const size_type n,
+ allocator_type *alloc) {
+ for (int j = 0; j < n; ++j) {
+ value_destroy(i + j, alloc);
+ }
+ }
+
+ template <typename P>
+ friend class btree;
+ template <typename N, typename R, typename P>
+ friend struct btree_iterator;
+ friend class BtreeNodePeer;
+ };
+
+ template <typename Node, typename Reference, typename Pointer>
+ struct btree_iterator {
+ private:
+ using key_type = typename Node::key_type;
+ using size_type = typename Node::size_type;
+ using params_type = typename Node::params_type;
+
+ using node_type = Node;
+ using normal_node = typename std::remove_const<Node>::type;
+ using const_node = const Node;
+ using normal_pointer = typename params_type::pointer;
+ using normal_reference = typename params_type::reference;
+ using const_pointer = typename params_type::const_pointer;
+ using const_reference = typename params_type::const_reference;
+ using slot_type = typename params_type::slot_type;
+
+ using iterator =
+ btree_iterator<normal_node, normal_reference, normal_pointer>;
+ using const_iterator =
+ btree_iterator<const_node, const_reference, const_pointer>;
+
+ public:
+ // These aliases are public for std::iterator_traits.
+ using difference_type = typename Node::difference_type;
+ using value_type = typename params_type::value_type;
+ using pointer = Pointer;
+ using reference = Reference;
+ using iterator_category = std::bidirectional_iterator_tag;
+
+ btree_iterator() : node(nullptr), position(-1) {}
+ btree_iterator(Node *n, int p) : node(n), position(p) {}
+
+ // NOTE: this SFINAE allows for implicit conversions from iterator to
+ // const_iterator, but it specifically avoids defining copy constructors so
+ // that btree_iterator can be trivially copyable. This is for performance and
+ // binary size reasons.
+ template <typename N, typename R, typename P,
+ phmap::enable_if_t<
+ std::is_same<btree_iterator<N, R, P>, iterator>::value &&
+ std::is_same<btree_iterator, const_iterator>::value,
+ int> = 0>
+ btree_iterator(const btree_iterator<N, R, P> &x) // NOLINT
+ : node(x.node), position(x.position) {}
+
+ private:
+ // This SFINAE allows explicit conversions from const_iterator to
+ // iterator, but also avoids defining a copy constructor.
+ // NOTE: the const_cast is safe because this constructor is only called by
+ // non-const methods and the container owns the nodes.
+ template <typename N, typename R, typename P,
+ phmap::enable_if_t<
+ std::is_same<btree_iterator<N, R, P>, const_iterator>::value &&
+ std::is_same<btree_iterator, iterator>::value,
+ int> = 0>
+ explicit btree_iterator(const btree_iterator<N, R, P> &x)
+ : node(const_cast<node_type *>(x.node)), position(x.position) {}
+
+ // Increment/decrement the iterator.
+ void increment() {
+ if (node->leaf() && ++position < node->count()) {
+ return;
+ }
+ increment_slow();
+ }
+ void increment_slow();
+
+ void decrement() {
+ if (node->leaf() && --position >= 0) {
+ return;
+ }
+ decrement_slow();
+ }
+ void decrement_slow();
+
+ public:
+ bool operator==(const const_iterator &x) const {
+ return node == x.node && position == x.position;
+ }
+ bool operator!=(const const_iterator &x) const {
+ return node != x.node || position != x.position;
+ }
+
+ // Accessors for the key/value the iterator is pointing at.
+ reference operator*() const {
+ return node->value(position);
+ }
+ pointer operator->() const {
+ return &node->value(position);
+ }
+
+ btree_iterator& operator++() {
+ increment();
+ return *this;
+ }
+ btree_iterator& operator--() {
+ decrement();
+ return *this;
+ }
+ btree_iterator operator++(int) {
+ btree_iterator tmp = *this;
+ ++*this;
+ return tmp;
+ }
+ btree_iterator operator--(int) {
+ btree_iterator tmp = *this;
+ --*this;
+ return tmp;
+ }
+
+ private:
+ template <typename Params>
+ friend class btree;
+ template <typename Tree>
+ friend class btree_container;
+ template <typename Tree>
+ friend class btree_set_container;
+ template <typename Tree>
+ friend class btree_map_container;
+ template <typename Tree>
+ friend class btree_multiset_container;
+ template <typename N, typename R, typename P>
+ friend struct btree_iterator;
+ template <typename TreeType, typename CheckerType>
+ friend class base_checker;
+
+ const key_type &key() const { return node->key(position); }
+ slot_type *slot() { return node->slot(position); }
+
+ // The node in the tree the iterator is pointing at.
+ Node *node;
+ // The position within the node of the tree the iterator is pointing at.
+ // TODO(ezb): make this a field_type
+ int position;
+ };
+
+ template <typename Params>
+ class btree {
+ using node_type = btree_node<Params>;
+ using is_key_compare_to = typename Params::is_key_compare_to;
+
+ // We use a static empty node for the root/leftmost/rightmost of empty btrees
+ // in order to avoid branching in begin()/end().
+ struct alignas(node_type::Alignment()) EmptyNodeType : node_type {
+ using field_type = typename node_type::field_type;
+ node_type *parent;
+ field_type position = 0;
+ field_type start = 0;
+ field_type count = 0;
+ // max_count must be != kInternalNodeMaxCount (so that this node is regarded
+ // as a leaf node). max_count() is never called when the tree is empty.
+ field_type max_count = node_type::kInternalNodeMaxCount + 1;
+
+#ifdef _MSC_VER
+ // MSVC has constexpr code generations bugs here.
+ EmptyNodeType() : parent(this) {}
+#else
+ constexpr EmptyNodeType(node_type *p) : parent(p) {}
+#endif
+ };
+
+ static node_type *EmptyNode() {
+#ifdef _MSC_VER
+ static EmptyNodeType empty_node;
+ // This assert fails on some other construction methods.
+ assert(empty_node.parent == &empty_node);
+ return &empty_node;
+#else
+ static constexpr EmptyNodeType empty_node(
+ const_cast<EmptyNodeType *>(&empty_node));
+ return const_cast<EmptyNodeType *>(&empty_node);
+#endif
+ }
+
+ enum {
+ kNodeValues = node_type::kNodeValues,
+ kMinNodeValues = kNodeValues / 2,
+ };
+
+ struct node_stats {
+ using size_type = typename Params::size_type;
+
+ node_stats(size_type l, size_type i)
+ : leaf_nodes(l),
+ internal_nodes(i) {
+ }
+
+ node_stats& operator+=(const node_stats &x) {
+ leaf_nodes += x.leaf_nodes;
+ internal_nodes += x.internal_nodes;
+ return *this;
+ }
+
+ size_type leaf_nodes;
+ size_type internal_nodes;
+ };
+
+ public:
+ using key_type = typename Params::key_type;
+ using value_type = typename Params::value_type;
+ using size_type = typename Params::size_type;
+ using difference_type = typename Params::difference_type;
+ using key_compare = typename Params::key_compare;
+ using value_compare = typename Params::value_compare;
+ using allocator_type = typename Params::allocator_type;
+ using reference = typename Params::reference;
+ using const_reference = typename Params::const_reference;
+ using pointer = typename Params::pointer;
+ using const_pointer = typename Params::const_pointer;
+ using iterator = btree_iterator<node_type, reference, pointer>;
+ using const_iterator = typename iterator::const_iterator;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using node_handle_type = node_handle<Params, Params, allocator_type>;
+
+ // Internal types made public for use by btree_container types.
+ using params_type = Params;
+ using slot_type = typename Params::slot_type;
+
+ private:
+ // For use in copy_or_move_values_in_order.
+ const value_type &maybe_move_from_iterator(const_iterator x) { return *x; }
+ value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); }
+
+ // Copies or moves (depending on the template parameter) the values in
+ // x into this btree in their order in x. This btree must be empty before this
+ // method is called. This method is used in copy construction, copy
+ // assignment, and move assignment.
+ template <typename Btree>
+ void copy_or_move_values_in_order(Btree *x);
+
+ // Validates that various assumptions/requirements are true at compile time.
+ constexpr static bool static_assert_validation();
+
+ public:
+ btree(const key_compare &comp, const allocator_type &alloc);
+
+ btree(const btree &x);
+ btree(btree &&x) noexcept
+ : root_(std::move(x.root_)),
+ rightmost_(phmap::exchange(x.rightmost_, EmptyNode())),
+ size_(phmap::exchange(x.size_, 0)) {
+ x.mutable_root() = EmptyNode();
+ }
+
+ ~btree() {
+ // Put static_asserts in destructor to avoid triggering them before the type
+ // is complete.
+ static_assert(static_assert_validation(), "This call must be elided.");
+ clear();
+ }
+
+ // Assign the contents of x to *this.
+ btree &operator=(const btree &x);
+ btree &operator=(btree &&x) noexcept;
+
+ iterator begin() {
+ return iterator(leftmost(), 0);
+ }
+ const_iterator begin() const {
+ return const_iterator(leftmost(), 0);
+ }
+ iterator end() { return iterator(rightmost_, rightmost_->count()); }
+ const_iterator end() const {
+ return const_iterator(rightmost_, rightmost_->count());
+ }
+ reverse_iterator rbegin() {
+ return reverse_iterator(end());
+ }
+ const_reverse_iterator rbegin() const {
+ return const_reverse_iterator(end());
+ }
+ reverse_iterator rend() {
+ return reverse_iterator(begin());
+ }
+ const_reverse_iterator rend() const {
+ return const_reverse_iterator(begin());
+ }
+
+ // Finds the first element whose key is not less than key.
+ template <typename K>
+ iterator lower_bound(const K &key) {
+ return internal_end(internal_lower_bound(key));
+ }
+ template <typename K>
+ const_iterator lower_bound(const K &key) const {
+ return internal_end(internal_lower_bound(key));
+ }
+
+ // Finds the first element whose key is greater than key.
+ template <typename K>
+ iterator upper_bound(const K &key) {
+ return internal_end(internal_upper_bound(key));
+ }
+ template <typename K>
+ const_iterator upper_bound(const K &key) const {
+ return internal_end(internal_upper_bound(key));
+ }
+
+ // Finds the range of values which compare equal to key. The first member of
+ // the returned pair is equal to lower_bound(key). The second member pair of
+ // the pair is equal to upper_bound(key).
+ template <typename K>
+ std::pair<iterator, iterator> equal_range(const K &key) {
+ return {lower_bound(key), upper_bound(key)};
+ }
+ template <typename K>
+ std::pair<const_iterator, const_iterator> equal_range(const K &key) const {
+ return {lower_bound(key), upper_bound(key)};
+ }
+
+ // Inserts a value into the btree only if it does not already exist. The
+ // boolean return value indicates whether insertion succeeded or failed.
+ // Requirement: if `key` already exists in the btree, does not consume `args`.
+ // Requirement: `key` is never referenced after consuming `args`.
+ template <typename... Args>
+ std::pair<iterator, bool> insert_unique(const key_type &key, Args &&... args);
+
+ // Inserts with hint. Checks to see if the value should be placed immediately
+ // before `position` in the tree. If so, then the insertion will take
+ // amortized constant time. If not, the insertion will take amortized
+ // logarithmic time as if a call to insert_unique() were made.
+ // Requirement: if `key` already exists in the btree, does not consume `args`.
+ // Requirement: `key` is never referenced after consuming `args`.
+ template <typename... Args>
+ std::pair<iterator, bool> insert_hint_unique(iterator position,
+ const key_type &key,
+ Args &&... args);
+
+ // Insert a range of values into the btree.
+ template <typename InputIterator>
+ void insert_iterator_unique(InputIterator b, InputIterator e);
+
+ // Inserts a value into the btree.
+ template <typename ValueType>
+ iterator insert_multi(const key_type &key, ValueType &&v);
+
+ // Inserts a value into the btree.
+ template <typename ValueType>
+ iterator insert_multi(ValueType &&v) {
+ return insert_multi(params_type::key(v), std::forward<ValueType>(v));
+ }
+
+ // Insert with hint. Check to see if the value should be placed immediately
+ // before position in the tree. If it does, then the insertion will take
+ // amortized constant time. If not, the insertion will take amortized
+ // logarithmic time as if a call to insert_multi(v) were made.
+ template <typename ValueType>
+ iterator insert_hint_multi(iterator position, ValueType &&v);
+
+ // Insert a range of values into the btree.
+ template <typename InputIterator>
+ void insert_iterator_multi(InputIterator b, InputIterator e);
+
+ // Erase the specified iterator from the btree. The iterator must be valid
+ // (i.e. not equal to end()). Return an iterator pointing to the node after
+ // the one that was erased (or end() if none exists).
+ // Requirement: does not read the value at `*iter`.
+ iterator erase(iterator iter);
+
+ // Erases range. Returns the number of keys erased and an iterator pointing
+ // to the element after the last erased element.
+ std::pair<size_type, iterator> erase(iterator begin, iterator end);
+
+ // Erases the specified key from the btree. Returns 1 if an element was
+ // erased and 0 otherwise.
+ template <typename K>
+ size_type erase_unique(const K &key);
+
+ // Erases all of the entries matching the specified key from the
+ // btree. Returns the number of elements erased.
+ template <typename K>
+ size_type erase_multi(const K &key);
+
+ // Finds the iterator corresponding to a key or returns end() if the key is
+ // not present.
+ template <typename K>
+ iterator find(const K &key) {
+ return internal_end(internal_find(key));
+ }
+ template <typename K>
+ const_iterator find(const K &key) const {
+ return internal_end(internal_find(key));
+ }
+
+ // Returns a count of the number of times the key appears in the btree.
+ template <typename K>
+ size_type count_unique(const K &key) const {
+ const iterator beg = internal_find(key);
+ if (beg.node == nullptr) {
+ // The key doesn't exist in the tree.
+ return 0;
+ }
+ return 1;
+ }
+ // Returns a count of the number of times the key appears in the btree.
+ template <typename K>
+ size_type count_multi(const K &key) const {
+ const auto range = equal_range(key);
+ return std::distance(range.first, range.second);
+ }
+
+ // Clear the btree, deleting all of the values it contains.
+ void clear();
+
+ // Swap the contents of *this and x.
+ void swap(btree &x);
+
+ const key_compare &key_comp() const noexcept {
+ return root_.template get<0>();
+ }
+ template <typename K, typename LK>
+ bool compare_keys(const K &x, const LK &y) const {
+ return compare_internal::compare_result_as_less_than(key_comp()(x, y));
+ }
+
+ value_compare value_comp() const { return value_compare(key_comp()); }
+
+ // Verifies the structure of the btree.
+ void verify() const;
+
+ // Size routines.
+ size_type size() const { return size_; }
+ size_type max_size() const { return (std::numeric_limits<size_type>::max)(); }
+ bool empty() const { return size_ == 0; }
+
+ // The height of the btree. An empty tree will have height 0.
+ size_type height() const {
+ size_type h = 0;
+ if (!empty()) {
+ // Count the length of the chain from the leftmost node up to the
+ // root. We actually count from the root back around to the level below
+ // the root, but the calculation is the same because of the circularity
+ // of that traversal.
+ const node_type *n = root();
+ do {
+ ++h;
+ n = n->parent();
+ } while (n != root());
+ }
+ return h;
+ }
+
+ // The number of internal, leaf and total nodes used by the btree.
+ size_type leaf_nodes() const {
+ return internal_stats(root()).leaf_nodes;
+ }
+ size_type internal_nodes() const {
+ return internal_stats(root()).internal_nodes;
+ }
+ size_type nodes() const {
+ node_stats stats = internal_stats(root());
+ return stats.leaf_nodes + stats.internal_nodes;
+ }
+
+ // The total number of bytes used by the btree.
+ size_type bytes_used() const {
+ node_stats stats = internal_stats(root());
+ if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) {
+ return sizeof(*this) +
+ node_type::LeafSize(root()->max_count());
+ } else {
+ return sizeof(*this) +
+ stats.leaf_nodes * node_type::LeafSize() +
+ stats.internal_nodes * node_type::InternalSize();
+ }
+ }
+
+ // The average number of bytes used per value stored in the btree.
+ static double average_bytes_per_value() {
+ // Returns the number of bytes per value on a leaf node that is 75%
+ // full. Experimentally, this matches up nicely with the computed number of
+ // bytes per value in trees that had their values inserted in random order.
+ return node_type::LeafSize() / (kNodeValues * 0.75);
+ }
+
+ // The fullness of the btree. Computed as the number of elements in the btree
+ // divided by the maximum number of elements a tree with the current number
+ // of nodes could hold. A value of 1 indicates perfect space
+ // utilization. Smaller values indicate space wastage.
+ // Returns 0 for empty trees.
+ double fullness() const {
+ if (empty()) return 0.0;
+ return static_cast<double>(size()) / (nodes() * kNodeValues);
+ }
+ // The overhead of the btree structure in bytes per node. Computed as the
+ // total number of bytes used by the btree minus the number of bytes used for
+ // storing elements divided by the number of elements.
+ // Returns 0 for empty trees.
+ double overhead() const {
+ if (empty()) return 0.0;
+ return (bytes_used() - size() * sizeof(value_type)) /
+ static_cast<double>(size());
+ }
+
+ // The allocator used by the btree.
+ allocator_type get_allocator() const {
+ return allocator();
+ }
+
+ private:
+ // Internal accessor routines.
+ node_type *root() { return root_.template get<2>(); }
+ const node_type *root() const { return root_.template get<2>(); }
+ node_type *&mutable_root() noexcept { return root_.template get<2>(); }
+ key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); }
+
+ // The leftmost node is stored as the parent of the root node.
+ node_type *leftmost() { return root()->parent(); }
+ const node_type *leftmost() const { return root()->parent(); }
+
+ // Allocator routines.
+ allocator_type *mutable_allocator() noexcept {
+ return &root_.template get<1>();
+ }
+ const allocator_type &allocator() const noexcept {
+ return root_.template get<1>();
+ }
+
+ // Allocates a correctly aligned node of at least size bytes using the
+ // allocator.
+ node_type *allocate(const size_type sz) {
+ return reinterpret_cast<node_type *>(
+ phmap::priv::Allocate<node_type::Alignment()>(
+ mutable_allocator(), (size_t)sz));
+ }
+
+ // Node creation/deletion routines.
+ node_type* new_internal_node(node_type *parent) {
+ node_type *p = allocate(node_type::InternalSize());
+ return node_type::init_internal(p, parent);
+ }
+ node_type* new_leaf_node(node_type *parent) {
+ node_type *p = allocate(node_type::LeafSize());
+ return node_type::init_leaf(p, parent, kNodeValues);
+ }
+ node_type *new_leaf_root_node(const int max_count) {
+ node_type *p = allocate(node_type::LeafSize(max_count));
+ return node_type::init_leaf(p, p, max_count);
+ }
+
+ // Deletion helper routines.
+ void erase_same_node(iterator begin, iterator end);
+ iterator erase_from_leaf_node(iterator begin, size_type to_erase);
+ iterator rebalance_after_delete(iterator iter);
+
+ // Deallocates a node of a certain size in bytes using the allocator.
+ void deallocate(const size_type sz, node_type *node) {
+ phmap::priv::Deallocate<node_type::Alignment()>(
+ mutable_allocator(), node, (size_t)sz);
+ }
+
+ void delete_internal_node(node_type *node) {
+ node->destroy(mutable_allocator());
+ deallocate(node_type::InternalSize(), node);
+ }
+ void delete_leaf_node(node_type *node) {
+ node->destroy(mutable_allocator());
+ deallocate(node_type::LeafSize(node->max_count()), node);
+ }
+
+ // Rebalances or splits the node iter points to.
+ void rebalance_or_split(iterator *iter);
+
+ // Merges the values of left, right and the delimiting key on their parent
+ // onto left, removing the delimiting key and deleting right.
+ void merge_nodes(node_type *left, node_type *right);
+
+ // Tries to merge node with its left or right sibling, and failing that,
+ // rebalance with its left or right sibling. Returns true if a merge
+ // occurred, at which point it is no longer valid to access node. Returns
+ // false if no merging took place.
+ bool try_merge_or_rebalance(iterator *iter);
+
+ // Tries to shrink the height of the tree by 1.
+ void try_shrink();
+
+ iterator internal_end(iterator iter) {
+ return iter.node != nullptr ? iter : end();
+ }
+ const_iterator internal_end(const_iterator iter) const {
+ return iter.node != nullptr ? iter : end();
+ }
+
+ // Emplaces a value into the btree immediately before iter. Requires that
+ // key(v) <= iter.key() and (--iter).key() <= key(v).
+ template <typename... Args>
+ iterator internal_emplace(iterator iter, Args &&... args);
+
+ // Returns an iterator pointing to the first value >= the value "iter" is
+ // pointing at. Note that "iter" might be pointing to an invalid location as
+ // iter.position == iter.node->count(). This routine simply moves iter up in
+ // the tree to a valid location.
+ // Requires: iter.node is non-null.
+ template <typename IterType>
+ static IterType internal_last(IterType iter);
+
+ // Returns an iterator pointing to the leaf position at which key would
+ // reside in the tree. We provide 2 versions of internal_locate. The first
+ // version uses a less-than comparator and is incapable of distinguishing when
+ // there is an exact match. The second version is for the key-compare-to
+ // specialization and distinguishes exact matches. The key-compare-to
+ // specialization allows the caller to avoid a subsequent comparison to
+ // determine if an exact match was made, which is important for keys with
+ // expensive comparison, such as strings.
+ template <typename K>
+ SearchResult<iterator, is_key_compare_to::value> internal_locate(
+ const K &key) const;
+
+ template <typename K>
+ SearchResult<iterator, false> internal_locate_impl(
+ const K &key, std::false_type /* IsCompareTo */) const;
+
+ template <typename K>
+ SearchResult<iterator, true> internal_locate_impl(
+ const K &key, std::true_type /* IsCompareTo */) const;
+
+ // Internal routine which implements lower_bound().
+ template <typename K>
+ iterator internal_lower_bound(const K &key) const;
+
+ // Internal routine which implements upper_bound().
+ template <typename K>
+ iterator internal_upper_bound(const K &key) const;
+
+ // Internal routine which implements find().
+ template <typename K>
+ iterator internal_find(const K &key) const;
+
+ // Deletes a node and all of its children.
+ void internal_clear(node_type *node);
+
+ // Verifies the tree structure of node.
+ int internal_verify(const node_type *node,
+ const key_type *lo, const key_type *hi) const;
+
+ node_stats internal_stats(const node_type *node) const {
+ // The root can be a static empty node.
+ if (node == nullptr || (node == root() && empty())) {
+ return node_stats(0, 0);
+ }
+ if (node->leaf()) {
+ return node_stats(1, 0);
+ }
+ node_stats res(0, 1);
+ for (int i = 0; i <= node->count(); ++i) {
+ res += internal_stats(node->child(i));
+ }
+ return res;
+ }
+
+ public:
+ // Exposed only for tests.
+ static bool testonly_uses_linear_node_search() {
+ return node_type::testonly_uses_linear_node_search();
+ }
+
+ private:
+ // We use compressed tuple in order to save space because key_compare and
+ // allocator_type are usually empty.
+ phmap::priv::CompressedTuple<key_compare, allocator_type,
+ node_type *>
+ root_;
+
+ // A pointer to the rightmost node. Note that the leftmost node is stored as
+ // the root's parent.
+ node_type *rightmost_;
+
+ // Number of values.
+ size_type size_;
+ };
+
+ ////
+ // btree_node methods
+ template <typename P>
+ template <typename... Args>
+ inline void btree_node<P>::emplace_value(const size_type i,
+ allocator_type *alloc,
+ Args &&... args) {
+ assert(i <= count());
+ // Shift old values to create space for new value and then construct it in
+ // place.
+ if (i < count()) {
+ value_init(count(), alloc, slot(count() - 1));
+ for (size_type j = count() - 1; j > i; --j)
+ params_type::move(alloc, slot(j - 1), slot(j));
+ value_destroy(i, alloc);
+ }
+ value_init(i, alloc, std::forward<Args>(args)...);
+ set_count((field_type)(count() + 1));
+
+ if (!leaf() && count() > i + 1) {
+ for (int j = count(); j > i + 1; --j) {
+ set_child(j, child(j - 1));
+ }
+ clear_child(i + 1);
+ }
+ }
+
+ template <typename P>
+ inline void btree_node<P>::remove_value(const int i, allocator_type *alloc) {
+ if (!leaf() && count() > i + 1) {
+ assert(child(i + 1)->count() == 0);
+ for (size_type j = i + 1; j < count(); ++j) {
+ set_child(j, child(j + 1));
+ }
+ clear_child(count());
+ }
+
+ remove_values_ignore_children(i, /*to_erase=*/1, alloc);
+ }
+
+ template <typename P>
+ inline void btree_node<P>::remove_values_ignore_children(
+ int i, size_type to_erase, allocator_type *alloc) {
+ params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i));
+ value_destroy_n(count() - to_erase, to_erase, alloc);
+ set_count((field_type)(count() - to_erase));
+ }
+
+ template <typename P>
+ void btree_node<P>::rebalance_right_to_left(const int to_move,
+ btree_node *right,
+ allocator_type *alloc) {
+ assert(parent() == right->parent());
+ assert(position() + 1 == right->position());
+ assert(right->count() >= count());
+ assert(to_move >= 1);
+ assert(to_move <= right->count());
+
+ // 1) Move the delimiting value in the parent to the left node.
+ value_init(count(), alloc, parent()->slot(position()));
+
+ // 2) Move the (to_move - 1) values from the right node to the left node.
+ right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc);
+
+ // 3) Move the new delimiting value to the parent from the right node.
+ params_type::move(alloc, right->slot(to_move - 1),
+ parent()->slot(position()));
+
+ // 4) Shift the values in the right node to their correct position.
+ params_type::move(alloc, right->slot(to_move), right->slot(right->count()),
+ right->slot(0));
+
+ // 5) Destroy the now-empty to_move entries in the right node.
+ right->value_destroy_n(right->count() - to_move, to_move, alloc);
+
+ if (!leaf()) {
+ // Move the child pointers from the right to the left node.
+ for (int i = 0; i < to_move; ++i) {
+ init_child(count() + i + 1, right->child(i));
+ }
+ for (int i = 0; i <= right->count() - to_move; ++i) {
+ assert(i + to_move <= right->max_count());
+ right->init_child(i, right->child(i + to_move));
+ right->clear_child(i + to_move);
+ }
+ }
+
+ // Fixup the counts on the left and right nodes.
+ set_count((field_type)(count() + to_move));
+ right->set_count((field_type)(right->count() - to_move));
+ }
+
+ template <typename P>
+ void btree_node<P>::rebalance_left_to_right(const int to_move,
+ btree_node *right,
+ allocator_type *alloc) {
+ assert(parent() == right->parent());
+ assert(position() + 1 == right->position());
+ assert(count() >= right->count());
+ assert(to_move >= 1);
+ assert(to_move <= count());
+
+ // Values in the right node are shifted to the right to make room for the
+ // new to_move values. Then, the delimiting value in the parent and the
+ // other (to_move - 1) values in the left node are moved into the right node.
+ // Lastly, a new delimiting value is moved from the left node into the
+ // parent, and the remaining empty left node entries are destroyed.
+
+ if (right->count() >= to_move) {
+ // The original location of the right->count() values are sufficient to hold
+ // the new to_move entries from the parent and left node.
+
+ // 1) Shift existing values in the right node to their correct positions.
+ right->uninitialized_move_n(to_move, right->count() - to_move,
+ right->count(), right, alloc);
+ for (slot_type *src = right->slot(right->count() - to_move - 1),
+ *dest = right->slot(right->count() - 1),
+ *end = right->slot(0);
+ src >= end; --src, --dest) {
+ params_type::move(alloc, src, dest);
+ }
+
+ // 2) Move the delimiting value in the parent to the right node.
+ params_type::move(alloc, parent()->slot(position()),
+ right->slot(to_move - 1));
+
+ // 3) Move the (to_move - 1) values from the left node to the right node.
+ params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()),
+ right->slot(0));
+ } else {
+ // The right node does not have enough initialized space to hold the new
+ // to_move entries, so part of them will move to uninitialized space.
+
+ // 1) Shift existing values in the right node to their correct positions.
+ right->uninitialized_move_n(right->count(), 0, to_move, right, alloc);
+
+ // 2) Move the delimiting value in the parent to the right node.
+ right->value_init(to_move - 1, alloc, parent()->slot(position()));
+
+ // 3) Move the (to_move - 1) values from the left node to the right node.
+ const size_type uninitialized_remaining = to_move - right->count() - 1;
+ uninitialized_move_n(uninitialized_remaining,
+ count() - uninitialized_remaining, right->count(),
+ right, alloc);
+ params_type::move(alloc, slot(count() - (to_move - 1)),
+ slot(count() - uninitialized_remaining), right->slot(0));
+ }
+
+ // 4) Move the new delimiting value to the parent from the left node.
+ params_type::move(alloc, slot(count() - to_move), parent()->slot(position()));
+
+ // 5) Destroy the now-empty to_move entries in the left node.
+ value_destroy_n(count() - to_move, to_move, alloc);
+
+ if (!leaf()) {
+ // Move the child pointers from the left to the right node.
+ for (int i = right->count(); i >= 0; --i) {
+ right->init_child(i + to_move, right->child(i));
+ right->clear_child(i);
+ }
+ for (int i = 1; i <= to_move; ++i) {
+ right->init_child(i - 1, child(count() - to_move + i));
+ clear_child(count() - to_move + i);
+ }
+ }
+
+ // Fixup the counts on the left and right nodes.
+ set_count((field_type)(count() - to_move));
+ right->set_count((field_type)(right->count() + to_move));
+ }
+
+ template <typename P>
+ void btree_node<P>::split(const int insert_position, btree_node *dest,
+ allocator_type *alloc) {
+ assert(dest->count() == 0);
+ assert(max_count() == kNodeValues);
+
+ // We bias the split based on the position being inserted. If we're
+ // inserting at the beginning of the left node then bias the split to put
+ // more values on the right node. If we're inserting at the end of the
+ // right node then bias the split to put more values on the left node.
+ if (insert_position == 0) {
+ dest->set_count((field_type)(count() - 1));
+ } else if (insert_position == kNodeValues) {
+ dest->set_count(0);
+ } else {
+ dest->set_count((field_type)(count() / 2));
+ }
+ set_count((field_type)(count() - dest->count()));
+ assert(count() >= 1);
+
+ // Move values from the left sibling to the right sibling.
+ uninitialized_move_n(dest->count(), count(), 0, dest, alloc);
+
+ // Destroy the now-empty entries in the left node.
+ value_destroy_n(count(), dest->count(), alloc);
+
+ // The split key is the largest value in the left sibling.
+ set_count((field_type)(count() - 1));
+ parent()->emplace_value(position(), alloc, slot(count()));
+ value_destroy(count(), alloc);
+ parent()->init_child(position() + 1, dest);
+
+ if (!leaf()) {
+ for (int i = 0; i <= dest->count(); ++i) {
+ assert(child(count() + i + 1) != nullptr);
+ dest->init_child(i, child(count() + i + 1));
+ clear_child(count() + i + 1);
+ }
+ }
+ }
+
+ template <typename P>
+ void btree_node<P>::merge(btree_node *src, allocator_type *alloc) {
+ assert(parent() == src->parent());
+ assert(position() + 1 == src->position());
+
+ // Move the delimiting value to the left node.
+ value_init(count(), alloc, parent()->slot(position()));
+
+ // Move the values from the right to the left node.
+ src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc);
+
+ // Destroy the now-empty entries in the right node.
+ src->value_destroy_n(0, src->count(), alloc);
+
+ if (!leaf()) {
+ // Move the child pointers from the right to the left node.
+ for (int i = 0; i <= src->count(); ++i) {
+ init_child(count() + i + 1, src->child(i));
+ src->clear_child(i);
+ }
+ }
+
+ // Fixup the counts on the src and dest nodes.
+ set_count((field_type)(1 + count() + src->count()));
+ src->set_count(0);
+
+ // Remove the value on the parent node.
+ parent()->remove_value(position(), alloc);
+ }
+
+ template <typename P>
+ void btree_node<P>::swap(btree_node *x, allocator_type *alloc) {
+ using std::swap;
+ assert(leaf() == x->leaf());
+
+ // Determine which is the smaller/larger node.
+ btree_node *smaller = this, *larger = x;
+ if (smaller->count() > larger->count()) {
+ swap(smaller, larger);
+ }
+
+ // Swap the values.
+ for (slot_type *a = smaller->slot(0), *b = larger->slot(0),
+ *end = a + smaller->count();
+ a != end; ++a, ++b) {
+ params_type::swap(alloc, a, b);
+ }
+
+ // Move values that can't be swapped.
+ const size_type to_move = larger->count() - smaller->count();
+ larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(),
+ smaller, alloc);
+ larger->value_destroy_n(smaller->count(), to_move, alloc);
+
+ if (!leaf()) {
+ // Swap the child pointers.
+ std::swap_ranges(&smaller->mutable_child(0),
+ &smaller->mutable_child(smaller->count() + 1),
+ &larger->mutable_child(0));
+ // Update swapped children's parent pointers.
+ int i = 0;
+ for (; i <= smaller->count(); ++i) {
+ smaller->child(i)->set_parent(smaller);
+ larger->child(i)->set_parent(larger);
+ }
+ // Move the child pointers that couldn't be swapped.
+ for (; i <= larger->count(); ++i) {
+ smaller->init_child(i, larger->child(i));
+ larger->clear_child(i);
+ }
+ }
+
+ // Swap the counts.
+ swap(mutable_count(), x->mutable_count());
+ }
+
+ ////
+ // btree_iterator methods
+ template <typename N, typename R, typename P>
+ void btree_iterator<N, R, P>::increment_slow() {
+ if (node->leaf()) {
+ assert(position >= node->count());
+ btree_iterator save(*this);
+ while (position == node->count() && !node->is_root()) {
+ assert(node->parent()->child(node->position()) == node);
+ position = node->position();
+ node = node->parent();
+ }
+ if (position == node->count()) {
+ *this = save;
+ }
+ } else {
+ assert(position < node->count());
+ node = node->child(position + 1);
+ while (!node->leaf()) {
+ node = node->child(0);
+ }
+ position = 0;
+ }
+ }
+
+ template <typename N, typename R, typename P>
+ void btree_iterator<N, R, P>::decrement_slow() {
+ if (node->leaf()) {
+ assert(position <= -1);
+ btree_iterator save(*this);
+ while (position < 0 && !node->is_root()) {
+ assert(node->parent()->child(node->position()) == node);
+ position = node->position() - 1;
+ node = node->parent();
+ }
+ if (position < 0) {
+ *this = save;
+ }
+ } else {
+ assert(position >= 0);
+ node = node->child(position);
+ while (!node->leaf()) {
+ node = node->child(node->count());
+ }
+ position = node->count() - 1;
+ }
+ }
+
+ ////
+ // btree methods
+ template <typename P>
+ template <typename Btree>
+ void btree<P>::copy_or_move_values_in_order(Btree *x) {
+ static_assert(std::is_same<btree, Btree>::value ||
+ std::is_same<const btree, Btree>::value,
+ "Btree type must be same or const.");
+ assert(empty());
+
+ // We can avoid key comparisons because we know the order of the
+ // values is the same order we'll store them in.
+ auto iter = x->begin();
+ if (iter == x->end()) return;
+ insert_multi(maybe_move_from_iterator(iter));
+ ++iter;
+ for (; iter != x->end(); ++iter) {
+ // If the btree is not empty, we can just insert the new value at the end
+ // of the tree.
+ internal_emplace(end(), maybe_move_from_iterator(iter));
+ }
+ }
+
+ template <typename P>
+ constexpr bool btree<P>::static_assert_validation() {
+ static_assert(std::is_nothrow_copy_constructible<key_compare>::value,
+ "Key comparison must be nothrow copy constructible");
+ static_assert(std::is_nothrow_copy_constructible<allocator_type>::value,
+ "Allocator must be nothrow copy constructible");
+ static_assert(type_traits_internal::is_trivially_copyable<iterator>::value,
+ "iterator not trivially copyable.");
+
+ // Note: We assert that kTargetValues, which is computed from
+ // Params::kTargetNodeSize, must fit the node_type::field_type.
+ static_assert(
+ kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))),
+ "target node size too large");
+
+ // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool.
+ using compare_result_type =
+ phmap::invoke_result_t<key_compare, key_type, key_type>;
+ static_assert(
+ std::is_same<compare_result_type, bool>::value ||
+ std::is_convertible<compare_result_type, phmap::weak_ordering>::value,
+ "key comparison function must return phmap::{weak,strong}_ordering or "
+ "bool.");
+
+ // Test the assumption made in setting kNodeValueSpace.
+ static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4,
+ "node space assumption incorrect");
+
+ return true;
+ }
+
+ template <typename P>
+ btree<P>::btree(const key_compare &comp, const allocator_type &alloc)
+ : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {}
+
+ template <typename P>
+ btree<P>::btree(const btree &x) : btree(x.key_comp(), x.allocator()) {
+ copy_or_move_values_in_order(&x);
+ }
+
+ template <typename P>
+ template <typename... Args>
+ auto btree<P>::insert_unique(const key_type &key, Args &&... args)
+ -> std::pair<iterator, bool> {
+ if (empty()) {
+ mutable_root() = rightmost_ = new_leaf_root_node(1);
+ }
+
+ auto res = internal_locate(key);
+ iterator &iter = res.value;
+
+ if (res.HasMatch()) {
+ if (res.IsEq()) {
+ // The key already exists in the tree, do nothing.
+ return {iter, false};
+ }
+ } else {
+ iterator last = internal_last(iter);
+ if (last.node && !compare_keys(key, last.key())) {
+ // The key already exists in the tree, do nothing.
+ return {last, false};
+ }
+ }
+ return {internal_emplace(iter, std::forward<Args>(args)...), true};
+ }
+
+ template <typename P>
+ template <typename... Args>
+ inline auto btree<P>::insert_hint_unique(iterator position, const key_type &key,
+ Args &&... args)
+ -> std::pair<iterator, bool> {
+ if (!empty()) {
+ if (position == end() || compare_keys(key, position.key())) {
+ iterator prev = position;
+ if (position == begin() || compare_keys((--prev).key(), key)) {
+ // prev.key() < key < position.key()
+ return {internal_emplace(position, std::forward<Args>(args)...), true};
+ }
+ } else if (compare_keys(position.key(), key)) {
+ ++position;
+ if (position == end() || compare_keys(key, position.key())) {
+ // {original `position`}.key() < key < {current `position`}.key()
+ return {internal_emplace(position, std::forward<Args>(args)...), true};
+ }
+ } else {
+ // position.key() == key
+ return {position, false};
+ }
+ }
+ return insert_unique(key, std::forward<Args>(args)...);
+ }
+
+ template <typename P>
+ template <typename InputIterator>
+ void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e) {
+ for (; b != e; ++b) {
+ insert_hint_unique(end(), params_type::key(*b), *b);
+ }
+ }
+
+ template <typename P>
+ template <typename ValueType>
+ auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator {
+ if (empty()) {
+ mutable_root() = rightmost_ = new_leaf_root_node(1);
+ }
+
+ iterator iter = internal_upper_bound(key);
+ if (iter.node == nullptr) {
+ iter = end();
+ }
+ return internal_emplace(iter, std::forward<ValueType>(v));
+ }
+
+ template <typename P>
+ template <typename ValueType>
+ auto btree<P>::insert_hint_multi(iterator position, ValueType &&v) -> iterator {
+ if (!empty()) {
+ const key_type &key = params_type::key(v);
+ if (position == end() || !compare_keys(position.key(), key)) {
+ iterator prev = position;
+ if (position == begin() || !compare_keys(key, (--prev).key())) {
+ // prev.key() <= key <= position.key()
+ return internal_emplace(position, std::forward<ValueType>(v));
+ }
+ } else {
+ iterator next = position;
+ ++next;
+ if (next == end() || !compare_keys(next.key(), key)) {
+ // position.key() < key <= next.key()
+ return internal_emplace(next, std::forward<ValueType>(v));
+ }
+ }
+ }
+ return insert_multi(std::forward<ValueType>(v));
+ }
+
+ template <typename P>
+ template <typename InputIterator>
+ void btree<P>::insert_iterator_multi(InputIterator b, InputIterator e) {
+ for (; b != e; ++b) {
+ insert_hint_multi(end(), *b);
+ }
+ }
+
+ template <typename P>
+ auto btree<P>::operator=(const btree &x) -> btree & {
+ if (this != &x) {
+ clear();
+
+ *mutable_key_comp() = x.key_comp();
+ if (phmap::allocator_traits<
+ allocator_type>::propagate_on_container_copy_assignment::value) {
+ *mutable_allocator() = x.allocator();
+ }
+
+ copy_or_move_values_in_order(&x);
+ }
+ return *this;
+ }
+
+ template <typename P>
+ auto btree<P>::operator=(btree &&x) noexcept -> btree & {
+ if (this != &x) {
+ clear();
+
+ using std::swap;
+ if (phmap::allocator_traits<
+ allocator_type>::propagate_on_container_copy_assignment::value) {
+ // Note: `root_` also contains the allocator and the key comparator.
+ swap(root_, x.root_);
+ swap(rightmost_, x.rightmost_);
+ swap(size_, x.size_);
+ } else {
+ if (allocator() == x.allocator()) {
+ swap(mutable_root(), x.mutable_root());
+ swap(*mutable_key_comp(), *x.mutable_key_comp());
+ swap(rightmost_, x.rightmost_);
+ swap(size_, x.size_);
+ } else {
+ // We aren't allowed to propagate the allocator and the allocator is
+ // different so we can't take over its memory. We must move each element
+ // individually. We need both `x` and `this` to have `x`s key comparator
+ // while moving the values so we can't swap the key comparators.
+ *mutable_key_comp() = x.key_comp();
+ copy_or_move_values_in_order(&x);
+ }
+ }
+ }
+ return *this;
+ }
+
+ template <typename P>
+ auto btree<P>::erase(iterator iter) -> iterator {
+ bool internal_delete = false;
+ if (!iter.node->leaf()) {
+ // Deletion of a value on an internal node. First, move the largest value
+ // from our left child here, then delete that position (in remove_value()
+ // below). We can get to the largest value from our left child by
+ // decrementing iter.
+ iterator internal_iter(iter);
+ --iter;
+ assert(iter.node->leaf());
+ params_type::move(mutable_allocator(), iter.node->slot(iter.position),
+ internal_iter.node->slot(internal_iter.position));
+ internal_delete = true;
+ }
+
+ // Delete the key from the leaf.
+ iter.node->remove_value(iter.position, mutable_allocator());
+ --size_;
+
+ // We want to return the next value after the one we just erased. If we
+ // erased from an internal node (internal_delete == true), then the next
+ // value is ++(++iter). If we erased from a leaf node (internal_delete ==
+ // false) then the next value is ++iter. Note that ++iter may point to an
+ // internal node and the value in the internal node may move to a leaf node
+ // (iter.node) when rebalancing is performed at the leaf level.
+
+ iterator res = rebalance_after_delete(iter);
+
+ // If we erased from an internal node, advance the iterator.
+ if (internal_delete) {
+ ++res;
+ }
+ return res;
+ }
+
+ template <typename P>
+ auto btree<P>::rebalance_after_delete(iterator iter) -> iterator {
+ // Merge/rebalance as we walk back up the tree.
+ iterator res(iter);
+ bool first_iteration = true;
+ for (;;) {
+ if (iter.node == root()) {
+ try_shrink();
+ if (empty()) {
+ return end();
+ }
+ break;
+ }
+ if (iter.node->count() >= kMinNodeValues) {
+ break;
+ }
+ bool merged = try_merge_or_rebalance(&iter);
+ // On the first iteration, we should update `res` with `iter` because `res`
+ // may have been invalidated.
+ if (first_iteration) {
+ res = iter;
+ first_iteration = false;
+ }
+ if (!merged) {
+ break;
+ }
+ iter.position = iter.node->position();
+ iter.node = iter.node->parent();
+ }
+
+ // Adjust our return value. If we're pointing at the end of a node, advance
+ // the iterator.
+ if (res.position == res.node->count()) {
+ res.position = res.node->count() - 1;
+ ++res;
+ }
+
+ return res;
+ }
+
+ template <typename P>
+ auto btree<P>::erase(iterator _begin, iterator _end)
+ -> std::pair<size_type, iterator> {
+ difference_type count = std::distance(_begin, _end);
+ assert(count >= 0);
+
+ if (count == 0) {
+ return {0, _begin};
+ }
+
+ if (count == (difference_type)size_) {
+ clear();
+ return {count, this->end()};
+ }
+
+ if (_begin.node == _end.node) {
+ erase_same_node(_begin, _end);
+ size_ -= count;
+ return {count, rebalance_after_delete(_begin)};
+ }
+
+ const size_type target_size = size_ - count;
+ while (size_ > target_size) {
+ if (_begin.node->leaf()) {
+ const size_type remaining_to_erase = size_ - target_size;
+ const size_type remaining_in_node = _begin.node->count() - _begin.position;
+ _begin = erase_from_leaf_node(
+ _begin, (std::min)(remaining_to_erase, remaining_in_node));
+ } else {
+ _begin = erase(_begin);
+ }
+ }
+ return {count, _begin};
+ }
+
+ template <typename P>
+ void btree<P>::erase_same_node(iterator _begin, iterator _end) {
+ assert(_begin.node == _end.node);
+ assert(_end.position > _begin.position);
+
+ node_type *node = _begin.node;
+ size_type to_erase = _end.position - _begin.position;
+ if (!node->leaf()) {
+ // Delete all children between _begin and _end.
+ for (size_type i = 0; i < to_erase; ++i) {
+ internal_clear(node->child(_begin.position + i + 1));
+ }
+ // Rotate children after _end into new positions.
+ for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) {
+ node->set_child(i - to_erase, node->child(i));
+ node->clear_child(i);
+ }
+ }
+ node->remove_values_ignore_children(_begin.position, to_erase,
+ mutable_allocator());
+
+ // Do not need to update rightmost_, because
+ // * either _end == this->end(), and therefore node == rightmost_, and still
+ // exists
+ // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since
+ // it wasn't covered in [_begin, _end)
+ }
+
+ template <typename P>
+ auto btree<P>::erase_from_leaf_node(iterator _begin, size_type to_erase)
+ -> iterator {
+ node_type *node = _begin.node;
+ assert(node->leaf());
+ assert(node->count() > _begin.position);
+ assert(_begin.position + to_erase <= node->count());
+
+ node->remove_values_ignore_children(_begin.position, to_erase,
+ mutable_allocator());
+
+ size_ -= to_erase;
+
+ return rebalance_after_delete(_begin);
+ }
+
+ template <typename P>
+ template <typename K>
+ auto btree<P>::erase_unique(const K &key) -> size_type {
+ const iterator iter = internal_find(key);
+ if (iter.node == nullptr) {
+ // The key doesn't exist in the tree, return nothing done.
+ return 0;
+ }
+ erase(iter);
+ return 1;
+ }
+
+ template <typename P>
+ template <typename K>
+ auto btree<P>::erase_multi(const K &key) -> size_type {
+ const iterator _begin = internal_lower_bound(key);
+ if (_begin.node == nullptr) {
+ // The key doesn't exist in the tree, return nothing done.
+ return 0;
+ }
+ // Delete all of the keys between _begin and upper_bound(key).
+ const iterator _end = internal_end(internal_upper_bound(key));
+ return erase(_begin, _end).first;
+ }
+
+ template <typename P>
+ void btree<P>::clear() {
+ if (!empty()) {
+ internal_clear(root());
+ }
+ mutable_root() = EmptyNode();
+ rightmost_ = EmptyNode();
+ size_ = 0;
+ }
+
+ template <typename P>
+ void btree<P>::swap(btree &x) {
+ using std::swap;
+ if (phmap::allocator_traits<
+ allocator_type>::propagate_on_container_swap::value) {
+ // Note: `root_` also contains the allocator and the key comparator.
+ swap(root_, x.root_);
+ } else {
+ // It's undefined behavior if the allocators are unequal here.
+ assert(allocator() == x.allocator());
+ swap(mutable_root(), x.mutable_root());
+ swap(*mutable_key_comp(), *x.mutable_key_comp());
+ }
+ swap(rightmost_, x.rightmost_);
+ swap(size_, x.size_);
+ }
+
+ template <typename P>
+ void btree<P>::verify() const {
+ assert(root() != nullptr);
+ assert(leftmost() != nullptr);
+ assert(rightmost_ != nullptr);
+ assert(empty() || size() == internal_verify(root(), nullptr, nullptr));
+ assert(leftmost() == (++const_iterator(root(), -1)).node);
+ assert(rightmost_ == (--const_iterator(root(), root()->count())).node);
+ assert(leftmost()->leaf());
+ assert(rightmost_->leaf());
+ }
+
+ template <typename P>
+ void btree<P>::rebalance_or_split(iterator *iter) {
+ node_type *&node = iter->node;
+ int &insert_position = iter->position;
+ assert(node->count() == node->max_count());
+ assert(kNodeValues == node->max_count());
+
+ // First try to make room on the node by rebalancing.
+ node_type *parent = node->parent();
+ if (node != root()) {
+ if (node->position() > 0) {
+ // Try rebalancing with our left sibling.
+ node_type *left = parent->child(node->position() - 1);
+ assert(left->max_count() == kNodeValues);
+ if (left->count() < kNodeValues) {
+ // We bias rebalancing based on the position being inserted. If we're
+ // inserting at the end of the right node then we bias rebalancing to
+ // fill up the left node.
+ int to_move = (kNodeValues - left->count()) /
+ (1 + (insert_position < kNodeValues));
+ to_move = (std::max)(1, to_move);
+
+ if (((insert_position - to_move) >= 0) ||
+ ((left->count() + to_move) < kNodeValues)) {
+ left->rebalance_right_to_left(to_move, node, mutable_allocator());
+
+ assert(node->max_count() - node->count() == to_move);
+ insert_position = insert_position - to_move;
+ if (insert_position < 0) {
+ insert_position = insert_position + left->count() + 1;
+ node = left;
+ }
+
+ assert(node->count() < node->max_count());
+ return;
+ }
+ }
+ }
+
+ if (node->position() < parent->count()) {
+ // Try rebalancing with our right sibling.
+ node_type *right = parent->child(node->position() + 1);
+ assert(right->max_count() == kNodeValues);
+ if (right->count() < kNodeValues) {
+ // We bias rebalancing based on the position being inserted. If we're
+ // inserting at the _beginning of the left node then we bias rebalancing
+ // to fill up the right node.
+ int to_move =
+ (kNodeValues - right->count()) / (1 + (insert_position > 0));
+ to_move = (std::max)(1, to_move);
+
+ if ((insert_position <= (node->count() - to_move)) ||
+ ((right->count() + to_move) < kNodeValues)) {
+ node->rebalance_left_to_right(to_move, right, mutable_allocator());
+
+ if (insert_position > node->count()) {
+ insert_position = insert_position - node->count() - 1;
+ node = right;
+ }
+
+ assert(node->count() < node->max_count());
+ return;
+ }
+ }
+ }
+
+ // Rebalancing failed, make sure there is room on the parent node for a new
+ // value.
+ assert(parent->max_count() == kNodeValues);
+ if (parent->count() == kNodeValues) {
+ iterator parent_iter(node->parent(), node->position());
+ rebalance_or_split(&parent_iter);
+ }
+ } else {
+ // Rebalancing not possible because this is the root node.
+ // Create a new root node and set the current root node as the child of the
+ // new root.
+ parent = new_internal_node(parent);
+ parent->init_child(0, root());
+ mutable_root() = parent;
+ // If the former root was a leaf node, then it's now the rightmost node.
+ assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_);
+ }
+
+ // Split the node.
+ node_type *split_node;
+ if (node->leaf()) {
+ split_node = new_leaf_node(parent);
+ node->split(insert_position, split_node, mutable_allocator());
+ if (rightmost_ == node) rightmost_ = split_node;
+ } else {
+ split_node = new_internal_node(parent);
+ node->split(insert_position, split_node, mutable_allocator());
+ }
+
+ if (insert_position > node->count()) {
+ insert_position = insert_position - node->count() - 1;
+ node = split_node;
+ }
+ }
+
+ template <typename P>
+ void btree<P>::merge_nodes(node_type *left, node_type *right) {
+ left->merge(right, mutable_allocator());
+ if (right->leaf()) {
+ if (rightmost_ == right) rightmost_ = left;
+ delete_leaf_node(right);
+ } else {
+ delete_internal_node(right);
+ }
+ }
+
+ template <typename P>
+ bool btree<P>::try_merge_or_rebalance(iterator *iter) {
+ node_type *parent = iter->node->parent();
+ if (iter->node->position() > 0) {
+ // Try merging with our left sibling.
+ node_type *left = parent->child(iter->node->position() - 1);
+ assert(left->max_count() == kNodeValues);
+ if ((1 + left->count() + iter->node->count()) <= kNodeValues) {
+ iter->position += 1 + left->count();
+ merge_nodes(left, iter->node);
+ iter->node = left;
+ return true;
+ }
+ }
+ if (iter->node->position() < parent->count()) {
+ // Try merging with our right sibling.
+ node_type *right = parent->child(iter->node->position() + 1);
+ assert(right->max_count() == kNodeValues);
+ if ((1 + iter->node->count() + right->count()) <= kNodeValues) {
+ merge_nodes(iter->node, right);
+ return true;
+ }
+ // Try rebalancing with our right sibling. We don't perform rebalancing if
+ // we deleted the first element from iter->node and the node is not
+ // empty. This is a small optimization for the common pattern of deleting
+ // from the front of the tree.
+ if ((right->count() > kMinNodeValues) &&
+ ((iter->node->count() == 0) ||
+ (iter->position > 0))) {
+ int to_move = (right->count() - iter->node->count()) / 2;
+ to_move = (std::min)(to_move, right->count() - 1);
+ iter->node->rebalance_right_to_left(to_move, right, mutable_allocator());
+ return false;
+ }
+ }
+ if (iter->node->position() > 0) {
+ // Try rebalancing with our left sibling. We don't perform rebalancing if
+ // we deleted the last element from iter->node and the node is not
+ // empty. This is a small optimization for the common pattern of deleting
+ // from the back of the tree.
+ node_type *left = parent->child(iter->node->position() - 1);
+ if ((left->count() > kMinNodeValues) &&
+ ((iter->node->count() == 0) ||
+ (iter->position < iter->node->count()))) {
+ int to_move = (left->count() - iter->node->count()) / 2;
+ to_move = (std::min)(to_move, left->count() - 1);
+ left->rebalance_left_to_right(to_move, iter->node, mutable_allocator());
+ iter->position += to_move;
+ return false;
+ }
+ }
+ return false;
+ }
+
+ template <typename P>
+ void btree<P>::try_shrink() {
+ if (root()->count() > 0) {
+ return;
+ }
+ // Deleted the last item on the root node, shrink the height of the tree.
+ if (root()->leaf()) {
+ assert(size() == 0);
+ delete_leaf_node(root());
+ mutable_root() = EmptyNode();
+ rightmost_ = EmptyNode();
+ } else {
+ node_type *child = root()->child(0);
+ child->make_root();
+ delete_internal_node(root());
+ mutable_root() = child;
+ }
+ }
+
+ template <typename P>
+ template <typename IterType>
+ inline IterType btree<P>::internal_last(IterType iter) {
+ assert(iter.node != nullptr);
+ while (iter.position == iter.node->count()) {
+ iter.position = iter.node->position();
+ iter.node = iter.node->parent();
+ if (iter.node->leaf()) {
+ iter.node = nullptr;
+ break;
+ }
+ }
+ return iter;
+ }
+
+ template <typename P>
+ template <typename... Args>
+ inline auto btree<P>::internal_emplace(iterator iter, Args &&... args)
+ -> iterator {
+ if (!iter.node->leaf()) {
+ // We can't insert on an internal node. Instead, we'll insert after the
+ // previous value which is guaranteed to be on a leaf node.
+ --iter;
+ ++iter.position;
+ }
+ const int max_count = iter.node->max_count();
+ if (iter.node->count() == max_count) {
+ // Make room in the leaf for the new item.
+ if (max_count < kNodeValues) {
+ // Insertion into the root where the root is smaller than the full node
+ // size. Simply grow the size of the root node.
+ assert(iter.node == root());
+ iter.node =
+ new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count));
+ iter.node->swap(root(), mutable_allocator());
+ delete_leaf_node(root());
+ mutable_root() = iter.node;
+ rightmost_ = iter.node;
+ } else {
+ rebalance_or_split(&iter);
+ }
+ }
+ iter.node->emplace_value(iter.position, mutable_allocator(),
+ std::forward<Args>(args)...);
+ ++size_;
+ return iter;
+ }
+
+ template <typename P>
+ template <typename K>
+ inline auto btree<P>::internal_locate(const K &key) const
+ -> SearchResult<iterator, is_key_compare_to::value> {
+ return internal_locate_impl(key, is_key_compare_to());
+ }
+
+ template <typename P>
+ template <typename K>
+ inline auto btree<P>::internal_locate_impl(
+ const K &key, std::false_type /* IsCompareTo */) const
+ -> SearchResult<iterator, false> {
+ iterator iter(const_cast<node_type *>(root()), 0);
+ for (;;) {
+ iter.position = iter.node->lower_bound(key, key_comp()).value;
+ // NOTE: we don't need to walk all the way down the tree if the keys are
+ // equal, but determining equality would require doing an extra comparison
+ // on each node on the way down, and we will need to go all the way to the
+ // leaf node in the expected case.
+ if (iter.node->leaf()) {
+ break;
+ }
+ iter.node = iter.node->child(iter.position);
+ }
+ return {iter};
+ }
+
+ template <typename P>
+ template <typename K>
+ inline auto btree<P>::internal_locate_impl(
+ const K &key, std::true_type /* IsCompareTo */) const
+ -> SearchResult<iterator, true> {
+ iterator iter(const_cast<node_type *>(root()), 0);
+ for (;;) {
+ SearchResult<int, true> res = iter.node->lower_bound(key, key_comp());
+ iter.position = res.value;
+ if (res.match == MatchKind::kEq) {
+ return {iter, MatchKind::kEq};
+ }
+ if (iter.node->leaf()) {
+ break;
+ }
+ iter.node = iter.node->child(iter.position);
+ }
+ return {iter, MatchKind::kNe};
+ }
+
+ template <typename P>
+ template <typename K>
+ auto btree<P>::internal_lower_bound(const K &key) const -> iterator {
+ iterator iter(const_cast<node_type *>(root()), 0);
+ for (;;) {
+ iter.position = iter.node->lower_bound(key, key_comp()).value;
+ if (iter.node->leaf()) {
+ break;
+ }
+ iter.node = iter.node->child(iter.position);
+ }
+ return internal_last(iter);
+ }
+
+ template <typename P>
+ template <typename K>
+ auto btree<P>::internal_upper_bound(const K &key) const -> iterator {
+ iterator iter(const_cast<node_type *>(root()), 0);
+ for (;;) {
+ iter.position = iter.node->upper_bound(key, key_comp());
+ if (iter.node->leaf()) {
+ break;
+ }
+ iter.node = iter.node->child(iter.position);
+ }
+ return internal_last(iter);
+ }
+
+ template <typename P>
+ template <typename K>
+ auto btree<P>::internal_find(const K &key) const -> iterator {
+ auto res = internal_locate(key);
+ if (res.HasMatch()) {
+ if (res.IsEq()) {
+ return res.value;
+ }
+ } else {
+ const iterator iter = internal_last(res.value);
+ if (iter.node != nullptr && !compare_keys(key, iter.key())) {
+ return iter;
+ }
+ }
+ return {nullptr, 0};
+ }
+
+ template <typename P>
+ void btree<P>::internal_clear(node_type *node) {
+ if (!node->leaf()) {
+ for (int i = 0; i <= node->count(); ++i) {
+ internal_clear(node->child(i));
+ }
+ delete_internal_node(node);
+ } else {
+ delete_leaf_node(node);
+ }
+ }
+
+ template <typename P>
+ int btree<P>::internal_verify(
+ const node_type *node, const key_type *lo, const key_type *hi) const {
+ assert(node->count() > 0);
+ assert(node->count() <= node->max_count());
+ if (lo) {
+ assert(!compare_keys(node->key(0), *lo));
+ }
+ if (hi) {
+ assert(!compare_keys(*hi, node->key(node->count() - 1)));
+ }
+ for (int i = 1; i < node->count(); ++i) {
+ assert(!compare_keys(node->key(i), node->key(i - 1)));
+ }
+ int count = node->count();
+ if (!node->leaf()) {
+ for (int i = 0; i <= node->count(); ++i) {
+ assert(node->child(i) != nullptr);
+ assert(node->child(i)->parent() == node);
+ assert(node->child(i)->position() == i);
+ count += internal_verify(
+ node->child(i),
+ (i == 0) ? lo : &node->key(i - 1),
+ (i == node->count()) ? hi : &node->key(i));
+ }
+ }
+ return count;
+ }
+
+ // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap.
+ // ---------------------------------------------------------------------------------
+ template <typename Tree>
+ class btree_container {
+ using params_type = typename Tree::params_type;
+
+ protected:
+ // Alias used for heterogeneous lookup functions.
+ // `key_arg<K>` evaluates to `K` when the functors are transparent and to
+ // `key_type` otherwise. It permits template argument deduction on `K` for the
+ // transparent case.
+ template <class K>
+ using key_arg =
+ typename KeyArg<IsTransparent<typename Tree::key_compare>::value>::
+ template type<K, typename Tree::key_type>;
+
+ public:
+ using key_type = typename Tree::key_type;
+ using value_type = typename Tree::value_type;
+ using size_type = typename Tree::size_type;
+ using difference_type = typename Tree::difference_type;
+ using key_compare = typename Tree::key_compare;
+ using value_compare = typename Tree::value_compare;
+ using allocator_type = typename Tree::allocator_type;
+ using reference = typename Tree::reference;
+ using const_reference = typename Tree::const_reference;
+ using pointer = typename Tree::pointer;
+ using const_pointer = typename Tree::const_pointer;
+ using iterator = typename Tree::iterator;
+ using const_iterator = typename Tree::const_iterator;
+ using reverse_iterator = typename Tree::reverse_iterator;
+ using const_reverse_iterator = typename Tree::const_reverse_iterator;
+ using node_type = typename Tree::node_handle_type;
+
+ // Constructors/assignments.
+ btree_container() : tree_(key_compare(), allocator_type()) {}
+ explicit btree_container(const key_compare &comp,
+ const allocator_type &alloc = allocator_type())
+ : tree_(comp, alloc) {}
+ btree_container(const btree_container &x) = default;
+ btree_container(btree_container &&x) noexcept = default;
+ btree_container &operator=(const btree_container &x) = default;
+ btree_container &operator=(btree_container &&x) noexcept(
+ std::is_nothrow_move_assignable<Tree>::value) = default;
+
+ // Iterator routines.
+ iterator begin() { return tree_.begin(); }
+ const_iterator begin() const { return tree_.begin(); }
+ const_iterator cbegin() const { return tree_.begin(); }
+ iterator end() { return tree_.end(); }
+ const_iterator end() const { return tree_.end(); }
+ const_iterator cend() const { return tree_.end(); }
+ reverse_iterator rbegin() { return tree_.rbegin(); }
+ const_reverse_iterator rbegin() const { return tree_.rbegin(); }
+ const_reverse_iterator crbegin() const { return tree_.rbegin(); }
+ reverse_iterator rend() { return tree_.rend(); }
+ const_reverse_iterator rend() const { return tree_.rend(); }
+ const_reverse_iterator crend() const { return tree_.rend(); }
+
+ // Lookup routines.
+ template <typename K = key_type>
+ iterator find(const key_arg<K> &key) {
+ return tree_.find(key);
+ }
+ template <typename K = key_type>
+ const_iterator find(const key_arg<K> &key) const { return tree_.find(key); }
+
+ template <typename K = key_type>
+ bool contains(const key_arg<K> &key) const { return find(key) != end(); }
+
+ template <typename K = key_type>
+ iterator lower_bound(const key_arg<K> &key) { return tree_.lower_bound(key); }
+
+ template <typename K = key_type>
+ const_iterator lower_bound(const key_arg<K> &key) const { return tree_.lower_bound(key); }
+
+ template <typename K = key_type>
+ iterator upper_bound(const key_arg<K> &key) { return tree_.upper_bound(key); }
+
+ template <typename K = key_type>
+ const_iterator upper_bound(const key_arg<K> &key) const { return tree_.upper_bound(key); }
+
+ template <typename K = key_type>
+ std::pair<iterator, iterator> equal_range(const key_arg<K> &key) { return tree_.equal_range(key); }
+
+ template <typename K = key_type>
+ std::pair<const_iterator, const_iterator> equal_range(
+ const key_arg<K> &key) const {
+ return tree_.equal_range(key);
+ }
+
+ iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); }
+ iterator erase(iterator iter) { return tree_.erase(iter); }
+ iterator erase(const_iterator first, const_iterator last) {
+ return tree_.erase(iterator(first), iterator(last)).second;
+ }
+
+ node_type extract(iterator position) {
+ // Use Move instead of Transfer, because the rebalancing code expects to
+ // have a valid object to scribble metadata bits on top of.
+ auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot());
+ erase(position);
+ return node;
+ }
+
+ node_type extract(const_iterator position) {
+ return extract(iterator(position));
+ }
+
+ public:
+ void clear() { tree_.clear(); }
+ void swap(btree_container &x) { tree_.swap(x.tree_); }
+ void verify() const { tree_.verify(); }
+
+ size_type size() const { return tree_.size(); }
+ size_type max_size() const { return tree_.max_size(); }
+ bool empty() const { return tree_.empty(); }
+
+ friend bool operator==(const btree_container &x, const btree_container &y) {
+ if (x.size() != y.size()) return false;
+ return std::equal(x.begin(), x.end(), y.begin());
+ }
+
+ friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); }
+
+ friend bool operator<(const btree_container &x, const btree_container &y) {
+ return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end());
+ }
+
+ friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; }
+
+ friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); }
+
+ friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); }
+
+ // The allocator used by the btree.
+ allocator_type get_allocator() const { return tree_.get_allocator(); }
+
+ // The key comparator used by the btree.
+ key_compare key_comp() const { return tree_.key_comp(); }
+ value_compare value_comp() const { return tree_.value_comp(); }
+
+ // Support absl::Hash.
+ template <typename State>
+ friend State AbslHashValue(State h, const btree_container &b) {
+ for (const auto &v : b) {
+ h = State::combine(std::move(h), v);
+ }
+ return State::combine(std::move(h), b.size());
+ }
+
+ protected:
+ Tree tree_;
+ };
+
+ // A common base class for btree_set and btree_map.
+ // -----------------------------------------------
+ template <typename Tree>
+ class btree_set_container : public btree_container<Tree> {
+ using super_type = btree_container<Tree>;
+ using params_type = typename Tree::params_type;
+ using init_type = typename params_type::init_type;
+ using is_key_compare_to = typename params_type::is_key_compare_to;
+ friend class BtreeNodePeer;
+
+ protected:
+ template <class K>
+ using key_arg = typename super_type::template key_arg<K>;
+
+ public:
+ using key_type = typename Tree::key_type;
+ using value_type = typename Tree::value_type;
+ using size_type = typename Tree::size_type;
+ using key_compare = typename Tree::key_compare;
+ using allocator_type = typename Tree::allocator_type;
+ using iterator = typename Tree::iterator;
+ using const_iterator = typename Tree::const_iterator;
+ using node_type = typename super_type::node_type;
+ using insert_return_type = InsertReturnType<iterator, node_type>;
+ using super_type::super_type;
+ btree_set_container() {}
+
+ template <class InputIterator>
+ btree_set_container(InputIterator b, InputIterator e,
+ const key_compare &comp = key_compare(),
+ const allocator_type &alloc = allocator_type())
+ : super_type(comp, alloc) {
+ insert(b, e);
+ }
+
+ btree_set_container(std::initializer_list<init_type> init,
+ const key_compare &comp = key_compare(),
+ const allocator_type &alloc = allocator_type())
+ : btree_set_container(init.begin(), init.end(), comp, alloc) {}
+
+ // Lookup routines.
+ template <typename K = key_type>
+ size_type count(const key_arg<K> &key) const {
+ return this->tree_.count_unique(key);
+ }
+
+ // Insertion routines.
+ std::pair<iterator, bool> insert(const value_type &x) {
+ return this->tree_.insert_unique(params_type::key(x), x);
+ }
+ std::pair<iterator, bool> insert(value_type &&x) {
+ return this->tree_.insert_unique(params_type::key(x), std::move(x));
+ }
+ template <typename... Args>
+ std::pair<iterator, bool> emplace(Args &&... args) {
+ init_type v(std::forward<Args>(args)...);
+ return this->tree_.insert_unique(params_type::key(v), std::move(v));
+ }
+ iterator insert(const_iterator position, const value_type &x) {
+ return this->tree_
+ .insert_hint_unique(iterator(position), params_type::key(x), x)
+ .first;
+ }
+ iterator insert(const_iterator position, value_type &&x) {
+ return this->tree_
+ .insert_hint_unique(iterator(position), params_type::key(x),
+ std::move(x))
+ .first;
+ }
+
+ template <typename... Args>
+ iterator emplace_hint(const_iterator position, Args &&... args) {
+ init_type v(std::forward<Args>(args)...);
+ return this->tree_
+ .insert_hint_unique(iterator(position), params_type::key(v),
+ std::move(v))
+ .first;
+ }
+
+ template <typename InputIterator>
+ void insert(InputIterator b, InputIterator e) {
+ this->tree_.insert_iterator_unique(b, e);
+ }
+
+ void insert(std::initializer_list<init_type> init) {
+ this->tree_.insert_iterator_unique(init.begin(), init.end());
+ }
+
+ insert_return_type insert(node_type &&node) {
+ if (!node) return {this->end(), false, node_type()};
+ std::pair<iterator, bool> res =
+ this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)),
+ CommonAccess::GetSlot(node));
+ if (res.second) {
+ CommonAccess::Destroy(&node);
+ return {res.first, true, node_type()};
+ } else {
+ return {res.first, false, std::move(node)};
+ }
+ }
+
+ iterator insert(const_iterator hint, node_type &&node) {
+ if (!node) return this->end();
+ std::pair<iterator, bool> res = this->tree_.insert_hint_unique(
+ iterator(hint), params_type::key(CommonAccess::GetSlot(node)),
+ CommonAccess::GetSlot(node));
+ if (res.second) CommonAccess::Destroy(&node);
+ return res.first;
+ }
+
+ template <typename K = key_type>
+ size_type erase(const key_arg<K> &key) { return this->tree_.erase_unique(key); }
+ using super_type::erase;
+
+ template <typename K = key_type>
+ node_type extract(const key_arg<K> &key) {
+ auto it = this->find(key);
+ return it == this->end() ? node_type() : extract(it);
+ }
+
+ using super_type::extract;
+
+ // Merge routines.
+ // Moves elements from `src` into `this`. If the element already exists in
+ // `this`, it is left unmodified in `src`.
+ template <
+ typename T,
+ typename phmap::enable_if_t<
+ phmap::conjunction<
+ std::is_same<value_type, typename T::value_type>,
+ std::is_same<allocator_type, typename T::allocator_type>,
+ std::is_same<typename params_type::is_map_container,
+ typename T::params_type::is_map_container>>::value,
+ int> = 0>
+ void merge(btree_container<T> &src) { // NOLINT
+ for (auto src_it = src.begin(); src_it != src.end();) {
+ if (insert(std::move(*src_it)).second) {
+ src_it = src.erase(src_it);
+ } else {
+ ++src_it;
+ }
+ }
+ }
+
+ template <
+ typename T,
+ typename phmap::enable_if_t<
+ phmap::conjunction<
+ std::is_same<value_type, typename T::value_type>,
+ std::is_same<allocator_type, typename T::allocator_type>,
+ std::is_same<typename params_type::is_map_container,
+ typename T::params_type::is_map_container>>::value,
+ int> = 0>
+ void merge(btree_container<T> &&src) {
+ merge(src);
+ }
+ };
+
+ // Base class for btree_map.
+ // -------------------------
+ template <typename Tree>
+ class btree_map_container : public btree_set_container<Tree> {
+ using super_type = btree_set_container<Tree>;
+ using params_type = typename Tree::params_type;
+
+ protected:
+ template <class K>
+ using key_arg = typename super_type::template key_arg<K>;
+
+ public:
+ using key_type = typename Tree::key_type;
+ using mapped_type = typename params_type::mapped_type;
+ using value_type = typename Tree::value_type;
+ using key_compare = typename Tree::key_compare;
+ using allocator_type = typename Tree::allocator_type;
+ using iterator = typename Tree::iterator;
+ using const_iterator = typename Tree::const_iterator;
+
+ // Inherit constructors.
+ using super_type::super_type;
+ btree_map_container() {}
+
+ // Insertion routines.
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(const key_type &k, Args &&... args) {
+ return this->tree_.insert_unique(
+ k, std::piecewise_construct, std::forward_as_tuple(k),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ }
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(key_type &&k, Args &&... args) {
+ // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k`
+ // and then using `k` unsequenced. This is safe because the move is into a
+ // forwarding reference and insert_unique guarantees that `key` is never
+ // referenced after consuming `args`.
+ const key_type& key_ref = k;
+ return this->tree_.insert_unique(
+ key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ }
+ template <typename... Args>
+ iterator try_emplace(const_iterator hint, const key_type &k,
+ Args &&... args) {
+ return this->tree_
+ .insert_hint_unique(iterator(hint), k, std::piecewise_construct,
+ std::forward_as_tuple(k),
+ std::forward_as_tuple(std::forward<Args>(args)...))
+ .first;
+ }
+ template <typename... Args>
+ iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) {
+ // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k`
+ // and then using `k` unsequenced. This is safe because the move is into a
+ // forwarding reference and insert_hint_unique guarantees that `key` is
+ // never referenced after consuming `args`.
+ const key_type& key_ref = k;
+ return this->tree_
+ .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct,
+ std::forward_as_tuple(std::move(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...))
+ .first;
+ }
+ mapped_type &operator[](const key_type &k) {
+ return try_emplace(k).first->second;
+ }
+ mapped_type &operator[](key_type &&k) {
+ return try_emplace(std::move(k)).first->second;
+ }
+
+ template <typename K = key_type>
+ mapped_type &at(const key_arg<K> &key) {
+ auto it = this->find(key);
+ if (it == this->end())
+ base_internal::ThrowStdOutOfRange("phmap::btree_map::at");
+ return it->second;
+ }
+ template <typename K = key_type>
+ const mapped_type &at(const key_arg<K> &key) const {
+ auto it = this->find(key);
+ if (it == this->end())
+ base_internal::ThrowStdOutOfRange("phmap::btree_map::at");
+ return it->second;
+ }
+ };
+
+ // A common base class for btree_multiset and btree_multimap.
+ template <typename Tree>
+ class btree_multiset_container : public btree_container<Tree> {
+ using super_type = btree_container<Tree>;
+ using params_type = typename Tree::params_type;
+ using init_type = typename params_type::init_type;
+ using is_key_compare_to = typename params_type::is_key_compare_to;
+
+ template <class K>
+ using key_arg = typename super_type::template key_arg<K>;
+
+ public:
+ using key_type = typename Tree::key_type;
+ using value_type = typename Tree::value_type;
+ using size_type = typename Tree::size_type;
+ using key_compare = typename Tree::key_compare;
+ using allocator_type = typename Tree::allocator_type;
+ using iterator = typename Tree::iterator;
+ using const_iterator = typename Tree::const_iterator;
+ using node_type = typename super_type::node_type;
+
+ // Inherit constructors.
+ using super_type::super_type;
+ btree_multiset_container() {}
+
+ // Range constructor.
+ template <class InputIterator>
+ btree_multiset_container(InputIterator b, InputIterator e,
+ const key_compare &comp = key_compare(),
+ const allocator_type &alloc = allocator_type())
+ : super_type(comp, alloc) {
+ insert(b, e);
+ }
+
+ // Initializer list constructor.
+ btree_multiset_container(std::initializer_list<init_type> init,
+ const key_compare &comp = key_compare(),
+ const allocator_type &alloc = allocator_type())
+ : btree_multiset_container(init.begin(), init.end(), comp, alloc) {}
+
+ // Lookup routines.
+ template <typename K = key_type>
+ size_type count(const key_arg<K> &key) const {
+ return this->tree_.count_multi(key);
+ }
+
+ // Insertion routines.
+ iterator insert(const value_type &x) { return this->tree_.insert_multi(x); }
+ iterator insert(value_type &&x) {
+ return this->tree_.insert_multi(std::move(x));
+ }
+ iterator insert(const_iterator position, const value_type &x) {
+ return this->tree_.insert_hint_multi(iterator(position), x);
+ }
+ iterator insert(const_iterator position, value_type &&x) {
+ return this->tree_.insert_hint_multi(iterator(position), std::move(x));
+ }
+ template <typename InputIterator>
+ void insert(InputIterator b, InputIterator e) {
+ this->tree_.insert_iterator_multi(b, e);
+ }
+ void insert(std::initializer_list<init_type> init) {
+ this->tree_.insert_iterator_multi(init.begin(), init.end());
+ }
+ template <typename... Args>
+ iterator emplace(Args &&... args) {
+ return this->tree_.insert_multi(init_type(std::forward<Args>(args)...));
+ }
+ template <typename... Args>
+ iterator emplace_hint(const_iterator position, Args &&... args) {
+ return this->tree_.insert_hint_multi(
+ iterator(position), init_type(std::forward<Args>(args)...));
+ }
+ iterator insert(node_type &&node) {
+ if (!node) return this->end();
+ iterator res =
+ this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)),
+ CommonAccess::GetSlot(node));
+ CommonAccess::Destroy(&node);
+ return res;
+ }
+ iterator insert(const_iterator hint, node_type &&node) {
+ if (!node) return this->end();
+ iterator res = this->tree_.insert_hint_multi(
+ iterator(hint),
+ std::move(params_type::element(CommonAccess::GetSlot(node))));
+ CommonAccess::Destroy(&node);
+ return res;
+ }
+
+ // Deletion routines.
+ template <typename K = key_type>
+ size_type erase(const key_arg<K> &key) {
+ return this->tree_.erase_multi(key);
+ }
+ using super_type::erase;
+
+ // Node extraction routines.
+ template <typename K = key_type>
+ node_type extract(const key_arg<K> &key) {
+ auto it = this->find(key);
+ return it == this->end() ? node_type() : extract(it);
+ }
+ using super_type::extract;
+
+ // Merge routines.
+ // Moves all elements from `src` into `this`.
+ template <
+ typename T,
+ typename phmap::enable_if_t<
+ phmap::conjunction<
+ std::is_same<value_type, typename T::value_type>,
+ std::is_same<allocator_type, typename T::allocator_type>,
+ std::is_same<typename params_type::is_map_container,
+ typename T::params_type::is_map_container>>::value,
+ int> = 0>
+ void merge(btree_container<T> &src) { // NOLINT
+ insert(std::make_move_iterator(src.begin()),
+ std::make_move_iterator(src.end()));
+ src.clear();
+ }
+
+ template <
+ typename T,
+ typename phmap::enable_if_t<
+ phmap::conjunction<
+ std::is_same<value_type, typename T::value_type>,
+ std::is_same<allocator_type, typename T::allocator_type>,
+ std::is_same<typename params_type::is_map_container,
+ typename T::params_type::is_map_container>>::value,
+ int> = 0>
+ void merge(btree_container<T> &&src) {
+ merge(src);
+ }
+ };
+
+ // A base class for btree_multimap.
+ template <typename Tree>
+ class btree_multimap_container : public btree_multiset_container<Tree> {
+ using super_type = btree_multiset_container<Tree>;
+ using params_type = typename Tree::params_type;
+
+ public:
+ using mapped_type = typename params_type::mapped_type;
+
+ // Inherit constructors.
+ using super_type::super_type;
+ btree_multimap_container() {}
+ };
+
+} // namespace priv
+
+
+
+ // ----------------------------------------------------------------------
+ // btree_set - default values in phmap_fwd_decl.h
+ // ----------------------------------------------------------------------
+ template <typename Key, typename Compare, typename Alloc>
+ class btree_set : public priv::btree_set_container<
+ priv::btree<priv::set_params<
+ Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>>
+ {
+ using Base = typename btree_set::btree_set_container;
+
+ public:
+ btree_set() {}
+ using Base::Base;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::end;
+ using Base::cend;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::get_allocator;
+ using Base::key_comp;
+ using Base::value_comp;
+ };
+
+ // Swaps the contents of two `phmap::btree_set` containers.
+ // -------------------------------------------------------
+ template <typename K, typename C, typename A>
+ void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) {
+ return x.swap(y);
+ }
+
+ // Erases all elements that satisfy the predicate pred from the container.
+ // ----------------------------------------------------------------------
+ template <typename K, typename C, typename A, typename Pred>
+ void erase_if(btree_set<K, C, A> &set, Pred pred) {
+ for (auto it = set.begin(); it != set.end();) {
+ if (pred(*it)) {
+ it = set.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+ // ----------------------------------------------------------------------
+ // btree_multiset - default values in phmap_fwd_decl.h
+ // ----------------------------------------------------------------------
+ template <typename Key, typename Compare, typename Alloc>
+ class btree_multiset : public priv::btree_multiset_container<
+ priv::btree<priv::set_params<
+ Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>>
+ {
+ using Base = typename btree_multiset::btree_multiset_container;
+
+ public:
+ btree_multiset() {}
+ using Base::Base;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::end;
+ using Base::cend;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::get_allocator;
+ using Base::key_comp;
+ using Base::value_comp;
+ };
+
+ // Swaps the contents of two `phmap::btree_multiset` containers.
+ // ------------------------------------------------------------
+ template <typename K, typename C, typename A>
+ void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) {
+ return x.swap(y);
+ }
+
+ // Erases all elements that satisfy the predicate pred from the container.
+ // ----------------------------------------------------------------------
+ template <typename K, typename C, typename A, typename Pred>
+ void erase_if(btree_multiset<K, C, A> &set, Pred pred) {
+ for (auto it = set.begin(); it != set.end();) {
+ if (pred(*it)) {
+ it = set.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+
+ // ----------------------------------------------------------------------
+ // btree_map - default values in phmap_fwd_decl.h
+ // ----------------------------------------------------------------------
+ template <typename Key, typename Value, typename Compare, typename Alloc>
+ class btree_map : public priv::btree_map_container<
+ priv::btree<priv::map_params<
+ Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>>
+ {
+ using Base = typename btree_map::btree_map_container;
+
+ public:
+ btree_map() {}
+ using Base::Base;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::end;
+ using Base::cend;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::try_emplace;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::at;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::operator[];
+ using Base::get_allocator;
+ using Base::key_comp;
+ using Base::value_comp;
+ };
+
+ // Swaps the contents of two `phmap::btree_map` containers.
+ // -------------------------------------------------------
+ template <typename K, typename V, typename C, typename A>
+ void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) {
+ return x.swap(y);
+ }
+
+ // ----------------------------------------------------------------------
+ template <typename K, typename V, typename C, typename A, typename Pred>
+ void erase_if(btree_map<K, V, C, A> &map, Pred pred) {
+ for (auto it = map.begin(); it != map.end();) {
+ if (pred(*it)) {
+ it = map.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+ // ----------------------------------------------------------------------
+ // btree_multimap - default values in phmap_fwd_decl.h
+ // ----------------------------------------------------------------------
+ template <typename Key, typename Value, typename Compare, typename Alloc>
+ class btree_multimap : public priv::btree_multimap_container<
+ priv::btree<priv::map_params<
+ Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>>
+ {
+ using Base = typename btree_multimap::btree_multimap_container;
+
+ public:
+ btree_multimap() {}
+ using Base::Base;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::end;
+ using Base::cend;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::get_allocator;
+ using Base::key_comp;
+ using Base::value_comp;
+ };
+
+ // Swaps the contents of two `phmap::btree_multimap` containers.
+ // ------------------------------------------------------------
+ template <typename K, typename V, typename C, typename A>
+ void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) {
+ return x.swap(y);
+ }
+
+ // Erases all elements that satisfy the predicate pred from the container.
+ // ----------------------------------------------------------------------
+ template <typename K, typename V, typename C, typename A, typename Pred>
+ void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) {
+ for (auto it = map.begin(); it != map.end();) {
+ if (pred(*it)) {
+ it = map.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+
+} // namespace btree
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+
+#endif // PHMAP_BTREE_BTREE_CONTAINER_H_
diff --git a/benchmarks/others/parallel_hashmap/conanfile.py b/benchmarks/others/parallel_hashmap/conanfile.py
new file mode 100644
index 00000000..c046377d
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/conanfile.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from conans import ConanFile, tools
+import os
+
+class SparseppConan(ConanFile):
+ name = "parallel_hashmap"
+ version = "1.27"
+ description = "A header-only, very fast and memory-friendly hash map"
+
+ # Indicates License type of the packaged library
+ license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE"
+
+ # Packages the license for the conanfile.py
+ exports = ["LICENSE"]
+
+ # Custom attributes for Bincrafters recipe conventions
+ source_subfolder = "source_subfolder"
+
+ def source(self):
+ source_url = "https://github.com/greg7mdp/parallel-hashmap"
+ tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version))
+ extracted_dir = self.name + "-" + self.version
+
+ #Rename to "source_folder" is a convention to simplify later steps
+ os.rename(extracted_dir, self.source_subfolder)
+
+
+ def package(self):
+ include_folder = os.path.join(self.source_subfolder, "parallel_hashmap")
+ self.copy(pattern="LICENSE")
+ self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder)
+
+ def package_id(self):
+ self.info.header_only()
diff --git a/benchmarks/others/sparsepp/spp_memory.h b/benchmarks/others/parallel_hashmap/meminfo.h
index cfaa108d..872f3c69 100644
--- a/benchmarks/others/sparsepp/spp_memory.h
+++ b/benchmarks/others/parallel_hashmap/meminfo.h
@@ -28,6 +28,11 @@
namespace spp
{
+ uint64_t GetSystemMemory();
+ uint64_t GetTotalMemoryUsed();
+ uint64_t GetProcessMemoryUsed();
+ uint64_t GetPhysicalMemory();
+
uint64_t GetSystemMemory()
{
#ifdef SPP_WIN
diff --git a/benchmarks/others/parallel_hashmap/phmap.h b/benchmarks/others/parallel_hashmap/phmap.h
new file mode 100644
index 00000000..653ae5ea
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap.h
@@ -0,0 +1,4788 @@
+#if !defined(phmap_h_guard_)
+#define phmap_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
+// with modifications.
+//
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+
+ #pragma warning(disable : 4127) // conditional expression is constant
+ #pragma warning(disable : 4324) // structure was padded due to alignment specifier
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+ #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted
+ #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted
+ #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted
+ #pragma warning(disable : 4710) // function not inlined
+ #pragma warning(disable : 4711) // selected for automatic inline expansion
+ #pragma warning(disable : 4820) // '6' bytes padding added after data member
+ #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list
+ #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted
+ #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified
+#endif
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <array>
+#include <cassert>
+#include <atomic>
+
+#include "phmap_fwd_decl.h"
+#include "phmap_utils.h"
+#include "phmap_base.h"
+
+#if PHMAP_HAVE_STD_STRING_VIEW
+ #include <string_view>
+#endif
+
+namespace phmap {
+
+namespace priv {
+
+// --------------------------------------------------------------------------
+template <size_t Width>
+class probe_seq
+{
+public:
+ probe_seq(size_t hashval, size_t mask) {
+ assert(((mask + 1) & mask) == 0 && "not a mask");
+ mask_ = mask;
+ offset_ = hashval & mask_;
+ }
+ size_t offset() const { return offset_; }
+ size_t offset(size_t i) const { return (offset_ + i) & mask_; }
+
+ void next() {
+ index_ += Width;
+ offset_ += index_;
+ offset_ &= mask_;
+ }
+ // 0-based probe index. The i-th probe in the probe sequence.
+ size_t getindex() const { return index_; }
+
+private:
+ size_t mask_;
+ size_t offset_;
+ size_t index_ = 0;
+};
+
+// --------------------------------------------------------------------------
+template <class ContainerKey, class Hash, class Eq>
+struct RequireUsableKey
+{
+ template <class PassedKey, class... Args>
+ std::pair<
+ decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())),
+ decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(),
+ std::declval<const PassedKey&>()))>*
+ operator()(const PassedKey&, const Args&...) const;
+};
+
+// --------------------------------------------------------------------------
+template <class E, class Policy, class Hash, class Eq, class... Ts>
+struct IsDecomposable : std::false_type {};
+
+template <class Policy, class Hash, class Eq, class... Ts>
+struct IsDecomposable<
+ phmap::void_t<decltype(
+ Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(),
+ std::declval<Ts>()...))>,
+ Policy, Hash, Eq, Ts...> : std::true_type {};
+
+// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it.
+// --------------------------------------------------------------------------
+template <class T>
+constexpr bool IsNoThrowSwappable() {
+ using std::swap;
+ return noexcept(swap(std::declval<T&>(), std::declval<T&>()));
+}
+
+// --------------------------------------------------------------------------
+template <typename T>
+int TrailingZeros(T x) {
+ PHMAP_IF_CONSTEXPR(sizeof(T) == 8)
+ return base_internal::CountTrailingZerosNonZero64(static_cast<uint64_t>(x));
+ else
+ return base_internal::CountTrailingZerosNonZero32(static_cast<uint32_t>(x));
+}
+
+// --------------------------------------------------------------------------
+template <typename T>
+int LeadingZeros(T x) {
+ PHMAP_IF_CONSTEXPR(sizeof(T) == 8)
+ return base_internal::CountLeadingZeros64(static_cast<uint64_t>(x));
+ else
+ return base_internal::CountLeadingZeros32(static_cast<uint32_t>(x));
+}
+
+// --------------------------------------------------------------------------
+// An abstraction over a bitmask. It provides an easy way to iterate through the
+// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE),
+// this is a true bitmask. On non-SSE, platforms the arithematic used to
+// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as
+// either 0x00 or 0x80.
+//
+// For example:
+// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2
+// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3
+// --------------------------------------------------------------------------
+template <class T, int SignificantBits, int Shift = 0>
+class BitMask
+{
+ static_assert(std::is_unsigned<T>::value, "");
+ static_assert(Shift == 0 || Shift == 3, "");
+
+public:
+ // These are useful for unit tests (gunit).
+ using value_type = int;
+ using iterator = BitMask;
+ using const_iterator = BitMask;
+
+ explicit BitMask(T mask) : mask_(mask) {}
+ BitMask& operator++() {
+ mask_ &= (mask_ - 1);
+ return *this;
+ }
+ explicit operator bool() const { return mask_ != 0; }
+ int operator*() const { return LowestBitSet(); }
+ int LowestBitSet() const {
+ return priv::TrailingZeros(mask_) >> Shift;
+ }
+ int HighestBitSet() const {
+ return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) -
+ 1) >>
+ Shift;
+ }
+
+ BitMask begin() const { return *this; }
+ BitMask end() const { return BitMask(0); }
+
+ int TrailingZeros() const {
+ return priv::TrailingZeros(mask_) >> Shift;
+ }
+
+ int LeadingZeros() const {
+ constexpr int total_significant_bits = SignificantBits << Shift;
+ constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits;
+ return priv::LeadingZeros(mask_ << extra_bits) >> Shift;
+ }
+
+private:
+ friend bool operator==(const BitMask& a, const BitMask& b) {
+ return a.mask_ == b.mask_;
+ }
+ friend bool operator!=(const BitMask& a, const BitMask& b) {
+ return a.mask_ != b.mask_;
+ }
+
+ T mask_;
+};
+
+// --------------------------------------------------------------------------
+using ctrl_t = signed char;
+using h2_t = uint8_t;
+
+// --------------------------------------------------------------------------
+// The values here are selected for maximum performance. See the static asserts
+// below for details.
+// --------------------------------------------------------------------------
+enum Ctrl : ctrl_t
+{
+ kEmpty = -128, // 0b10000000
+ kDeleted = -2, // 0b11111110
+ kSentinel = -1, // 0b11111111
+};
+
+static_assert(
+ kEmpty & kDeleted & kSentinel & 0x80,
+ "Special markers need to have the MSB to make checking for them efficient");
+static_assert(kEmpty < kSentinel && kDeleted < kSentinel,
+ "kEmpty and kDeleted must be smaller than kSentinel to make the "
+ "SIMD test of IsEmptyOrDeleted() efficient");
+static_assert(kSentinel == -1,
+ "kSentinel must be -1 to elide loading it from memory into SIMD "
+ "registers (pcmpeqd xmm, xmm)");
+static_assert(kEmpty == -128,
+ "kEmpty must be -128 to make the SIMD check for its "
+ "existence efficient (psignb xmm, xmm)");
+static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F,
+ "kEmpty and kDeleted must share an unset bit that is not shared "
+ "by kSentinel to make the scalar test for MatchEmptyOrDeleted() "
+ "efficient");
+static_assert(kDeleted == -2,
+ "kDeleted must be -2 to make the implementation of "
+ "ConvertSpecialToEmptyAndFullToDeleted efficient");
+
+// --------------------------------------------------------------------------
+// A single block of empty control bytes for tables without any slots allocated.
+// This enables removing a branch in the hot path of find().
+// --------------------------------------------------------------------------
+inline ctrl_t* EmptyGroup() {
+ alignas(16) static constexpr ctrl_t empty_group[] = {
+ kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
+ kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
+ return const_cast<ctrl_t*>(empty_group);
+}
+
+// --------------------------------------------------------------------------
+inline size_t HashSeed(const ctrl_t* ctrl) {
+ // The low bits of the pointer have little or no entropy because of
+ // alignment. We shift the pointer to try to use higher entropy bits. A
+ // good number seems to be 12 bits, because that aligns with page size.
+ return reinterpret_cast<uintptr_t>(ctrl) >> 12;
+}
+
+#ifdef PHMAP_NON_DETERMINISTIC
+
+inline size_t H1(size_t hashval, const ctrl_t* ctrl) {
+ // use ctrl_ pointer to add entropy to ensure
+ // non-deterministic iteration order.
+ return (hashval >> 7) ^ HashSeed(ctrl);
+}
+
+#else
+
+inline size_t H1(size_t hashval, const ctrl_t* ) {
+ return (hashval >> 7);
+}
+
+#endif
+
+
+inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); }
+
+inline bool IsEmpty(ctrl_t c) { return c == kEmpty; }
+inline bool IsFull(ctrl_t c) { return c >= 0; }
+inline bool IsDeleted(ctrl_t c) { return c == kDeleted; }
+inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; }
+
+#if PHMAP_HAVE_SSE2
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch
+#endif
+
+// --------------------------------------------------------------------------
+// https://github.com/abseil/abseil-cpp/issues/209
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853
+// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char
+// Work around this by using the portable implementation of Group
+// when using -funsigned-char under GCC.
+// --------------------------------------------------------------------------
+inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) {
+#if defined(__GNUC__) && !defined(__clang__)
+ #pragma GCC diagnostic push
+ #pragma GCC diagnostic ignored "-Woverflow"
+
+ if (std::is_unsigned<char>::value) {
+ const __m128i mask = _mm_set1_epi8(static_cast<char>(0x80));
+ const __m128i diff = _mm_subs_epi8(b, a);
+ return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask);
+ }
+
+ #pragma GCC diagnostic pop
+#endif
+ return _mm_cmpgt_epi8(a, b);
+}
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+struct GroupSse2Impl
+{
+ enum { kWidth = 16 }; // the number of slots per group
+
+ explicit GroupSse2Impl(const ctrl_t* pos) {
+ ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos));
+ }
+
+ // Returns a bitmask representing the positions of slots that match hash.
+ // ----------------------------------------------------------------------
+ BitMask<uint32_t, kWidth> Match(h2_t hash) const {
+ auto match = _mm_set1_epi8((char)hash);
+ return BitMask<uint32_t, kWidth>(
+ _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)));
+ }
+
+ // Returns a bitmask representing the positions of empty slots.
+ // ------------------------------------------------------------
+ BitMask<uint32_t, kWidth> MatchEmpty() const {
+#if PHMAP_HAVE_SSSE3
+ // This only works because kEmpty is -128.
+ return BitMask<uint32_t, kWidth>(
+ _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)));
+#else
+ return Match(static_cast<h2_t>(kEmpty));
+#endif
+ }
+
+ // Returns a bitmask representing the positions of empty or deleted slots.
+ // -----------------------------------------------------------------------
+ BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const {
+ auto special = _mm_set1_epi8(kSentinel);
+ return BitMask<uint32_t, kWidth>(
+ _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)));
+ }
+
+ // Returns the number of trailing empty or deleted elements in the group.
+ // ----------------------------------------------------------------------
+ uint32_t CountLeadingEmptyOrDeleted() const {
+ auto special = _mm_set1_epi8(kSentinel);
+ return TrailingZeros(
+ _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1);
+ }
+
+ // ----------------------------------------------------------------------
+ void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
+ auto msbs = _mm_set1_epi8(static_cast<char>(-128));
+ auto x126 = _mm_set1_epi8(126);
+#if PHMAP_HAVE_SSSE3
+ auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs);
+#else
+ auto zero = _mm_setzero_si128();
+ auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl);
+ auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126));
+#endif
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res);
+ }
+
+ __m128i ctrl;
+};
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+#endif // PHMAP_HAVE_SSE2
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+struct GroupPortableImpl
+{
+ enum { kWidth = 8 };
+
+ explicit GroupPortableImpl(const ctrl_t* pos)
+ : ctrl(little_endian::Load64(pos)) {}
+
+ BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const {
+ // For the technique, see:
+ // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord
+ // (Determine if a word has a byte equal to n).
+ //
+ // Caveat: there are false positives but:
+ // - they only occur if there is a real match
+ // - they never occur on kEmpty, kDeleted, kSentinel
+ // - they will be handled gracefully by subsequent checks in code
+ //
+ // Example:
+ // v = 0x1716151413121110
+ // hash = 0x12
+ // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000
+ constexpr uint64_t msbs = 0x8080808080808080ULL;
+ constexpr uint64_t lsbs = 0x0101010101010101ULL;
+ auto x = ctrl ^ (lsbs * hash);
+ return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs);
+ }
+
+ BitMask<uint64_t, kWidth, 3> MatchEmpty() const {
+ constexpr uint64_t msbs = 0x8080808080808080ULL;
+ return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs);
+ }
+
+ BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const {
+ constexpr uint64_t msbs = 0x8080808080808080ULL;
+ return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs);
+ }
+
+ uint32_t CountLeadingEmptyOrDeleted() const {
+ constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL;
+ return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3);
+ }
+
+ void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
+ constexpr uint64_t msbs = 0x8080808080808080ULL;
+ constexpr uint64_t lsbs = 0x0101010101010101ULL;
+ auto x = ctrl & msbs;
+ auto res = (~x + (x >> 7)) & ~lsbs;
+ little_endian::Store64(dst, res);
+ }
+
+ uint64_t ctrl;
+};
+
+#if PHMAP_HAVE_SSE2
+ using Group = GroupSse2Impl;
+#else
+ using Group = GroupPortableImpl;
+#endif
+
+template <class Policy, class Hash, class Eq, class Alloc>
+class raw_hash_set;
+
+inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; }
+
+// --------------------------------------------------------------------------
+// PRECONDITION:
+// IsValidCapacity(capacity)
+// ctrl[capacity] == kSentinel
+// ctrl[i] != kSentinel for all i < capacity
+// Applies mapping for every byte in ctrl:
+// DELETED -> EMPTY
+// EMPTY -> EMPTY
+// FULL -> DELETED
+// --------------------------------------------------------------------------
+inline void ConvertDeletedToEmptyAndFullToDeleted(
+ ctrl_t* ctrl, size_t capacity)
+{
+ assert(ctrl[capacity] == kSentinel);
+ assert(IsValidCapacity(capacity));
+ for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) {
+ Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos);
+ }
+ // Copy the cloned ctrl bytes.
+ std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth);
+ ctrl[capacity] = kSentinel;
+}
+
+// --------------------------------------------------------------------------
+// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1.
+// --------------------------------------------------------------------------
+inline size_t NormalizeCapacity(size_t n)
+{
+ return n ? ~size_t{} >> LeadingZeros(n) : 1;
+}
+
+// --------------------------------------------------------------------------
+// We use 7/8th as maximum load factor.
+// For 16-wide groups, that gives an average of two empty slots per group.
+// --------------------------------------------------------------------------
+inline size_t CapacityToGrowth(size_t capacity)
+{
+ assert(IsValidCapacity(capacity));
+ // `capacity*7/8`
+ PHMAP_IF_CONSTEXPR (Group::kWidth == 8) {
+ if (capacity == 7)
+ {
+ // x-x/8 does not work when x==7.
+ return 6;
+ }
+ }
+ return capacity - capacity / 8;
+}
+
+// --------------------------------------------------------------------------
+// From desired "growth" to a lowerbound of the necessary capacity.
+// Might not be a valid one and required NormalizeCapacity().
+// --------------------------------------------------------------------------
+inline size_t GrowthToLowerboundCapacity(size_t growth)
+{
+ // `growth*8/7`
+ PHMAP_IF_CONSTEXPR (Group::kWidth == 8) {
+ if (growth == 7)
+ {
+ // x+(x-1)/7 does not work when x==7.
+ return 8;
+ }
+ }
+ return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7);
+}
+
+namespace hashtable_debug_internal {
+
+// If it is a map, call get<0>().
+using std::get;
+template <typename T, typename = typename T::mapped_type>
+auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) {
+ return get<0>(pair);
+}
+
+// If it is not a map, return the value directly.
+template <typename T>
+const typename T::key_type& GetKey(const typename T::key_type& key, char) {
+ return key;
+}
+
+// --------------------------------------------------------------------------
+// Containers should specialize this to provide debug information for that
+// container.
+// --------------------------------------------------------------------------
+template <class Container, typename Enabler = void>
+struct HashtableDebugAccess
+{
+ // Returns the number of probes required to find `key` in `c`. The "number of
+ // probes" is a concept that can vary by container. Implementations should
+ // return 0 when `key` was found in the minimum number of operations and
+ // should increment the result for each non-trivial operation required to find
+ // `key`.
+ //
+ // The default implementation uses the bucket api from the standard and thus
+ // works for `std::unordered_*` containers.
+ // --------------------------------------------------------------------------
+ static size_t GetNumProbes(const Container& c,
+ const typename Container::key_type& key) {
+ if (!c.bucket_count()) return {};
+ size_t num_probes = 0;
+ size_t bucket = c.bucket(key);
+ for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) {
+ if (it == e) return num_probes;
+ if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes;
+ }
+ }
+};
+
+} // namespace hashtable_debug_internal
+
+// ----------------------------------------------------------------------------
+// I N F O Z S T U B S
+// ----------------------------------------------------------------------------
+struct HashtablezInfo
+{
+ void PrepareForSampling() {}
+};
+
+inline void RecordRehashSlow(HashtablezInfo*, size_t ) {}
+
+static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {}
+
+static inline void RecordEraseSlow(HashtablezInfo*) {}
+
+static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; }
+static inline void UnsampleSlow(HashtablezInfo* ) {}
+
+class HashtablezInfoHandle
+{
+public:
+ inline void RecordStorageChanged(size_t , size_t ) {}
+ inline void RecordRehash(size_t ) {}
+ inline void RecordInsert(size_t , size_t ) {}
+ inline void RecordErase() {}
+ friend inline void swap(HashtablezInfoHandle& ,
+ HashtablezInfoHandle& ) noexcept {}
+};
+
+static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); }
+
+class HashtablezSampler
+{
+public:
+ // Returns a global Sampler.
+ static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; }
+ HashtablezInfo* Register() { static HashtablezInfo info; return &info; }
+ void Unregister(HashtablezInfo* ) {}
+
+ using DisposeCallback = void (*)(const HashtablezInfo&);
+ DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; }
+ int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& ) { return 0; }
+};
+
+static inline void SetHashtablezEnabled(bool ) {}
+static inline void SetHashtablezSampleParameter(int32_t ) {}
+static inline void SetHashtablezMaxSamples(int32_t ) {}
+
+
+namespace memory_internal {
+
+// Constructs T into uninitialized storage pointed by `ptr` using the args
+// specified in the tuple.
+// ----------------------------------------------------------------------------
+template <class Alloc, class T, class Tuple, size_t... I>
+void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t,
+ phmap::index_sequence<I...>) {
+ phmap::allocator_traits<Alloc>::construct(
+ *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...);
+}
+
+template <class T, class F>
+struct WithConstructedImplF {
+ template <class... Args>
+ decltype(std::declval<F>()(std::declval<T>())) operator()(
+ Args&&... args) const {
+ return std::forward<F>(f)(T(std::forward<Args>(args)...));
+ }
+ F&& f;
+};
+
+template <class T, class Tuple, size_t... Is, class F>
+decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl(
+ Tuple&& t, phmap::index_sequence<Is...>, F&& f) {
+ return WithConstructedImplF<T, F>{std::forward<F>(f)}(
+ std::get<Is>(std::forward<Tuple>(t))...);
+}
+
+template <class T, size_t... Is>
+auto TupleRefImpl(T&& t, phmap::index_sequence<Is...>)
+ -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) {
+ return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...);
+}
+
+// Returns a tuple of references to the elements of the input tuple. T must be a
+// tuple.
+// ----------------------------------------------------------------------------
+template <class T>
+auto TupleRef(T&& t) -> decltype(
+ TupleRefImpl(std::forward<T>(t),
+ phmap::make_index_sequence<
+ std::tuple_size<typename std::decay<T>::type>::value>())) {
+ return TupleRefImpl(
+ std::forward<T>(t),
+ phmap::make_index_sequence<
+ std::tuple_size<typename std::decay<T>::type>::value>());
+}
+
+template <class F, class K, class V>
+decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct,
+ std::declval<std::tuple<K>>(), std::declval<V>()))
+DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) {
+ const auto& key = std::get<0>(p.first);
+ return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first),
+ std::move(p.second));
+}
+
+} // namespace memory_internal
+
+
+// ----------------------------------------------------------------------------
+// R A W _ H A S H _ S E T
+// ----------------------------------------------------------------------------
+// An open-addressing
+// hashtable with quadratic probing.
+//
+// This is a low level hashtable on top of which different interfaces can be
+// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc.
+//
+// The table interface is similar to that of std::unordered_set. Notable
+// differences are that most member functions support heterogeneous keys when
+// BOTH the hash and eq functions are marked as transparent. They do so by
+// providing a typedef called `is_transparent`.
+//
+// When heterogeneous lookup is enabled, functions that take key_type act as if
+// they have an overload set like:
+//
+// iterator find(const key_type& key);
+// template <class K>
+// iterator find(const K& key);
+//
+// size_type erase(const key_type& key);
+// template <class K>
+// size_type erase(const K& key);
+//
+// std::pair<iterator, iterator> equal_range(const key_type& key);
+// template <class K>
+// std::pair<iterator, iterator> equal_range(const K& key);
+//
+// When heterogeneous lookup is disabled, only the explicit `key_type` overloads
+// exist.
+//
+// find() also supports passing the hash explicitly:
+//
+// iterator find(const key_type& key, size_t hash);
+// template <class U>
+// iterator find(const U& key, size_t hash);
+//
+// In addition the pointer to element and iterator stability guarantees are
+// weaker: all iterators and pointers are invalidated after a new element is
+// inserted.
+//
+// IMPLEMENTATION DETAILS
+//
+// The table stores elements inline in a slot array. In addition to the slot
+// array the table maintains some control state per slot. The extra state is one
+// byte per slot and stores empty or deleted marks, or alternatively 7 bits from
+// the hash of an occupied slot. The table is split into logical groups of
+// slots, like so:
+//
+// Group 1 Group 2 Group 3
+// +---------------+---------------+---------------+
+// | | | | | | | | | | | | | | | | | | | | | | | | |
+// +---------------+---------------+---------------+
+//
+// On lookup the hash is split into two parts:
+// - H2: 7 bits (those stored in the control bytes)
+// - H1: the rest of the bits
+// The groups are probed using H1. For each group the slots are matched to H2 in
+// parallel. Because H2 is 7 bits (128 states) and the number of slots per group
+// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit.
+//
+// On insert, once the right group is found (as in lookup), its slots are
+// filled in order.
+//
+// On erase a slot is cleared. In case the group did not have any empty slots
+// before the erase, the erased slot is marked as deleted.
+//
+// Groups without empty slots (but maybe with deleted slots) extend the probe
+// sequence. The probing algorithm is quadratic. Given N the number of groups,
+// the probing function for the i'th probe is:
+//
+// P(0) = H1 % N
+//
+// P(i) = (P(i - 1) + i) % N
+//
+// This probing function guarantees that after N probes, all the groups of the
+// table will be probed exactly once.
+// ----------------------------------------------------------------------------
+template <class Policy, class Hash, class Eq, class Alloc>
+class raw_hash_set
+{
+ using PolicyTraits = hash_policy_traits<Policy>;
+ using KeyArgImpl =
+ KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
+
+public:
+ using init_type = typename PolicyTraits::init_type;
+ using key_type = typename PolicyTraits::key_type;
+ // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user
+ // code fixes!
+ using slot_type = typename PolicyTraits::slot_type;
+ using allocator_type = Alloc;
+ using size_type = size_t;
+ using difference_type = ptrdiff_t;
+ using hasher = Hash;
+ using key_equal = Eq;
+ using policy_type = Policy;
+ using value_type = typename PolicyTraits::value_type;
+ using reference = value_type&;
+ using const_reference = const value_type&;
+ using pointer = typename phmap::allocator_traits<
+ allocator_type>::template rebind_traits<value_type>::pointer;
+ using const_pointer = typename phmap::allocator_traits<
+ allocator_type>::template rebind_traits<value_type>::const_pointer;
+
+ // Alias used for heterogeneous lookup functions.
+ // `key_arg<K>` evaluates to `K` when the functors are transparent and to
+ // `key_type` otherwise. It permits template argument deduction on `K` for the
+ // transparent case.
+ template <class K>
+ using key_arg = typename KeyArgImpl::template type<K, key_type>;
+
+private:
+ // Give an early error when key_type is not hashable/eq.
+ auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
+ auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k));
+
+ using Layout = phmap::priv::Layout<ctrl_t, slot_type>;
+
+ static Layout MakeLayout(size_t capacity) {
+ assert(IsValidCapacity(capacity));
+ return Layout(capacity + Group::kWidth + 1, capacity);
+ }
+
+ using AllocTraits = phmap::allocator_traits<allocator_type>;
+ using SlotAlloc = typename phmap::allocator_traits<
+ allocator_type>::template rebind_alloc<slot_type>;
+ using SlotAllocTraits = typename phmap::allocator_traits<
+ allocator_type>::template rebind_traits<slot_type>;
+
+ static_assert(std::is_lvalue_reference<reference>::value,
+ "Policy::element() must return a reference");
+
+ template <typename T>
+ struct SameAsElementReference
+ : std::is_same<typename std::remove_cv<
+ typename std::remove_reference<reference>::type>::type,
+ typename std::remove_cv<
+ typename std::remove_reference<T>::type>::type> {};
+
+ // An enabler for insert(T&&): T must be convertible to init_type or be the
+ // same as [cv] value_type [ref].
+ // Note: we separate SameAsElementReference into its own type to avoid using
+ // reference unless we need to. MSVC doesn't seem to like it in some
+ // cases.
+ template <class T>
+ using RequiresInsertable = typename std::enable_if<
+ phmap::disjunction<std::is_convertible<T, init_type>,
+ SameAsElementReference<T>>::value,
+ int>::type;
+
+ // RequiresNotInit is a workaround for gcc prior to 7.1.
+ // See https://godbolt.org/g/Y4xsUh.
+ template <class T>
+ using RequiresNotInit =
+ typename std::enable_if<!std::is_same<T, init_type>::value, int>::type;
+
+ template <class... Ts>
+ using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>;
+
+public:
+ static_assert(std::is_same<pointer, value_type*>::value,
+ "Allocators with custom pointer types are not supported");
+ static_assert(std::is_same<const_pointer, const value_type*>::value,
+ "Allocators with custom pointer types are not supported");
+
+ class iterator
+ {
+ friend class raw_hash_set;
+
+ public:
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = typename raw_hash_set::value_type;
+ using reference =
+ phmap::conditional_t<PolicyTraits::constant_iterators::value,
+ const value_type&, value_type&>;
+ using pointer = phmap::remove_reference_t<reference>*;
+ using difference_type = typename raw_hash_set::difference_type;
+
+ iterator() {}
+
+ // PRECONDITION: not an end() iterator.
+ reference operator*() const { return PolicyTraits::element(slot_); }
+
+ // PRECONDITION: not an end() iterator.
+ pointer operator->() const { return &operator*(); }
+
+ // PRECONDITION: not an end() iterator.
+ iterator& operator++() {
+ ++ctrl_;
+ ++slot_;
+ skip_empty_or_deleted();
+ return *this;
+ }
+ // PRECONDITION: not an end() iterator.
+ iterator operator++(int) {
+ auto tmp = *this;
+ ++*this;
+ return tmp;
+ }
+
+#if PHMAP_BIDIRECTIONAL
+ // PRECONDITION: not a begin() iterator.
+ iterator& operator--() {
+ assert(ctrl_);
+ do {
+ --ctrl_;
+ --slot_;
+ } while (IsEmptyOrDeleted(*ctrl_));
+ return *this;
+ }
+
+ // PRECONDITION: not a begin() iterator.
+ iterator operator--(int) {
+ auto tmp = *this;
+ --*this;
+ return tmp;
+ }
+#endif
+
+ friend bool operator==(const iterator& a, const iterator& b) {
+ return a.ctrl_ == b.ctrl_;
+ }
+ friend bool operator!=(const iterator& a, const iterator& b) {
+ return !(a == b);
+ }
+
+ private:
+ iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end()
+ iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {}
+
+ void skip_empty_or_deleted() {
+ while (IsEmptyOrDeleted(*ctrl_)) {
+ // ctrl is not necessarily aligned to Group::kWidth. It is also likely
+ // to read past the space for ctrl bytes and into slots. This is ok
+ // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there
+ // is no way to read outside the combined slot array.
+ uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted();
+ ctrl_ += shift;
+ slot_ += shift;
+ }
+ }
+
+ ctrl_t* ctrl_ = nullptr;
+ // To avoid uninitialized member warnings, put slot_ in an anonymous union.
+ // The member is not initialized on singleton and end iterators.
+ union {
+ slot_type* slot_;
+ };
+ };
+
+ class const_iterator
+ {
+ friend class raw_hash_set;
+
+ public:
+ using iterator_category = typename iterator::iterator_category;
+ using value_type = typename raw_hash_set::value_type;
+ using reference = typename raw_hash_set::const_reference;
+ using pointer = typename raw_hash_set::const_pointer;
+ using difference_type = typename raw_hash_set::difference_type;
+
+ const_iterator() {}
+ // Implicit construction from iterator.
+ const_iterator(iterator i) : inner_(std::move(i)) {}
+
+ reference operator*() const { return *inner_; }
+ pointer operator->() const { return inner_.operator->(); }
+
+ const_iterator& operator++() {
+ ++inner_;
+ return *this;
+ }
+ const_iterator operator++(int) { return inner_++; }
+
+ friend bool operator==(const const_iterator& a, const const_iterator& b) {
+ return a.inner_ == b.inner_;
+ }
+ friend bool operator!=(const const_iterator& a, const const_iterator& b) {
+ return !(a == b);
+ }
+
+ private:
+ const_iterator(const ctrl_t* ctrl, const slot_type* slot)
+ : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {}
+
+ iterator inner_;
+ };
+
+ using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>;
+ using insert_return_type = InsertReturnType<iterator, node_type>;
+
+ raw_hash_set() noexcept(
+ std::is_nothrow_default_constructible<hasher>::value&&
+ std::is_nothrow_default_constructible<key_equal>::value&&
+ std::is_nothrow_default_constructible<allocator_type>::value) {}
+
+ explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(),
+ const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) {
+ if (bucket_cnt) {
+ capacity_ = NormalizeCapacity(bucket_cnt);
+ reset_growth_left();
+ initialize_slots();
+ }
+ }
+
+ raw_hash_set(size_t bucket_cnt, const hasher& hashfn,
+ const allocator_type& alloc)
+ : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {}
+
+ raw_hash_set(size_t bucket_cnt, const allocator_type& alloc)
+ : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ explicit raw_hash_set(const allocator_type& alloc)
+ : raw_hash_set(0, hasher(), key_equal(), alloc) {}
+
+ template <class InputIter>
+ raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0,
+ const hasher& hashfn = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : raw_hash_set(bucket_cnt, hashfn, eq, alloc) {
+ insert(first, last);
+ }
+
+ template <class InputIter>
+ raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt,
+ const hasher& hashfn, const allocator_type& alloc)
+ : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {}
+
+ template <class InputIter>
+ raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ template <class InputIter>
+ raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc)
+ : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {}
+
+ // Instead of accepting std::initializer_list<value_type> as the first
+ // argument like std::unordered_set<value_type> does, we have two overloads
+ // that accept std::initializer_list<T> and std::initializer_list<init_type>.
+ // This is advantageous for performance.
+ //
+ // // Turns {"abc", "def"} into std::initializer_list<std::string>, then
+ // // copies the strings into the set.
+ // std::unordered_set<std::string> s = {"abc", "def"};
+ //
+ // // Turns {"abc", "def"} into std::initializer_list<const char*>, then
+ // // copies the strings into the set.
+ // phmap::flat_hash_set<std::string> s = {"abc", "def"};
+ //
+ // The same trick is used in insert().
+ //
+ // The enabler is necessary to prevent this constructor from triggering where
+ // the copy constructor is meant to be called.
+ //
+ // phmap::flat_hash_set<int> a, b{a};
+ //
+ // RequiresNotInit<T> is a workaround for gcc prior to 7.1.
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0,
+ const hasher& hashfn = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {}
+
+ raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0,
+ const hasher& hashfn = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt,
+ const hasher& hashfn, const allocator_type& alloc)
+ : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {}
+
+ raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt,
+ const hasher& hashfn, const allocator_type& alloc)
+ : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc)
+ : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {}
+
+ raw_hash_set(std::initializer_list<init_type> init,
+ const allocator_type& alloc)
+ : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {}
+
+ raw_hash_set(const raw_hash_set& that)
+ : raw_hash_set(that, AllocTraits::select_on_container_copy_construction(
+ that.alloc_ref())) {}
+
+ raw_hash_set(const raw_hash_set& that, const allocator_type& a)
+ : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) {
+ reserve(that.size());
+ // Because the table is guaranteed to be empty, we can do something faster
+ // than a full `insert`.
+ for (const auto& v : that) {
+ const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v);
+ auto target = find_first_non_full(hashval);
+ set_ctrl(target.offset, H2(hashval));
+ emplace_at(target.offset, v);
+ infoz_.RecordInsert(hashval, target.probe_length);
+ }
+ size_ = that.size();
+ growth_left() -= that.size();
+ }
+
+ raw_hash_set(raw_hash_set&& that) noexcept(
+ std::is_nothrow_copy_constructible<hasher>::value&&
+ std::is_nothrow_copy_constructible<key_equal>::value&&
+ std::is_nothrow_copy_constructible<allocator_type>::value)
+ : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())),
+ slots_(phmap::exchange(that.slots_, nullptr)),
+ size_(phmap::exchange(that.size_, 0)),
+ capacity_(phmap::exchange(that.capacity_, 0)),
+ infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())),
+ // Hash, equality and allocator are copied instead of moved because
+ // `that` must be left valid. If Hash is std::function<Key>, moving it
+ // would create a nullptr functor that cannot be called.
+ settings_(that.settings_) {
+ // growth_left was copied above, reset the one from `that`.
+ that.growth_left() = 0;
+ }
+
+ raw_hash_set(raw_hash_set&& that, const allocator_type& a)
+ : ctrl_(EmptyGroup()),
+ slots_(nullptr),
+ size_(0),
+ capacity_(0),
+ settings_(0, that.hash_ref(), that.eq_ref(), a) {
+ if (a == that.alloc_ref()) {
+ std::swap(ctrl_, that.ctrl_);
+ std::swap(slots_, that.slots_);
+ std::swap(size_, that.size_);
+ std::swap(capacity_, that.capacity_);
+ std::swap(growth_left(), that.growth_left());
+ std::swap(infoz_, that.infoz_);
+ } else {
+ reserve(that.size());
+ // Note: this will copy elements of dense_set and unordered_set instead of
+ // moving them. This can be fixed if it ever becomes an issue.
+ for (auto& elem : that) insert(std::move(elem));
+ }
+ }
+
+ raw_hash_set& operator=(const raw_hash_set& that) {
+ raw_hash_set tmp(that,
+ AllocTraits::propagate_on_container_copy_assignment::value
+ ? that.alloc_ref()
+ : alloc_ref());
+ swap(tmp);
+ return *this;
+ }
+
+ raw_hash_set& operator=(raw_hash_set&& that) noexcept(
+ phmap::allocator_traits<allocator_type>::is_always_equal::value&&
+ std::is_nothrow_move_assignable<hasher>::value&&
+ std::is_nothrow_move_assignable<key_equal>::value) {
+ // TODO(sbenza): We should only use the operations from the noexcept clause
+ // to make sure we actually adhere to that contract.
+ return move_assign(
+ std::move(that),
+ typename AllocTraits::propagate_on_container_move_assignment());
+ }
+
+ ~raw_hash_set() { destroy_slots(); }
+
+ iterator begin() {
+ auto it = iterator_at(0);
+ it.skip_empty_or_deleted();
+ return it;
+ }
+ iterator end()
+ {
+#if PHMAP_BIDIRECTIONAL
+ return iterator_at(capacity_);
+#else
+ return {ctrl_ + capacity_};
+#endif
+ }
+
+ const_iterator begin() const {
+ return const_cast<raw_hash_set*>(this)->begin();
+ }
+ const_iterator end() const { return const_cast<raw_hash_set*>(this)->end(); }
+ const_iterator cbegin() const { return begin(); }
+ const_iterator cend() const { return end(); }
+
+ bool empty() const { return !size(); }
+ size_t size() const { return size_; }
+ size_t capacity() const { return capacity_; }
+ size_t max_size() const { return (std::numeric_limits<size_t>::max)(); }
+
+ PHMAP_ATTRIBUTE_REINITIALIZES void clear() {
+ // Iterating over this container is O(bucket_count()). When bucket_count()
+ // is much greater than size(), iteration becomes prohibitively expensive.
+ // For clear() it is more important to reuse the allocated array when the
+ // container is small because allocation takes comparatively long time
+ // compared to destruction of the elements of the container. So we pick the
+ // largest bucket_count() threshold for which iteration is still fast and
+ // past that we simply deallocate the array.
+ if (empty())
+ return;
+ if (capacity_ > 127) {
+ destroy_slots();
+ } else if (capacity_) {
+ for (size_t i = 0; i != capacity_; ++i) {
+ if (IsFull(ctrl_[i])) {
+ PolicyTraits::destroy(&alloc_ref(), slots_ + i);
+ }
+ }
+ size_ = 0;
+ reset_ctrl();
+ reset_growth_left();
+ }
+ assert(empty());
+ infoz_.RecordStorageChanged(0, capacity_);
+ }
+
+ // This overload kicks in when the argument is an rvalue of insertable and
+ // decomposable type other than init_type.
+ //
+ // flat_hash_map<std::string, int> m;
+ // m.insert(std::make_pair("abc", 42));
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<T>::value, int>::type = 0,
+ T* = nullptr>
+ std::pair<iterator, bool> insert(T&& value) {
+ return emplace(std::forward<T>(value));
+ }
+
+ // This overload kicks in when the argument is a bitfield or an lvalue of
+ // insertable and decomposable type.
+ //
+ // union { int n : 1; };
+ // flat_hash_set<int> s;
+ // s.insert(n);
+ //
+ // flat_hash_set<std::string> s;
+ // const char* p = "hello";
+ // s.insert(p);
+ //
+ // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace
+ // RequiresInsertable<T> with RequiresInsertable<const T&>.
+ // We are hitting this bug: https://godbolt.org/g/1Vht4f.
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
+ std::pair<iterator, bool> insert(const T& value) {
+ return emplace(value);
+ }
+
+ // This overload kicks in when the argument is an rvalue of init_type. Its
+ // purpose is to handle brace-init-list arguments.
+ //
+ // flat_hash_set<std::string, int> s;
+ // s.insert({"abc", 42});
+ std::pair<iterator, bool> insert(init_type&& value) {
+ return emplace(std::move(value));
+ }
+
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<T>::value, int>::type = 0,
+ T* = nullptr>
+ iterator insert(const_iterator, T&& value) {
+ return insert(std::forward<T>(value)).first;
+ }
+
+ // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace
+ // RequiresInsertable<T> with RequiresInsertable<const T&>.
+ // We are hitting this bug: https://godbolt.org/g/1Vht4f.
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
+ iterator insert(const_iterator, const T& value) {
+ return insert(value).first;
+ }
+
+ iterator insert(const_iterator, init_type&& value) {
+ return insert(std::move(value)).first;
+ }
+
+ template <typename It>
+ using IsRandomAccess = std::is_same<typename std::iterator_traits<It>::iterator_category,
+ std::random_access_iterator_tag>;
+
+
+ template<typename T>
+ struct has_difference_operator
+ {
+ private:
+ using yes = std::true_type;
+ using no = std::false_type;
+
+ template<typename U> static auto test(int) -> decltype(std::declval<U>() - std::declval<U>() == 1, yes());
+ template<typename> static no test(...);
+
+ public:
+ static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value;
+ };
+
+ template <class InputIt, typename phmap::enable_if_t<has_difference_operator<InputIt>::value, int> = 0>
+ void insert(InputIt first, InputIt last) {
+ this->reserve(this->size() + (last - first));
+ for (; first != last; ++first)
+ emplace(*first);
+ }
+
+ template <class InputIt, typename phmap::enable_if_t<!has_difference_operator<InputIt>::value, int> = 0>
+ void insert(InputIt first, InputIt last) {
+ for (; first != last; ++first)
+ emplace(*first);
+ }
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0>
+ void insert(std::initializer_list<T> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ void insert(std::initializer_list<init_type> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ insert_return_type insert(node_type&& node) {
+ if (!node) return {end(), false, node_type()};
+ const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node));
+ auto res = PolicyTraits::apply(
+ InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))},
+ elem);
+ if (res.second) {
+ CommonAccess::Reset(&node);
+ return {res.first, true, node_type()};
+ } else {
+ return {res.first, false, std::move(node)};
+ }
+ }
+
+ insert_return_type insert(node_type&& node, size_t hashval) {
+ if (!node) return {end(), false, node_type()};
+ const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node));
+ auto res = PolicyTraits::apply(
+ InsertSlotWithHash<false>{*this, std::move(*CommonAccess::GetSlot(node)), hashval},
+ elem);
+ if (res.second) {
+ CommonAccess::Reset(&node);
+ return {res.first, true, node_type()};
+ } else {
+ return {res.first, false, std::move(node)};
+ }
+ }
+
+ iterator insert(const_iterator, node_type&& node) {
+ return insert(std::move(node)).first;
+ }
+
+ // This overload kicks in if we can deduce the key from args. This enables us
+ // to avoid constructing value_type if an entry with the same key already
+ // exists.
+ //
+ // For example:
+ //
+ // flat_hash_map<std::string, std::string> m = {{"abc", "def"}};
+ // // Creates no std::string copies and makes no heap allocations.
+ // m.emplace("abc", "xyz");
+ template <class... Args, typename std::enable_if<
+ IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace(Args&&... args) {
+ return PolicyTraits::apply(EmplaceDecomposable{*this},
+ std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename std::enable_if<IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) {
+ return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward<Args>(args)...);
+ }
+
+ // This overload kicks in if we cannot deduce the key from args. It constructs
+ // value_type unconditionally and then either moves it into the table or
+ // destroys.
+ template <class... Args, typename std::enable_if<
+ !IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace(Args&&... args) {
+ typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type
+ raw;
+ slot_type* slot = reinterpret_cast<slot_type*>(&raw);
+
+ PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
+ const auto& elem = PolicyTraits::element(slot);
+ return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem);
+ }
+
+ template <class... Args, typename std::enable_if<!IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) {
+ typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw;
+ slot_type* slot = reinterpret_cast<slot_type*>(&raw);
+
+ PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
+ const auto& elem = PolicyTraits::element(slot);
+ return PolicyTraits::apply(InsertSlotWithHash<true>{*this, std::move(*slot), hashval}, elem);
+ }
+
+ template <class... Args>
+ iterator emplace_hint(const_iterator, Args&&... args) {
+ return emplace(std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args>
+ iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) {
+ return emplace_with_hash(hashval, std::forward<Args>(args)...).first;
+ }
+
+ // Extension API: support for lazy emplace.
+ //
+ // Looks up key in the table. If found, returns the iterator to the element.
+ // Otherwise calls f with one argument of type raw_hash_set::constructor. f
+ // MUST call raw_hash_set::constructor with arguments as if a
+ // raw_hash_set::value_type is constructed, otherwise the behavior is
+ // undefined.
+ //
+ // For example:
+ //
+ // std::unordered_set<ArenaString> s;
+ // // Makes ArenaStr even if "abc" is in the map.
+ // s.insert(ArenaString(&arena, "abc"));
+ //
+ // flat_hash_set<ArenaStr> s;
+ // // Makes ArenaStr only if "abc" is not in the map.
+ // s.lazy_emplace("abc", [&](const constructor& ctor) {
+ // ctor(&arena, "abc");
+ // });
+ //
+ // WARNING: This API is currently experimental. If there is a way to implement
+ // the same thing with the rest of the API, prefer that.
+ class constructor
+ {
+ friend class raw_hash_set;
+
+ public:
+ template <class... Args>
+ void operator()(Args&&... args) const {
+ assert(*slot_);
+ PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...);
+ *slot_ = nullptr;
+ }
+
+ private:
+ constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {}
+
+ allocator_type* alloc_;
+ slot_type** slot_;
+ };
+
+ template <class K = key_type, class F>
+ iterator lazy_emplace(const key_arg<K>& key, F&& f) {
+ auto res = find_or_prepare_insert(key);
+ if (res.second) {
+ lazy_emplace_at(res.first, std::forward<F>(f));
+ }
+ return iterator_at(res.first);
+ }
+
+ template <class K = key_type, class F>
+ iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t &hashval, F&& f) {
+ auto res = find_or_prepare_insert(key, hashval);
+ if (res.second) {
+ lazy_emplace_at(res.first, std::forward<F>(f));
+ }
+ return iterator_at(res.first);
+ }
+
+ template <class K = key_type, class F>
+ void lazy_emplace_at(size_t& idx, F&& f) {
+ slot_type* slot = slots_ + idx;
+ std::forward<F>(f)(constructor(&alloc_ref(), &slot));
+ assert(!slot);
+ }
+
+
+ // Extension API: support for heterogeneous keys.
+ //
+ // std::unordered_set<std::string> s;
+ // // Turns "abc" into std::string.
+ // s.erase("abc");
+ //
+ // flat_hash_set<std::string> s;
+ // // Uses "abc" directly without copying it into std::string.
+ // s.erase("abc");
+ template <class K = key_type>
+ size_type erase(const key_arg<K>& key) {
+ auto it = find(key);
+ if (it == end()) return 0;
+ _erase(it);
+ return 1;
+ }
+
+
+ iterator erase(const_iterator cit) { return erase(cit.inner_); }
+
+ // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`,
+ // this method returns void to reduce algorithmic complexity to O(1). In
+ // order to erase while iterating across a map, use the following idiom (which
+ // also works for standard containers):
+ //
+ // for (auto it = m.begin(), end = m.end(); it != end;) {
+ // if (<pred>) {
+ // m._erase(it++);
+ // } else {
+ // ++it;
+ // }
+ // }
+ void _erase(iterator it) {
+ assert(it != end());
+ PolicyTraits::destroy(&alloc_ref(), it.slot_);
+ erase_meta_only(it);
+ }
+ void _erase(const_iterator cit) { _erase(cit.inner_); }
+
+ // This overload is necessary because otherwise erase<K>(const K&) would be
+ // a better match if non-const iterator is passed as an argument.
+ iterator erase(iterator it) {
+ auto res = it;
+ ++res;
+ _erase(it);
+ return res;
+ }
+
+ iterator erase(const_iterator first, const_iterator last) {
+ while (first != last) {
+ _erase(first++);
+ }
+ return last.inner_;
+ }
+
+ // Moves elements from `src` into `this`.
+ // If the element already exists in `this`, it is left unmodified in `src`.
+ template <typename H, typename E>
+ void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT
+ assert(this != &src);
+ for (auto it = src.begin(), e = src.end(); it != e; ++it) {
+ if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)},
+ PolicyTraits::element(it.slot_))
+ .second) {
+ src.erase_meta_only(it);
+ }
+ }
+ }
+
+ template <typename H, typename E>
+ void merge(raw_hash_set<Policy, H, E, Alloc>&& src) {
+ merge(src);
+ }
+
+ node_type extract(const_iterator position) {
+ auto node =
+ CommonAccess::Make<node_type>(alloc_ref(), position.inner_.slot_);
+ erase_meta_only(position);
+ return node;
+ }
+
+ template <
+ class K = key_type,
+ typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0>
+ node_type extract(const key_arg<K>& key) {
+ auto it = find(key);
+ return it == end() ? node_type() : extract(const_iterator{it});
+ }
+
+ void swap(raw_hash_set& that) noexcept(
+ IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() &&
+ (!AllocTraits::propagate_on_container_swap::value ||
+ IsNoThrowSwappable<allocator_type>())) {
+ using std::swap;
+ swap(ctrl_, that.ctrl_);
+ swap(slots_, that.slots_);
+ swap(size_, that.size_);
+ swap(capacity_, that.capacity_);
+ swap(growth_left(), that.growth_left());
+ swap(hash_ref(), that.hash_ref());
+ swap(eq_ref(), that.eq_ref());
+ swap(infoz_, that.infoz_);
+ if (AllocTraits::propagate_on_container_swap::value) {
+ swap(alloc_ref(), that.alloc_ref());
+ } else {
+ // If the allocators do not compare equal it is officially undefined
+ // behavior. We choose to do nothing.
+ }
+ }
+
+#ifndef PHMAP_NON_DETERMINISTIC
+ template<typename OutputArchive>
+ bool dump(OutputArchive&) const;
+
+ template<typename InputArchive>
+ bool load(InputArchive&);
+#endif
+
+ void rehash(size_t n) {
+ if (n == 0 && capacity_ == 0) return;
+ if (n == 0 && size_ == 0) {
+ destroy_slots();
+ infoz_.RecordStorageChanged(0, 0);
+ return;
+ }
+ // bitor is a faster way of doing `max` here. We will round up to the next
+ // power-of-2-minus-1, so bitor is good enough.
+ auto m = NormalizeCapacity((std::max)(n, size()));
+ // n == 0 unconditionally rehashes as per the standard.
+ if (n == 0 || m > capacity_) {
+ resize(m);
+ }
+ }
+
+ void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); }
+
+ // Extension API: support for heterogeneous keys.
+ //
+ // std::unordered_set<std::string> s;
+ // // Turns "abc" into std::string.
+ // s.count("abc");
+ //
+ // ch_set<std::string> s;
+ // // Uses "abc" directly without copying it into std::string.
+ // s.count("abc");
+ template <class K = key_type>
+ size_t count(const key_arg<K>& key) const {
+ return find(key) == end() ? size_t(0) : size_t(1);
+ }
+
+ // Issues CPU prefetch instructions for the memory needed to find or insert
+ // a key. Like all lookup functions, this support heterogeneous keys.
+ //
+ // NOTE: This is a very low level operation and should not be used without
+ // specific benchmarks indicating its importance.
+ void prefetch_hash(size_t hashval) const {
+ (void)hashval;
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+ auto seq = probe(hashval);
+ _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA);
+ _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA);
+#elif defined(__GNUC__)
+ auto seq = probe(hashval);
+ __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset()));
+ __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset()));
+#endif // __GNUC__
+ }
+
+ template <class K = key_type>
+ void prefetch(const key_arg<K>& key) const {
+ prefetch_hash(this->hash(key));
+ }
+
+ // The API of find() has two extensions.
+ //
+ // 1. The hash can be passed by the user. It must be equal to the hash of the
+ // key.
+ //
+ // 2. The type of the key argument doesn't have to be key_type. This is so
+ // called heterogeneous key support.
+ template <class K = key_type>
+ iterator find(const key_arg<K>& key, size_t hashval) {
+ size_t offset;
+ if (find_impl(key, hashval, offset))
+ return iterator_at(offset);
+ else
+ return end();
+ }
+
+ template <class K = key_type>
+ pointer find_ptr(const key_arg<K>& key, size_t hashval) {
+ size_t offset;
+ if (find_impl(key, hashval, offset))
+ return &PolicyTraits::element(slots_ + offset);
+ else
+ return nullptr;
+ }
+
+ template <class K = key_type>
+ iterator find(const key_arg<K>& key) {
+ return find(key, this->hash(key));
+ }
+
+ template <class K = key_type>
+ const_iterator find(const key_arg<K>& key, size_t hashval) const {
+ return const_cast<raw_hash_set*>(this)->find(key, hashval);
+ }
+ template <class K = key_type>
+ const_iterator find(const key_arg<K>& key) const {
+ return find(key, this->hash(key));
+ }
+
+ template <class K = key_type>
+ bool contains(const key_arg<K>& key) const {
+ return find(key) != end();
+ }
+
+ template <class K = key_type>
+ bool contains(const key_arg<K>& key, size_t hashval) const {
+ return find(key, hashval) != end();
+ }
+
+ template <class K = key_type>
+ std::pair<iterator, iterator> equal_range(const key_arg<K>& key) {
+ auto it = find(key);
+ if (it != end()) return {it, std::next(it)};
+ return {it, it};
+ }
+ template <class K = key_type>
+ std::pair<const_iterator, const_iterator> equal_range(
+ const key_arg<K>& key) const {
+ auto it = find(key);
+ if (it != end()) return {it, std::next(it)};
+ return {it, it};
+ }
+
+ size_t bucket_count() const { return capacity_; }
+ float load_factor() const {
+ return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0;
+ }
+ float max_load_factor() const { return 1.0f; }
+ void max_load_factor(float) {
+ // Does nothing.
+ }
+
+ hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function
+ key_equal key_eq() const { return eq_ref(); }
+ allocator_type get_allocator() const { return alloc_ref(); }
+
+ friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) {
+ if (a.size() != b.size()) return false;
+ const raw_hash_set* outer = &a;
+ const raw_hash_set* inner = &b;
+ if (outer->capacity() > inner->capacity())
+ std::swap(outer, inner);
+ for (const value_type& elem : *outer)
+ if (!inner->has_element(elem)) return false;
+ return true;
+ }
+
+ friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) {
+ return !(a == b);
+ }
+
+ friend void swap(raw_hash_set& a,
+ raw_hash_set& b) noexcept(noexcept(a.swap(b))) {
+ a.swap(b);
+ }
+
+ template <class K>
+ size_t hash(const K& key) const {
+ return HashElement{hash_ref()}(key);
+ }
+
+private:
+ template <class Container, typename Enabler>
+ friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess;
+
+ template <class K = key_type>
+ bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) {
+ auto seq = probe(hashval);
+ while (true) {
+ Group g{ ctrl_ + seq.offset() };
+ for (int i : g.Match((h2_t)H2(hashval))) {
+ offset = seq.offset((size_t)i);
+ if (PHMAP_PREDICT_TRUE(PolicyTraits::apply(
+ EqualElement<K>{key, eq_ref()},
+ PolicyTraits::element(slots_ + offset))))
+ return true;
+ }
+ if (PHMAP_PREDICT_TRUE(g.MatchEmpty()))
+ return false;
+ seq.next();
+ }
+ }
+
+ struct FindElement
+ {
+ template <class K, class... Args>
+ const_iterator operator()(const K& key, Args&&...) const {
+ return s.find(key);
+ }
+ const raw_hash_set& s;
+ };
+
+ struct HashElement
+ {
+ template <class K, class... Args>
+ size_t operator()(const K& key, Args&&...) const {
+ return phmap_mix<sizeof(size_t)>()(h(key));
+ }
+ const hasher& h;
+ };
+
+ template <class K1>
+ struct EqualElement
+ {
+ template <class K2, class... Args>
+ bool operator()(const K2& lhs, Args&&...) const {
+ return eq(lhs, rhs);
+ }
+ const K1& rhs;
+ const key_equal& eq;
+ };
+
+ template <class K, class... Args>
+ std::pair<iterator, bool> emplace_decomposable(const K& key, size_t hashval,
+ Args&&... args)
+ {
+ auto res = find_or_prepare_insert(key, hashval);
+ if (res.second) {
+ emplace_at(res.first, std::forward<Args>(args)...);
+ }
+ return {iterator_at(res.first), res.second};
+ }
+
+ struct EmplaceDecomposable
+ {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&... args) const {
+ return s.emplace_decomposable(key, s.hash(key), std::forward<Args>(args)...);
+ }
+ raw_hash_set& s;
+ };
+
+ struct EmplaceDecomposableHashval {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&... args) const {
+ return s.emplace_decomposable(key, hashval, std::forward<Args>(args)...);
+ }
+ raw_hash_set& s;
+ size_t hashval;
+ };
+
+ template <bool do_destroy>
+ struct InsertSlot
+ {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&...) && {
+ auto res = s.find_or_prepare_insert(key);
+ if (res.second) {
+ PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot);
+ } else if (do_destroy) {
+ PolicyTraits::destroy(&s.alloc_ref(), &slot);
+ }
+ return {s.iterator_at(res.first), res.second};
+ }
+ raw_hash_set& s;
+ // Constructed slot. Either moved into place or destroyed.
+ slot_type&& slot;
+ };
+
+ template <bool do_destroy>
+ struct InsertSlotWithHash
+ {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&...) && {
+ auto res = s.find_or_prepare_insert(key, hashval);
+ if (res.second) {
+ PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot);
+ } else if (do_destroy) {
+ PolicyTraits::destroy(&s.alloc_ref(), &slot);
+ }
+ return {s.iterator_at(res.first), res.second};
+ }
+ raw_hash_set& s;
+ // Constructed slot. Either moved into place or destroyed.
+ slot_type&& slot;
+ size_t &hashval;
+ };
+
+ // "erases" the object from the container, except that it doesn't actually
+ // destroy the object. It only updates all the metadata of the class.
+ // This can be used in conjunction with Policy::transfer to move the object to
+ // another place.
+ void erase_meta_only(const_iterator it) {
+ assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator");
+ --size_;
+ const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_);
+ const size_t index_before = (index - Group::kWidth) & capacity_;
+ const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty();
+ const auto empty_before = Group(ctrl_ + index_before).MatchEmpty();
+
+ // We count how many consecutive non empties we have to the right and to the
+ // left of `it`. If the sum is >= kWidth then there is at least one probe
+ // window that might have seen a full group.
+ bool was_never_full =
+ empty_before && empty_after &&
+ static_cast<size_t>(empty_after.TrailingZeros() +
+ empty_before.LeadingZeros()) < Group::kWidth;
+
+ set_ctrl(index, was_never_full ? kEmpty : kDeleted);
+ growth_left() += was_never_full;
+ infoz_.RecordErase();
+ }
+
+ void initialize_slots() {
+ assert(capacity_);
+ if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value &&
+ slots_ == nullptr) {
+ infoz_ = Sample();
+ }
+
+ auto layout = MakeLayout(capacity_);
+ char* mem = static_cast<char*>(
+ Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize()));
+ ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem));
+ slots_ = layout.template Pointer<1>(mem);
+ reset_ctrl();
+ reset_growth_left();
+ infoz_.RecordStorageChanged(size_, capacity_);
+ }
+
+ void destroy_slots() {
+ if (!capacity_) return;
+ for (size_t i = 0; i != capacity_; ++i) {
+ if (IsFull(ctrl_[i])) {
+ PolicyTraits::destroy(&alloc_ref(), slots_ + i);
+ }
+ }
+ auto layout = MakeLayout(capacity_);
+ // Unpoison before returning the memory to the allocator.
+ SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
+ Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize());
+ ctrl_ = EmptyGroup();
+ slots_ = nullptr;
+ size_ = 0;
+ capacity_ = 0;
+ growth_left() = 0;
+ }
+
+ void resize(size_t new_capacity) {
+ assert(IsValidCapacity(new_capacity));
+ auto* old_ctrl = ctrl_;
+ auto* old_slots = slots_;
+ const size_t old_capacity = capacity_;
+ capacity_ = new_capacity;
+ initialize_slots();
+
+ for (size_t i = 0; i != old_capacity; ++i) {
+ if (IsFull(old_ctrl[i])) {
+ size_t hashval = PolicyTraits::apply(HashElement{hash_ref()},
+ PolicyTraits::element(old_slots + i));
+ auto target = find_first_non_full(hashval);
+ size_t new_i = target.offset;
+ set_ctrl(new_i, H2(hashval));
+ PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i);
+ }
+ }
+ if (old_capacity) {
+ SanitizerUnpoisonMemoryRegion(old_slots,
+ sizeof(slot_type) * old_capacity);
+ auto layout = MakeLayout(old_capacity);
+ Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl,
+ layout.AllocSize());
+ }
+ }
+
+ void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE {
+ assert(IsValidCapacity(capacity_));
+ assert(!is_small());
+ // Algorithm:
+ // - mark all DELETED slots as EMPTY
+ // - mark all FULL slots as DELETED
+ // - for each slot marked as DELETED
+ // hash = Hash(element)
+ // target = find_first_non_full(hash)
+ // if target is in the same group
+ // mark slot as FULL
+ // else if target is EMPTY
+ // transfer element to target
+ // mark slot as EMPTY
+ // mark target as FULL
+ // else if target is DELETED
+ // swap current element with target element
+ // mark target as FULL
+ // repeat procedure for current slot with moved from element (target)
+ ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_);
+ typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type
+ raw;
+ slot_type* slot = reinterpret_cast<slot_type*>(&raw);
+ for (size_t i = 0; i != capacity_; ++i) {
+ if (!IsDeleted(ctrl_[i])) continue;
+ size_t hashval = PolicyTraits::apply(HashElement{hash_ref()},
+ PolicyTraits::element(slots_ + i));
+ auto target = find_first_non_full(hashval);
+ size_t new_i = target.offset;
+
+ // Verify if the old and new i fall within the same group wrt the hashval.
+ // If they do, we don't need to move the object as it falls already in the
+ // best probe we can.
+ const auto probe_index = [&](size_t pos) {
+ return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth;
+ };
+
+ // Element doesn't move.
+ if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) {
+ set_ctrl(i, H2(hashval));
+ continue;
+ }
+ if (IsEmpty(ctrl_[new_i])) {
+ // Transfer element to the empty spot.
+ // set_ctrl poisons/unpoisons the slots so we have to call it at the
+ // right time.
+ set_ctrl(new_i, H2(hashval));
+ PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i);
+ set_ctrl(i, kEmpty);
+ } else {
+ assert(IsDeleted(ctrl_[new_i]));
+ set_ctrl(new_i, H2(hashval));
+ // Until we are done rehashing, DELETED marks previously FULL slots.
+ // Swap i and new_i elements.
+ PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i);
+ PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i);
+ PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot);
+ --i; // repeat
+ }
+ }
+ reset_growth_left();
+ }
+
+ void rehash_and_grow_if_necessary() {
+ if (capacity_ == 0) {
+ resize(1);
+ } else if (size() <= CapacityToGrowth(capacity()) / 2) {
+ // Squash DELETED without growing if there is enough capacity.
+ drop_deletes_without_resize();
+ } else {
+ // Otherwise grow the container.
+ resize(capacity_ * 2 + 1);
+ }
+ }
+
+ bool has_element(const value_type& elem, size_t hashval) const {
+ auto seq = probe(hashval);
+ while (true) {
+ Group g{ctrl_ + seq.offset()};
+ for (int i : g.Match((h2_t)H2(hashval))) {
+ if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) ==
+ elem))
+ return true;
+ }
+ if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false;
+ seq.next();
+ assert(seq.getindex() < capacity_ && "full table!");
+ }
+ return false;
+ }
+
+ bool has_element(const value_type& elem) const {
+ size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem);
+ return has_element(elem, hashval);
+ }
+
+ // Probes the raw_hash_set with the probe sequence for hash and returns the
+ // pointer to the first empty or deleted slot.
+ // NOTE: this function must work with tables having both kEmpty and kDelete
+ // in one group. Such tables appears during drop_deletes_without_resize.
+ //
+ // This function is very useful when insertions happen and:
+ // - the input is already a set
+ // - there are enough slots
+ // - the element with the hash is not in the table
+ struct FindInfo
+ {
+ size_t offset;
+ size_t probe_length;
+ };
+ FindInfo find_first_non_full(size_t hashval) {
+ auto seq = probe(hashval);
+ while (true) {
+ Group g{ctrl_ + seq.offset()};
+ auto mask = g.MatchEmptyOrDeleted();
+ if (mask) {
+ return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()};
+ }
+ assert(seq.getindex() < capacity_ && "full table!");
+ seq.next();
+ }
+ }
+
+ // TODO(alkis): Optimize this assuming *this and that don't overlap.
+ raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) {
+ raw_hash_set tmp(std::move(that));
+ swap(tmp);
+ return *this;
+ }
+ raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) {
+ raw_hash_set tmp(std::move(that), alloc_ref());
+ swap(tmp);
+ return *this;
+ }
+
+protected:
+ template <class K>
+ std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) {
+ auto seq = probe(hashval);
+ while (true) {
+ Group g{ctrl_ + seq.offset()};
+ for (int i : g.Match((h2_t)H2(hashval))) {
+ if (PHMAP_PREDICT_TRUE(PolicyTraits::apply(
+ EqualElement<K>{key, eq_ref()},
+ PolicyTraits::element(slots_ + seq.offset((size_t)i)))))
+ return {seq.offset((size_t)i), false};
+ }
+ if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break;
+ seq.next();
+ }
+ return {prepare_insert(hashval), true};
+ }
+
+ template <class K>
+ std::pair<size_t, bool> find_or_prepare_insert(const K& key) {
+ return find_or_prepare_insert(key, this->hash(key));
+ }
+
+ size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE {
+ auto target = find_first_non_full(hashval);
+ if (PHMAP_PREDICT_FALSE(growth_left() == 0 &&
+ !IsDeleted(ctrl_[target.offset]))) {
+ rehash_and_grow_if_necessary();
+ target = find_first_non_full(hashval);
+ }
+ ++size_;
+ growth_left() -= IsEmpty(ctrl_[target.offset]);
+ set_ctrl(target.offset, H2(hashval));
+ infoz_.RecordInsert(hashval, target.probe_length);
+ return target.offset;
+ }
+
+ // Constructs the value in the space pointed by the iterator. This only works
+ // after an unsuccessful find_or_prepare_insert() and before any other
+ // modifications happen in the raw_hash_set.
+ //
+ // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where
+ // k is the key decomposed from `forward<Args>(args)...`, and the bool
+ // returned by find_or_prepare_insert(k) was true.
+ // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...).
+ template <class... Args>
+ void emplace_at(size_t i, Args&&... args) {
+ PolicyTraits::construct(&alloc_ref(), slots_ + i,
+ std::forward<Args>(args)...);
+
+ assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) ==
+ iterator_at(i) &&
+ "constructed value does not match the lookup key");
+ }
+
+ iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; }
+ const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; }
+
+private:
+ friend struct RawHashSetTestOnlyAccess;
+
+ probe_seq<Group::kWidth> probe(size_t hashval) const {
+ return probe_seq<Group::kWidth>(H1(hashval, ctrl_), capacity_);
+ }
+
+ // Reset all ctrl bytes back to kEmpty, except the sentinel.
+ void reset_ctrl() {
+ std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth);
+ ctrl_[capacity_] = kSentinel;
+ SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
+ }
+
+ void reset_growth_left() {
+ growth_left() = CapacityToGrowth(capacity()) - size_;
+ }
+
+ // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at
+ // the end too.
+ void set_ctrl(size_t i, ctrl_t h) {
+ assert(i < capacity_);
+
+ if (IsFull(h)) {
+ SanitizerUnpoisonObject(slots_ + i);
+ } else {
+ SanitizerPoisonObject(slots_ + i);
+ }
+
+ ctrl_[i] = h;
+ ctrl_[((i - Group::kWidth) & capacity_) + 1 +
+ ((Group::kWidth - 1) & capacity_)] = h;
+ }
+
+ size_t& growth_left() { return settings_.template get<0>(); }
+
+ template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class M, class P, class H, class E, class A>
+ friend class parallel_hash_set;
+
+ template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class M, class P, class H, class E, class A>
+ friend class parallel_hash_map;
+
+ // The representation of the object has two modes:
+ // - small: For capacities < kWidth-1
+ // - large: For the rest.
+ //
+ // Differences:
+ // - In small mode we are able to use the whole capacity. The extra control
+ // bytes give us at least one "empty" control byte to stop the iteration.
+ // This is important to make 1 a valid capacity.
+ //
+ // - In small mode only the first `capacity()` control bytes after the
+ // sentinel are valid. The rest contain dummy kEmpty values that do not
+ // represent a real slot. This is important to take into account on
+ // find_first_non_full(), where we never try ShouldInsertBackwards() for
+ // small tables.
+ bool is_small() const { return capacity_ < Group::kWidth - 1; }
+
+ hasher& hash_ref() { return settings_.template get<1>(); }
+ const hasher& hash_ref() const { return settings_.template get<1>(); }
+ key_equal& eq_ref() { return settings_.template get<2>(); }
+ const key_equal& eq_ref() const { return settings_.template get<2>(); }
+ allocator_type& alloc_ref() { return settings_.template get<3>(); }
+ const allocator_type& alloc_ref() const {
+ return settings_.template get<3>();
+ }
+
+ // TODO(alkis): Investigate removing some of these fields:
+ // - ctrl/slots can be derived from each other
+ // - size can be moved into the slot array
+ ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t]
+ slot_type* slots_ = nullptr; // [capacity * slot_type]
+ size_t size_ = 0; // number of full slots
+ size_t capacity_ = 0; // total number of slots
+ HashtablezInfoHandle infoz_;
+ phmap::priv::CompressedTuple<size_t /* growth_left */, hasher,
+ key_equal, allocator_type>
+ settings_{0, hasher{}, key_equal{}, allocator_type{}};
+};
+
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <class Policy, class Hash, class Eq, class Alloc>
+class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc>
+{
+ // P is Policy. It's passed as a template argument to support maps that have
+ // incomplete types as values, as in unordered_map<K, IncompleteType>.
+ // MappedReference<> may be a non-reference type.
+ template <class P>
+ using MappedReference = decltype(P::value(
+ std::addressof(std::declval<typename raw_hash_map::reference>())));
+
+ // MappedConstReference<> may be a non-reference type.
+ template <class P>
+ using MappedConstReference = decltype(P::value(
+ std::addressof(std::declval<typename raw_hash_map::const_reference>())));
+
+ using KeyArgImpl =
+ KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
+
+ using Base = raw_hash_set<Policy, Hash, Eq, Alloc>;
+
+public:
+ using key_type = typename Policy::key_type;
+ using mapped_type = typename Policy::mapped_type;
+ template <class K>
+ using key_arg = typename KeyArgImpl::template type<K, key_type>;
+
+ static_assert(!std::is_reference<key_type>::value, "");
+ // TODO(alkis): remove this assertion and verify that reference mapped_type is
+ // supported.
+ static_assert(!std::is_reference<mapped_type>::value, "");
+
+ using iterator = typename raw_hash_map::raw_hash_set::iterator;
+ using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator;
+
+ raw_hash_map() {}
+ using Base::raw_hash_set; // use raw_hash_set constructor
+
+ // The last two template parameters ensure that both arguments are rvalues
+ // (lvalue arguments are handled by the overloads below). This is necessary
+ // for supporting bitfield arguments.
+ //
+ // union { int n : 1; };
+ // flat_hash_map<int, int> m;
+ // m.insert_or_assign(n, n);
+ template <class K = key_type, class V = mapped_type, K* = nullptr,
+ V* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) {
+ return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v));
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) {
+ return insert_or_assign_impl(std::forward<K>(k), v);
+ }
+
+ template <class K = key_type, class V = mapped_type, V* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) {
+ return insert_or_assign_impl(k, std::forward<V>(v));
+ }
+
+ template <class K = key_type, class V = mapped_type>
+ std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) {
+ return insert_or_assign_impl(k, v);
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr,
+ V* = nullptr>
+ iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) {
+ return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first;
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr>
+ iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) {
+ return insert_or_assign(std::forward<K>(k), v).first;
+ }
+
+ template <class K = key_type, class V = mapped_type, V* = nullptr>
+ iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) {
+ return insert_or_assign(k, std::forward<V>(v)).first;
+ }
+
+ template <class K = key_type, class V = mapped_type>
+ iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) {
+ return insert_or_assign(k, v).first;
+ }
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0,
+ K* = nullptr>
+ std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) {
+ return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0>
+ std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) {
+ return try_emplace_impl(k, std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args, K* = nullptr>
+ iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) {
+ return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class... Args>
+ iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) {
+ return try_emplace(k, std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedReference<P> at(const key_arg<K>& key) {
+ auto it = this->find(key);
+ if (it == this->end())
+ phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key");
+ return Policy::value(&*it);
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedConstReference<P> at(const key_arg<K>& key) const {
+ auto it = this->find(key);
+ if (it == this->end())
+ phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key");
+ return Policy::value(&*it);
+ }
+
+ template <class K = key_type, class P = Policy, K* = nullptr>
+ MappedReference<P> operator[](key_arg<K>&& key) {
+ return Policy::value(&*try_emplace(std::forward<K>(key)).first);
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedReference<P> operator[](const key_arg<K>& key) {
+ return Policy::value(&*try_emplace(key).first);
+ }
+
+private:
+ template <class K, class V>
+ std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
+ auto res = this->find_or_prepare_insert(k);
+ if (res.second)
+ this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v));
+ else
+ Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v);
+ return {this->iterator_at(res.first), res.second};
+ }
+
+ template <class K = key_type, class... Args>
+ std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) {
+ auto res = this->find_or_prepare_insert(k);
+ if (res.second)
+ this->emplace_at(res.first, std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ return {this->iterator_at(res.first), res.second};
+ }
+};
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+// Returns "random" seed.
+inline size_t RandomSeed()
+{
+#if PHMAP_HAVE_THREAD_LOCAL
+ static thread_local size_t counter = 0;
+ size_t value = ++counter;
+#else // PHMAP_HAVE_THREAD_LOCAL
+ static std::atomic<size_t> counter(0);
+ size_t value = counter.fetch_add(1, std::memory_order_relaxed);
+#endif // PHMAP_HAVE_THREAD_LOCAL
+ return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter));
+}
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class Mtx_,
+ class Policy, class Hash, class Eq, class Alloc>
+class parallel_hash_set
+{
+ using PolicyTraits = hash_policy_traits<Policy>;
+ using KeyArgImpl =
+ KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
+
+ static_assert(N <= 12, "N = 12 means 4096 hash tables!");
+ constexpr static size_t num_tables = 1 << N;
+ constexpr static size_t mask = num_tables - 1;
+
+public:
+ using EmbeddedSet = RefSet<Policy, Hash, Eq, Alloc>;
+ using EmbeddedIterator= typename EmbeddedSet::iterator;
+ using EmbeddedConstIterator= typename EmbeddedSet::const_iterator;
+ using constructor = typename EmbeddedSet::constructor;
+ using init_type = typename PolicyTraits::init_type;
+ using key_type = typename PolicyTraits::key_type;
+ using slot_type = typename PolicyTraits::slot_type;
+ using allocator_type = Alloc;
+ using size_type = size_t;
+ using difference_type = ptrdiff_t;
+ using hasher = Hash;
+ using key_equal = Eq;
+ using policy_type = Policy;
+ using value_type = typename PolicyTraits::value_type;
+ using reference = value_type&;
+ using const_reference = const value_type&;
+ using pointer = typename phmap::allocator_traits<
+ allocator_type>::template rebind_traits<value_type>::pointer;
+ using const_pointer = typename phmap::allocator_traits<
+ allocator_type>::template rebind_traits<value_type>::const_pointer;
+
+ // Alias used for heterogeneous lookup functions.
+ // `key_arg<K>` evaluates to `K` when the functors are transparent and to
+ // `key_type` otherwise. It permits template argument deduction on `K` for the
+ // transparent case.
+ // --------------------------------------------------------------------
+ template <class K>
+ using key_arg = typename KeyArgImpl::template type<K, key_type>;
+
+protected:
+ using Lockable = phmap::LockableImpl<Mtx_>;
+
+ // --------------------------------------------------------------------
+ struct Inner : public Lockable
+ {
+ bool operator==(const Inner& o) const
+ {
+ typename Lockable::SharedLocks l(const_cast<Inner &>(*this), const_cast<Inner &>(o));
+ return set_ == o.set_;
+ }
+
+ EmbeddedSet set_;
+ };
+
+private:
+ // Give an early error when key_type is not hashable/eq.
+ // --------------------------------------------------------------------
+ auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
+ auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k));
+
+ using AllocTraits = phmap::allocator_traits<allocator_type>;
+
+ static_assert(std::is_lvalue_reference<reference>::value,
+ "Policy::element() must return a reference");
+
+ template <typename T>
+ struct SameAsElementReference : std::is_same<
+ typename std::remove_cv<typename std::remove_reference<reference>::type>::type,
+ typename std::remove_cv<typename std::remove_reference<T>::type>::type> {};
+
+ // An enabler for insert(T&&): T must be convertible to init_type or be the
+ // same as [cv] value_type [ref].
+ // Note: we separate SameAsElementReference into its own type to avoid using
+ // reference unless we need to. MSVC doesn't seem to like it in some
+ // cases.
+ // --------------------------------------------------------------------
+ template <class T>
+ using RequiresInsertable = typename std::enable_if<
+ phmap::disjunction<std::is_convertible<T, init_type>,
+ SameAsElementReference<T>>::value,
+ int>::type;
+
+ // RequiresNotInit is a workaround for gcc prior to 7.1.
+ // See https://godbolt.org/g/Y4xsUh.
+ template <class T>
+ using RequiresNotInit =
+ typename std::enable_if<!std::is_same<T, init_type>::value, int>::type;
+
+ template <class... Ts>
+ using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>;
+
+public:
+ static_assert(std::is_same<pointer, value_type*>::value,
+ "Allocators with custom pointer types are not supported");
+ static_assert(std::is_same<const_pointer, const value_type*>::value,
+ "Allocators with custom pointer types are not supported");
+
+ // --------------------- i t e r a t o r ------------------------------
+ class iterator
+ {
+ friend class parallel_hash_set;
+
+ public:
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = typename parallel_hash_set::value_type;
+ using reference =
+ phmap::conditional_t<PolicyTraits::constant_iterators::value,
+ const value_type&, value_type&>;
+ using pointer = phmap::remove_reference_t<reference>*;
+ using difference_type = typename parallel_hash_set::difference_type;
+ using Inner = typename parallel_hash_set::Inner;
+ using EmbeddedSet = typename parallel_hash_set::EmbeddedSet;
+ using EmbeddedIterator = typename EmbeddedSet::iterator;
+
+ iterator() {}
+
+ reference operator*() const { return *it_; }
+ pointer operator->() const { return &operator*(); }
+
+ iterator& operator++() {
+ assert(inner_); // null inner means we are already at the end
+ ++it_;
+ skip_empty();
+ return *this;
+ }
+
+ iterator operator++(int) {
+ assert(inner_); // null inner means we are already at the end
+ auto tmp = *this;
+ ++*this;
+ return tmp;
+ }
+
+ friend bool operator==(const iterator& a, const iterator& b) {
+ return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_);
+ }
+
+ friend bool operator!=(const iterator& a, const iterator& b) {
+ return !(a == b);
+ }
+
+ private:
+ iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) :
+ inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end()
+ if (inner)
+ it_end_ = inner->set_.end();
+ }
+
+ void skip_empty() {
+ while (it_ == it_end_) {
+ ++inner_;
+ if (inner_ == inner_end_) {
+ inner_ = nullptr; // marks end()
+ break;
+ }
+ else {
+ it_ = inner_->set_.begin();
+ it_end_ = inner_->set_.end();
+ }
+ }
+ }
+
+ Inner *inner_ = nullptr;
+ Inner *inner_end_ = nullptr;
+ EmbeddedIterator it_, it_end_;
+ };
+
+ // --------------------- c o n s t i t e r a t o r -----------------
+ class const_iterator
+ {
+ friend class parallel_hash_set;
+
+ public:
+ using iterator_category = typename iterator::iterator_category;
+ using value_type = typename parallel_hash_set::value_type;
+ using reference = typename parallel_hash_set::const_reference;
+ using pointer = typename parallel_hash_set::const_pointer;
+ using difference_type = typename parallel_hash_set::difference_type;
+ using Inner = typename parallel_hash_set::Inner;
+
+ const_iterator() {}
+ // Implicit construction from iterator.
+ const_iterator(iterator i) : iter_(std::move(i)) {}
+
+ reference operator*() const { return *(iter_); }
+ pointer operator->() const { return iter_.operator->(); }
+
+ const_iterator& operator++() {
+ ++iter_;
+ return *this;
+ }
+ const_iterator operator++(int) { return iter_++; }
+
+ friend bool operator==(const const_iterator& a, const const_iterator& b) {
+ return a.iter_ == b.iter_;
+ }
+ friend bool operator!=(const const_iterator& a, const const_iterator& b) {
+ return !(a == b);
+ }
+
+ private:
+ const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it)
+ : iter_(const_cast<Inner**>(inner),
+ const_cast<Inner**>(inner_end),
+ const_cast<EmbeddedIterator*>(it)) {}
+
+ iterator iter_;
+ };
+
+ using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>;
+ using insert_return_type = InsertReturnType<iterator, node_type>;
+
+ // ------------------------- c o n s t r u c t o r s ------------------
+
+ parallel_hash_set() noexcept(
+ std::is_nothrow_default_constructible<hasher>::value&&
+ std::is_nothrow_default_constructible<key_equal>::value&&
+ std::is_nothrow_default_constructible<allocator_type>::value) {}
+
+ explicit parallel_hash_set(size_t bucket_cnt,
+ const hasher& hash_param = hasher(),
+ const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type()) {
+ for (auto& inner : sets_)
+ inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc);
+ }
+
+ parallel_hash_set(size_t bucket_cnt,
+ const hasher& hash_param,
+ const allocator_type& alloc)
+ : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {}
+
+ parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc)
+ : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ explicit parallel_hash_set(const allocator_type& alloc)
+ : parallel_hash_set(0, hasher(), key_equal(), alloc) {}
+
+ template <class InputIter>
+ parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0,
+ const hasher& hash_param = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) {
+ insert(first, last);
+ }
+
+ template <class InputIter>
+ parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt,
+ const hasher& hash_param, const allocator_type& alloc)
+ : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {}
+
+ template <class InputIter>
+ parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ template <class InputIter>
+ parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc)
+ : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {}
+
+ // Instead of accepting std::initializer_list<value_type> as the first
+ // argument like std::unordered_set<value_type> does, we have two overloads
+ // that accept std::initializer_list<T> and std::initializer_list<init_type>.
+ // This is advantageous for performance.
+ //
+ // // Turns {"abc", "def"} into std::initializer_list<std::string>, then copies
+ // // the strings into the set.
+ // std::unordered_set<std::string> s = {"abc", "def"};
+ //
+ // // Turns {"abc", "def"} into std::initializer_list<const char*>, then
+ // // copies the strings into the set.
+ // phmap::flat_hash_set<std::string> s = {"abc", "def"};
+ //
+ // The same trick is used in insert().
+ //
+ // The enabler is necessary to prevent this constructor from triggering where
+ // the copy constructor is meant to be called.
+ //
+ // phmap::flat_hash_set<int> a, b{a};
+ //
+ // RequiresNotInit<T> is a workaround for gcc prior to 7.1.
+ // --------------------------------------------------------------------
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0,
+ const hasher& hash_param = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {}
+
+ parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0,
+ const hasher& hash_param = hasher(), const key_equal& eq = key_equal(),
+ const allocator_type& alloc = allocator_type())
+ : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt,
+ const hasher& hash_param, const allocator_type& alloc)
+ : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {}
+
+ parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt,
+ const hasher& hash_param, const allocator_type& alloc)
+ : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt,
+ const allocator_type& alloc)
+ : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {}
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
+ parallel_hash_set(std::initializer_list<T> init, const allocator_type& alloc)
+ : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {}
+
+ parallel_hash_set(std::initializer_list<init_type> init,
+ const allocator_type& alloc)
+ : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {}
+
+ parallel_hash_set(const parallel_hash_set& that)
+ : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction(
+ that.alloc_ref())) {}
+
+ parallel_hash_set(const parallel_hash_set& that, const allocator_type& a)
+ : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) {
+ for (size_t i=0; i<num_tables; ++i)
+ sets_[i].set_ = { that.sets_[i].set_, a };
+ }
+
+ parallel_hash_set(parallel_hash_set&& that) noexcept(
+ std::is_nothrow_copy_constructible<hasher>::value&&
+ std::is_nothrow_copy_constructible<key_equal>::value&&
+ std::is_nothrow_copy_constructible<allocator_type>::value)
+ : parallel_hash_set(std::move(that), that.alloc_ref()) {
+ }
+
+ parallel_hash_set(parallel_hash_set&& that, const allocator_type& a)
+ {
+ for (size_t i=0; i<num_tables; ++i)
+ sets_[i].set_ = { std::move(that.sets_[i]).set_, a };
+ }
+
+ parallel_hash_set& operator=(const parallel_hash_set& that) {
+ for (size_t i=0; i<num_tables; ++i)
+ sets_[i].set_ = that.sets_[i].set_;
+ return *this;
+ }
+
+ parallel_hash_set& operator=(parallel_hash_set&& that) noexcept(
+ phmap::allocator_traits<allocator_type>::is_always_equal::value &&
+ std::is_nothrow_move_assignable<hasher>::value &&
+ std::is_nothrow_move_assignable<key_equal>::value) {
+ for (size_t i=0; i<num_tables; ++i)
+ sets_[i].set_ = std::move(that.sets_[i].set_);
+ return *this;
+ }
+
+ ~parallel_hash_set() {}
+
+ iterator begin() {
+ auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin());
+ it.skip_empty();
+ return it;
+ }
+
+ iterator end() { return iterator(); }
+ const_iterator begin() const { return const_cast<parallel_hash_set *>(this)->begin(); }
+ const_iterator end() const { return const_cast<parallel_hash_set *>(this)->end(); }
+ const_iterator cbegin() const { return begin(); }
+ const_iterator cend() const { return end(); }
+
+ bool empty() const { return !size(); }
+
+ size_t size() const {
+ size_t sz = 0;
+ for (const auto& inner : sets_)
+ sz += inner.set_.size();
+ return sz;
+ }
+
+ size_t capacity() const {
+ size_t c = 0;
+ for (const auto& inner : sets_)
+ c += inner.set_.capacity();
+ return c;
+ }
+
+ size_t max_size() const { return (std::numeric_limits<size_t>::max)(); }
+
+ PHMAP_ATTRIBUTE_REINITIALIZES void clear() {
+ for (auto& inner : sets_)
+ {
+ typename Lockable::UniqueLock m(inner);
+ inner.set_.clear();
+ }
+ }
+
+ // extension - clears only soecified submap
+ // ----------------------------------------
+ void clear(std::size_t submap_index) {
+ Inner& inner = sets_[submap_index];
+ typename Lockable::UniqueLock m(inner);
+ inner.set_.clear();
+ }
+
+ // This overload kicks in when the argument is an rvalue of insertable and
+ // decomposable type other than init_type.
+ //
+ // flat_hash_map<std::string, int> m;
+ // m.insert(std::make_pair("abc", 42));
+ // --------------------------------------------------------------------
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<T>::value, int>::type = 0,
+ T* = nullptr>
+ std::pair<iterator, bool> insert(T&& value) {
+ return emplace(std::forward<T>(value));
+ }
+
+ // This overload kicks in when the argument is a bitfield or an lvalue of
+ // insertable and decomposable type.
+ //
+ // union { int n : 1; };
+ // flat_hash_set<int> s;
+ // s.insert(n);
+ //
+ // flat_hash_set<std::string> s;
+ // const char* p = "hello";
+ // s.insert(p);
+ //
+ // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace
+ // RequiresInsertable<T> with RequiresInsertable<const T&>.
+ // We are hitting this bug: https://godbolt.org/g/1Vht4f.
+ // --------------------------------------------------------------------
+ template <
+ class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
+ std::pair<iterator, bool> insert(const T& value) {
+ return emplace(value);
+ }
+
+ // This overload kicks in when the argument is an rvalue of init_type. Its
+ // purpose is to handle brace-init-list arguments.
+ //
+ // flat_hash_set<std::pair<std::string, int>> s;
+ // s.insert({"abc", 42});
+ // --------------------------------------------------------------------
+ std::pair<iterator, bool> insert(init_type&& value) {
+ return emplace(std::move(value));
+ }
+
+ template <class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<T>::value, int>::type = 0,
+ T* = nullptr>
+ iterator insert(const_iterator, T&& value) {
+ return insert(std::forward<T>(value)).first;
+ }
+
+ // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace
+ // RequiresInsertable<T> with RequiresInsertable<const T&>.
+ // We are hitting this bug: https://godbolt.org/g/1Vht4f.
+ // --------------------------------------------------------------------
+ template <
+ class T, RequiresInsertable<T> = 0,
+ typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
+ iterator insert(const_iterator, const T& value) {
+ return insert(value).first;
+ }
+
+ iterator insert(const_iterator, init_type&& value) {
+ return insert(std::move(value)).first;
+ }
+
+ template <class InputIt>
+ void insert(InputIt first, InputIt last) {
+ for (; first != last; ++first) insert(*first);
+ }
+
+ template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0>
+ void insert(std::initializer_list<T> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ void insert(std::initializer_list<init_type> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ insert_return_type insert(node_type&& node) {
+ if (!node)
+ return {end(), false, node_type()};
+ auto& key = node.key();
+ size_t hashval = this->hash(key);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+
+ typename Lockable::UniqueLock m(inner);
+ auto res = set.insert(std::move(node), hashval);
+ return { make_iterator(&inner, res.position),
+ res.inserted,
+ res.inserted ? node_type() : std::move(res.node) };
+ }
+
+ iterator insert(const_iterator, node_type&& node) {
+ return insert(std::move(node)).first;
+ }
+
+ struct ReturnKey_
+ {
+ template <class Key, class... Args>
+ Key operator()(Key&& k, const Args&...) const {
+ return std::forward<Key>(k);
+ }
+ };
+
+ // --------------------------------------------------------------------
+ // phmap expension: emplace_with_hash
+ // ----------------------------------
+ // same as emplace, but hashval is provided
+ // --------------------------------------------------------------------
+ template <class K, class... Args>
+ std::pair<iterator, bool> emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args)
+ {
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...));
+ }
+
+ struct EmplaceDecomposableHashval
+ {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&... args) const {
+ return s.emplace_decomposable_with_hash(key, hashval, std::forward<Args>(args)...);
+ }
+ parallel_hash_set& s;
+ size_t hashval;
+ };
+
+ // This overload kicks in if we can deduce the key from args. This enables us
+ // to avoid constructing value_type if an entry with the same key already
+ // exists.
+ //
+ // For example:
+ //
+ // flat_hash_map<std::string, std::string> m = {{"abc", "def"}};
+ // // Creates no std::string copies and makes no heap allocations.
+ // m.emplace("abc", "xyz");
+ // --------------------------------------------------------------------
+ template <class... Args, typename std::enable_if<
+ IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) {
+ return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval},
+ std::forward<Args>(args)...);
+ }
+
+ // This overload kicks in if we cannot deduce the key from args. It constructs
+ // value_type unconditionally and then either moves it into the table or
+ // destroys.
+ // --------------------------------------------------------------------
+ template <class... Args, typename std::enable_if<
+ !IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) {
+ typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw;
+ slot_type* slot = reinterpret_cast<slot_type*>(&raw);
+
+ PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
+ const auto& elem = PolicyTraits::element(slot);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ typename EmbeddedSet::template InsertSlotWithHash<true> f {
+ inner, std::move(*slot), hashval};
+ return make_rv(PolicyTraits::apply(f, elem));
+ }
+
+ template <class... Args>
+ iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) {
+ return emplace_with_hash(hashval, std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class F>
+ iterator lazy_emplace_with_hash(size_t hashval, const key_arg<K>& key, F&& f) {
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f)));
+ }
+
+ // --------------------------------------------------------------------
+ // end of phmap expension
+ // --------------------------------------------------------------------
+
+ template <class K, class... Args>
+ std::pair<iterator, bool> emplace_decomposable(const K& key, Args&&... args)
+ {
+ size_t hashval = this->hash(key);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...));
+ }
+
+ struct EmplaceDecomposable
+ {
+ template <class K, class... Args>
+ std::pair<iterator, bool> operator()(const K& key, Args&&... args) const {
+ return s.emplace_decomposable(key, std::forward<Args>(args)...);
+ }
+ parallel_hash_set& s;
+ };
+
+ // This overload kicks in if we can deduce the key from args. This enables us
+ // to avoid constructing value_type if an entry with the same key already
+ // exists.
+ //
+ // For example:
+ //
+ // flat_hash_map<std::string, std::string> m = {{"abc", "def"}};
+ // // Creates no std::string copies and makes no heap allocations.
+ // m.emplace("abc", "xyz");
+ // --------------------------------------------------------------------
+ template <class... Args, typename std::enable_if<
+ IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace(Args&&... args) {
+ return PolicyTraits::apply(EmplaceDecomposable{*this},
+ std::forward<Args>(args)...);
+ }
+
+ // This overload kicks in if we cannot deduce the key from args. It constructs
+ // value_type unconditionally and then either moves it into the table or
+ // destroys.
+ // --------------------------------------------------------------------
+ template <class... Args, typename std::enable_if<
+ !IsDecomposable<Args...>::value, int>::type = 0>
+ std::pair<iterator, bool> emplace(Args&&... args) {
+ typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw;
+ slot_type* slot = reinterpret_cast<slot_type*>(&raw);
+ size_t hashval = this->hash(PolicyTraits::key(slot));
+
+ PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
+ const auto& elem = PolicyTraits::element(slot);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ typename EmbeddedSet::template InsertSlotWithHash<true> f {
+ inner, std::move(*slot), hashval};
+ return make_rv(PolicyTraits::apply(f, elem));
+ }
+
+ template <class... Args>
+ iterator emplace_hint(const_iterator, Args&&... args) {
+ return emplace(std::forward<Args>(args)...).first;
+ }
+
+ iterator make_iterator(Inner* inner, const EmbeddedIterator it)
+ {
+ if (it == inner->set_.end())
+ return iterator();
+ return iterator(inner, &sets_[0] + num_tables, it);
+ }
+
+ std::pair<iterator, bool> make_rv(Inner* inner,
+ const std::pair<EmbeddedIterator, bool>& res)
+ {
+ return {iterator(inner, &sets_[0] + num_tables, res.first), res.second};
+ }
+
+ template <class K = key_type, class F>
+ iterator lazy_emplace(const key_arg<K>& key, F&& f) {
+ auto hashval = this->hash(key);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UniqueLock m(inner);
+ return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f)));
+ }
+
+ template <class K = key_type, class FExists, class FEmplace>
+ bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) {
+ typename Lockable::UniqueLock m;
+ auto res = this->find_or_prepare_insert(key, m);
+ Inner* inner = std::get<0>(res);
+ if (std::get<2>(res))
+ inner->set_.lazy_emplace_at(std::get<1>(res), std::forward<FEmplace>(fEmplace));
+ else {
+ auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res)));
+ std::forward<FExists>(fExists)(Policy::value(&*it));
+ }
+ return std::get<2>(res);
+ }
+
+ // Extension API: support for heterogeneous keys.
+ //
+ // std::unordered_set<std::string> s;
+ // // Turns "abc" into std::string.
+ // s.erase("abc");
+ //
+ // flat_hash_set<std::string> s;
+ // // Uses "abc" directly without copying it into std::string.
+ // s.erase("abc");
+ // --------------------------------------------------------------------
+ template <class K = key_type>
+ size_type erase(const key_arg<K>& key) {
+ auto hashval = this->hash(key);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::UpgradeLock m(inner);
+ auto it = set.find(key, hashval);
+ if (it == set.end())
+ return 0;
+
+ typename Lockable::UpgradeToUnique unique(m);
+ set._erase(it);
+ return 1;
+ }
+
+ // --------------------------------------------------------------------
+ iterator erase(const_iterator cit) { return erase(cit.iter_); }
+
+ // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`,
+ // this method returns void to reduce algorithmic complexity to O(1). In
+ // order to erase while iterating across a map, use the following idiom (which
+ // also works for standard containers):
+ //
+ // for (auto it = m.begin(), end = m.end(); it != end;) {
+ // if (<pred>) {
+ // m._erase(it++);
+ // } else {
+ // ++it;
+ // }
+ // }
+ // --------------------------------------------------------------------
+ void _erase(iterator it) {
+ assert(it.inner_ != nullptr);
+ it.inner_->set_._erase(it.it_);
+ }
+ void _erase(const_iterator cit) { _erase(cit.iter_); }
+
+ // This overload is necessary because otherwise erase<K>(const K&) would be
+ // a better match if non-const iterator is passed as an argument.
+ // --------------------------------------------------------------------
+ iterator erase(iterator it) { _erase(it++); return it; }
+
+ iterator erase(const_iterator first, const_iterator last) {
+ while (first != last) {
+ _erase(first++);
+ }
+ return last.iter_;
+ }
+
+ // Moves elements from `src` into `this`.
+ // If the element already exists in `this`, it is left unmodified in `src`.
+ // --------------------------------------------------------------------
+ template <typename E = Eq>
+ void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>& src) { // NOLINT
+ assert(this != &src);
+ if (this != &src)
+ {
+ for (size_t i=0; i<num_tables; ++i)
+ {
+ typename Lockable::UniqueLocks l(sets_[i], src.sets_[i]);
+ sets_[i].set_.merge(src.sets_[i].set_);
+ }
+ }
+ }
+
+ template <typename E = Eq>
+ void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>&& src) {
+ merge(src);
+ }
+
+ node_type extract(const_iterator position) {
+ return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_));
+ }
+
+ template <
+ class K = key_type,
+ typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0>
+ node_type extract(const key_arg<K>& key) {
+ auto it = find(key);
+ return it == end() ? node_type() : extract(const_iterator{it});
+ }
+
+ void swap(parallel_hash_set& that) noexcept(
+ IsNoThrowSwappable<EmbeddedSet>() &&
+ (!AllocTraits::propagate_on_container_swap::value ||
+ IsNoThrowSwappable<allocator_type>())) {
+ using std::swap;
+ for (size_t i=0; i<num_tables; ++i)
+ {
+ typename Lockable::UniqueLocks l(sets_[i], that.sets_[i]);
+ swap(sets_[i].set_, that.sets_[i].set_);
+ }
+ }
+
+ void rehash(size_t n) {
+ size_t nn = n / num_tables;
+ for (auto& inner : sets_)
+ {
+ typename Lockable::UniqueLock m(inner);
+ inner.set_.rehash(nn);
+ }
+ }
+
+ void reserve(size_t n)
+ {
+ size_t target = GrowthToLowerboundCapacity(n);
+ size_t normalized = 16 * NormalizeCapacity(n / num_tables);
+ rehash(normalized > target ? normalized : target);
+ }
+
+ // Extension API: support for heterogeneous keys.
+ //
+ // std::unordered_set<std::string> s;
+ // // Turns "abc" into std::string.
+ // s.count("abc");
+ //
+ // ch_set<std::string> s;
+ // // Uses "abc" directly without copying it into std::string.
+ // s.count("abc");
+ // --------------------------------------------------------------------
+ template <class K = key_type>
+ size_t count(const key_arg<K>& key) const {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ // Issues CPU prefetch instructions for the memory needed to find or insert
+ // a key. Like all lookup functions, this support heterogeneous keys.
+ //
+ // NOTE: This is a very low level operation and should not be used without
+ // specific benchmarks indicating its importance.
+ // --------------------------------------------------------------------
+ void prefetch_hash(size_t hashval) const {
+ const Inner& inner = sets_[subidx(hashval)];
+ const auto& set = inner.set_;
+ typename Lockable::SharedLock m(const_cast<Inner&>(inner));
+ set.prefetch_hash(hashval);
+ }
+
+ template <class K = key_type>
+ void prefetch(const key_arg<K>& key) const {
+ prefetch_hash(this->hash(key));
+ }
+
+ // The API of find() has two extensions.
+ //
+ // 1. The hash can be passed by the user. It must be equal to the hash of the
+ // key.
+ //
+ // 2. The type of the key argument doesn't have to be key_type. This is so
+ // called heterogeneous key support.
+ // --------------------------------------------------------------------
+ template <class K = key_type>
+ iterator find(const key_arg<K>& key, size_t hashval) {
+ typename Lockable::SharedLock m;
+ return find(key, hashval, m);
+ }
+
+ template <class K = key_type>
+ iterator find(const key_arg<K>& key) {
+ return find(key, this->hash(key));
+ }
+
+ template <class K = key_type>
+ const_iterator find(const key_arg<K>& key, size_t hashval) const {
+ return const_cast<parallel_hash_set*>(this)->find(key, hashval);
+ }
+
+ template <class K = key_type>
+ const_iterator find(const key_arg<K>& key) const {
+ return find(key, this->hash(key));
+ }
+
+ template <class K = key_type>
+ bool contains(const key_arg<K>& key) const {
+ return find(key) != end();
+ }
+
+ template <class K = key_type>
+ bool contains(const key_arg<K>& key, size_t hashval) const {
+ return find(key, hashval) != end();
+ }
+
+ template <class K = key_type>
+ std::pair<iterator, iterator> equal_range(const key_arg<K>& key) {
+ auto it = find(key);
+ if (it != end()) return {it, std::next(it)};
+ return {it, it};
+ }
+
+ template <class K = key_type>
+ std::pair<const_iterator, const_iterator> equal_range(
+ const key_arg<K>& key) const {
+ auto it = find(key);
+ if (it != end()) return {it, std::next(it)};
+ return {it, it};
+ }
+
+ size_t bucket_count() const {
+ size_t sz = 0;
+ for (const auto& inner : sets_)
+ {
+ typename Lockable::SharedLock m(const_cast<Inner&>(inner));
+ sz += inner.set_.bucket_count();
+ }
+ return sz;
+ }
+
+ float load_factor() const {
+ size_t _capacity = bucket_count();
+ return _capacity ? static_cast<float>(static_cast<double>(size()) / _capacity) : 0;
+ }
+
+ float max_load_factor() const { return 1.0f; }
+ void max_load_factor(float) {
+ // Does nothing.
+ }
+
+ hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function
+ key_equal key_eq() const { return eq_ref(); }
+ allocator_type get_allocator() const { return alloc_ref(); }
+
+ friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) {
+ return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin());
+ }
+
+ friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) {
+ return !(a == b);
+ }
+
+ friend void swap(parallel_hash_set& a,
+ parallel_hash_set& b) noexcept(noexcept(a.swap(b))) {
+ a.swap(b);
+ }
+
+ template <class K>
+ size_t hash(const K& key) const {
+ return HashElement{hash_ref()}(key);
+ }
+
+#ifndef PHMAP_NON_DETERMINISTIC
+ template<typename OutputArchive>
+ bool dump(OutputArchive& ar) const;
+
+ template<typename InputArchive>
+ bool load(InputArchive& ar);
+#endif
+
+private:
+ template <class Container, typename Enabler>
+ friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess;
+
+ struct FindElement
+ {
+ template <class K, class... Args>
+ const_iterator operator()(const K& key, Args&&...) const {
+ return s.find(key);
+ }
+ const parallel_hash_set& s;
+ };
+
+ struct HashElement
+ {
+ template <class K, class... Args>
+ size_t operator()(const K& key, Args&&...) const {
+ return phmap_mix<sizeof(size_t)>()(h(key));
+ }
+ const hasher& h;
+ };
+
+ template <class K1>
+ struct EqualElement
+ {
+ template <class K2, class... Args>
+ bool operator()(const K2& lhs, Args&&...) const {
+ return eq(lhs, rhs);
+ }
+ const K1& rhs;
+ const key_equal& eq;
+ };
+
+ // "erases" the object from the container, except that it doesn't actually
+ // destroy the object. It only updates all the metadata of the class.
+ // This can be used in conjunction with Policy::transfer to move the object to
+ // another place.
+ // --------------------------------------------------------------------
+ void erase_meta_only(const_iterator cit) {
+ auto &it = cit.iter_;
+ assert(it.set_ != nullptr);
+ it.set_.erase_meta_only(const_iterator(it.it_));
+ }
+
+ void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE {
+ for (auto& inner : sets_)
+ {
+ typename Lockable::UniqueLock m(inner);
+ inner.set_.drop_deletes_without_resize();
+ }
+ }
+
+ bool has_element(const value_type& elem) const {
+ size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem);
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ typename Lockable::SharedLock m(const_cast<Inner&>(inner));
+ return set.has_element(elem, hashval);
+ }
+
+ // TODO(alkis): Optimize this assuming *this and that don't overlap.
+ // --------------------------------------------------------------------
+ parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) {
+ parallel_hash_set tmp(std::move(that));
+ swap(tmp);
+ return *this;
+ }
+
+ parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) {
+ parallel_hash_set tmp(std::move(that), alloc_ref());
+ swap(tmp);
+ return *this;
+ }
+
+protected:
+ template <class K = key_type, class L = typename Lockable::SharedLock>
+ pointer find_ptr(const key_arg<K>& key, size_t hashval, L& mutexlock)
+ {
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ mutexlock = std::move(L(inner));
+ return set.find_ptr(key, hashval);
+ }
+
+ template <class K = key_type, class L = typename Lockable::SharedLock>
+ iterator find(const key_arg<K>& key, size_t hashval, L& mutexlock) {
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ mutexlock = std::move(L(inner));
+ return make_iterator(&inner, set.find(key, hashval));
+ }
+
+ template <class K>
+ std::tuple<Inner*, size_t, bool>
+ find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) {
+ Inner& inner = sets_[subidx(hashval)];
+ auto& set = inner.set_;
+ mutexlock = std::move(typename Lockable::UniqueLock(inner));
+ auto p = set.find_or_prepare_insert(key, hashval); // std::pair<size_t, bool>
+ return std::make_tuple(&inner, p.first, p.second);
+ }
+
+ template <class K>
+ std::tuple<Inner*, size_t, bool>
+ find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) {
+ return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock);
+ }
+
+ iterator iterator_at(Inner *inner,
+ const EmbeddedIterator& it) {
+ return {inner, &sets_[0] + num_tables, it};
+ }
+ const_iterator iterator_at(Inner *inner,
+ const EmbeddedIterator& it) const {
+ return {inner, &sets_[0] + num_tables, it};
+ }
+
+ static size_t subidx(size_t hashval) {
+ return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask;
+ }
+
+ static size_t subcnt() {
+ return num_tables;
+ }
+
+private:
+ friend struct RawHashSetTestOnlyAccess;
+
+ size_t growth_left() {
+ size_t sz = 0;
+ for (const auto& set : sets_)
+ sz += set.growth_left();
+ return sz;
+ }
+
+ hasher& hash_ref() { return sets_[0].set_.hash_ref(); }
+ const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); }
+ key_equal& eq_ref() { return sets_[0].set_.eq_ref(); }
+ const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); }
+ allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); }
+ const allocator_type& alloc_ref() const {
+ return sets_[0].set_.alloc_ref();
+ }
+
+protected: // protected in case users want to derive fromm this
+ std::array<Inner, num_tables> sets_;
+};
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class Mtx_,
+ class Policy, class Hash, class Eq, class Alloc>
+class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>
+{
+ // P is Policy. It's passed as a template argument to support maps that have
+ // incomplete types as values, as in unordered_map<K, IncompleteType>.
+ // MappedReference<> may be a non-reference type.
+ template <class P>
+ using MappedReference = decltype(P::value(
+ std::addressof(std::declval<typename parallel_hash_map::reference>())));
+
+ // MappedConstReference<> may be a non-reference type.
+ template <class P>
+ using MappedConstReference = decltype(P::value(
+ std::addressof(std::declval<typename parallel_hash_map::const_reference>())));
+
+ using KeyArgImpl =
+ KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
+
+ using Base = typename parallel_hash_map::parallel_hash_set;
+ using Lockable = phmap::LockableImpl<Mtx_>;
+
+public:
+ using key_type = typename Policy::key_type;
+ using mapped_type = typename Policy::mapped_type;
+ template <class K>
+ using key_arg = typename KeyArgImpl::template type<K, key_type>;
+
+ static_assert(!std::is_reference<key_type>::value, "");
+ // TODO(alkis): remove this assertion and verify that reference mapped_type is
+ // supported.
+ static_assert(!std::is_reference<mapped_type>::value, "");
+
+ using iterator = typename parallel_hash_map::parallel_hash_set::iterator;
+ using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator;
+
+ parallel_hash_map() {}
+
+#ifdef __INTEL_COMPILER
+ using Base::parallel_hash_set;
+#else
+ using parallel_hash_map::parallel_hash_set::parallel_hash_set;
+#endif
+
+ // The last two template parameters ensure that both arguments are rvalues
+ // (lvalue arguments are handled by the overloads below). This is necessary
+ // for supporting bitfield arguments.
+ //
+ // union { int n : 1; };
+ // flat_hash_map<int, int> m;
+ // m.insert_or_assign(n, n);
+ template <class K = key_type, class V = mapped_type, K* = nullptr,
+ V* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) {
+ return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v));
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) {
+ return insert_or_assign_impl(std::forward<K>(k), v);
+ }
+
+ template <class K = key_type, class V = mapped_type, V* = nullptr>
+ std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) {
+ return insert_or_assign_impl(k, std::forward<V>(v));
+ }
+
+ template <class K = key_type, class V = mapped_type>
+ std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) {
+ return insert_or_assign_impl(k, v);
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr,
+ V* = nullptr>
+ iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) {
+ return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first;
+ }
+
+ template <class K = key_type, class V = mapped_type, K* = nullptr>
+ iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) {
+ return insert_or_assign(std::forward<K>(k), v).first;
+ }
+
+ template <class K = key_type, class V = mapped_type, V* = nullptr>
+ iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) {
+ return insert_or_assign(k, std::forward<V>(v)).first;
+ }
+
+ template <class K = key_type, class V = mapped_type>
+ iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) {
+ return insert_or_assign(k, v).first;
+ }
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0,
+ K* = nullptr>
+ std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) {
+ return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0>
+ std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) {
+ return try_emplace_impl(k, std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args, K* = nullptr>
+ iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) {
+ return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class... Args>
+ iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) {
+ return try_emplace(k, std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedReference<P> at(const key_arg<K>& key) {
+ auto it = this->find(key);
+ if (it == this->end())
+ phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key");
+ return Policy::value(&*it);
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedConstReference<P> at(const key_arg<K>& key) const {
+ auto it = this->find(key);
+ if (it == this->end())
+ phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key");
+ return Policy::value(&*it);
+ }
+
+ // ----------- phmap extensions --------------------------
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0,
+ K* = nullptr>
+ std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, key_arg<K>&& k, Args&&... args) {
+ return try_emplace_impl_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args,
+ typename std::enable_if<
+ !std::is_convertible<K, const_iterator>::value, int>::type = 0>
+ std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, const key_arg<K>& k, Args&&... args) {
+ return try_emplace_impl_with_hash(hashval, k, std::forward<Args>(args)...);
+ }
+
+ template <class K = key_type, class... Args, K* = nullptr>
+ iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg<K>&& k, Args&&... args) {
+ return try_emplace_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...).first;
+ }
+
+ template <class K = key_type, class... Args>
+ iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg<K>& k, Args&&... args) {
+ return try_emplace_with_hash(hashval, k, std::forward<Args>(args)...).first;
+ }
+
+ // if map contains key, lambda is called with the mapped value (under read lock protection),
+ // and if_contains returns true. This is a const API and lambda should not modify the value
+ // -----------------------------------------------------------------------------------------
+ template <class K = key_type, class F>
+ bool if_contains(const key_arg<K>& key, F&& f) const {
+ return const_cast<parallel_hash_map*>(this)->template
+ modify_if_impl<K, F, typename Lockable::SharedLock>(key, std::forward<F>(f));
+ }
+
+ // if map contains key, lambda is called with the mapped value without read lock protection,
+ // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value
+ // This should be used only if we know that no other thread may be mutating the map at the time.
+ // -----------------------------------------------------------------------------------------
+ template <class K = key_type, class F>
+ bool if_contains_unsafe(const key_arg<K>& key, F&& f) const {
+ return const_cast<parallel_hash_map*>(this)->template
+ modify_if_impl<K, F, LockableBaseImpl<phmap::NullMutex>::DoNothing>(key, std::forward<F>(f));
+ }
+
+ // if map contains key, lambda is called with the mapped value (under write lock protection),
+ // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value
+ // ----------------------------------------------------------------------------------------------------
+ template <class K = key_type, class F>
+ bool modify_if(const key_arg<K>& key, F&& f) {
+ return modify_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f));
+ }
+
+
+ // if map contains key, lambda is called with the mapped value (under write lock protection).
+ // If the lambda returns true, the key is subsequently erased from the map (the write lock
+ // is only released after erase).
+ // returns true if key was erased, false otherwise.
+ // ----------------------------------------------------------------------------------------------------
+ template <class K = key_type, class F>
+ bool erase_if(const key_arg<K>& key, F&& f) {
+ return erase_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f));
+ }
+
+ // if map does not contains key, it is inserted and the mapped value is value-constructed
+ // with the provided arguments (if any), as with try_emplace.
+ // if map already contains key, then the lambda is called with the mapped value (under
+ // write lock protection) and can update the mapped value.
+ // returns true if key was not already present, false otherwise.
+ // ---------------------------------------------------------------------------------------
+ template <class K = key_type, class F, class... Args>
+ bool try_emplace_l(K&& k, F&& f, Args&&... args) {
+ typename Lockable::UniqueLock m;
+ auto res = this->find_or_prepare_insert(k, m);
+ typename Base::Inner *inner = std::get<0>(res);
+ if (std::get<2>(res))
+ inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ else {
+ auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res)));
+ std::forward<F>(f)(Policy::value(&*it));
+ }
+ return std::get<2>(res);
+ }
+
+ // ----------- end of phmap extensions --------------------------
+
+ template <class K = key_type, class P = Policy, K* = nullptr>
+ MappedReference<P> operator[](key_arg<K>&& key) {
+ return Policy::value(&*try_emplace(std::forward<K>(key)).first);
+ }
+
+ template <class K = key_type, class P = Policy>
+ MappedReference<P> operator[](const key_arg<K>& key) {
+ return Policy::value(&*try_emplace(key).first);
+ }
+
+private:
+ template <class K = key_type, class F, class L>
+ bool modify_if_impl(const key_arg<K>& key, F&& f) {
+#if __cplusplus >= 201703L
+ static_assert(std::is_invocable<F, mapped_type&>::value);
+#endif
+ L m;
+ auto ptr = this->template find_ptr<K, L>(key, this->hash(key), m);
+ if (ptr == nullptr)
+ return false;
+ std::forward<F>(f)(Policy::value(ptr));
+ return true;
+ }
+
+ template <class K = key_type, class F, class L>
+ bool erase_if_impl(const key_arg<K>& key, F&& f) {
+#if __cplusplus >= 201703L
+ static_assert(std::is_invocable<F, mapped_type&>::value);
+#endif
+ L m;
+ auto it = this->template find<K, L>(key, this->hash(key), m);
+ if (it == this->end()) return false;
+ if (std::forward<F>(f)(Policy::value(&*it)))
+ {
+ this->erase(it);
+ return true;
+ }
+ return false;
+ }
+
+
+ template <class K, class V>
+ std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
+ typename Lockable::UniqueLock m;
+ auto res = this->find_or_prepare_insert(k, m);
+ typename Base::Inner *inner = std::get<0>(res);
+ if (std::get<2>(res))
+ inner->set_.emplace_at(std::get<1>(res), std::forward<K>(k), std::forward<V>(v));
+ else
+ Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward<V>(v);
+ return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))),
+ std::get<2>(res)};
+ }
+
+ template <class K = key_type, class... Args>
+ std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) {
+ typename Lockable::UniqueLock m;
+ auto res = this->find_or_prepare_insert(k, m);
+ typename Base::Inner *inner = std::get<0>(res);
+ if (std::get<2>(res))
+ inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))),
+ std::get<2>(res)};
+ }
+
+ template <class K = key_type, class... Args>
+ std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) {
+ typename Lockable::UniqueLock m;
+ auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
+ typename Base::Inner *inner = std::get<0>(res);
+ if (std::get<2>(res))
+ inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(k)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))),
+ std::get<2>(res)};
+ }
+
+
+};
+
+
+// Constructs T into uninitialized storage pointed by `ptr` using the args
+// specified in the tuple.
+// ----------------------------------------------------------------------------
+template <class Alloc, class T, class Tuple>
+void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) {
+ memory_internal::ConstructFromTupleImpl(
+ alloc, ptr, std::forward<Tuple>(t),
+ phmap::make_index_sequence<
+ std::tuple_size<typename std::decay<Tuple>::type>::value>());
+}
+
+// Constructs T using the args specified in the tuple and calls F with the
+// constructed value.
+// ----------------------------------------------------------------------------
+template <class T, class Tuple, class F>
+decltype(std::declval<F>()(std::declval<T>())) WithConstructed(
+ Tuple&& t, F&& f) {
+ return memory_internal::WithConstructedImpl<T>(
+ std::forward<Tuple>(t),
+ phmap::make_index_sequence<
+ std::tuple_size<typename std::decay<Tuple>::type>::value>(),
+ std::forward<F>(f));
+}
+
+// ----------------------------------------------------------------------------
+// Given arguments of an std::pair's consructor, PairArgs() returns a pair of
+// tuples with references to the passed arguments. The tuples contain
+// constructor arguments for the first and the second elements of the pair.
+//
+// The following two snippets are equivalent.
+//
+// 1. std::pair<F, S> p(args...);
+//
+// 2. auto a = PairArgs(args...);
+// std::pair<F, S> p(std::piecewise_construct,
+// std::move(p.first), std::move(p.second));
+// ----------------------------------------------------------------------------
+inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; }
+
+template <class F, class S>
+std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) {
+ return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)),
+ std::forward_as_tuple(std::forward<S>(s))};
+}
+
+template <class F, class S>
+std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs(
+ const std::pair<F, S>& p) {
+ return PairArgs(p.first, p.second);
+}
+
+template <class F, class S>
+std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) {
+ return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second));
+}
+
+template <class F, class S>
+auto PairArgs(std::piecewise_construct_t, F&& f, S&& s)
+ -> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)),
+ memory_internal::TupleRef(std::forward<S>(s)))) {
+ return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)),
+ memory_internal::TupleRef(std::forward<S>(s)));
+}
+
+// A helper function for implementing apply() in map policies.
+// ----------------------------------------------------------------------------
+template <class F, class... Args>
+auto DecomposePair(F&& f, Args&&... args)
+ -> decltype(memory_internal::DecomposePairImpl(
+ std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) {
+ return memory_internal::DecomposePairImpl(
+ std::forward<F>(f), PairArgs(std::forward<Args>(args)...));
+}
+
+// A helper function for implementing apply() in set policies.
+// ----------------------------------------------------------------------------
+template <class F, class Arg>
+decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>()))
+DecomposeValue(F&& f, Arg&& arg) {
+ const auto& key = arg;
+ return std::forward<F>(f)(key, std::forward<Arg>(arg));
+}
+
+
+// --------------------------------------------------------------------------
+// Policy: a policy defines how to perform different operations on
+// the slots of the hashtable (see hash_policy_traits.h for the full interface
+// of policy).
+//
+// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The
+// functor should accept a key and return size_t as hash. For best performance
+// it is important that the hash function provides high entropy across all bits
+// of the hash.
+//
+// Eq: a (possibly polymorphic) functor that compares two keys for equality. It
+// should accept two (of possibly different type) keys and return a bool: true
+// if they are equal, false if they are not. If two keys compare equal, then
+// their hash values as defined by Hash MUST be equal.
+//
+// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which
+// the storage of the hashtable will be allocated and the elements will be
+// constructed and destroyed.
+// --------------------------------------------------------------------------
+template <class T>
+struct FlatHashSetPolicy
+{
+ using slot_type = T;
+ using key_type = T;
+ using init_type = T;
+ using constant_iterators = std::true_type;
+
+ template <class Allocator, class... Args>
+ static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
+ phmap::allocator_traits<Allocator>::construct(*alloc, slot,
+ std::forward<Args>(args)...);
+ }
+
+ template <class Allocator>
+ static void destroy(Allocator* alloc, slot_type* slot) {
+ phmap::allocator_traits<Allocator>::destroy(*alloc, slot);
+ }
+
+ template <class Allocator>
+ static void transfer(Allocator* alloc, slot_type* new_slot,
+ slot_type* old_slot) {
+ construct(alloc, new_slot, std::move(*old_slot));
+ destroy(alloc, old_slot);
+ }
+
+ static T& element(slot_type* slot) { return *slot; }
+
+ template <class F, class... Args>
+ static decltype(phmap::priv::DecomposeValue(
+ std::declval<F>(), std::declval<Args>()...))
+ apply(F&& f, Args&&... args) {
+ return phmap::priv::DecomposeValue(
+ std::forward<F>(f), std::forward<Args>(args)...);
+ }
+
+ static size_t space_used(const T*) { return 0; }
+};
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <class K, class V>
+struct FlatHashMapPolicy
+{
+ using slot_policy = priv::map_slot_policy<K, V>;
+ using slot_type = typename slot_policy::slot_type;
+ using key_type = K;
+ using mapped_type = V;
+ using init_type = std::pair</*non const*/ key_type, mapped_type>;
+
+ template <class Allocator, class... Args>
+ static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
+ slot_policy::construct(alloc, slot, std::forward<Args>(args)...);
+ }
+
+ template <class Allocator>
+ static void destroy(Allocator* alloc, slot_type* slot) {
+ slot_policy::destroy(alloc, slot);
+ }
+
+ template <class Allocator>
+ static void transfer(Allocator* alloc, slot_type* new_slot,
+ slot_type* old_slot) {
+ slot_policy::transfer(alloc, new_slot, old_slot);
+ }
+
+ template <class F, class... Args>
+ static decltype(phmap::priv::DecomposePair(
+ std::declval<F>(), std::declval<Args>()...))
+ apply(F&& f, Args&&... args) {
+ return phmap::priv::DecomposePair(std::forward<F>(f),
+ std::forward<Args>(args)...);
+ }
+
+ static size_t space_used(const slot_type*) { return 0; }
+
+ static std::pair<const K, V>& element(slot_type* slot) { return slot->value; }
+
+ static V& value(std::pair<const K, V>* kv) { return kv->second; }
+ static const V& value(const std::pair<const K, V>* kv) { return kv->second; }
+};
+
+template <class Reference, class Policy>
+struct node_hash_policy {
+ static_assert(std::is_lvalue_reference<Reference>::value, "");
+
+ using slot_type = typename std::remove_cv<
+ typename std::remove_reference<Reference>::type>::type*;
+
+ template <class Alloc, class... Args>
+ static void construct(Alloc* alloc, slot_type* slot, Args&&... args) {
+ *slot = Policy::new_element(alloc, std::forward<Args>(args)...);
+ }
+
+ template <class Alloc>
+ static void destroy(Alloc* alloc, slot_type* slot) {
+ Policy::delete_element(alloc, *slot);
+ }
+
+ template <class Alloc>
+ static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) {
+ *new_slot = *old_slot;
+ }
+
+ static size_t space_used(const slot_type* slot) {
+ if (slot == nullptr) return Policy::element_space_used(nullptr);
+ return Policy::element_space_used(*slot);
+ }
+
+ static Reference element(slot_type* slot) { return **slot; }
+
+ template <class T, class P = Policy>
+ static auto value(T* elem) -> decltype(P::value(elem)) {
+ return P::value(elem);
+ }
+
+ template <class... Ts, class P = Policy>
+ static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward<Ts>(ts)...)) {
+ return P::apply(std::forward<Ts>(ts)...);
+ }
+};
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <class T>
+struct NodeHashSetPolicy
+ : phmap::priv::node_hash_policy<T&, NodeHashSetPolicy<T>>
+{
+ using key_type = T;
+ using init_type = T;
+ using constant_iterators = std::true_type;
+
+ template <class Allocator, class... Args>
+ static T* new_element(Allocator* alloc, Args&&... args) {
+ using ValueAlloc =
+ typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>;
+ ValueAlloc value_alloc(*alloc);
+ T* res = phmap::allocator_traits<ValueAlloc>::allocate(value_alloc, 1);
+ phmap::allocator_traits<ValueAlloc>::construct(value_alloc, res,
+ std::forward<Args>(args)...);
+ return res;
+ }
+
+ template <class Allocator>
+ static void delete_element(Allocator* alloc, T* elem) {
+ using ValueAlloc =
+ typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>;
+ ValueAlloc value_alloc(*alloc);
+ phmap::allocator_traits<ValueAlloc>::destroy(value_alloc, elem);
+ phmap::allocator_traits<ValueAlloc>::deallocate(value_alloc, elem, 1);
+ }
+
+ template <class F, class... Args>
+ static decltype(phmap::priv::DecomposeValue(
+ std::declval<F>(), std::declval<Args>()...))
+ apply(F&& f, Args&&... args) {
+ return phmap::priv::DecomposeValue(
+ std::forward<F>(f), std::forward<Args>(args)...);
+ }
+
+ static size_t element_space_used(const T*) { return sizeof(T); }
+};
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <class Key, class Value>
+class NodeHashMapPolicy
+ : public phmap::priv::node_hash_policy<
+ std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>>
+{
+ using value_type = std::pair<const Key, Value>;
+
+public:
+ using key_type = Key;
+ using mapped_type = Value;
+ using init_type = std::pair</*non const*/ key_type, mapped_type>;
+
+ template <class Allocator, class... Args>
+ static value_type* new_element(Allocator* alloc, Args&&... args) {
+ using PairAlloc = typename phmap::allocator_traits<
+ Allocator>::template rebind_alloc<value_type>;
+ PairAlloc pair_alloc(*alloc);
+ value_type* res =
+ phmap::allocator_traits<PairAlloc>::allocate(pair_alloc, 1);
+ phmap::allocator_traits<PairAlloc>::construct(pair_alloc, res,
+ std::forward<Args>(args)...);
+ return res;
+ }
+
+ template <class Allocator>
+ static void delete_element(Allocator* alloc, value_type* pair) {
+ using PairAlloc = typename phmap::allocator_traits<
+ Allocator>::template rebind_alloc<value_type>;
+ PairAlloc pair_alloc(*alloc);
+ phmap::allocator_traits<PairAlloc>::destroy(pair_alloc, pair);
+ phmap::allocator_traits<PairAlloc>::deallocate(pair_alloc, pair, 1);
+ }
+
+ template <class F, class... Args>
+ static decltype(phmap::priv::DecomposePair(
+ std::declval<F>(), std::declval<Args>()...))
+ apply(F&& f, Args&&... args) {
+ return phmap::priv::DecomposePair(std::forward<F>(f),
+ std::forward<Args>(args)...);
+ }
+
+ static size_t element_space_used(const value_type*) {
+ return sizeof(value_type);
+ }
+
+ static Value& value(value_type* elem) { return elem->second; }
+ static const Value& value(const value_type* elem) { return elem->second; }
+};
+
+
+// --------------------------------------------------------------------------
+// hash_default
+// --------------------------------------------------------------------------
+
+#if PHMAP_HAVE_STD_STRING_VIEW
+
+// support char16_t wchar_t ....
+template<class CharT>
+struct StringHashT
+{
+ using is_transparent = void;
+
+ size_t operator()(std::basic_string_view<CharT> v) const {
+ std::string_view bv{reinterpret_cast<const char*>(v.data()), v.size() * sizeof(CharT)};
+ return std::hash<std::string_view>()(bv);
+ }
+};
+
+// Supports heterogeneous lookup for basic_string<T>-like elements.
+template<class CharT>
+struct StringHashEqT
+{
+ using Hash = StringHashT<CharT>;
+
+ struct Eq {
+ using is_transparent = void;
+
+ bool operator()(std::basic_string_view<CharT> lhs, std::basic_string_view<CharT> rhs) const {
+ return lhs == rhs;
+ }
+ };
+};
+
+template <>
+struct HashEq<std::string> : StringHashEqT<char> {};
+
+template <>
+struct HashEq<std::string_view> : StringHashEqT<char> {};
+
+// char16_t
+template <>
+struct HashEq<std::u16string> : StringHashEqT<char16_t> {};
+
+template <>
+struct HashEq<std::u16string_view> : StringHashEqT<char16_t> {};
+
+// wchar_t
+template <>
+struct HashEq<std::wstring> : StringHashEqT<wchar_t> {};
+
+template <>
+struct HashEq<std::wstring_view> : StringHashEqT<wchar_t> {};
+
+#endif
+
+// Supports heterogeneous lookup for pointers and smart pointers.
+// -------------------------------------------------------------
+template <class T>
+struct HashEq<T*>
+{
+ struct Hash {
+ using is_transparent = void;
+ template <class U>
+ size_t operator()(const U& ptr) const {
+ return phmap::Hash<const T*>{}(HashEq::ToPtr(ptr));
+ }
+ };
+
+ struct Eq {
+ using is_transparent = void;
+ template <class A, class B>
+ bool operator()(const A& a, const B& b) const {
+ return HashEq::ToPtr(a) == HashEq::ToPtr(b);
+ }
+ };
+
+private:
+ static const T* ToPtr(const T* ptr) { return ptr; }
+
+ template <class U, class D>
+ static const T* ToPtr(const std::unique_ptr<U, D>& ptr) {
+ return ptr.get();
+ }
+
+ template <class U>
+ static const T* ToPtr(const std::shared_ptr<U>& ptr) {
+ return ptr.get();
+ }
+};
+
+template <class T, class D>
+struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {};
+
+template <class T>
+struct HashEq<std::shared_ptr<T>> : HashEq<T*> {};
+
+namespace hashtable_debug_internal {
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <typename Set>
+struct HashtableDebugAccess<Set, phmap::void_t<typename Set::raw_hash_set>>
+{
+ using Traits = typename Set::PolicyTraits;
+ using Slot = typename Traits::slot_type;
+
+ static size_t GetNumProbes(const Set& set,
+ const typename Set::key_type& key) {
+ size_t num_probes = 0;
+ size_t hashval = set.hash(key);
+ auto seq = set.probe(hashval);
+ while (true) {
+ priv::Group g{set.ctrl_ + seq.offset()};
+ for (int i : g.Match(priv::H2(hashval))) {
+ if (Traits::apply(
+ typename Set::template EqualElement<typename Set::key_type>{
+ key, set.eq_ref()},
+ Traits::element(set.slots_ + seq.offset((size_t)i))))
+ return num_probes;
+ ++num_probes;
+ }
+ if (g.MatchEmpty()) return num_probes;
+ seq.next();
+ ++num_probes;
+ }
+ }
+
+ static size_t AllocatedByteSize(const Set& c) {
+ size_t capacity = c.capacity_;
+ if (capacity == 0) return 0;
+ auto layout = Set::MakeLayout(capacity);
+ size_t m = layout.AllocSize();
+
+ size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr));
+ if (per_slot != ~size_t{}) {
+ m += per_slot * c.size();
+ } else {
+ for (size_t i = 0; i != capacity; ++i) {
+ if (priv::IsFull(c.ctrl_[i])) {
+ m += Traits::space_used(c.slots_ + i);
+ }
+ }
+ }
+ return m;
+ }
+
+ static size_t LowerBoundAllocatedByteSize(size_t size) {
+ size_t capacity = GrowthToLowerboundCapacity(size);
+ if (capacity == 0) return 0;
+ auto layout = Set::MakeLayout(NormalizeCapacity(capacity));
+ size_t m = layout.AllocSize();
+ size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr));
+ if (per_slot != ~size_t{}) {
+ m += per_slot * size;
+ }
+ return m;
+ }
+};
+
+} // namespace hashtable_debug_internal
+} // namespace priv
+
+// -----------------------------------------------------------------------------
+// phmap::flat_hash_set
+// -----------------------------------------------------------------------------
+// An `phmap::flat_hash_set<T>` is an unordered associative container which has
+// been optimized for both speed and memory footprint in most common use cases.
+// Its interface is similar to that of `std::unordered_set<T>` with the
+// following notable differences:
+//
+// * Supports heterogeneous lookup, through `find()`, `operator[]()` and
+// `insert()`, provided that the set is provided a compatible heterogeneous
+// hashing function and equality operator.
+// * Invalidates any references and pointers to elements within the table after
+// `rehash()`.
+// * Contains a `capacity()` member function indicating the number of element
+// slots (open, deleted, and empty) within the hash set.
+// * Returns `void` from the `_erase(iterator)` overload.
+// -----------------------------------------------------------------------------
+template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h
+class flat_hash_set
+ : public phmap::priv::raw_hash_set<
+ phmap::priv::FlatHashSetPolicy<T>, Hash, Eq, Alloc>
+{
+ using Base = typename flat_hash_set::raw_hash_set;
+
+public:
+ flat_hash_set() {}
+#ifdef __INTEL_COMPILER
+ using Base::raw_hash_set;
+#else
+ using Base::Base;
+#endif
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())`
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::hash;
+ using Base::key_eq;
+};
+
+// -----------------------------------------------------------------------------
+// phmap::flat_hash_map
+// -----------------------------------------------------------------------------
+//
+// An `phmap::flat_hash_map<K, V>` is an unordered associative container which
+// has been optimized for both speed and memory footprint in most common use
+// cases. Its interface is similar to that of `std::unordered_map<K, V>` with
+// the following notable differences:
+//
+// * Supports heterogeneous lookup, through `find()`, `operator[]()` and
+// `insert()`, provided that the map is provided a compatible heterogeneous
+// hashing function and equality operator.
+// * Invalidates any references and pointers to elements within the table after
+// `rehash()`.
+// * Contains a `capacity()` member function indicating the number of element
+// slots (open, deleted, and empty) within the hash map.
+// * Returns `void` from the `_erase(iterator)` overload.
+// -----------------------------------------------------------------------------
+template <class K, class V, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h
+class flat_hash_map : public phmap::priv::raw_hash_map<
+ phmap::priv::FlatHashMapPolicy<K, V>,
+ Hash, Eq, Alloc> {
+ using Base = typename flat_hash_map::raw_hash_map;
+
+public:
+ flat_hash_map() {}
+#ifdef __INTEL_COMPILER
+ using Base::raw_hash_map;
+#else
+ using Base::Base;
+#endif
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::insert_or_assign;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::try_emplace;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::at;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::operator[];
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::hash;
+ using Base::key_eq;
+};
+
+// -----------------------------------------------------------------------------
+// phmap::node_hash_set
+// -----------------------------------------------------------------------------
+// An `phmap::node_hash_set<T>` is an unordered associative container which
+// has been optimized for both speed and memory footprint in most common use
+// cases. Its interface is similar to that of `std::unordered_set<T>` with the
+// following notable differences:
+//
+// * Supports heterogeneous lookup, through `find()`, `operator[]()` and
+// `insert()`, provided that the map is provided a compatible heterogeneous
+// hashing function and equality operator.
+// * Contains a `capacity()` member function indicating the number of element
+// slots (open, deleted, and empty) within the hash set.
+// * Returns `void` from the `erase(iterator)` overload.
+// -----------------------------------------------------------------------------
+template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h
+class node_hash_set
+ : public phmap::priv::raw_hash_set<
+ phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc>
+{
+ using Base = typename node_hash_set::raw_hash_set;
+
+public:
+ node_hash_set() {}
+#ifdef __INTEL_COMPILER
+ using Base::raw_hash_set;
+#else
+ using Base::Base;
+#endif
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::emplace_with_hash;
+ using Base::emplace_hint_with_hash;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::hash;
+ using Base::key_eq;
+ typename Base::hasher hash_funct() { return this->hash_function(); }
+ void resize(typename Base::size_type hint) { this->rehash(hint); }
+};
+
+// -----------------------------------------------------------------------------
+// phmap::node_hash_map
+// -----------------------------------------------------------------------------
+//
+// An `phmap::node_hash_map<K, V>` is an unordered associative container which
+// has been optimized for both speed and memory footprint in most common use
+// cases. Its interface is similar to that of `std::unordered_map<K, V>` with
+// the following notable differences:
+//
+// * Supports heterogeneous lookup, through `find()`, `operator[]()` and
+// `insert()`, provided that the map is provided a compatible heterogeneous
+// hashing function and equality operator.
+// * Contains a `capacity()` member function indicating the number of element
+// slots (open, deleted, and empty) within the hash map.
+// * Returns `void` from the `erase(iterator)` overload.
+// -----------------------------------------------------------------------------
+template <class Key, class Value, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h
+class node_hash_map
+ : public phmap::priv::raw_hash_map<
+ phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq,
+ Alloc>
+{
+ using Base = typename node_hash_map::raw_hash_map;
+
+public:
+ node_hash_map() {}
+#ifdef __INTEL_COMPILER
+ using Base::raw_hash_map;
+#else
+ using Base::Base;
+#endif
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::insert_or_assign;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::try_emplace;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::at;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::operator[];
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::hash;
+ using Base::key_eq;
+ typename Base::hasher hash_funct() { return this->hash_function(); }
+ void resize(typename Base::size_type hint) { this->rehash(hint); }
+};
+
+// -----------------------------------------------------------------------------
+// phmap::parallel_flat_hash_set
+// -----------------------------------------------------------------------------
+template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> // default values in phmap_fwd_decl.h
+class parallel_flat_hash_set
+ : public phmap::priv::parallel_hash_set<
+ N, phmap::priv::raw_hash_set, Mtx_,
+ phmap::priv::FlatHashSetPolicy<T>,
+ Hash, Eq, Alloc>
+{
+ using Base = typename parallel_flat_hash_set::parallel_hash_set;
+
+public:
+ parallel_flat_hash_set() {}
+#ifdef __INTEL_COMPILER
+ using Base::parallel_hash_set;
+#else
+ using Base::Base;
+#endif
+ using Base::hash;
+ using Base::subidx;
+ using Base::subcnt;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::emplace_with_hash;
+ using Base::emplace_hint_with_hash;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::key_eq;
+};
+
+// -----------------------------------------------------------------------------
+// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h
+// -----------------------------------------------------------------------------
+template <class K, class V, class Hash, class Eq, class Alloc, size_t N, class Mtx_>
+class parallel_flat_hash_map : public phmap::priv::parallel_hash_map<
+ N, phmap::priv::raw_hash_set, Mtx_,
+ phmap::priv::FlatHashMapPolicy<K, V>,
+ Hash, Eq, Alloc>
+{
+ using Base = typename parallel_flat_hash_map::parallel_hash_map;
+
+public:
+ parallel_flat_hash_map() {}
+#ifdef __INTEL_COMPILER
+ using Base::parallel_hash_map;
+#else
+ using Base::Base;
+#endif
+ using Base::hash;
+ using Base::subidx;
+ using Base::subcnt;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::insert_or_assign;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::try_emplace;
+ using Base::emplace_with_hash;
+ using Base::emplace_hint_with_hash;
+ using Base::try_emplace_with_hash;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::at;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::operator[];
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::key_eq;
+};
+
+// -----------------------------------------------------------------------------
+// phmap::parallel_node_hash_set
+// -----------------------------------------------------------------------------
+template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_>
+class parallel_node_hash_set
+ : public phmap::priv::parallel_hash_set<
+ N, phmap::priv::raw_hash_set, Mtx_,
+ phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc>
+{
+ using Base = typename parallel_node_hash_set::parallel_hash_set;
+
+public:
+ parallel_node_hash_set() {}
+#ifdef __INTEL_COMPILER
+ using Base::parallel_hash_set;
+#else
+ using Base::Base;
+#endif
+ using Base::hash;
+ using Base::subidx;
+ using Base::subcnt;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::emplace_with_hash;
+ using Base::emplace_hint_with_hash;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::key_eq;
+ typename Base::hasher hash_funct() { return this->hash_function(); }
+ void resize(typename Base::size_type hint) { this->rehash(hint); }
+};
+
+// -----------------------------------------------------------------------------
+// phmap::parallel_node_hash_map
+// -----------------------------------------------------------------------------
+template <class Key, class Value, class Hash, class Eq, class Alloc, size_t N, class Mtx_>
+class parallel_node_hash_map
+ : public phmap::priv::parallel_hash_map<
+ N, phmap::priv::raw_hash_set, Mtx_,
+ phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq,
+ Alloc>
+{
+ using Base = typename parallel_node_hash_map::parallel_hash_map;
+
+public:
+ parallel_node_hash_map() {}
+#ifdef __INTEL_COMPILER
+ using Base::parallel_hash_map;
+#else
+ using Base::Base;
+#endif
+ using Base::hash;
+ using Base::subidx;
+ using Base::subcnt;
+ using Base::begin;
+ using Base::cbegin;
+ using Base::cend;
+ using Base::end;
+ using Base::capacity;
+ using Base::empty;
+ using Base::max_size;
+ using Base::size;
+ using Base::clear;
+ using Base::erase;
+ using Base::insert;
+ using Base::insert_or_assign;
+ using Base::emplace;
+ using Base::emplace_hint;
+ using Base::try_emplace;
+ using Base::emplace_with_hash;
+ using Base::emplace_hint_with_hash;
+ using Base::try_emplace_with_hash;
+ using Base::extract;
+ using Base::merge;
+ using Base::swap;
+ using Base::rehash;
+ using Base::reserve;
+ using Base::at;
+ using Base::contains;
+ using Base::count;
+ using Base::equal_range;
+ using Base::find;
+ using Base::operator[];
+ using Base::bucket_count;
+ using Base::load_factor;
+ using Base::max_load_factor;
+ using Base::get_allocator;
+ using Base::hash_function;
+ using Base::key_eq;
+ typename Base::hasher hash_funct() { return this->hash_function(); }
+ void resize(typename Base::size_type hint) { this->rehash(hint); }
+};
+
+} // namespace phmap
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+
+#endif // phmap_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_base.h b/benchmarks/others/parallel_hashmap/phmap_base.h
new file mode 100644
index 00000000..6b9ea9ee
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_base.h
@@ -0,0 +1,5171 @@
+#if !defined(phmap_base_h_guard_)
+#define phmap_base_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
+// with modifications.
+//
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <initializer_list>
+#include <iterator>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <functional>
+#include <tuple>
+#include <utility>
+#include <memory>
+#include <mutex> // for std::lock
+
+#include "phmap_config.h"
+
+#ifdef PHMAP_HAVE_SHARED_MUTEX
+ #include <shared_mutex> // after "phmap_config.h"
+#endif
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+ #pragma warning(disable : 4582) // constructor is not implicitly called
+ #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted
+ #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted
+ #pragma warning(disable : 4710) // function not inlined
+ #pragma warning(disable : 4711) // selected for automatic inline expansion
+ #pragma warning(disable : 4820) // '6' bytes padding added after data member
+#endif // _MSC_VER
+
+namespace phmap {
+
+template <class T> using Allocator = typename std::allocator<T>;
+
+template<class T1, class T2> using Pair = typename std::pair<T1, T2>;
+
+template <class T>
+struct EqualTo
+{
+ inline bool operator()(const T& a, const T& b) const
+ {
+ return std::equal_to<T>()(a, b);
+ }
+};
+
+template <class T>
+struct Less
+{
+ inline bool operator()(const T& a, const T& b) const
+ {
+ return std::less<T>()(a, b);
+ }
+};
+
+namespace type_traits_internal {
+
+template <typename... Ts>
+struct VoidTImpl {
+ using type = void;
+};
+
+// This trick to retrieve a default alignment is necessary for our
+// implementation of aligned_storage_t to be consistent with any implementation
+// of std::aligned_storage.
+// ---------------------------------------------------------------------------
+template <size_t Len, typename T = std::aligned_storage<Len>>
+struct default_alignment_of_aligned_storage;
+
+template <size_t Len, size_t Align>
+struct default_alignment_of_aligned_storage<Len,
+ std::aligned_storage<Len, Align>> {
+ static constexpr size_t value = Align;
+};
+
+// NOTE: The `is_detected` family of templates here differ from the library
+// fundamentals specification in that for library fundamentals, `Op<Args...>` is
+// evaluated as soon as the type `is_detected<Op, Args...>` undergoes
+// substitution, regardless of whether or not the `::value` is accessed. That
+// is inconsistent with all other standard traits and prevents lazy evaluation
+// in larger contexts (such as if the `is_detected` check is a trailing argument
+// of a `conjunction`. This implementation opts to instead be lazy in the same
+// way that the standard traits are (this "defect" of the detection idiom
+// specifications has been reported).
+// ---------------------------------------------------------------------------
+
+template <class Enabler, template <class...> class Op, class... Args>
+struct is_detected_impl {
+ using type = std::false_type;
+};
+
+template <template <class...> class Op, class... Args>
+struct is_detected_impl<typename VoidTImpl<Op<Args...>>::type, Op, Args...> {
+ using type = std::true_type;
+};
+
+template <template <class...> class Op, class... Args>
+struct is_detected : is_detected_impl<void, Op, Args...>::type {};
+
+template <class Enabler, class To, template <class...> class Op, class... Args>
+struct is_detected_convertible_impl {
+ using type = std::false_type;
+};
+
+template <class To, template <class...> class Op, class... Args>
+struct is_detected_convertible_impl<
+ typename std::enable_if<std::is_convertible<Op<Args...>, To>::value>::type,
+ To, Op, Args...> {
+ using type = std::true_type;
+};
+
+template <class To, template <class...> class Op, class... Args>
+struct is_detected_convertible
+ : is_detected_convertible_impl<void, To, Op, Args...>::type {};
+
+template <typename T>
+using IsCopyAssignableImpl =
+ decltype(std::declval<T&>() = std::declval<const T&>());
+
+template <typename T>
+using IsMoveAssignableImpl = decltype(std::declval<T&>() = std::declval<T&&>());
+
+} // namespace type_traits_internal
+
+template <typename T>
+struct is_copy_assignable : type_traits_internal::is_detected<
+ type_traits_internal::IsCopyAssignableImpl, T> {
+};
+
+template <typename T>
+struct is_move_assignable : type_traits_internal::is_detected<
+ type_traits_internal::IsMoveAssignableImpl, T> {
+};
+
+// ---------------------------------------------------------------------------
+// void_t()
+//
+// Ignores the type of any its arguments and returns `void`. In general, this
+// metafunction allows you to create a general case that maps to `void` while
+// allowing specializations that map to specific types.
+//
+// This metafunction is designed to be a drop-in replacement for the C++17
+// `std::void_t` metafunction.
+//
+// NOTE: `phmap::void_t` does not use the standard-specified implementation so
+// that it can remain compatible with gcc < 5.1. This can introduce slightly
+// different behavior, such as when ordering partial specializations.
+// ---------------------------------------------------------------------------
+template <typename... Ts>
+using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type;
+
+// ---------------------------------------------------------------------------
+// conjunction
+//
+// Performs a compile-time logical AND operation on the passed types (which
+// must have `::value` members convertible to `bool`. Short-circuits if it
+// encounters any `false` members (and does not compare the `::value` members
+// of any remaining arguments).
+//
+// This metafunction is designed to be a drop-in replacement for the C++17
+// `std::conjunction` metafunction.
+// ---------------------------------------------------------------------------
+template <typename... Ts>
+struct conjunction;
+
+template <typename T, typename... Ts>
+struct conjunction<T, Ts...>
+ : std::conditional<T::value, conjunction<Ts...>, T>::type {};
+
+template <typename T>
+struct conjunction<T> : T {};
+
+template <>
+struct conjunction<> : std::true_type {};
+
+// ---------------------------------------------------------------------------
+// disjunction
+//
+// Performs a compile-time logical OR operation on the passed types (which
+// must have `::value` members convertible to `bool`. Short-circuits if it
+// encounters any `true` members (and does not compare the `::value` members
+// of any remaining arguments).
+//
+// This metafunction is designed to be a drop-in replacement for the C++17
+// `std::disjunction` metafunction.
+// ---------------------------------------------------------------------------
+template <typename... Ts>
+struct disjunction;
+
+template <typename T, typename... Ts>
+struct disjunction<T, Ts...> :
+ std::conditional<T::value, T, disjunction<Ts...>>::type {};
+
+template <typename T>
+struct disjunction<T> : T {};
+
+template <>
+struct disjunction<> : std::false_type {};
+
+template <typename T>
+struct negation : std::integral_constant<bool, !T::value> {};
+
+template <typename T>
+struct is_trivially_destructible
+ : std::integral_constant<bool, __has_trivial_destructor(T) &&
+ std::is_destructible<T>::value> {};
+
+template <typename T>
+struct is_trivially_default_constructible
+ : std::integral_constant<bool, __has_trivial_constructor(T) &&
+ std::is_default_constructible<T>::value &&
+ is_trivially_destructible<T>::value> {};
+
+template <typename T>
+struct is_trivially_copy_constructible
+ : std::integral_constant<bool, __has_trivial_copy(T) &&
+ std::is_copy_constructible<T>::value &&
+ is_trivially_destructible<T>::value> {};
+
+template <typename T>
+struct is_trivially_copy_assignable
+ : std::integral_constant<
+ bool, __has_trivial_assign(typename std::remove_reference<T>::type) &&
+ phmap::is_copy_assignable<T>::value> {};
+
+// -----------------------------------------------------------------------------
+// C++14 "_t" trait aliases
+// -----------------------------------------------------------------------------
+
+template <typename T>
+using remove_cv_t = typename std::remove_cv<T>::type;
+
+template <typename T>
+using remove_const_t = typename std::remove_const<T>::type;
+
+template <typename T>
+using remove_volatile_t = typename std::remove_volatile<T>::type;
+
+template <typename T>
+using add_cv_t = typename std::add_cv<T>::type;
+
+template <typename T>
+using add_const_t = typename std::add_const<T>::type;
+
+template <typename T>
+using add_volatile_t = typename std::add_volatile<T>::type;
+
+template <typename T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+
+template <typename T>
+using add_lvalue_reference_t = typename std::add_lvalue_reference<T>::type;
+
+template <typename T>
+using add_rvalue_reference_t = typename std::add_rvalue_reference<T>::type;
+
+template <typename T>
+using remove_pointer_t = typename std::remove_pointer<T>::type;
+
+template <typename T>
+using add_pointer_t = typename std::add_pointer<T>::type;
+
+template <typename T>
+using make_signed_t = typename std::make_signed<T>::type;
+
+template <typename T>
+using make_unsigned_t = typename std::make_unsigned<T>::type;
+
+template <typename T>
+using remove_extent_t = typename std::remove_extent<T>::type;
+
+template <typename T>
+using remove_all_extents_t = typename std::remove_all_extents<T>::type;
+
+template <size_t Len, size_t Align = type_traits_internal::
+ default_alignment_of_aligned_storage<Len>::value>
+using aligned_storage_t = typename std::aligned_storage<Len, Align>::type;
+
+template <typename T>
+using decay_t = typename std::decay<T>::type;
+
+template <bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template <bool B, typename T, typename F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+
+
+template <typename... T>
+using common_type_t = typename std::common_type<T...>::type;
+
+template <typename T>
+using underlying_type_t = typename std::underlying_type<T>::type;
+
+template< class F, class... ArgTypes>
+#if PHMAP_HAVE_CC17
+ using invoke_result_t = typename std::invoke_result_t<F, ArgTypes...>;
+#else
+ using invoke_result_t = typename std::result_of<F(ArgTypes...)>::type;
+#endif
+
+namespace type_traits_internal {
+
+// ----------------------------------------------------------------------
+// In MSVC we can't probe std::hash or stdext::hash because it triggers a
+// static_assert instead of failing substitution. Libc++ prior to 4.0
+// also used a static_assert.
+// ----------------------------------------------------------------------
+#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \
+ _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11)
+ #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0
+#else
+ #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1
+#endif
+
+#if !PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_
+ template <typename Key, typename = size_t>
+ struct IsHashable : std::true_type {};
+#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_
+ template <typename Key, typename = void>
+ struct IsHashable : std::false_type {};
+
+ template <typename Key>
+ struct IsHashable<Key,
+ phmap::enable_if_t<std::is_convertible<
+ decltype(std::declval<std::hash<Key>&>()(std::declval<Key const&>())),
+ std::size_t>::value>> : std::true_type {};
+#endif
+
+struct AssertHashEnabledHelper
+{
+private:
+ static void Sink(...) {}
+ struct NAT {};
+
+ template <class Key>
+ static auto GetReturnType(int)
+ -> decltype(std::declval<std::hash<Key>>()(std::declval<Key const&>()));
+ template <class Key>
+ static NAT GetReturnType(...);
+
+ template <class Key>
+ static std::nullptr_t DoIt() {
+ static_assert(IsHashable<Key>::value,
+ "std::hash<Key> does not provide a call operator");
+ static_assert(
+ std::is_default_constructible<std::hash<Key>>::value,
+ "std::hash<Key> must be default constructible when it is enabled");
+ static_assert(
+ std::is_copy_constructible<std::hash<Key>>::value,
+ "std::hash<Key> must be copy constructible when it is enabled");
+ static_assert(phmap::is_copy_assignable<std::hash<Key>>::value,
+ "std::hash<Key> must be copy assignable when it is enabled");
+ // is_destructible is unchecked as it's implied by each of the
+ // is_constructible checks.
+ using ReturnType = decltype(GetReturnType<Key>(0));
+ static_assert(std::is_same<ReturnType, NAT>::value ||
+ std::is_same<ReturnType, size_t>::value,
+ "std::hash<Key> must return size_t");
+ return nullptr;
+ }
+
+ template <class... Ts>
+ friend void AssertHashEnabled();
+};
+
+template <class... Ts>
+inline void AssertHashEnabled
+()
+{
+ using Helper = AssertHashEnabledHelper;
+ Helper::Sink(Helper::DoIt<Ts>()...);
+}
+
+} // namespace type_traits_internal
+
+} // namespace phmap
+
+
+// -----------------------------------------------------------------------------
+// hash_policy_traits
+// -----------------------------------------------------------------------------
+namespace phmap {
+namespace priv {
+
+// Defines how slots are initialized/destroyed/moved.
+template <class Policy, class = void>
+struct hash_policy_traits
+{
+private:
+ struct ReturnKey
+ {
+ // We return `Key` here.
+ // When Key=T&, we forward the lvalue reference.
+ // When Key=T, we return by value to avoid a dangling reference.
+ // eg, for string_hash_map.
+ template <class Key, class... Args>
+ Key operator()(Key&& k, const Args&...) const {
+ return std::forward<Key>(k);
+ }
+ };
+
+ template <class P = Policy, class = void>
+ struct ConstantIteratorsImpl : std::false_type {};
+
+ template <class P>
+ struct ConstantIteratorsImpl<P, phmap::void_t<typename P::constant_iterators>>
+ : P::constant_iterators {};
+
+public:
+ // The actual object stored in the hash table.
+ using slot_type = typename Policy::slot_type;
+
+ // The type of the keys stored in the hashtable.
+ using key_type = typename Policy::key_type;
+
+ // The argument type for insertions into the hashtable. This is different
+ // from value_type for increased performance. See initializer_list constructor
+ // and insert() member functions for more details.
+ using init_type = typename Policy::init_type;
+
+ using reference = decltype(Policy::element(std::declval<slot_type*>()));
+ using pointer = typename std::remove_reference<reference>::type*;
+ using value_type = typename std::remove_reference<reference>::type;
+
+ // Policies can set this variable to tell raw_hash_set that all iterators
+ // should be constant, even `iterator`. This is useful for set-like
+ // containers.
+ // Defaults to false if not provided by the policy.
+ using constant_iterators = ConstantIteratorsImpl<>;
+
+ // PRECONDITION: `slot` is UNINITIALIZED
+ // POSTCONDITION: `slot` is INITIALIZED
+ template <class Alloc, class... Args>
+ static void construct(Alloc* alloc, slot_type* slot, Args&&... args) {
+ Policy::construct(alloc, slot, std::forward<Args>(args)...);
+ }
+
+ // PRECONDITION: `slot` is INITIALIZED
+ // POSTCONDITION: `slot` is UNINITIALIZED
+ template <class Alloc>
+ static void destroy(Alloc* alloc, slot_type* slot) {
+ Policy::destroy(alloc, slot);
+ }
+
+ // Transfers the `old_slot` to `new_slot`. Any memory allocated by the
+ // allocator inside `old_slot` to `new_slot` can be transferred.
+ //
+ // OPTIONAL: defaults to:
+ //
+ // clone(new_slot, std::move(*old_slot));
+ // destroy(old_slot);
+ //
+ // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED
+ // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is
+ // UNINITIALIZED
+ template <class Alloc>
+ static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) {
+ transfer_impl(alloc, new_slot, old_slot, 0);
+ }
+
+ // PRECONDITION: `slot` is INITIALIZED
+ // POSTCONDITION: `slot` is INITIALIZED
+ template <class P = Policy>
+ static auto element(slot_type* slot) -> decltype(P::element(slot)) {
+ return P::element(slot);
+ }
+
+ // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`.
+ //
+ // If `slot` is nullptr, returns the constant amount of memory owned by any
+ // full slot or -1 if slots own variable amounts of memory.
+ //
+ // PRECONDITION: `slot` is INITIALIZED or nullptr
+ template <class P = Policy>
+ static size_t space_used(const slot_type* slot) {
+ return P::space_used(slot);
+ }
+
+ // Provides generalized access to the key for elements, both for elements in
+ // the table and for elements that have not yet been inserted (or even
+ // constructed). We would like an API that allows us to say: `key(args...)`
+ // but we cannot do that for all cases, so we use this more general API that
+ // can be used for many things, including the following:
+ //
+ // - Given an element in a table, get its key.
+ // - Given an element initializer, get its key.
+ // - Given `emplace()` arguments, get the element key.
+ //
+ // Implementations of this must adhere to a very strict technical
+ // specification around aliasing and consuming arguments:
+ //
+ // Let `value_type` be the result type of `element()` without ref- and
+ // cv-qualifiers. The first argument is a functor, the rest are constructor
+ // arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where
+ // `k` is the element key, and `xs...` are the new constructor arguments for
+ // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias
+ // `ts...`. The key won't be touched once `xs...` are used to construct an
+ // element; `ts...` won't be touched at all, which allows `apply()` to consume
+ // any rvalues among them.
+ //
+ // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not
+ // trigger a hard compile error unless it originates from `f`. In other words,
+ // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not
+ // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK.
+ //
+ // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`,
+ // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not.
+ template <class F, class... Ts, class P = Policy>
+ static auto apply(F&& f, Ts&&... ts)
+ -> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) {
+ return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...);
+ }
+
+ // Returns the "key" portion of the slot.
+ // Used for node handle manipulation.
+ template <class P = Policy>
+ static auto key(slot_type* slot)
+ -> decltype(P::apply(ReturnKey(), element(slot))) {
+ return P::apply(ReturnKey(), element(slot));
+ }
+
+ // Returns the "value" (as opposed to the "key") portion of the element. Used
+ // by maps to implement `operator[]`, `at()` and `insert_or_assign()`.
+ template <class T, class P = Policy>
+ static auto value(T* elem) -> decltype(P::value(elem)) {
+ return P::value(elem);
+ }
+
+private:
+
+ // Use auto -> decltype as an enabler.
+ template <class Alloc, class P = Policy>
+ static auto transfer_impl(Alloc* alloc, slot_type* new_slot,
+ slot_type* old_slot, int)
+ -> decltype((void)P::transfer(alloc, new_slot, old_slot)) {
+ P::transfer(alloc, new_slot, old_slot);
+ }
+
+ template <class Alloc>
+ static void transfer_impl(Alloc* alloc, slot_type* new_slot,
+ slot_type* old_slot, char) {
+ construct(alloc, new_slot, std::move(element(old_slot)));
+ destroy(alloc, old_slot);
+ }
+};
+
+} // namespace priv
+} // namespace phmap
+
+// -----------------------------------------------------------------------------
+// file utility.h
+// -----------------------------------------------------------------------------
+
+// --------- identity.h
+namespace phmap {
+namespace internal {
+
+template <typename T>
+struct identity {
+ typedef T type;
+};
+
+template <typename T>
+using identity_t = typename identity<T>::type;
+
+} // namespace internal
+} // namespace phmap
+
+
+// --------- inline_variable.h
+
+#ifdef __cpp_inline_variables
+
+#if defined(__clang__)
+ #define PHMAP_INTERNAL_EXTERN_DECL(type, name) \
+ extern const ::phmap::internal::identity_t<type> name;
+#else // Otherwise, just define the macro to do nothing.
+ #define PHMAP_INTERNAL_EXTERN_DECL(type, name)
+#endif // defined(__clang__)
+
+// See above comment at top of file for details.
+#define PHMAP_INTERNAL_INLINE_CONSTEXPR(type, name, init) \
+ PHMAP_INTERNAL_EXTERN_DECL(type, name) \
+ inline constexpr ::phmap::internal::identity_t<type> name = init
+
+#else
+
+// See above comment at top of file for details.
+//
+// Note:
+// identity_t is used here so that the const and name are in the
+// appropriate place for pointer types, reference types, function pointer
+// types, etc..
+#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \
+ template <class /*PhmapInternalDummy*/ = void> \
+ struct PhmapInternalInlineVariableHolder##name { \
+ static constexpr ::phmap::internal::identity_t<var_type> kInstance = init; \
+ }; \
+ \
+ template <class PhmapInternalDummy> \
+ constexpr ::phmap::internal::identity_t<var_type> \
+ PhmapInternalInlineVariableHolder##name<PhmapInternalDummy>::kInstance; \
+ \
+ static constexpr const ::phmap::internal::identity_t<var_type>& \
+ name = /* NOLINT */ \
+ PhmapInternalInlineVariableHolder##name<>::kInstance; \
+ static_assert(sizeof(void (*)(decltype(name))) != 0, \
+ "Silence unused variable warnings.")
+
+#endif // __cpp_inline_variables
+
+// ----------- throw_delegate
+
+namespace phmap {
+namespace base_internal {
+
+namespace {
+template <typename T>
+#ifdef PHMAP_HAVE_EXCEPTIONS
+[[noreturn]] void Throw(const T& error) {
+ throw error;
+}
+#else
+[[noreturn]] void Throw(const T&) {
+ std::abort();
+}
+#endif
+} // namespace
+
+static inline void ThrowStdLogicError(const std::string& what_arg) {
+ Throw(std::logic_error(what_arg));
+}
+static inline void ThrowStdLogicError(const char* what_arg) {
+ Throw(std::logic_error(what_arg));
+}
+static inline void ThrowStdInvalidArgument(const std::string& what_arg) {
+ Throw(std::invalid_argument(what_arg));
+}
+static inline void ThrowStdInvalidArgument(const char* what_arg) {
+ Throw(std::invalid_argument(what_arg));
+}
+
+static inline void ThrowStdDomainError(const std::string& what_arg) {
+ Throw(std::domain_error(what_arg));
+}
+static inline void ThrowStdDomainError(const char* what_arg) {
+ Throw(std::domain_error(what_arg));
+}
+
+static inline void ThrowStdLengthError(const std::string& what_arg) {
+ Throw(std::length_error(what_arg));
+}
+static inline void ThrowStdLengthError(const char* what_arg) {
+ Throw(std::length_error(what_arg));
+}
+
+static inline void ThrowStdOutOfRange(const std::string& what_arg) {
+ Throw(std::out_of_range(what_arg));
+}
+static inline void ThrowStdOutOfRange(const char* what_arg) {
+ Throw(std::out_of_range(what_arg));
+}
+
+static inline void ThrowStdRuntimeError(const std::string& what_arg) {
+ Throw(std::runtime_error(what_arg));
+}
+static inline void ThrowStdRuntimeError(const char* what_arg) {
+ Throw(std::runtime_error(what_arg));
+}
+
+static inline void ThrowStdRangeError(const std::string& what_arg) {
+ Throw(std::range_error(what_arg));
+}
+static inline void ThrowStdRangeError(const char* what_arg) {
+ Throw(std::range_error(what_arg));
+}
+
+static inline void ThrowStdOverflowError(const std::string& what_arg) {
+ Throw(std::overflow_error(what_arg));
+}
+static inline void ThrowStdOverflowError(const char* what_arg) {
+ Throw(std::overflow_error(what_arg));
+}
+
+static inline void ThrowStdUnderflowError(const std::string& what_arg) {
+ Throw(std::underflow_error(what_arg));
+}
+static inline void ThrowStdUnderflowError(const char* what_arg) {
+ Throw(std::underflow_error(what_arg));
+}
+
+static inline void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); }
+
+static inline void ThrowStdBadAlloc() { Throw(std::bad_alloc()); }
+
+} // namespace base_internal
+} // namespace phmap
+
+// ----------- invoke.h
+
+namespace phmap {
+namespace base_internal {
+
+template <typename Derived>
+struct StrippedAccept
+{
+ template <typename... Args>
+ struct Accept : Derived::template AcceptImpl<typename std::remove_cv<
+ typename std::remove_reference<Args>::type>::type...> {};
+};
+
+// (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T
+// and t1 is an object of type T or a reference to an object of type T or a
+// reference to an object of a type derived from T.
+struct MemFunAndRef : StrippedAccept<MemFunAndRef>
+{
+ template <typename... Args>
+ struct AcceptImpl : std::false_type {};
+
+ template <typename R, typename C, typename... Params, typename Obj,
+ typename... Args>
+ struct AcceptImpl<R (C::*)(Params...), Obj, Args...>
+ : std::is_base_of<C, Obj> {};
+
+ template <typename R, typename C, typename... Params, typename Obj,
+ typename... Args>
+ struct AcceptImpl<R (C::*)(Params...) const, Obj, Args...>
+ : std::is_base_of<C, Obj> {};
+
+ template <typename MemFun, typename Obj, typename... Args>
+ static decltype((std::declval<Obj>().*
+ std::declval<MemFun>())(std::declval<Args>()...))
+ Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) {
+ return (std::forward<Obj>(obj).*
+ std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...);
+ }
+};
+
+// ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a
+// class T and t1 is not one of the types described in the previous item.
+struct MemFunAndPtr : StrippedAccept<MemFunAndPtr>
+{
+ template <typename... Args>
+ struct AcceptImpl : std::false_type {};
+
+ template <typename R, typename C, typename... Params, typename Ptr,
+ typename... Args>
+ struct AcceptImpl<R (C::*)(Params...), Ptr, Args...>
+ : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {};
+
+ template <typename R, typename C, typename... Params, typename Ptr,
+ typename... Args>
+ struct AcceptImpl<R (C::*)(Params...) const, Ptr, Args...>
+ : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {};
+
+ template <typename MemFun, typename Ptr, typename... Args>
+ static decltype(((*std::declval<Ptr>()).*
+ std::declval<MemFun>())(std::declval<Args>()...))
+ Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) {
+ return ((*std::forward<Ptr>(ptr)).*
+ std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...);
+ }
+};
+
+// t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is
+// an object of type T or a reference to an object of type T or a reference
+// to an object of a type derived from T.
+struct DataMemAndRef : StrippedAccept<DataMemAndRef>
+{
+ template <typename... Args>
+ struct AcceptImpl : std::false_type {};
+
+ template <typename R, typename C, typename Obj>
+ struct AcceptImpl<R C::*, Obj> : std::is_base_of<C, Obj> {};
+
+ template <typename DataMem, typename Ref>
+ static decltype(std::declval<Ref>().*std::declval<DataMem>()) Invoke(
+ DataMem&& data_mem, Ref&& ref) {
+ return std::forward<Ref>(ref).*std::forward<DataMem>(data_mem);
+ }
+};
+
+// (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1
+// is not one of the types described in the previous item.
+struct DataMemAndPtr : StrippedAccept<DataMemAndPtr>
+{
+ template <typename... Args>
+ struct AcceptImpl : std::false_type {};
+
+ template <typename R, typename C, typename Ptr>
+ struct AcceptImpl<R C::*, Ptr>
+ : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {};
+
+ template <typename DataMem, typename Ptr>
+ static decltype((*std::declval<Ptr>()).*std::declval<DataMem>()) Invoke(
+ DataMem&& data_mem, Ptr&& ptr) {
+ return (*std::forward<Ptr>(ptr)).*std::forward<DataMem>(data_mem);
+ }
+};
+
+// f(t1, t2, ..., tN) in all other cases.
+struct Callable
+{
+ // Callable doesn't have Accept because it's the last clause that gets picked
+ // when none of the previous clauses are applicable.
+ template <typename F, typename... Args>
+ static decltype(std::declval<F>()(std::declval<Args>()...)) Invoke(
+ F&& f, Args&&... args) {
+ return std::forward<F>(f)(std::forward<Args>(args)...);
+ }
+};
+
+// Resolves to the first matching clause.
+template <typename... Args>
+struct Invoker
+{
+ typedef typename std::conditional<
+ MemFunAndRef::Accept<Args...>::value, MemFunAndRef,
+ typename std::conditional<
+ MemFunAndPtr::Accept<Args...>::value, MemFunAndPtr,
+ typename std::conditional<
+ DataMemAndRef::Accept<Args...>::value, DataMemAndRef,
+ typename std::conditional<DataMemAndPtr::Accept<Args...>::value,
+ DataMemAndPtr, Callable>::type>::type>::
+ type>::type type;
+};
+
+// The result type of Invoke<F, Args...>.
+template <typename F, typename... Args>
+using InvokeT = decltype(Invoker<F, Args...>::type::Invoke(
+ std::declval<F>(), std::declval<Args>()...));
+
+// Invoke(f, args...) is an implementation of INVOKE(f, args...) from section
+// [func.require] of the C++ standard.
+template <typename F, typename... Args>
+InvokeT<F, Args...> Invoke(F&& f, Args&&... args) {
+ return Invoker<F, Args...>::type::Invoke(std::forward<F>(f),
+ std::forward<Args>(args)...);
+}
+} // namespace base_internal
+} // namespace phmap
+
+
+// ----------- utility.h
+
+namespace phmap {
+
+// integer_sequence
+//
+// Class template representing a compile-time integer sequence. An instantiation
+// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its
+// type through its template arguments (which is a common need when
+// working with C++11 variadic templates). `phmap::integer_sequence` is designed
+// to be a drop-in replacement for C++14's `std::integer_sequence`.
+//
+// Example:
+//
+// template< class T, T... Ints >
+// void user_function(integer_sequence<T, Ints...>);
+//
+// int main()
+// {
+// // user_function's `T` will be deduced to `int` and `Ints...`
+// // will be deduced to `0, 1, 2, 3, 4`.
+// user_function(make_integer_sequence<int, 5>());
+// }
+template <typename T, T... Ints>
+struct integer_sequence
+{
+ using value_type = T;
+ static constexpr size_t size() noexcept { return sizeof...(Ints); }
+};
+
+// index_sequence
+//
+// A helper template for an `integer_sequence` of `size_t`,
+// `phmap::index_sequence` is designed to be a drop-in replacement for C++14's
+// `std::index_sequence`.
+template <size_t... Ints>
+using index_sequence = integer_sequence<size_t, Ints...>;
+
+namespace utility_internal {
+
+template <typename Seq, size_t SeqSize, size_t Rem>
+struct Extend;
+
+// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency.
+template <typename T, T... Ints, size_t SeqSize>
+struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> {
+ using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>;
+};
+
+template <typename T, T... Ints, size_t SeqSize>
+struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> {
+ using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>;
+};
+
+// Recursion helper for 'make_integer_sequence<T, N>'.
+// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'.
+template <typename T, size_t N>
+struct Gen {
+ using type =
+ typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type;
+};
+
+template <typename T>
+struct Gen<T, 0> {
+ using type = integer_sequence<T>;
+};
+
+} // namespace utility_internal
+
+// Compile-time sequences of integers
+
+// make_integer_sequence
+//
+// This template alias is equivalent to
+// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in
+// replacement for C++14's `std::make_integer_sequence`.
+template <typename T, T N>
+using make_integer_sequence = typename utility_internal::Gen<T, N>::type;
+
+// make_index_sequence
+//
+// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`,
+// and is designed to be a drop-in replacement for C++14's
+// `std::make_index_sequence`.
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+
+// index_sequence_for
+//
+// Converts a typename pack into an index sequence of the same length, and
+// is designed to be a drop-in replacement for C++14's
+// `std::index_sequence_for()`
+template <typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+// Tag types
+
+#ifdef PHMAP_HAVE_STD_OPTIONAL
+
+using std::in_place_t;
+using std::in_place;
+
+#else // PHMAP_HAVE_STD_OPTIONAL
+
+// in_place_t
+//
+// Tag type used to specify in-place construction, such as with
+// `phmap::optional`, designed to be a drop-in replacement for C++17's
+// `std::in_place_t`.
+struct in_place_t {};
+
+PHMAP_INTERNAL_INLINE_CONSTEXPR(in_place_t, in_place, {});
+
+#endif // PHMAP_HAVE_STD_OPTIONAL
+
+#if defined(PHMAP_HAVE_STD_ANY) || defined(PHMAP_HAVE_STD_VARIANT)
+using std::in_place_type_t;
+#else
+
+// in_place_type_t
+//
+// Tag type used for in-place construction when the type to construct needs to
+// be specified, such as with `phmap::any`, designed to be a drop-in replacement
+// for C++17's `std::in_place_type_t`.
+template <typename T>
+struct in_place_type_t {};
+#endif // PHMAP_HAVE_STD_ANY || PHMAP_HAVE_STD_VARIANT
+
+#ifdef PHMAP_HAVE_STD_VARIANT
+using std::in_place_index_t;
+#else
+
+// in_place_index_t
+//
+// Tag type used for in-place construction when the type to construct needs to
+// be specified, such as with `phmap::any`, designed to be a drop-in replacement
+// for C++17's `std::in_place_index_t`.
+template <size_t I>
+struct in_place_index_t {};
+#endif // PHMAP_HAVE_STD_VARIANT
+
+// Constexpr move and forward
+
+// move()
+//
+// A constexpr version of `std::move()`, designed to be a drop-in replacement
+// for C++14's `std::move()`.
+template <typename T>
+constexpr phmap::remove_reference_t<T>&& move(T&& t) noexcept {
+ return static_cast<phmap::remove_reference_t<T>&&>(t);
+}
+
+// forward()
+//
+// A constexpr version of `std::forward()`, designed to be a drop-in replacement
+// for C++14's `std::forward()`.
+template <typename T>
+constexpr T&& forward(
+ phmap::remove_reference_t<T>& t) noexcept { // NOLINT(runtime/references)
+ return static_cast<T&&>(t);
+}
+
+namespace utility_internal {
+// Helper method for expanding tuple into a called method.
+template <typename Functor, typename Tuple, std::size_t... Indexes>
+auto apply_helper(Functor&& functor, Tuple&& t, index_sequence<Indexes...>)
+ -> decltype(phmap::base_internal::Invoke(
+ phmap::forward<Functor>(functor),
+ std::get<Indexes>(phmap::forward<Tuple>(t))...)) {
+ return phmap::base_internal::Invoke(
+ phmap::forward<Functor>(functor),
+ std::get<Indexes>(phmap::forward<Tuple>(t))...);
+}
+
+} // namespace utility_internal
+
+// apply
+//
+// Invokes a Callable using elements of a tuple as its arguments.
+// Each element of the tuple corresponds to an argument of the call (in order).
+// Both the Callable argument and the tuple argument are perfect-forwarded.
+// For member-function Callables, the first tuple element acts as the `this`
+// pointer. `phmap::apply` is designed to be a drop-in replacement for C++17's
+// `std::apply`. Unlike C++17's `std::apply`, this is not currently `constexpr`.
+//
+// Example:
+//
+// class Foo {
+// public:
+// void Bar(int);
+// };
+// void user_function1(int, std::string);
+// void user_function2(std::unique_ptr<Foo>);
+// auto user_lambda = [](int, int) {};
+//
+// int main()
+// {
+// std::tuple<int, std::string> tuple1(42, "bar");
+// // Invokes the first user function on int, std::string.
+// phmap::apply(&user_function1, tuple1);
+//
+// std::tuple<std::unique_ptr<Foo>> tuple2(phmap::make_unique<Foo>());
+// // Invokes the user function that takes ownership of the unique
+// // pointer.
+// phmap::apply(&user_function2, std::move(tuple2));
+//
+// auto foo = phmap::make_unique<Foo>();
+// std::tuple<Foo*, int> tuple3(foo.get(), 42);
+// // Invokes the method Bar on foo with one argument, 42.
+// phmap::apply(&Foo::Bar, tuple3);
+//
+// std::tuple<int, int> tuple4(8, 9);
+// // Invokes a lambda.
+// phmap::apply(user_lambda, tuple4);
+// }
+template <typename Functor, typename Tuple>
+auto apply(Functor&& functor, Tuple&& t)
+ -> decltype(utility_internal::apply_helper(
+ phmap::forward<Functor>(functor), phmap::forward<Tuple>(t),
+ phmap::make_index_sequence<std::tuple_size<
+ typename std::remove_reference<Tuple>::type>::value>{})) {
+ return utility_internal::apply_helper(
+ phmap::forward<Functor>(functor), phmap::forward<Tuple>(t),
+ phmap::make_index_sequence<std::tuple_size<
+ typename std::remove_reference<Tuple>::type>::value>{});
+}
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch
+#endif // _MSC_VER
+
+// exchange
+//
+// Replaces the value of `obj` with `new_value` and returns the old value of
+// `obj`. `phmap::exchange` is designed to be a drop-in replacement for C++14's
+// `std::exchange`.
+//
+// Example:
+//
+// Foo& operator=(Foo&& other) {
+// ptr1_ = phmap::exchange(other.ptr1_, nullptr);
+// int1_ = phmap::exchange(other.int1_, -1);
+// return *this;
+// }
+template <typename T, typename U = T>
+T exchange(T& obj, U&& new_value)
+{
+ T old_value = phmap::move(obj);
+ obj = phmap::forward<U>(new_value);
+ return old_value;
+}
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif // _MSC_VER
+
+
+} // namespace phmap
+
+// -----------------------------------------------------------------------------
+// memory.h
+// -----------------------------------------------------------------------------
+
+namespace phmap {
+
+template <typename T>
+std::unique_ptr<T> WrapUnique(T* ptr)
+{
+ static_assert(!std::is_array<T>::value, "array types are unsupported");
+ static_assert(std::is_object<T>::value, "non-object types are unsupported");
+ return std::unique_ptr<T>(ptr);
+}
+
+namespace memory_internal {
+
+// Traits to select proper overload and return type for `phmap::make_unique<>`.
+template <typename T>
+struct MakeUniqueResult {
+ using scalar = std::unique_ptr<T>;
+};
+template <typename T>
+struct MakeUniqueResult<T[]> {
+ using array = std::unique_ptr<T[]>;
+};
+template <typename T, size_t N>
+struct MakeUniqueResult<T[N]> {
+ using invalid = void;
+};
+
+} // namespace memory_internal
+
+#if (__cplusplus > 201103L || defined(_MSC_VER)) && \
+ !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8)
+ using std::make_unique;
+#else
+
+ template <typename T, typename... Args>
+ typename memory_internal::MakeUniqueResult<T>::scalar make_unique(
+ Args&&... args) {
+ return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+ }
+
+ template <typename T>
+ typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) {
+ return std::unique_ptr<T>(new typename phmap::remove_extent_t<T>[n]());
+ }
+
+ template <typename T, typename... Args>
+ typename memory_internal::MakeUniqueResult<T>::invalid make_unique(
+ Args&&... /* args */) = delete;
+#endif
+
+template <typename T>
+auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr))
+{
+ // ptr is a forwarding reference to support Ts with non-const operators.
+ return (ptr != nullptr) ? std::addressof(*ptr) : nullptr;
+}
+
+inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; }
+
+template <typename T, typename D>
+std::shared_ptr<T> ShareUniquePtr(std::unique_ptr<T, D>&& ptr) {
+ return ptr ? std::shared_ptr<T>(std::move(ptr)) : std::shared_ptr<T>();
+}
+
+template <typename T>
+std::weak_ptr<T> WeakenPtr(const std::shared_ptr<T>& ptr) {
+ return std::weak_ptr<T>(ptr);
+}
+
+namespace memory_internal {
+
+// ExtractOr<E, O, D>::type evaluates to E<O> if possible. Otherwise, D.
+template <template <typename> class Extract, typename Obj, typename Default,
+ typename>
+struct ExtractOr {
+ using type = Default;
+};
+
+template <template <typename> class Extract, typename Obj, typename Default>
+struct ExtractOr<Extract, Obj, Default, void_t<Extract<Obj>>> {
+ using type = Extract<Obj>;
+};
+
+template <template <typename> class Extract, typename Obj, typename Default>
+using ExtractOrT = typename ExtractOr<Extract, Obj, Default, void>::type;
+
+// Extractors for the features of allocators.
+template <typename T>
+using GetPointer = typename T::pointer;
+
+template <typename T>
+using GetConstPointer = typename T::const_pointer;
+
+template <typename T>
+using GetVoidPointer = typename T::void_pointer;
+
+template <typename T>
+using GetConstVoidPointer = typename T::const_void_pointer;
+
+template <typename T>
+using GetDifferenceType = typename T::difference_type;
+
+template <typename T>
+using GetSizeType = typename T::size_type;
+
+template <typename T>
+using GetPropagateOnContainerCopyAssignment =
+ typename T::propagate_on_container_copy_assignment;
+
+template <typename T>
+using GetPropagateOnContainerMoveAssignment =
+ typename T::propagate_on_container_move_assignment;
+
+template <typename T>
+using GetPropagateOnContainerSwap = typename T::propagate_on_container_swap;
+
+template <typename T>
+using GetIsAlwaysEqual = typename T::is_always_equal;
+
+template <typename T>
+struct GetFirstArg;
+
+template <template <typename...> class Class, typename T, typename... Args>
+struct GetFirstArg<Class<T, Args...>> {
+ using type = T;
+};
+
+template <typename Ptr, typename = void>
+struct ElementType {
+ using type = typename GetFirstArg<Ptr>::type;
+};
+
+template <typename T>
+struct ElementType<T, void_t<typename T::element_type>> {
+ using type = typename T::element_type;
+};
+
+template <typename T, typename U>
+struct RebindFirstArg;
+
+template <template <typename...> class Class, typename T, typename... Args,
+ typename U>
+struct RebindFirstArg<Class<T, Args...>, U> {
+ using type = Class<U, Args...>;
+};
+
+template <typename T, typename U, typename = void>
+struct RebindPtr {
+ using type = typename RebindFirstArg<T, U>::type;
+};
+
+template <typename T, typename U>
+struct RebindPtr<T, U, void_t<typename T::template rebind<U>>> {
+ using type = typename T::template rebind<U>;
+};
+
+template <typename T, typename U>
+constexpr bool HasRebindAlloc(...) {
+ return false;
+}
+
+template <typename T, typename U>
+constexpr bool HasRebindAlloc(typename std::allocator_traits<T>::template rebind_alloc<U>*) {
+ return true;
+}
+
+template <typename T, typename U, bool = HasRebindAlloc<T, U>(nullptr)>
+struct RebindAlloc {
+ using type = typename RebindFirstArg<T, U>::type;
+};
+
+template <typename A, typename U>
+struct RebindAlloc<A, U, true> {
+ using type = typename std::allocator_traits<A>::template rebind_alloc<U>;
+};
+
+
+} // namespace memory_internal
+
+template <typename Ptr>
+struct pointer_traits
+{
+ using pointer = Ptr;
+
+ // element_type:
+ // Ptr::element_type if present. Otherwise T if Ptr is a template
+ // instantiation Template<T, Args...>
+ using element_type = typename memory_internal::ElementType<Ptr>::type;
+
+ // difference_type:
+ // Ptr::difference_type if present, otherwise std::ptrdiff_t
+ using difference_type =
+ memory_internal::ExtractOrT<memory_internal::GetDifferenceType, Ptr,
+ std::ptrdiff_t>;
+
+ // rebind:
+ // Ptr::rebind<U> if exists, otherwise Template<U, Args...> if Ptr is a
+ // template instantiation Template<T, Args...>
+ template <typename U>
+ using rebind = typename memory_internal::RebindPtr<Ptr, U>::type;
+
+ // pointer_to:
+ // Calls Ptr::pointer_to(r)
+ static pointer pointer_to(element_type& r) { // NOLINT(runtime/references)
+ return Ptr::pointer_to(r);
+ }
+};
+
+// Specialization for T*.
+template <typename T>
+struct pointer_traits<T*>
+{
+ using pointer = T*;
+ using element_type = T;
+ using difference_type = std::ptrdiff_t;
+
+ template <typename U>
+ using rebind = U*;
+
+ // pointer_to:
+ // Calls std::addressof(r)
+ static pointer pointer_to(
+ element_type& r) noexcept { // NOLINT(runtime/references)
+ return std::addressof(r);
+ }
+};
+
+// -----------------------------------------------------------------------------
+// Class Template: allocator_traits
+// -----------------------------------------------------------------------------
+//
+// A C++11 compatible implementation of C++17's std::allocator_traits.
+//
+template <typename Alloc>
+struct allocator_traits
+{
+ using allocator_type = Alloc;
+
+ // value_type:
+ // Alloc::value_type
+ using value_type = typename Alloc::value_type;
+
+ // pointer:
+ // Alloc::pointer if present, otherwise value_type*
+ using pointer = memory_internal::ExtractOrT<memory_internal::GetPointer,
+ Alloc, value_type*>;
+
+ // const_pointer:
+ // Alloc::const_pointer if present, otherwise
+ // phmap::pointer_traits<pointer>::rebind<const value_type>
+ using const_pointer =
+ memory_internal::ExtractOrT<memory_internal::GetConstPointer, Alloc,
+ typename phmap::pointer_traits<pointer>::
+ template rebind<const value_type>>;
+
+ // void_pointer:
+ // Alloc::void_pointer if present, otherwise
+ // phmap::pointer_traits<pointer>::rebind<void>
+ using void_pointer = memory_internal::ExtractOrT<
+ memory_internal::GetVoidPointer, Alloc,
+ typename phmap::pointer_traits<pointer>::template rebind<void>>;
+
+ // const_void_pointer:
+ // Alloc::const_void_pointer if present, otherwise
+ // phmap::pointer_traits<pointer>::rebind<const void>
+ using const_void_pointer = memory_internal::ExtractOrT<
+ memory_internal::GetConstVoidPointer, Alloc,
+ typename phmap::pointer_traits<pointer>::template rebind<const void>>;
+
+ // difference_type:
+ // Alloc::difference_type if present, otherwise
+ // phmap::pointer_traits<pointer>::difference_type
+ using difference_type = memory_internal::ExtractOrT<
+ memory_internal::GetDifferenceType, Alloc,
+ typename phmap::pointer_traits<pointer>::difference_type>;
+
+ // size_type:
+ // Alloc::size_type if present, otherwise
+ // std::make_unsigned<difference_type>::type
+ using size_type = memory_internal::ExtractOrT<
+ memory_internal::GetSizeType, Alloc,
+ typename std::make_unsigned<difference_type>::type>;
+
+ // propagate_on_container_copy_assignment:
+ // Alloc::propagate_on_container_copy_assignment if present, otherwise
+ // std::false_type
+ using propagate_on_container_copy_assignment = memory_internal::ExtractOrT<
+ memory_internal::GetPropagateOnContainerCopyAssignment, Alloc,
+ std::false_type>;
+
+ // propagate_on_container_move_assignment:
+ // Alloc::propagate_on_container_move_assignment if present, otherwise
+ // std::false_type
+ using propagate_on_container_move_assignment = memory_internal::ExtractOrT<
+ memory_internal::GetPropagateOnContainerMoveAssignment, Alloc,
+ std::false_type>;
+
+ // propagate_on_container_swap:
+ // Alloc::propagate_on_container_swap if present, otherwise std::false_type
+ using propagate_on_container_swap =
+ memory_internal::ExtractOrT<memory_internal::GetPropagateOnContainerSwap,
+ Alloc, std::false_type>;
+
+ // is_always_equal:
+ // Alloc::is_always_equal if present, otherwise std::is_empty<Alloc>::type
+ using is_always_equal =
+ memory_internal::ExtractOrT<memory_internal::GetIsAlwaysEqual, Alloc,
+ typename std::is_empty<Alloc>::type>;
+
+ // rebind_alloc:
+ // Alloc::rebind<T>::other if present, otherwise Alloc<T, Args> if this Alloc
+ // is Alloc<U, Args>
+ template <typename T>
+ using rebind_alloc = typename memory_internal::RebindAlloc<Alloc, T>::type;
+
+ // rebind_traits:
+ // phmap::allocator_traits<rebind_alloc<T>>
+ template <typename T>
+ using rebind_traits = phmap::allocator_traits<rebind_alloc<T>>;
+
+ // allocate(Alloc& a, size_type n):
+ // Calls a.allocate(n)
+ static pointer allocate(Alloc& a, // NOLINT(runtime/references)
+ size_type n) {
+ return a.allocate(n);
+ }
+
+ // allocate(Alloc& a, size_type n, const_void_pointer hint):
+ // Calls a.allocate(n, hint) if possible.
+ // If not possible, calls a.allocate(n)
+ static pointer allocate(Alloc& a, size_type n, // NOLINT(runtime/references)
+ const_void_pointer hint) {
+ return allocate_impl(0, a, n, hint);
+ }
+
+ // deallocate(Alloc& a, pointer p, size_type n):
+ // Calls a.deallocate(p, n)
+ static void deallocate(Alloc& a, pointer p, // NOLINT(runtime/references)
+ size_type n) {
+ a.deallocate(p, n);
+ }
+
+ // construct(Alloc& a, T* p, Args&&... args):
+ // Calls a.construct(p, std::forward<Args>(args)...) if possible.
+ // If not possible, calls
+ // ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...)
+ template <typename T, typename... Args>
+ static void construct(Alloc& a, T* p, // NOLINT(runtime/references)
+ Args&&... args) {
+ construct_impl(0, a, p, std::forward<Args>(args)...);
+ }
+
+ // destroy(Alloc& a, T* p):
+ // Calls a.destroy(p) if possible. If not possible, calls p->~T().
+ template <typename T>
+ static void destroy(Alloc& a, T* p) { // NOLINT(runtime/references)
+ destroy_impl(0, a, p);
+ }
+
+ // max_size(const Alloc& a):
+ // Returns a.max_size() if possible. If not possible, returns
+ // std::numeric_limits<size_type>::max() / sizeof(value_type)
+ static size_type max_size(const Alloc& a) { return max_size_impl(0, a); }
+
+ // select_on_container_copy_construction(const Alloc& a):
+ // Returns a.select_on_container_copy_construction() if possible.
+ // If not possible, returns a.
+ static Alloc select_on_container_copy_construction(const Alloc& a) {
+ return select_on_container_copy_construction_impl(0, a);
+ }
+
+private:
+ template <typename A>
+ static auto allocate_impl(int, A& a, // NOLINT(runtime/references)
+ size_type n, const_void_pointer hint)
+ -> decltype(a.allocate(n, hint)) {
+ return a.allocate(n, hint);
+ }
+ static pointer allocate_impl(char, Alloc& a, // NOLINT(runtime/references)
+ size_type n, const_void_pointer) {
+ return a.allocate(n);
+ }
+
+ template <typename A, typename... Args>
+ static auto construct_impl(int, A& a, // NOLINT(runtime/references)
+ Args&&... args)
+ -> decltype(std::allocator_traits<A>::construct(a, std::forward<Args>(args)...)) {
+ std::allocator_traits<A>::construct(a, std::forward<Args>(args)...);
+ }
+
+ template <typename T, typename... Args>
+ static void construct_impl(char, Alloc&, T* p, Args&&... args) {
+ ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...);
+ }
+
+ template <typename A, typename T>
+ static auto destroy_impl(int, A& a, // NOLINT(runtime/references)
+ T* p) -> decltype(std::allocator_traits<A>::destroy(a, p)) {
+ std::allocator_traits<A>::destroy(a, p);
+ }
+ template <typename T>
+ static void destroy_impl(char, Alloc&, T* p) {
+ p->~T();
+ }
+
+ template <typename A>
+ static auto max_size_impl(int, const A& a) -> decltype(a.max_size()) {
+ return a.max_size();
+ }
+ static size_type max_size_impl(char, const Alloc&) {
+ return (std::numeric_limits<size_type>::max)() / sizeof(value_type);
+ }
+
+ template <typename A>
+ static auto select_on_container_copy_construction_impl(int, const A& a)
+ -> decltype(a.select_on_container_copy_construction()) {
+ return a.select_on_container_copy_construction();
+ }
+ static Alloc select_on_container_copy_construction_impl(char,
+ const Alloc& a) {
+ return a;
+ }
+};
+
+namespace memory_internal {
+
+// This template alias transforms Alloc::is_nothrow into a metafunction with
+// Alloc as a parameter so it can be used with ExtractOrT<>.
+template <typename Alloc>
+using GetIsNothrow = typename Alloc::is_nothrow;
+
+} // namespace memory_internal
+
+// PHMAP_ALLOCATOR_NOTHROW is a build time configuration macro for user to
+// specify whether the default allocation function can throw or never throws.
+// If the allocation function never throws, user should define it to a non-zero
+// value (e.g. via `-DPHMAP_ALLOCATOR_NOTHROW`).
+// If the allocation function can throw, user should leave it undefined or
+// define it to zero.
+//
+// allocator_is_nothrow<Alloc> is a traits class that derives from
+// Alloc::is_nothrow if present, otherwise std::false_type. It's specialized
+// for Alloc = std::allocator<T> for any type T according to the state of
+// PHMAP_ALLOCATOR_NOTHROW.
+//
+// default_allocator_is_nothrow is a class that derives from std::true_type
+// when the default allocator (global operator new) never throws, and
+// std::false_type when it can throw. It is a convenience shorthand for writing
+// allocator_is_nothrow<std::allocator<T>> (T can be any type).
+// NOTE: allocator_is_nothrow<std::allocator<T>> is guaranteed to derive from
+// the same type for all T, because users should specialize neither
+// allocator_is_nothrow nor std::allocator.
+template <typename Alloc>
+struct allocator_is_nothrow
+ : memory_internal::ExtractOrT<memory_internal::GetIsNothrow, Alloc,
+ std::false_type> {};
+
+#if defined(PHMAP_ALLOCATOR_NOTHROW) && PHMAP_ALLOCATOR_NOTHROW
+ template <typename T>
+ struct allocator_is_nothrow<std::allocator<T>> : std::true_type {};
+ struct default_allocator_is_nothrow : std::true_type {};
+#else
+ struct default_allocator_is_nothrow : std::false_type {};
+#endif
+
+namespace memory_internal {
+template <typename Allocator, typename Iterator, typename... Args>
+void ConstructRange(Allocator& alloc, Iterator first, Iterator last,
+ const Args&... args)
+{
+ for (Iterator cur = first; cur != last; ++cur) {
+ PHMAP_INTERNAL_TRY {
+ std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur),
+ args...);
+ }
+ PHMAP_INTERNAL_CATCH_ANY {
+ while (cur != first) {
+ --cur;
+ std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur));
+ }
+ PHMAP_INTERNAL_RETHROW;
+ }
+ }
+}
+
+template <typename Allocator, typename Iterator, typename InputIterator>
+void CopyRange(Allocator& alloc, Iterator destination, InputIterator first,
+ InputIterator last)
+{
+ for (Iterator cur = destination; first != last;
+ static_cast<void>(++cur), static_cast<void>(++first)) {
+ PHMAP_INTERNAL_TRY {
+ std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur),
+ *first);
+ }
+ PHMAP_INTERNAL_CATCH_ANY {
+ while (cur != destination) {
+ --cur;
+ std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur));
+ }
+ PHMAP_INTERNAL_RETHROW;
+ }
+ }
+}
+} // namespace memory_internal
+} // namespace phmap
+
+
+// -----------------------------------------------------------------------------
+// optional.h
+// -----------------------------------------------------------------------------
+#ifdef PHMAP_HAVE_STD_OPTIONAL
+
+#include <optional> // IWYU pragma: export
+
+namespace phmap {
+using std::bad_optional_access;
+using std::optional;
+using std::make_optional;
+using std::nullopt_t;
+using std::nullopt;
+} // namespace phmap
+
+#else
+
+#if defined(__clang__)
+ #if __has_feature(cxx_inheriting_constructors)
+ #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1
+ #endif
+#elif (defined(__GNUC__) && \
+ (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8)) || \
+ (__cpp_inheriting_constructors >= 200802) || \
+ (defined(_MSC_VER) && _MSC_VER >= 1910)
+
+ #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1
+#endif
+
+namespace phmap {
+
+class bad_optional_access : public std::exception
+{
+public:
+ bad_optional_access() = default;
+ ~bad_optional_access() override;
+ const char* what() const noexcept override;
+};
+
+template <typename T>
+class optional;
+
+// --------------------------------
+struct nullopt_t
+{
+ struct init_t {};
+ static init_t init;
+
+ explicit constexpr nullopt_t(init_t& /*unused*/) {}
+};
+
+constexpr nullopt_t nullopt(nullopt_t::init);
+
+namespace optional_internal {
+
+// throw delegator
+[[noreturn]] void throw_bad_optional_access();
+
+
+struct empty_struct {};
+
+// This class stores the data in optional<T>.
+// It is specialized based on whether T is trivially destructible.
+// This is the specialization for non trivially destructible type.
+template <typename T, bool unused = std::is_trivially_destructible<T>::value>
+class optional_data_dtor_base
+{
+ struct dummy_type {
+ static_assert(sizeof(T) % sizeof(empty_struct) == 0, "");
+ // Use an array to avoid GCC 6 placement-new warning.
+ empty_struct data[sizeof(T) / sizeof(empty_struct)];
+ };
+
+protected:
+ // Whether there is data or not.
+ bool engaged_;
+ // Data storage
+ union {
+ dummy_type dummy_;
+ T data_;
+ };
+
+ void destruct() noexcept {
+ if (engaged_) {
+ data_.~T();
+ engaged_ = false;
+ }
+ }
+
+ // dummy_ must be initialized for constexpr constructor.
+ constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {}
+
+ template <typename... Args>
+ constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args)
+ : engaged_(true), data_(phmap::forward<Args>(args)...) {}
+
+ ~optional_data_dtor_base() { destruct(); }
+};
+
+// Specialization for trivially destructible type.
+template <typename T>
+class optional_data_dtor_base<T, true>
+{
+ struct dummy_type {
+ static_assert(sizeof(T) % sizeof(empty_struct) == 0, "");
+ // Use array to avoid GCC 6 placement-new warning.
+ empty_struct data[sizeof(T) / sizeof(empty_struct)];
+ };
+
+protected:
+ // Whether there is data or not.
+ bool engaged_;
+ // Data storage
+ union {
+ dummy_type dummy_;
+ T data_;
+ };
+ void destruct() noexcept { engaged_ = false; }
+
+ // dummy_ must be initialized for constexpr constructor.
+ constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {}
+
+ template <typename... Args>
+ constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args)
+ : engaged_(true), data_(phmap::forward<Args>(args)...) {}
+};
+
+template <typename T>
+class optional_data_base : public optional_data_dtor_base<T>
+{
+protected:
+ using base = optional_data_dtor_base<T>;
+#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS
+ using base::base;
+#else
+ optional_data_base() = default;
+
+ template <typename... Args>
+ constexpr explicit optional_data_base(in_place_t t, Args&&... args)
+ : base(t, phmap::forward<Args>(args)...) {}
+#endif
+
+ template <typename... Args>
+ void construct(Args&&... args) {
+ // Use dummy_'s address to work around casting cv-qualified T* to void*.
+ ::new (static_cast<void*>(&this->dummy_)) T(std::forward<Args>(args)...);
+ this->engaged_ = true;
+ }
+
+ template <typename U>
+ void assign(U&& u) {
+ if (this->engaged_) {
+ this->data_ = std::forward<U>(u);
+ } else {
+ construct(std::forward<U>(u));
+ }
+ }
+};
+
+// TODO: Add another class using
+// std::is_trivially_move_constructible trait when available to match
+// http://cplusplus.github.io/LWG/lwg-defects.html#2900, for types that
+// have trivial move but nontrivial copy.
+// Also, we should be checking is_trivially_copyable here, which is not
+// supported now, so we use is_trivially_* traits instead.
+template <typename T,
+ bool unused = phmap::is_trivially_copy_constructible<T>::value&&
+ phmap::is_trivially_copy_assignable<typename std::remove_cv<
+ T>::type>::value&& std::is_trivially_destructible<T>::value>
+class optional_data;
+
+// Trivially copyable types
+template <typename T>
+class optional_data<T, true> : public optional_data_base<T>
+{
+protected:
+#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS
+ using optional_data_base<T>::optional_data_base;
+#else
+ optional_data() = default;
+
+ template <typename... Args>
+ constexpr explicit optional_data(in_place_t t, Args&&... args)
+ : optional_data_base<T>(t, phmap::forward<Args>(args)...) {}
+#endif
+};
+
+template <typename T>
+class optional_data<T, false> : public optional_data_base<T>
+{
+protected:
+#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS
+ using optional_data_base<T>::optional_data_base;
+#else
+ template <typename... Args>
+ constexpr explicit optional_data(in_place_t t, Args&&... args)
+ : optional_data_base<T>(t, phmap::forward<Args>(args)...) {}
+#endif
+
+ optional_data() = default;
+
+ optional_data(const optional_data& rhs) : optional_data_base<T>() {
+ if (rhs.engaged_) {
+ this->construct(rhs.data_);
+ }
+ }
+
+ optional_data(optional_data&& rhs) noexcept(
+ phmap::default_allocator_is_nothrow::value ||
+ std::is_nothrow_move_constructible<T>::value)
+ : optional_data_base<T>() {
+ if (rhs.engaged_) {
+ this->construct(std::move(rhs.data_));
+ }
+ }
+
+ optional_data& operator=(const optional_data& rhs) {
+ if (rhs.engaged_) {
+ this->assign(rhs.data_);
+ } else {
+ this->destruct();
+ }
+ return *this;
+ }
+
+ optional_data& operator=(optional_data&& rhs) noexcept(
+ std::is_nothrow_move_assignable<T>::value&&
+ std::is_nothrow_move_constructible<T>::value) {
+ if (rhs.engaged_) {
+ this->assign(std::move(rhs.data_));
+ } else {
+ this->destruct();
+ }
+ return *this;
+ }
+};
+
+// Ordered by level of restriction, from low to high.
+// Copyable implies movable.
+enum class copy_traits { copyable = 0, movable = 1, non_movable = 2 };
+
+// Base class for enabling/disabling copy/move constructor.
+template <copy_traits>
+class optional_ctor_base;
+
+template <>
+class optional_ctor_base<copy_traits::copyable>
+{
+public:
+ constexpr optional_ctor_base() = default;
+ optional_ctor_base(const optional_ctor_base&) = default;
+ optional_ctor_base(optional_ctor_base&&) = default;
+ optional_ctor_base& operator=(const optional_ctor_base&) = default;
+ optional_ctor_base& operator=(optional_ctor_base&&) = default;
+};
+
+template <>
+class optional_ctor_base<copy_traits::movable>
+{
+public:
+ constexpr optional_ctor_base() = default;
+ optional_ctor_base(const optional_ctor_base&) = delete;
+ optional_ctor_base(optional_ctor_base&&) = default;
+ optional_ctor_base& operator=(const optional_ctor_base&) = default;
+ optional_ctor_base& operator=(optional_ctor_base&&) = default;
+};
+
+template <>
+class optional_ctor_base<copy_traits::non_movable>
+{
+public:
+ constexpr optional_ctor_base() = default;
+ optional_ctor_base(const optional_ctor_base&) = delete;
+ optional_ctor_base(optional_ctor_base&&) = delete;
+ optional_ctor_base& operator=(const optional_ctor_base&) = default;
+ optional_ctor_base& operator=(optional_ctor_base&&) = default;
+};
+
+// Base class for enabling/disabling copy/move assignment.
+template <copy_traits>
+class optional_assign_base;
+
+template <>
+class optional_assign_base<copy_traits::copyable>
+{
+public:
+ constexpr optional_assign_base() = default;
+ optional_assign_base(const optional_assign_base&) = default;
+ optional_assign_base(optional_assign_base&&) = default;
+ optional_assign_base& operator=(const optional_assign_base&) = default;
+ optional_assign_base& operator=(optional_assign_base&&) = default;
+};
+
+template <>
+class optional_assign_base<copy_traits::movable>
+{
+public:
+ constexpr optional_assign_base() = default;
+ optional_assign_base(const optional_assign_base&) = default;
+ optional_assign_base(optional_assign_base&&) = default;
+ optional_assign_base& operator=(const optional_assign_base&) = delete;
+ optional_assign_base& operator=(optional_assign_base&&) = default;
+};
+
+template <>
+class optional_assign_base<copy_traits::non_movable>
+{
+public:
+ constexpr optional_assign_base() = default;
+ optional_assign_base(const optional_assign_base&) = default;
+ optional_assign_base(optional_assign_base&&) = default;
+ optional_assign_base& operator=(const optional_assign_base&) = delete;
+ optional_assign_base& operator=(optional_assign_base&&) = delete;
+};
+
+template <typename T>
+constexpr copy_traits get_ctor_copy_traits()
+{
+ return std::is_copy_constructible<T>::value
+ ? copy_traits::copyable
+ : std::is_move_constructible<T>::value ? copy_traits::movable
+ : copy_traits::non_movable;
+}
+
+template <typename T>
+constexpr copy_traits get_assign_copy_traits()
+{
+ return phmap::is_copy_assignable<T>::value &&
+ std::is_copy_constructible<T>::value
+ ? copy_traits::copyable
+ : phmap::is_move_assignable<T>::value &&
+ std::is_move_constructible<T>::value
+ ? copy_traits::movable
+ : copy_traits::non_movable;
+}
+
+// Whether T is constructible or convertible from optional<U>.
+template <typename T, typename U>
+struct is_constructible_convertible_from_optional
+ : std::integral_constant<
+ bool, std::is_constructible<T, optional<U>&>::value ||
+ std::is_constructible<T, optional<U>&&>::value ||
+ std::is_constructible<T, const optional<U>&>::value ||
+ std::is_constructible<T, const optional<U>&&>::value ||
+ std::is_convertible<optional<U>&, T>::value ||
+ std::is_convertible<optional<U>&&, T>::value ||
+ std::is_convertible<const optional<U>&, T>::value ||
+ std::is_convertible<const optional<U>&&, T>::value> {};
+
+// Whether T is constructible or convertible or assignable from optional<U>.
+template <typename T, typename U>
+struct is_constructible_convertible_assignable_from_optional
+ : std::integral_constant<
+ bool, is_constructible_convertible_from_optional<T, U>::value ||
+ std::is_assignable<T&, optional<U>&>::value ||
+ std::is_assignable<T&, optional<U>&&>::value ||
+ std::is_assignable<T&, const optional<U>&>::value ||
+ std::is_assignable<T&, const optional<U>&&>::value> {};
+
+// Helper function used by [optional.relops], [optional.comp_with_t],
+// for checking whether an expression is convertible to bool.
+bool convertible_to_bool(bool);
+
+// Base class for std::hash<phmap::optional<T>>:
+// If std::hash<std::remove_const_t<T>> is enabled, it provides operator() to
+// compute the hash; Otherwise, it is disabled.
+// Reference N4659 23.14.15 [unord.hash].
+template <typename T, typename = size_t>
+struct optional_hash_base
+{
+ optional_hash_base() = delete;
+ optional_hash_base(const optional_hash_base&) = delete;
+ optional_hash_base(optional_hash_base&&) = delete;
+ optional_hash_base& operator=(const optional_hash_base&) = delete;
+ optional_hash_base& operator=(optional_hash_base&&) = delete;
+};
+
+template <typename T>
+struct optional_hash_base<T, decltype(std::hash<phmap::remove_const_t<T> >()(
+ std::declval<phmap::remove_const_t<T> >()))>
+{
+ using argument_type = phmap::optional<T>;
+ using result_type = size_t;
+ size_t operator()(const phmap::optional<T>& opt) const {
+ phmap::type_traits_internal::AssertHashEnabled<phmap::remove_const_t<T>>();
+ if (opt) {
+ return std::hash<phmap::remove_const_t<T> >()(*opt);
+ } else {
+ return static_cast<size_t>(0x297814aaad196e6dULL);
+ }
+ }
+};
+
+} // namespace optional_internal
+
+
+// -----------------------------------------------------------------------------
+// phmap::optional class definition
+// -----------------------------------------------------------------------------
+
+template <typename T>
+class optional : private optional_internal::optional_data<T>,
+ private optional_internal::optional_ctor_base<
+ optional_internal::get_ctor_copy_traits<T>()>,
+ private optional_internal::optional_assign_base<
+ optional_internal::get_assign_copy_traits<T>()>
+{
+ using data_base = optional_internal::optional_data<T>;
+
+public:
+ typedef T value_type;
+
+ // Constructors
+
+ // Constructs an `optional` holding an empty value, NOT a default constructed
+ // `T`.
+ constexpr optional() noexcept {}
+
+ // Constructs an `optional` initialized with `nullopt` to hold an empty value.
+ constexpr optional(nullopt_t) noexcept {} // NOLINT(runtime/explicit)
+
+ // Copy constructor, standard semantics
+ optional(const optional& src) = default;
+
+ // Move constructor, standard semantics
+ optional(optional&& src) = default;
+
+ // Constructs a non-empty `optional` direct-initialized value of type `T` from
+ // the arguments `std::forward<Args>(args)...` within the `optional`.
+ // (The `in_place_t` is a tag used to indicate that the contained object
+ // should be constructed in-place.)
+ template <typename InPlaceT, typename... Args,
+ phmap::enable_if_t<phmap::conjunction<
+ std::is_same<InPlaceT, in_place_t>,
+ std::is_constructible<T, Args&&...> >::value>* = nullptr>
+ constexpr explicit optional(InPlaceT, Args&&... args)
+ : data_base(in_place_t(), phmap::forward<Args>(args)...) {}
+
+ // Constructs a non-empty `optional` direct-initialized value of type `T` from
+ // the arguments of an initializer_list and `std::forward<Args>(args)...`.
+ // (The `in_place_t` is a tag used to indicate that the contained object
+ // should be constructed in-place.)
+ template <typename U, typename... Args,
+ typename = typename std::enable_if<std::is_constructible<
+ T, std::initializer_list<U>&, Args&&...>::value>::type>
+ constexpr explicit optional(in_place_t, std::initializer_list<U> il,
+ Args&&... args)
+ : data_base(in_place_t(), il, phmap::forward<Args>(args)...) {
+ }
+
+ // Value constructor (implicit)
+ template <
+ typename U = T,
+ typename std::enable_if<
+ phmap::conjunction<phmap::negation<std::is_same<
+ in_place_t, typename std::decay<U>::type> >,
+ phmap::negation<std::is_same<
+ optional<T>, typename std::decay<U>::type> >,
+ std::is_convertible<U&&, T>,
+ std::is_constructible<T, U&&> >::value,
+ bool>::type = false>
+ constexpr optional(U&& v) : data_base(in_place_t(), phmap::forward<U>(v)) {}
+
+ // Value constructor (explicit)
+ template <
+ typename U = T,
+ typename std::enable_if<
+ phmap::conjunction<phmap::negation<std::is_same<
+ in_place_t, typename std::decay<U>::type>>,
+ phmap::negation<std::is_same<
+ optional<T>, typename std::decay<U>::type>>,
+ phmap::negation<std::is_convertible<U&&, T>>,
+ std::is_constructible<T, U&&>>::value,
+ bool>::type = false>
+ explicit constexpr optional(U&& v)
+ : data_base(in_place_t(), phmap::forward<U>(v)) {}
+
+ // Converting copy constructor (implicit)
+ template <typename U,
+ typename std::enable_if<
+ phmap::conjunction<
+ phmap::negation<std::is_same<T, U> >,
+ std::is_constructible<T, const U&>,
+ phmap::negation<
+ optional_internal::
+ is_constructible_convertible_from_optional<T, U> >,
+ std::is_convertible<const U&, T> >::value,
+ bool>::type = false>
+ optional(const optional<U>& rhs) {
+ if (rhs) {
+ this->construct(*rhs);
+ }
+ }
+
+ // Converting copy constructor (explicit)
+ template <typename U,
+ typename std::enable_if<
+ phmap::conjunction<
+ phmap::negation<std::is_same<T, U>>,
+ std::is_constructible<T, const U&>,
+ phmap::negation<
+ optional_internal::
+ is_constructible_convertible_from_optional<T, U>>,
+ phmap::negation<std::is_convertible<const U&, T>>>::value,
+ bool>::type = false>
+ explicit optional(const optional<U>& rhs) {
+ if (rhs) {
+ this->construct(*rhs);
+ }
+ }
+
+ // Converting move constructor (implicit)
+ template <typename U,
+ typename std::enable_if<
+ phmap::conjunction<
+ phmap::negation<std::is_same<T, U> >,
+ std::is_constructible<T, U&&>,
+ phmap::negation<
+ optional_internal::
+ is_constructible_convertible_from_optional<T, U> >,
+ std::is_convertible<U&&, T> >::value,
+ bool>::type = false>
+ optional(optional<U>&& rhs) {
+ if (rhs) {
+ this->construct(std::move(*rhs));
+ }
+ }
+
+ // Converting move constructor (explicit)
+ template <
+ typename U,
+ typename std::enable_if<
+ phmap::conjunction<
+ phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>,
+ phmap::negation<
+ optional_internal::is_constructible_convertible_from_optional<
+ T, U>>,
+ phmap::negation<std::is_convertible<U&&, T>>>::value,
+ bool>::type = false>
+ explicit optional(optional<U>&& rhs) {
+ if (rhs) {
+ this->construct(std::move(*rhs));
+ }
+ }
+
+ // Destructor. Trivial if `T` is trivially destructible.
+ ~optional() = default;
+
+ // Assignment Operators
+
+ // Assignment from `nullopt`
+ //
+ // Example:
+ //
+ // struct S { int value; };
+ // optional<S> opt = phmap::nullopt; // Could also use opt = { };
+ optional& operator=(nullopt_t) noexcept {
+ this->destruct();
+ return *this;
+ }
+
+ // Copy assignment operator, standard semantics
+ optional& operator=(const optional& src) = default;
+
+ // Move assignment operator, standard semantics
+ optional& operator=(optional&& src) = default;
+
+ // Value assignment operators
+ template <
+ typename U = T,
+ typename = typename std::enable_if<phmap::conjunction<
+ phmap::negation<
+ std::is_same<optional<T>, typename std::decay<U>::type>>,
+ phmap::negation<
+ phmap::conjunction<std::is_scalar<T>,
+ std::is_same<T, typename std::decay<U>::type>>>,
+ std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type>
+ optional& operator=(U&& v) {
+ this->assign(std::forward<U>(v));
+ return *this;
+ }
+
+ template <
+ typename U,
+ typename = typename std::enable_if<phmap::conjunction<
+ phmap::negation<std::is_same<T, U>>,
+ std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>,
+ phmap::negation<
+ optional_internal::
+ is_constructible_convertible_assignable_from_optional<
+ T, U>>>::value>::type>
+ optional& operator=(const optional<U>& rhs) {
+ if (rhs) {
+ this->assign(*rhs);
+ } else {
+ this->destruct();
+ }
+ return *this;
+ }
+
+ template <typename U,
+ typename = typename std::enable_if<phmap::conjunction<
+ phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U>,
+ std::is_assignable<T&, U>,
+ phmap::negation<
+ optional_internal::
+ is_constructible_convertible_assignable_from_optional<
+ T, U>>>::value>::type>
+ optional& operator=(optional<U>&& rhs) {
+ if (rhs) {
+ this->assign(std::move(*rhs));
+ } else {
+ this->destruct();
+ }
+ return *this;
+ }
+
+ // Modifiers
+
+ // optional::reset()
+ //
+ // Destroys the inner `T` value of an `phmap::optional` if one is present.
+ PHMAP_ATTRIBUTE_REINITIALIZES void reset() noexcept { this->destruct(); }
+
+ // optional::emplace()
+ //
+ // (Re)constructs the underlying `T` in-place with the given forwarded
+ // arguments.
+ //
+ // Example:
+ //
+ // optional<Foo> opt;
+ // opt.emplace(arg1,arg2,arg3); // Constructs Foo(arg1,arg2,arg3)
+ //
+ // If the optional is non-empty, and the `args` refer to subobjects of the
+ // current object, then behaviour is undefined, because the current object
+ // will be destructed before the new object is constructed with `args`.
+ template <typename... Args,
+ typename = typename std::enable_if<
+ std::is_constructible<T, Args&&...>::value>::type>
+ T& emplace(Args&&... args) {
+ this->destruct();
+ this->construct(std::forward<Args>(args)...);
+ return reference();
+ }
+
+ // Emplace reconstruction overload for an initializer list and the given
+ // forwarded arguments.
+ //
+ // Example:
+ //
+ // struct Foo {
+ // Foo(std::initializer_list<int>);
+ // };
+ //
+ // optional<Foo> opt;
+ // opt.emplace({1,2,3}); // Constructs Foo({1,2,3})
+ template <typename U, typename... Args,
+ typename = typename std::enable_if<std::is_constructible<
+ T, std::initializer_list<U>&, Args&&...>::value>::type>
+ T& emplace(std::initializer_list<U> il, Args&&... args) {
+ this->destruct();
+ this->construct(il, std::forward<Args>(args)...);
+ return reference();
+ }
+
+ // Swaps
+
+ // Swap, standard semantics
+ void swap(optional& rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value&&
+ std::is_trivial<T>::value) {
+ if (*this) {
+ if (rhs) {
+ using std::swap;
+ swap(**this, *rhs);
+ } else {
+ rhs.construct(std::move(**this));
+ this->destruct();
+ }
+ } else {
+ if (rhs) {
+ this->construct(std::move(*rhs));
+ rhs.destruct();
+ } else {
+ // No effect (swap(disengaged, disengaged)).
+ }
+ }
+ }
+
+ // Observers
+
+ // optional::operator->()
+ //
+ // Accesses the underlying `T` value's member `m` of an `optional`. If the
+ // `optional` is empty, behavior is undefined.
+ //
+ // If you need myOpt->foo in constexpr, use (*myOpt).foo instead.
+ const T* operator->() const {
+ assert(this->engaged_);
+ return std::addressof(this->data_);
+ }
+ T* operator->() {
+ assert(this->engaged_);
+ return std::addressof(this->data_);
+ }
+
+ // optional::operator*()
+ //
+ // Accesses the underlying `T` value of an `optional`. If the `optional` is
+ // empty, behavior is undefined.
+ constexpr const T& operator*() const & { return reference(); }
+ T& operator*() & {
+ assert(this->engaged_);
+ return reference();
+ }
+ constexpr const T&& operator*() const && {
+ return phmap::move(reference());
+ }
+ T&& operator*() && {
+ assert(this->engaged_);
+ return std::move(reference());
+ }
+
+ // optional::operator bool()
+ //
+ // Returns false if and only if the `optional` is empty.
+ //
+ // if (opt) {
+ // // do something with opt.value();
+ // } else {
+ // // opt is empty.
+ // }
+ //
+ constexpr explicit operator bool() const noexcept { return this->engaged_; }
+
+ // optional::has_value()
+ //
+ // Determines whether the `optional` contains a value. Returns `false` if and
+ // only if `*this` is empty.
+ constexpr bool has_value() const noexcept { return this->engaged_; }
+
+// Suppress bogus warning on MSVC: MSVC complains call to reference() after
+// throw_bad_optional_access() is unreachable.
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4702)
+#endif // _MSC_VER
+ // optional::value()
+ //
+ // Returns a reference to an `optional`s underlying value. The constness
+ // and lvalue/rvalue-ness of the `optional` is preserved to the view of
+ // the `T` sub-object. Throws `phmap::bad_optional_access` when the `optional`
+ // is empty.
+ constexpr const T& value() const & {
+ return static_cast<bool>(*this)
+ ? reference()
+ : (optional_internal::throw_bad_optional_access(), reference());
+ }
+ T& value() & {
+ return static_cast<bool>(*this)
+ ? reference()
+ : (optional_internal::throw_bad_optional_access(), reference());
+ }
+ T&& value() && { // NOLINT(build/c++11)
+ return std::move(
+ static_cast<bool>(*this)
+ ? reference()
+ : (optional_internal::throw_bad_optional_access(), reference()));
+ }
+ constexpr const T&& value() const && { // NOLINT(build/c++11)
+ return phmap::move(
+ static_cast<bool>(*this)
+ ? reference()
+ : (optional_internal::throw_bad_optional_access(), reference()));
+ }
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif // _MSC_VER
+
+ // optional::value_or()
+ //
+ // Returns either the value of `T` or a passed default `v` if the `optional`
+ // is empty.
+ template <typename U>
+ constexpr T value_or(U&& v) const& {
+ static_assert(std::is_copy_constructible<value_type>::value,
+ "optional<T>::value_or: T must by copy constructible");
+ static_assert(std::is_convertible<U&&, value_type>::value,
+ "optional<T>::value_or: U must be convertible to T");
+ return static_cast<bool>(*this)
+ ? **this
+ : static_cast<T>(phmap::forward<U>(v));
+ }
+ template <typename U>
+ T value_or(U&& v) && { // NOLINT(build/c++11)
+ static_assert(std::is_move_constructible<value_type>::value,
+ "optional<T>::value_or: T must by move constructible");
+ static_assert(std::is_convertible<U&&, value_type>::value,
+ "optional<T>::value_or: U must be convertible to T");
+ return static_cast<bool>(*this) ? std::move(**this)
+ : static_cast<T>(std::forward<U>(v));
+ }
+
+private:
+ // Private accessors for internal storage viewed as reference to T.
+ constexpr const T& reference() const { return this->data_; }
+ T& reference() { return this->data_; }
+
+ // T constraint checks. You can't have an optional of nullopt_t, in_place_t
+ // or a reference.
+ static_assert(
+ !std::is_same<nullopt_t, typename std::remove_cv<T>::type>::value,
+ "optional<nullopt_t> is not allowed.");
+ static_assert(
+ !std::is_same<in_place_t, typename std::remove_cv<T>::type>::value,
+ "optional<in_place_t> is not allowed.");
+ static_assert(!std::is_reference<T>::value,
+ "optional<reference> is not allowed.");
+};
+
+// Non-member functions
+
+// swap()
+//
+// Performs a swap between two `phmap::optional` objects, using standard
+// semantics.
+//
+// NOTE: we assume `is_swappable()` is always `true`. A compile error will
+// result if this is not the case.
+template <typename T,
+ typename std::enable_if<std::is_move_constructible<T>::value,
+ bool>::type = false>
+void swap(optional<T>& a, optional<T>& b) noexcept(noexcept(a.swap(b))) {
+ a.swap(b);
+}
+
+// make_optional()
+//
+// Creates a non-empty `optional<T>` where the type of `T` is deduced. An
+// `phmap::optional` can also be explicitly instantiated with
+// `make_optional<T>(v)`.
+//
+// Note: `make_optional()` constructions may be declared `constexpr` for
+// trivially copyable types `T`. Non-trivial types require copy elision
+// support in C++17 for `make_optional` to support `constexpr` on such
+// non-trivial types.
+//
+// Example:
+//
+// constexpr phmap::optional<int> opt = phmap::make_optional(1);
+// static_assert(opt.value() == 1, "");
+template <typename T>
+constexpr optional<typename std::decay<T>::type> make_optional(T&& v) {
+ return optional<typename std::decay<T>::type>(phmap::forward<T>(v));
+}
+
+template <typename T, typename... Args>
+constexpr optional<T> make_optional(Args&&... args) {
+ return optional<T>(in_place_t(), phmap::forward<Args>(args)...);
+}
+
+template <typename T, typename U, typename... Args>
+constexpr optional<T> make_optional(std::initializer_list<U> il,
+ Args&&... args) {
+ return optional<T>(in_place_t(), il,
+ phmap::forward<Args>(args)...);
+}
+
+// Relational operators [optional.relops]
+
+// Empty optionals are considered equal to each other and less than non-empty
+// optionals. Supports relations between optional<T> and optional<U>, between
+// optional<T> and U, and between optional<T> and nullopt.
+//
+// Note: We're careful to support T having non-bool relationals.
+
+// Requires: The expression, e.g. "*x == *y" shall be well-formed and its result
+// shall be convertible to bool.
+// The C++17 (N4606) "Returns:" statements are translated into
+// code in an obvious way here, and the original text retained as function docs.
+// Returns: If bool(x) != bool(y), false; otherwise if bool(x) == false, true;
+// otherwise *x == *y.
+template <typename T, typename U>
+constexpr auto operator==(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x == *y)) {
+ return static_cast<bool>(x) != static_cast<bool>(y)
+ ? false
+ : static_cast<bool>(x) == false ? true
+ : static_cast<bool>(*x == *y);
+}
+
+// Returns: If bool(x) != bool(y), true; otherwise, if bool(x) == false, false;
+// otherwise *x != *y.
+template <typename T, typename U>
+constexpr auto operator!=(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x != *y)) {
+ return static_cast<bool>(x) != static_cast<bool>(y)
+ ? true
+ : static_cast<bool>(x) == false ? false
+ : static_cast<bool>(*x != *y);
+}
+// Returns: If !y, false; otherwise, if !x, true; otherwise *x < *y.
+template <typename T, typename U>
+constexpr auto operator<(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x < *y)) {
+ return !y ? false : !x ? true : static_cast<bool>(*x < *y);
+}
+// Returns: If !x, false; otherwise, if !y, true; otherwise *x > *y.
+template <typename T, typename U>
+constexpr auto operator>(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x > *y)) {
+ return !x ? false : !y ? true : static_cast<bool>(*x > *y);
+}
+// Returns: If !x, true; otherwise, if !y, false; otherwise *x <= *y.
+template <typename T, typename U>
+constexpr auto operator<=(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x <= *y)) {
+ return !x ? true : !y ? false : static_cast<bool>(*x <= *y);
+}
+// Returns: If !y, true; otherwise, if !x, false; otherwise *x >= *y.
+template <typename T, typename U>
+constexpr auto operator>=(const optional<T>& x, const optional<U>& y)
+ -> decltype(optional_internal::convertible_to_bool(*x >= *y)) {
+ return !y ? true : !x ? false : static_cast<bool>(*x >= *y);
+}
+
+// Comparison with nullopt [optional.nullops]
+// The C++17 (N4606) "Returns:" statements are used directly here.
+template <typename T>
+constexpr bool operator==(const optional<T>& x, nullopt_t) noexcept {
+ return !x;
+}
+template <typename T>
+constexpr bool operator==(nullopt_t, const optional<T>& x) noexcept {
+ return !x;
+}
+template <typename T>
+constexpr bool operator!=(const optional<T>& x, nullopt_t) noexcept {
+ return static_cast<bool>(x);
+}
+template <typename T>
+constexpr bool operator!=(nullopt_t, const optional<T>& x) noexcept {
+ return static_cast<bool>(x);
+}
+template <typename T>
+constexpr bool operator<(const optional<T>&, nullopt_t) noexcept {
+ return false;
+}
+template <typename T>
+constexpr bool operator<(nullopt_t, const optional<T>& x) noexcept {
+ return static_cast<bool>(x);
+}
+template <typename T>
+constexpr bool operator<=(const optional<T>& x, nullopt_t) noexcept {
+ return !x;
+}
+template <typename T>
+constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept {
+ return true;
+}
+template <typename T>
+constexpr bool operator>(const optional<T>& x, nullopt_t) noexcept {
+ return static_cast<bool>(x);
+}
+template <typename T>
+constexpr bool operator>(nullopt_t, const optional<T>&) noexcept {
+ return false;
+}
+template <typename T>
+constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept {
+ return true;
+}
+template <typename T>
+constexpr bool operator>=(nullopt_t, const optional<T>& x) noexcept {
+ return !x;
+}
+
+// Comparison with T [optional.comp_with_t]
+
+// Requires: The expression, e.g. "*x == v" shall be well-formed and its result
+// shall be convertible to bool.
+// The C++17 (N4606) "Equivalent to:" statements are used directly here.
+template <typename T, typename U>
+constexpr auto operator==(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x == v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x == v) : false;
+}
+template <typename T, typename U>
+constexpr auto operator==(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v == *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v == *x) : false;
+}
+template <typename T, typename U>
+constexpr auto operator!=(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x != v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x != v) : true;
+}
+template <typename T, typename U>
+constexpr auto operator!=(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v != *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v != *x) : true;
+}
+template <typename T, typename U>
+constexpr auto operator<(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x < v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x < v) : true;
+}
+template <typename T, typename U>
+constexpr auto operator<(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v < *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v < *x) : false;
+}
+template <typename T, typename U>
+constexpr auto operator<=(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x <= v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x <= v) : true;
+}
+template <typename T, typename U>
+constexpr auto operator<=(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v <= *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v <= *x) : false;
+}
+template <typename T, typename U>
+constexpr auto operator>(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x > v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x > v) : false;
+}
+template <typename T, typename U>
+constexpr auto operator>(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v > *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v > *x) : true;
+}
+template <typename T, typename U>
+constexpr auto operator>=(const optional<T>& x, const U& v)
+ -> decltype(optional_internal::convertible_to_bool(*x >= v)) {
+ return static_cast<bool>(x) ? static_cast<bool>(*x >= v) : false;
+}
+template <typename T, typename U>
+constexpr auto operator>=(const U& v, const optional<T>& x)
+ -> decltype(optional_internal::convertible_to_bool(v >= *x)) {
+ return static_cast<bool>(x) ? static_cast<bool>(v >= *x) : true;
+}
+
+} // namespace phmap
+
+namespace std {
+
+// std::hash specialization for phmap::optional.
+template <typename T>
+struct hash<phmap::optional<T> >
+ : phmap::optional_internal::optional_hash_base<T> {};
+
+} // namespace std
+
+#endif
+
+// -----------------------------------------------------------------------------
+// common.h
+// -----------------------------------------------------------------------------
+namespace phmap {
+namespace priv {
+
+template <class, class = void>
+struct IsTransparent : std::false_type {};
+template <class T>
+struct IsTransparent<T, phmap::void_t<typename T::is_transparent>>
+ : std::true_type {};
+
+template <bool is_transparent>
+struct KeyArg
+{
+ // Transparent. Forward `K`.
+ template <typename K, typename key_type>
+ using type = K;
+};
+
+template <>
+struct KeyArg<false>
+{
+ // Not transparent. Always use `key_type`.
+ template <typename K, typename key_type>
+ using type = key_type;
+};
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ // warning C4820: '6' bytes padding added after data member
+ #pragma warning(disable : 4820)
+#endif
+
+// The node_handle concept from C++17.
+// We specialize node_handle for sets and maps. node_handle_base holds the
+// common API of both.
+// -----------------------------------------------------------------------
+template <typename PolicyTraits, typename Alloc>
+class node_handle_base
+{
+protected:
+ using slot_type = typename PolicyTraits::slot_type;
+
+public:
+ using allocator_type = Alloc;
+
+ constexpr node_handle_base() {}
+
+ node_handle_base(node_handle_base&& other) noexcept {
+ *this = std::move(other);
+ }
+
+ ~node_handle_base() { destroy(); }
+
+ node_handle_base& operator=(node_handle_base&& other) noexcept {
+ destroy();
+ if (!other.empty()) {
+ alloc_ = other.alloc_;
+ PolicyTraits::transfer(alloc(), slot(), other.slot());
+ other.reset();
+ }
+ return *this;
+ }
+
+ bool empty() const noexcept { return !alloc_; }
+ explicit operator bool() const noexcept { return !empty(); }
+ allocator_type get_allocator() const { return *alloc_; }
+
+protected:
+ friend struct CommonAccess;
+
+ struct transfer_tag_t {};
+ node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s)
+ : alloc_(a) {
+ PolicyTraits::transfer(alloc(), slot(), s);
+ }
+
+ struct move_tag_t {};
+ node_handle_base(move_tag_t, const allocator_type& a, slot_type* s)
+ : alloc_(a) {
+ PolicyTraits::construct(alloc(), slot(), s);
+ }
+
+ node_handle_base(const allocator_type& a, slot_type* s) : alloc_(a) {
+ PolicyTraits::transfer(alloc(), slot(), s);
+ }
+
+ //node_handle_base(const node_handle_base&) = delete;
+ //node_handle_base& operator=(const node_handle_base&) = delete;
+
+ void destroy() {
+ if (!empty()) {
+ PolicyTraits::destroy(alloc(), slot());
+ reset();
+ }
+ }
+
+ void reset() {
+ assert(alloc_.has_value());
+ alloc_ = phmap::nullopt;
+ }
+
+ slot_type* slot() const {
+ assert(!empty());
+ return reinterpret_cast<slot_type*>(std::addressof(slot_space_));
+ }
+
+ allocator_type* alloc() { return std::addressof(*alloc_); }
+
+private:
+ phmap::optional<allocator_type> alloc_;
+ mutable phmap::aligned_storage_t<sizeof(slot_type), alignof(slot_type)> slot_space_;
+};
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+// For sets.
+// ---------
+template <typename Policy, typename PolicyTraits, typename Alloc,
+ typename = void>
+class node_handle : public node_handle_base<PolicyTraits, Alloc>
+{
+ using Base = node_handle_base<PolicyTraits, Alloc>;
+
+public:
+ using value_type = typename PolicyTraits::value_type;
+
+ constexpr node_handle() {}
+
+ value_type& value() const { return PolicyTraits::element(this->slot()); }
+
+ value_type& key() const { return PolicyTraits::element(this->slot()); }
+
+private:
+ friend struct CommonAccess;
+
+ using Base::Base;
+};
+
+// For maps.
+// ---------
+template <typename Policy, typename PolicyTraits, typename Alloc>
+class node_handle<Policy, PolicyTraits, Alloc,
+ phmap::void_t<typename Policy::mapped_type>>
+ : public node_handle_base<PolicyTraits, Alloc>
+{
+ using Base = node_handle_base<PolicyTraits, Alloc>;
+
+public:
+ using key_type = typename Policy::key_type;
+ using mapped_type = typename Policy::mapped_type;
+
+ constexpr node_handle() {}
+
+ auto key() const -> decltype(PolicyTraits::key(this->slot())) {
+ return PolicyTraits::key(this->slot());
+ }
+
+ mapped_type& mapped() const {
+ return PolicyTraits::value(&PolicyTraits::element(this->slot()));
+ }
+
+private:
+ friend struct CommonAccess;
+
+ using Base::Base;
+};
+
+// Provide access to non-public node-handle functions.
+struct CommonAccess
+{
+ template <typename Node>
+ static auto GetSlot(const Node& node) -> decltype(node.slot()) {
+ return node.slot();
+ }
+
+ template <typename Node>
+ static void Destroy(Node* node) {
+ node->destroy();
+ }
+
+ template <typename Node>
+ static void Reset(Node* node) {
+ node->reset();
+ }
+
+ template <typename T, typename... Args>
+ static T Make(Args&&... args) {
+ return T(std::forward<Args>(args)...);
+ }
+
+ template <typename T, typename... Args>
+ static T Transfer(Args&&... args) {
+ return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...);
+ }
+
+ template <typename T, typename... Args>
+ static T Move(Args&&... args) {
+ return T(typename T::move_tag_t{}, std::forward<Args>(args)...);
+ }
+};
+
+// Implement the insert_return_type<> concept of C++17.
+template <class Iterator, class NodeType>
+struct InsertReturnType
+{
+ Iterator position;
+ bool inserted;
+ NodeType node;
+};
+
+} // namespace priv
+} // namespace phmap
+
+
+#ifdef ADDRESS_SANITIZER
+ #include <sanitizer/asan_interface.h>
+#endif
+
+// ---------------------------------------------------------------------------
+// span.h
+// ---------------------------------------------------------------------------
+
+namespace phmap {
+
+template <typename T>
+class Span;
+
+namespace span_internal {
+// A constexpr min function
+constexpr size_t Min(size_t a, size_t b) noexcept { return a < b ? a : b; }
+
+// Wrappers for access to container data pointers.
+template <typename C>
+constexpr auto GetDataImpl(C& c, char) noexcept // NOLINT(runtime/references)
+ -> decltype(c.data()) {
+ return c.data();
+}
+
+// Before C++17, std::string::data returns a const char* in all cases.
+inline char* GetDataImpl(std::string& s, // NOLINT(runtime/references)
+ int) noexcept {
+ return &s[0];
+}
+
+template <typename C>
+constexpr auto GetData(C& c) noexcept // NOLINT(runtime/references)
+ -> decltype(GetDataImpl(c, 0)) {
+ return GetDataImpl(c, 0);
+}
+
+// Detection idioms for size() and data().
+template <typename C>
+using HasSize =
+ std::is_integral<phmap::decay_t<decltype(std::declval<C&>().size())>>;
+
+// We want to enable conversion from vector<T*> to Span<const T* const> but
+// disable conversion from vector<Derived> to Span<Base>. Here we use
+// the fact that U** is convertible to Q* const* if and only if Q is the same
+// type or a more cv-qualified version of U. We also decay the result type of
+// data() to avoid problems with classes which have a member function data()
+// which returns a reference.
+template <typename T, typename C>
+using HasData =
+ std::is_convertible<phmap::decay_t<decltype(GetData(std::declval<C&>()))>*,
+ T* const*>;
+
+// Extracts value type from a Container
+template <typename C>
+struct ElementType {
+ using type = typename phmap::remove_reference_t<C>::value_type;
+};
+
+template <typename T, size_t N>
+struct ElementType<T (&)[N]> {
+ using type = T;
+};
+
+template <typename C>
+using ElementT = typename ElementType<C>::type;
+
+template <typename T>
+using EnableIfMutable =
+ typename std::enable_if<!std::is_const<T>::value, int>::type;
+
+template <typename T>
+bool EqualImpl(Span<T> a, Span<T> b) {
+ static_assert(std::is_const<T>::value, "");
+ return std::equal(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T>
+bool LessThanImpl(Span<T> a, Span<T> b) {
+ static_assert(std::is_const<T>::value, "");
+ return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
+}
+
+// The `IsConvertible` classes here are needed because of the
+// `std::is_convertible` bug in libcxx when compiled with GCC. This build
+// configuration is used by Android NDK toolchain. Reference link:
+// https://bugs.llvm.org/show_bug.cgi?id=27538.
+template <typename From, typename To>
+struct IsConvertibleHelper {
+ static std::true_type testval(To);
+ static std::false_type testval(...);
+
+ using type = decltype(testval(std::declval<From>()));
+};
+
+template <typename From, typename To>
+struct IsConvertible : IsConvertibleHelper<From, To>::type {};
+
+// TODO(zhangxy): replace `IsConvertible` with `std::is_convertible` once the
+// older version of libcxx is not supported.
+template <typename From, typename To>
+using EnableIfConvertibleToSpanConst =
+ typename std::enable_if<IsConvertible<From, Span<const To>>::value>::type;
+} // namespace span_internal
+
+//------------------------------------------------------------------------------
+// Span
+//------------------------------------------------------------------------------
+//
+// A `Span` is an "array view" type for holding a view of a contiguous data
+// array; the `Span` object does not and cannot own such data itself. A span
+// provides an easy way to provide overloads for anything operating on
+// contiguous sequences without needing to manage pointers and array lengths
+// manually.
+
+// A span is conceptually a pointer (ptr) and a length (size) into an already
+// existing array of contiguous memory; the array it represents references the
+// elements "ptr[0] .. ptr[size-1]". Passing a properly-constructed `Span`
+// instead of raw pointers avoids many issues related to index out of bounds
+// errors.
+//
+// Spans may also be constructed from containers holding contiguous sequences.
+// Such containers must supply `data()` and `size() const` methods (e.g
+// `std::vector<T>`, `phmap::InlinedVector<T, N>`). All implicit conversions to
+// `phmap::Span` from such containers will create spans of type `const T`;
+// spans which can mutate their values (of type `T`) must use explicit
+// constructors.
+//
+// A `Span<T>` is somewhat analogous to an `phmap::string_view`, but for an array
+// of elements of type `T`. A user of `Span` must ensure that the data being
+// pointed to outlives the `Span` itself.
+//
+// You can construct a `Span<T>` in several ways:
+//
+// * Explicitly from a reference to a container type
+// * Explicitly from a pointer and size
+// * Implicitly from a container type (but only for spans of type `const T`)
+// * Using the `MakeSpan()` or `MakeConstSpan()` factory functions.
+//
+// Examples:
+//
+// // Construct a Span explicitly from a container:
+// std::vector<int> v = {1, 2, 3, 4, 5};
+// auto span = phmap::Span<const int>(v);
+//
+// // Construct a Span explicitly from a C-style array:
+// int a[5] = {1, 2, 3, 4, 5};
+// auto span = phmap::Span<const int>(a);
+//
+// // Construct a Span implicitly from a container
+// void MyRoutine(phmap::Span<const int> a) {
+// ...
+// }
+// std::vector v = {1,2,3,4,5};
+// MyRoutine(v) // convert to Span<const T>
+//
+// Note that `Span` objects, in addition to requiring that the memory they
+// point to remains alive, must also ensure that such memory does not get
+// reallocated. Therefore, to avoid undefined behavior, containers with
+// associated span views should not invoke operations that may reallocate memory
+// (such as resizing) or invalidate iterators into the container.
+//
+// One common use for a `Span` is when passing arguments to a routine that can
+// accept a variety of array types (e.g. a `std::vector`, `phmap::InlinedVector`,
+// a C-style array, etc.). Instead of creating overloads for each case, you
+// can simply specify a `Span` as the argument to such a routine.
+//
+// Example:
+//
+// void MyRoutine(phmap::Span<const int> a) {
+// ...
+// }
+//
+// std::vector v = {1,2,3,4,5};
+// MyRoutine(v);
+//
+// phmap::InlinedVector<int, 4> my_inline_vector;
+// MyRoutine(my_inline_vector);
+//
+// // Explicit constructor from pointer,size
+// int* my_array = new int[10];
+// MyRoutine(phmap::Span<const int>(my_array, 10));
+template <typename T>
+class Span
+{
+private:
+ // Used to determine whether a Span can be constructed from a container of
+ // type C.
+ template <typename C>
+ using EnableIfConvertibleFrom =
+ typename std::enable_if<span_internal::HasData<T, C>::value &&
+ span_internal::HasSize<C>::value>::type;
+
+ // Used to SFINAE-enable a function when the slice elements are const.
+ template <typename U>
+ using EnableIfConstView =
+ typename std::enable_if<std::is_const<T>::value, U>::type;
+
+ // Used to SFINAE-enable a function when the slice elements are mutable.
+ template <typename U>
+ using EnableIfMutableView =
+ typename std::enable_if<!std::is_const<T>::value, U>::type;
+
+public:
+ using value_type = phmap::remove_cv_t<T>;
+ using pointer = T*;
+ using const_pointer = const T*;
+ using reference = T&;
+ using const_reference = const T&;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using size_type = size_t;
+ using difference_type = ptrdiff_t;
+
+ static const size_type npos = ~(size_type(0));
+
+ constexpr Span() noexcept : Span(nullptr, 0) {}
+ constexpr Span(pointer array, size_type lgth) noexcept
+ : ptr_(array), len_(lgth) {}
+
+ // Implicit conversion constructors
+ template <size_t N>
+ constexpr Span(T (&a)[N]) noexcept // NOLINT(runtime/explicit)
+ : Span(a, N) {}
+
+ // Explicit reference constructor for a mutable `Span<T>` type. Can be
+ // replaced with MakeSpan() to infer the type parameter.
+ template <typename V, typename = EnableIfConvertibleFrom<V>,
+ typename = EnableIfMutableView<V>>
+ explicit Span(V& v) noexcept // NOLINT(runtime/references)
+ : Span(span_internal::GetData(v), v.size()) {}
+
+ // Implicit reference constructor for a read-only `Span<const T>` type
+ template <typename V, typename = EnableIfConvertibleFrom<V>,
+ typename = EnableIfConstView<V>>
+ constexpr Span(const V& v) noexcept // NOLINT(runtime/explicit)
+ : Span(span_internal::GetData(v), v.size()) {}
+
+ // Implicit constructor from an initializer list, making it possible to pass a
+ // brace-enclosed initializer list to a function expecting a `Span`. Such
+ // spans constructed from an initializer list must be of type `Span<const T>`.
+ //
+ // void Process(phmap::Span<const int> x);
+ // Process({1, 2, 3});
+ //
+ // Note that as always the array referenced by the span must outlive the span.
+ // Since an initializer list constructor acts as if it is fed a temporary
+ // array (cf. C++ standard [dcl.init.list]/5), it's safe to use this
+ // constructor only when the `std::initializer_list` itself outlives the span.
+ // In order to meet this requirement it's sufficient to ensure that neither
+ // the span nor a copy of it is used outside of the expression in which it's
+ // created:
+ //
+ // // Assume that this function uses the array directly, not retaining any
+ // // copy of the span or pointer to any of its elements.
+ // void Process(phmap::Span<const int> ints);
+ //
+ // // Okay: the std::initializer_list<int> will reference a temporary array
+ // // that isn't destroyed until after the call to Process returns.
+ // Process({ 17, 19 });
+ //
+ // // Not okay: the storage used by the std::initializer_list<int> is not
+ // // allowed to be referenced after the first line.
+ // phmap::Span<const int> ints = { 17, 19 };
+ // Process(ints);
+ //
+ // // Not okay for the same reason as above: even when the elements of the
+ // // initializer list expression are not temporaries the underlying array
+ // // is, so the initializer list must still outlive the span.
+ // const int foo = 17;
+ // phmap::Span<const int> ints = { foo };
+ // Process(ints);
+ //
+ template <typename LazyT = T,
+ typename = EnableIfConstView<LazyT>>
+ Span(
+ std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit)
+ : Span(v.begin(), v.size()) {}
+
+ // Accessors
+
+ // Span::data()
+ //
+ // Returns a pointer to the span's underlying array of data (which is held
+ // outside the span).
+ constexpr pointer data() const noexcept { return ptr_; }
+
+ // Span::size()
+ //
+ // Returns the size of this span.
+ constexpr size_type size() const noexcept { return len_; }
+
+ // Span::length()
+ //
+ // Returns the length (size) of this span.
+ constexpr size_type length() const noexcept { return size(); }
+
+ // Span::empty()
+ //
+ // Returns a boolean indicating whether or not this span is considered empty.
+ constexpr bool empty() const noexcept { return size() == 0; }
+
+ // Span::operator[]
+ //
+ // Returns a reference to the i'th element of this span.
+ constexpr reference operator[](size_type i) const noexcept {
+ // MSVC 2015 accepts this as constexpr, but not ptr_[i]
+ return *(data() + i);
+ }
+
+ // Span::at()
+ //
+ // Returns a reference to the i'th element of this span.
+ constexpr reference at(size_type i) const {
+ return PHMAP_PREDICT_TRUE(i < size()) //
+ ? *(data() + i)
+ : (base_internal::ThrowStdOutOfRange(
+ "Span::at failed bounds check"),
+ *(data() + i));
+ }
+
+ // Span::front()
+ //
+ // Returns a reference to the first element of this span.
+ constexpr reference front() const noexcept {
+ return PHMAP_ASSERT(size() > 0), *data();
+ }
+
+ // Span::back()
+ //
+ // Returns a reference to the last element of this span.
+ constexpr reference back() const noexcept {
+ return PHMAP_ASSERT(size() > 0), *(data() + size() - 1);
+ }
+
+ // Span::begin()
+ //
+ // Returns an iterator to the first element of this span.
+ constexpr iterator begin() const noexcept { return data(); }
+
+ // Span::cbegin()
+ //
+ // Returns a const iterator to the first element of this span.
+ constexpr const_iterator cbegin() const noexcept { return begin(); }
+
+ // Span::end()
+ //
+ // Returns an iterator to the last element of this span.
+ constexpr iterator end() const noexcept { return data() + size(); }
+
+ // Span::cend()
+ //
+ // Returns a const iterator to the last element of this span.
+ constexpr const_iterator cend() const noexcept { return end(); }
+
+ // Span::rbegin()
+ //
+ // Returns a reverse iterator starting at the last element of this span.
+ constexpr reverse_iterator rbegin() const noexcept {
+ return reverse_iterator(end());
+ }
+
+ // Span::crbegin()
+ //
+ // Returns a reverse const iterator starting at the last element of this span.
+ constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); }
+
+ // Span::rend()
+ //
+ // Returns a reverse iterator starting at the first element of this span.
+ constexpr reverse_iterator rend() const noexcept {
+ return reverse_iterator(begin());
+ }
+
+ // Span::crend()
+ //
+ // Returns a reverse iterator starting at the first element of this span.
+ constexpr const_reverse_iterator crend() const noexcept { return rend(); }
+
+ // Span mutations
+
+ // Span::remove_prefix()
+ //
+ // Removes the first `n` elements from the span.
+ void remove_prefix(size_type n) noexcept {
+ assert(size() >= n);
+ ptr_ += n;
+ len_ -= n;
+ }
+
+ // Span::remove_suffix()
+ //
+ // Removes the last `n` elements from the span.
+ void remove_suffix(size_type n) noexcept {
+ assert(size() >= n);
+ len_ -= n;
+ }
+
+ // Span::subspan()
+ //
+ // Returns a `Span` starting at element `pos` and of length `len`. Both `pos`
+ // and `len` are of type `size_type` and thus non-negative. Parameter `pos`
+ // must be <= size(). Any `len` value that points past the end of the span
+ // will be trimmed to at most size() - `pos`. A default `len` value of `npos`
+ // ensures the returned subspan continues until the end of the span.
+ //
+ // Examples:
+ //
+ // std::vector<int> vec = {10, 11, 12, 13};
+ // phmap::MakeSpan(vec).subspan(1, 2); // {11, 12}
+ // phmap::MakeSpan(vec).subspan(2, 8); // {12, 13}
+ // phmap::MakeSpan(vec).subspan(1); // {11, 12, 13}
+ // phmap::MakeSpan(vec).subspan(4); // {}
+ // phmap::MakeSpan(vec).subspan(5); // throws std::out_of_range
+ constexpr Span subspan(size_type pos = 0, size_type len = npos) const {
+ return (pos <= size())
+ ? Span(data() + pos, span_internal::Min(size() - pos, len))
+ : (base_internal::ThrowStdOutOfRange("pos > size()"), Span());
+ }
+
+ // Span::first()
+ //
+ // Returns a `Span` containing first `len` elements. Parameter `len` is of
+ // type `size_type` and thus non-negative. `len` value must be <= size().
+ //
+ // Examples:
+ //
+ // std::vector<int> vec = {10, 11, 12, 13};
+ // phmap::MakeSpan(vec).first(1); // {10}
+ // phmap::MakeSpan(vec).first(3); // {10, 11, 12}
+ // phmap::MakeSpan(vec).first(5); // throws std::out_of_range
+ constexpr Span first(size_type len) const {
+ return (len <= size())
+ ? Span(data(), len)
+ : (base_internal::ThrowStdOutOfRange("len > size()"), Span());
+ }
+
+ // Span::last()
+ //
+ // Returns a `Span` containing last `len` elements. Parameter `len` is of
+ // type `size_type` and thus non-negative. `len` value must be <= size().
+ //
+ // Examples:
+ //
+ // std::vector<int> vec = {10, 11, 12, 13};
+ // phmap::MakeSpan(vec).last(1); // {13}
+ // phmap::MakeSpan(vec).last(3); // {11, 12, 13}
+ // phmap::MakeSpan(vec).last(5); // throws std::out_of_range
+ constexpr Span last(size_type len) const {
+ return (len <= size())
+ ? Span(size() - len + data(), len)
+ : (base_internal::ThrowStdOutOfRange("len > size()"), Span());
+ }
+
+ // Support for phmap::Hash.
+ template <typename H>
+ friend H AbslHashValue(H h, Span v) {
+ return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()),
+ v.size());
+ }
+
+private:
+ pointer ptr_;
+ size_type len_;
+};
+
+template <typename T>
+const typename Span<T>::size_type Span<T>::npos;
+
+// Span relationals
+
+// Equality is compared element-by-element, while ordering is lexicographical.
+// We provide three overloads for each operator to cover any combination on the
+// left or right hand side of mutable Span<T>, read-only Span<const T>, and
+// convertible-to-read-only Span<T>.
+// TODO(zhangxy): Due to MSVC overload resolution bug with partial ordering
+// template functions, 5 overloads per operator is needed as a workaround. We
+// should update them to 3 overloads per operator using non-deduced context like
+// string_view, i.e.
+// - (Span<T>, Span<T>)
+// - (Span<T>, non_deduced<Span<const T>>)
+// - (non_deduced<Span<const T>>, Span<T>)
+
+// operator==
+template <typename T>
+bool operator==(Span<T> a, Span<T> b) {
+ return span_internal::EqualImpl<const T>(a, b);
+}
+
+template <typename T>
+bool operator==(Span<const T> a, Span<T> b) {
+ return span_internal::EqualImpl<const T>(a, b);
+}
+
+template <typename T>
+bool operator==(Span<T> a, Span<const T> b) {
+ return span_internal::EqualImpl<const T>(a, b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator==(const U& a, Span<T> b) {
+ return span_internal::EqualImpl<const T>(a, b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator==(Span<T> a, const U& b) {
+ return span_internal::EqualImpl<const T>(a, b);
+}
+
+// operator!=
+template <typename T>
+bool operator!=(Span<T> a, Span<T> b) {
+ return !(a == b);
+}
+
+template <typename T>
+bool operator!=(Span<const T> a, Span<T> b) {
+ return !(a == b);
+}
+
+template <typename T>
+bool operator!=(Span<T> a, Span<const T> b) {
+ return !(a == b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator!=(const U& a, Span<T> b) {
+ return !(a == b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator!=(Span<T> a, const U& b) {
+ return !(a == b);
+}
+
+// operator<
+template <typename T>
+bool operator<(Span<T> a, Span<T> b) {
+ return span_internal::LessThanImpl<const T>(a, b);
+}
+
+template <typename T>
+bool operator<(Span<const T> a, Span<T> b) {
+ return span_internal::LessThanImpl<const T>(a, b);
+}
+
+template <typename T>
+bool operator<(Span<T> a, Span<const T> b) {
+ return span_internal::LessThanImpl<const T>(a, b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator<(const U& a, Span<T> b) {
+ return span_internal::LessThanImpl<const T>(a, b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator<(Span<T> a, const U& b) {
+ return span_internal::LessThanImpl<const T>(a, b);
+}
+
+// operator>
+template <typename T>
+bool operator>(Span<T> a, Span<T> b) {
+ return b < a;
+}
+
+template <typename T>
+bool operator>(Span<const T> a, Span<T> b) {
+ return b < a;
+}
+
+template <typename T>
+bool operator>(Span<T> a, Span<const T> b) {
+ return b < a;
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator>(const U& a, Span<T> b) {
+ return b < a;
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator>(Span<T> a, const U& b) {
+ return b < a;
+}
+
+// operator<=
+template <typename T>
+bool operator<=(Span<T> a, Span<T> b) {
+ return !(b < a);
+}
+
+template <typename T>
+bool operator<=(Span<const T> a, Span<T> b) {
+ return !(b < a);
+}
+
+template <typename T>
+bool operator<=(Span<T> a, Span<const T> b) {
+ return !(b < a);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator<=(const U& a, Span<T> b) {
+ return !(b < a);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator<=(Span<T> a, const U& b) {
+ return !(b < a);
+}
+
+// operator>=
+template <typename T>
+bool operator>=(Span<T> a, Span<T> b) {
+ return !(a < b);
+}
+
+template <typename T>
+bool operator>=(Span<const T> a, Span<T> b) {
+ return !(a < b);
+}
+
+template <typename T>
+bool operator>=(Span<T> a, Span<const T> b) {
+ return !(a < b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator>=(const U& a, Span<T> b) {
+ return !(a < b);
+}
+
+template <typename T, typename U,
+ typename = span_internal::EnableIfConvertibleToSpanConst<U, T>>
+bool operator>=(Span<T> a, const U& b) {
+ return !(a < b);
+}
+
+// MakeSpan()
+//
+// Constructs a mutable `Span<T>`, deducing `T` automatically from either a
+// container or pointer+size.
+//
+// Because a read-only `Span<const T>` is implicitly constructed from container
+// types regardless of whether the container itself is a const container,
+// constructing mutable spans of type `Span<T>` from containers requires
+// explicit constructors. The container-accepting version of `MakeSpan()`
+// deduces the type of `T` by the constness of the pointer received from the
+// container's `data()` member. Similarly, the pointer-accepting version returns
+// a `Span<const T>` if `T` is `const`, and a `Span<T>` otherwise.
+//
+// Examples:
+//
+// void MyRoutine(phmap::Span<MyComplicatedType> a) {
+// ...
+// };
+// // my_vector is a container of non-const types
+// std::vector<MyComplicatedType> my_vector;
+//
+// // Constructing a Span implicitly attempts to create a Span of type
+// // `Span<const T>`
+// MyRoutine(my_vector); // error, type mismatch
+//
+// // Explicitly constructing the Span is verbose
+// MyRoutine(phmap::Span<MyComplicatedType>(my_vector));
+//
+// // Use MakeSpan() to make an phmap::Span<T>
+// MyRoutine(phmap::MakeSpan(my_vector));
+//
+// // Construct a span from an array ptr+size
+// phmap::Span<T> my_span() {
+// return phmap::MakeSpan(&array[0], num_elements_);
+// }
+//
+template <int&... ExplicitArgumentBarrier, typename T>
+constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept {
+ return Span<T>(ptr, size);
+}
+
+template <int&... ExplicitArgumentBarrier, typename T>
+Span<T> MakeSpan(T* begin, T* end) noexcept {
+ return PHMAP_ASSERT(begin <= end), Span<T>(begin, end - begin);
+}
+
+template <int&... ExplicitArgumentBarrier, typename C>
+constexpr auto MakeSpan(C& c) noexcept // NOLINT(runtime/references)
+ -> decltype(phmap::MakeSpan(span_internal::GetData(c), c.size())) {
+ return MakeSpan(span_internal::GetData(c), c.size());
+}
+
+template <int&... ExplicitArgumentBarrier, typename T, size_t N>
+constexpr Span<T> MakeSpan(T (&array)[N]) noexcept {
+ return Span<T>(array, N);
+}
+
+// MakeConstSpan()
+//
+// Constructs a `Span<const T>` as with `MakeSpan`, deducing `T` automatically,
+// but always returning a `Span<const T>`.
+//
+// Examples:
+//
+// void ProcessInts(phmap::Span<const int> some_ints);
+//
+// // Call with a pointer and size.
+// int array[3] = { 0, 0, 0 };
+// ProcessInts(phmap::MakeConstSpan(&array[0], 3));
+//
+// // Call with a [begin, end) pair.
+// ProcessInts(phmap::MakeConstSpan(&array[0], &array[3]));
+//
+// // Call directly with an array.
+// ProcessInts(phmap::MakeConstSpan(array));
+//
+// // Call with a contiguous container.
+// std::vector<int> some_ints = ...;
+// ProcessInts(phmap::MakeConstSpan(some_ints));
+// ProcessInts(phmap::MakeConstSpan(std::vector<int>{ 0, 0, 0 }));
+//
+template <int&... ExplicitArgumentBarrier, typename T>
+constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept {
+ return Span<const T>(ptr, size);
+}
+
+template <int&... ExplicitArgumentBarrier, typename T>
+Span<const T> MakeConstSpan(T* begin, T* end) noexcept {
+ return PHMAP_ASSERT(begin <= end), Span<const T>(begin, end - begin);
+}
+
+template <int&... ExplicitArgumentBarrier, typename C>
+constexpr auto MakeConstSpan(const C& c) noexcept -> decltype(MakeSpan(c)) {
+ return MakeSpan(c);
+}
+
+template <int&... ExplicitArgumentBarrier, typename T, size_t N>
+constexpr Span<const T> MakeConstSpan(const T (&array)[N]) noexcept {
+ return Span<const T>(array, N);
+}
+} // namespace phmap
+
+// ---------------------------------------------------------------------------
+// layout.h
+// ---------------------------------------------------------------------------
+#if defined(__GXX_RTTI)
+ #define PHMAP_INTERNAL_HAS_CXA_DEMANGLE
+#endif
+
+#ifdef PHMAP_INTERNAL_HAS_CXA_DEMANGLE
+ #include <cxxabi.h>
+#endif
+
+namespace phmap {
+namespace priv {
+
+// A type wrapper that instructs `Layout` to use the specific alignment for the
+// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API
+// and behavior as `Layout<..., T, ...>` except that the first element of the
+// array of `T` is aligned to `N` (the rest of the elements follow without
+// padding).
+//
+// Requires: `N >= alignof(T)` and `N` is a power of 2.
+template <class T, size_t N>
+struct Aligned;
+
+namespace internal_layout {
+
+template <class T>
+struct NotAligned {};
+
+template <class T, size_t N>
+struct NotAligned<const Aligned<T, N>> {
+ static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified");
+};
+
+template <size_t>
+using IntToSize = size_t;
+
+template <class>
+using TypeToSize = size_t;
+
+template <class T>
+struct Type : NotAligned<T> {
+ using type = T;
+};
+
+template <class T, size_t N>
+struct Type<Aligned<T, N>> {
+ using type = T;
+};
+
+template <class T>
+struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {};
+
+template <class T, size_t N>
+struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {};
+
+// Note: workaround for https://gcc.gnu.org/PR88115
+template <class T>
+struct AlignOf : NotAligned<T> {
+ static constexpr size_t value = alignof(T);
+};
+
+template <class T, size_t N>
+struct AlignOf<Aligned<T, N>> {
+ static_assert(N % alignof(T) == 0,
+ "Custom alignment can't be lower than the type's alignment");
+ static constexpr size_t value = N;
+};
+
+// Does `Ts...` contain `T`?
+template <class T, class... Ts>
+using Contains = phmap::disjunction<std::is_same<T, Ts>...>;
+
+template <class From, class To>
+using CopyConst =
+ typename std::conditional<std::is_const<From>::value, const To, To>::type;
+
+// Note: We're not qualifying this with phmap:: because it doesn't compile under
+// MSVC.
+template <class T>
+using SliceType = Span<T>;
+
+// This namespace contains no types. It prevents functions defined in it from
+// being found by ADL.
+namespace adl_barrier {
+
+template <class Needle, class... Ts>
+constexpr size_t Find(Needle, Needle, Ts...) {
+ static_assert(!Contains<Needle, Ts...>(), "Duplicate element type");
+ return 0;
+}
+
+template <class Needle, class T, class... Ts>
+constexpr size_t Find(Needle, T, Ts...) {
+ return adl_barrier::Find(Needle(), Ts()...) + 1;
+}
+
+constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); }
+
+// Returns `q * m` for the smallest `q` such that `q * m >= n`.
+// Requires: `m` is a power of two. It's enforced by IsLegalElementType below.
+constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }
+
+constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; }
+
+constexpr size_t Max(size_t a) { return a; }
+
+template <class... Ts>
+constexpr size_t Max(size_t a, size_t b, Ts... rest) {
+ return adl_barrier::Max(b < a ? a : b, rest...);
+}
+
+} // namespace adl_barrier
+
+template <bool C>
+using EnableIf = typename std::enable_if<C, int>::type;
+
+// Can `T` be a template argument of `Layout`?
+// ---------------------------------------------------------------------------
+template <class T>
+using IsLegalElementType = std::integral_constant<
+ bool, !std::is_reference<T>::value && !std::is_volatile<T>::value &&
+ !std::is_reference<typename Type<T>::type>::value &&
+ !std::is_volatile<typename Type<T>::type>::value &&
+ adl_barrier::IsPow2(AlignOf<T>::value)>;
+
+template <class Elements, class SizeSeq, class OffsetSeq>
+class LayoutImpl;
+
+// ---------------------------------------------------------------------------
+// Public base class of `Layout` and the result type of `Layout::Partial()`.
+//
+// `Elements...` contains all template arguments of `Layout` that created this
+// instance.
+//
+// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments
+// passed to `Layout::Partial()` or `Layout::Layout()`.
+//
+// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is
+// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we
+// can compute offsets).
+// ---------------------------------------------------------------------------
+template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq>
+class LayoutImpl<std::tuple<Elements...>, phmap::index_sequence<SizeSeq...>,
+ phmap::index_sequence<OffsetSeq...>>
+{
+private:
+ static_assert(sizeof...(Elements) > 0, "At least one field is required");
+ static_assert(phmap::conjunction<IsLegalElementType<Elements>...>::value,
+ "Invalid element type (see IsLegalElementType)");
+
+ enum {
+ NumTypes = sizeof...(Elements),
+ NumSizes = sizeof...(SizeSeq),
+ NumOffsets = sizeof...(OffsetSeq),
+ };
+
+ // These are guaranteed by `Layout`.
+ static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1),
+ "Internal error");
+ static_assert(NumTypes > 0, "Internal error");
+
+ // Returns the index of `T` in `Elements...`. Results in a compilation error
+ // if `Elements...` doesn't contain exactly one instance of `T`.
+ template <class T>
+ static constexpr size_t ElementIndex() {
+ static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(),
+ "Type not found");
+ return adl_barrier::Find(Type<T>(),
+ Type<typename Type<Elements>::type>()...);
+ }
+
+ template <size_t N>
+ using ElementAlignment =
+ AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>;
+
+public:
+ // Element types of all arrays packed in a tuple.
+ using ElementTypes = std::tuple<typename Type<Elements>::type...>;
+
+ // Element type of the Nth array.
+ template <size_t N>
+ using ElementType = typename std::tuple_element<N, ElementTypes>::type;
+
+ constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes)
+ : size_{sizes...} {}
+
+ // Alignment of the layout, equal to the strictest alignment of all elements.
+ // All pointers passed to the methods of layout must be aligned to this value.
+ static constexpr size_t Alignment() {
+ return adl_barrier::Max(AlignOf<Elements>::value...);
+ }
+
+ // Offset in bytes of the Nth array.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // assert(x.Offset<0>() == 0); // The ints starts from 0.
+ // assert(x.Offset<1>() == 16); // The doubles starts from 16.
+ //
+ // Requires: `N <= NumSizes && N < sizeof...(Ts)`.
+ template <size_t N, EnableIf<N == 0> = 0>
+ constexpr size_t Offset() const {
+ return 0;
+ }
+
+ template <size_t N, EnableIf<N != 0> = 0>
+ constexpr size_t Offset() const {
+ static_assert(N < NumOffsets, "Index out of bounds");
+ return adl_barrier::Align(
+ Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1],
+ ElementAlignment<N>::value);
+ }
+
+ // Offset in bytes of the array with the specified element type. There must
+ // be exactly one such array and its zero-based index must be at most
+ // `NumSizes`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // assert(x.Offset<int>() == 0); // The ints starts from 0.
+ // assert(x.Offset<double>() == 16); // The doubles starts from 16.
+ template <class T>
+ constexpr size_t Offset() const {
+ return Offset<ElementIndex<T>()>();
+ }
+
+ // Offsets in bytes of all arrays for which the offsets are known.
+ constexpr std::array<size_t, NumOffsets> Offsets() const {
+ return {{Offset<OffsetSeq>()...}};
+ }
+
+ // The number of elements in the Nth array. This is the Nth argument of
+ // `Layout::Partial()` or `Layout::Layout()` (zero-based).
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // assert(x.Size<0>() == 3);
+ // assert(x.Size<1>() == 4);
+ //
+ // Requires: `N < NumSizes`.
+ template <size_t N>
+ constexpr size_t Size() const {
+ static_assert(N < NumSizes, "Index out of bounds");
+ return size_[N];
+ }
+
+ // The number of elements in the array with the specified element type.
+ // There must be exactly one such array and its zero-based index must be
+ // at most `NumSizes`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // assert(x.Size<int>() == 3);
+ // assert(x.Size<double>() == 4);
+ template <class T>
+ constexpr size_t Size() const {
+ return Size<ElementIndex<T>()>();
+ }
+
+ // The number of elements of all arrays for which they are known.
+ constexpr std::array<size_t, NumSizes> Sizes() const {
+ return {{Size<SizeSeq>()...}};
+ }
+
+ // Pointer to the beginning of the Nth array.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ // int* ints = x.Pointer<0>(p);
+ // double* doubles = x.Pointer<1>(p);
+ //
+ // Requires: `N <= NumSizes && N < sizeof...(Ts)`.
+ // Requires: `p` is aligned to `Alignment()`.
+ template <size_t N, class Char>
+ CopyConst<Char, ElementType<N>>* Pointer(Char* p) const {
+ using C = typename std::remove_const<Char>::type;
+ static_assert(
+ std::is_same<C, char>() || std::is_same<C, unsigned char>() ||
+ std::is_same<C, signed char>(),
+ "The argument must be a pointer to [const] [signed|unsigned] char");
+ constexpr size_t alignment = Alignment();
+ (void)alignment;
+ assert(reinterpret_cast<uintptr_t>(p) % alignment == 0);
+ return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>());
+ }
+
+ // Pointer to the beginning of the array with the specified element type.
+ // There must be exactly one such array and its zero-based index must be at
+ // most `NumSizes`.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ // int* ints = x.Pointer<int>(p);
+ // double* doubles = x.Pointer<double>(p);
+ //
+ // Requires: `p` is aligned to `Alignment()`.
+ template <class T, class Char>
+ CopyConst<Char, T>* Pointer(Char* p) const {
+ return Pointer<ElementIndex<T>()>(p);
+ }
+
+ // Pointers to all arrays for which pointers are known.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ //
+ // int* ints;
+ // double* doubles;
+ // std::tie(ints, doubles) = x.Pointers(p);
+ //
+ // Requires: `p` is aligned to `Alignment()`.
+ //
+ // Note: We're not using ElementType alias here because it does not compile
+ // under MSVC.
+ template <class Char>
+ std::tuple<CopyConst<
+ Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...>
+ Pointers(Char* p) const {
+ return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>(
+ Pointer<OffsetSeq>(p)...);
+ }
+
+ // The Nth array.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ // Span<int> ints = x.Slice<0>(p);
+ // Span<double> doubles = x.Slice<1>(p);
+ //
+ // Requires: `N < NumSizes`.
+ // Requires: `p` is aligned to `Alignment()`.
+ template <size_t N, class Char>
+ SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const {
+ return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>());
+ }
+
+ // The array with the specified element type. There must be exactly one
+ // such array and its zero-based index must be less than `NumSizes`.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ // Span<int> ints = x.Slice<int>(p);
+ // Span<double> doubles = x.Slice<double>(p);
+ //
+ // Requires: `p` is aligned to `Alignment()`.
+ template <class T, class Char>
+ SliceType<CopyConst<Char, T>> Slice(Char* p) const {
+ return Slice<ElementIndex<T>()>(p);
+ }
+
+ // All arrays with known sizes.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()];
+ //
+ // Span<int> ints;
+ // Span<double> doubles;
+ // std::tie(ints, doubles) = x.Slices(p);
+ //
+ // Requires: `p` is aligned to `Alignment()`.
+ //
+ // Note: We're not using ElementType alias here because it does not compile
+ // under MSVC.
+ template <class Char>
+ std::tuple<SliceType<CopyConst<
+ Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...>
+ Slices(Char* p) const {
+ // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed
+ // in 6.1).
+ (void)p;
+ return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>(
+ Slice<SizeSeq>(p)...);
+ }
+
+ // The size of the allocation that fits all arrays.
+ //
+ // // int[3], 4 bytes of padding, double[4].
+ // Layout<int, double> x(3, 4);
+ // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes
+ //
+ // Requires: `NumSizes == sizeof...(Ts)`.
+ constexpr size_t AllocSize() const {
+ static_assert(NumTypes == NumSizes, "You must specify sizes of all fields");
+ return Offset<NumTypes - 1>() +
+ SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1];
+ }
+
+ // If built with --config=asan, poisons padding bytes (if any) in the
+ // allocation. The pointer must point to a memory block at least
+ // `AllocSize()` bytes in length.
+ //
+ // `Char` must be `[const] [signed|unsigned] char`.
+ //
+ // Requires: `p` is aligned to `Alignment()`.
+ template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0>
+ void PoisonPadding(const Char* p) const {
+ Pointer<0>(p); // verify the requirements on `Char` and `p`
+ }
+
+ template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0>
+ void PoisonPadding(const Char* p) const {
+ static_assert(N < NumOffsets, "Index out of bounds");
+ (void)p;
+#ifdef ADDRESS_SANITIZER
+ PoisonPadding<Char, N - 1>(p);
+ // The `if` is an optimization. It doesn't affect the observable behaviour.
+ if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) {
+ size_t start =
+ Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1];
+ ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start);
+ }
+#endif
+ }
+
+private:
+ // Arguments of `Layout::Partial()` or `Layout::Layout()`.
+ size_t size_[NumSizes > 0 ? NumSizes : 1];
+};
+
+template <size_t NumSizes, class... Ts>
+using LayoutType = LayoutImpl<
+ std::tuple<Ts...>, phmap::make_index_sequence<NumSizes>,
+ phmap::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>;
+
+} // namespace internal_layout
+
+// ---------------------------------------------------------------------------
+// Descriptor of arrays of various types and sizes laid out in memory one after
+// another. See the top of the file for documentation.
+//
+// Check out the public API of internal_layout::LayoutImpl above. The type is
+// internal to the library but its methods are public, and they are inherited
+// by `Layout`.
+// ---------------------------------------------------------------------------
+template <class... Ts>
+class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...>
+{
+public:
+ static_assert(sizeof...(Ts) > 0, "At least one field is required");
+ static_assert(
+ phmap::conjunction<internal_layout::IsLegalElementType<Ts>...>::value,
+ "Invalid element type (see IsLegalElementType)");
+
+ template <size_t NumSizes>
+ using PartialType = internal_layout::LayoutType<NumSizes, Ts...>;
+
+ template <class... Sizes>
+ static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) {
+ static_assert(sizeof...(Sizes) <= sizeof...(Ts), "");
+ return PartialType<sizeof...(Sizes)>(phmap::forward<Sizes>(sizes)...);
+ }
+
+ // Creates a layout with the sizes of all arrays specified. If you know
+ // only the sizes of the first N arrays (where N can be zero), you can use
+ // `Partial()` defined above. The constructor is essentially equivalent to
+ // calling `Partial()` and passing in all array sizes; the constructor is
+ // provided as a convenient abbreviation.
+ //
+ // Note: The sizes of the arrays must be specified in number of elements,
+ // not in bytes.
+ constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes)
+ : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {}
+};
+
+} // namespace priv
+} // namespace phmap
+
+// ---------------------------------------------------------------------------
+// compressed_tuple.h
+// ---------------------------------------------------------------------------
+
+#ifdef _MSC_VER
+ // We need to mark these classes with this declspec to ensure that
+ // CompressedTuple happens.
+ #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases)
+#else // _MSC_VER
+ #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC
+#endif // _MSC_VER
+
+namespace phmap {
+namespace priv {
+
+template <typename... Ts>
+class CompressedTuple;
+
+namespace internal_compressed_tuple {
+
+template <typename D, size_t I>
+struct Elem;
+template <typename... B, size_t I>
+struct Elem<CompressedTuple<B...>, I>
+ : std::tuple_element<I, std::tuple<B...>> {};
+template <typename D, size_t I>
+using ElemT = typename Elem<D, I>::type;
+
+// ---------------------------------------------------------------------------
+// Use the __is_final intrinsic if available. Where it's not available, classes
+// declared with the 'final' specifier cannot be used as CompressedTuple
+// elements.
+// TODO(sbenza): Replace this with std::is_final in C++14.
+// ---------------------------------------------------------------------------
+template <typename T>
+constexpr bool IsFinal() {
+#if defined(__clang__) || defined(__GNUC__)
+ return __is_final(T);
+#else
+ return false;
+#endif
+}
+
+template <typename T>
+constexpr bool ShouldUseBase() {
+#ifdef __INTEL_COMPILER
+ // avoid crash in Intel compiler
+ // assertion failed at: "shared/cfe/edgcpfe/lower_init.c", line 7013
+ return false;
+#else
+ return std::is_class<T>::value && std::is_empty<T>::value && !IsFinal<T>();
+#endif
+}
+
+// The storage class provides two specializations:
+// - For empty classes, it stores T as a base class.
+// - For everything else, it stores T as a member.
+// ------------------------------------------------
+template <typename D, size_t I, bool = ShouldUseBase<ElemT<D, I>>()>
+struct Storage
+{
+ using T = ElemT<D, I>;
+ T value;
+ constexpr Storage() = default;
+ explicit constexpr Storage(T&& v) : value(phmap::forward<T>(v)) {}
+ constexpr const T& get() const& { return value; }
+ T& get() & { return value; }
+ constexpr const T&& get() const&& { return phmap::move(*this).value; }
+ T&& get() && { return std::move(*this).value; }
+};
+
+template <typename D, size_t I>
+struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<D, I, true>
+ : ElemT<D, I>
+{
+ using T = internal_compressed_tuple::ElemT<D, I>;
+ constexpr Storage() = default;
+ explicit constexpr Storage(T&& v) : T(phmap::forward<T>(v)) {}
+ constexpr const T& get() const& { return *this; }
+ T& get() & { return *this; }
+ constexpr const T&& get() const&& { return phmap::move(*this); }
+ T&& get() && { return std::move(*this); }
+};
+
+template <typename D, typename I>
+struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl;
+
+template <typename... Ts, size_t... I>
+struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC
+ CompressedTupleImpl<CompressedTuple<Ts...>, phmap::index_sequence<I...>>
+ // We use the dummy identity function through std::integral_constant to
+ // convince MSVC of accepting and expanding I in that context. Without it
+ // you would get:
+ // error C3548: 'I': parameter pack cannot be used in this context
+ : Storage<CompressedTuple<Ts...>,
+ std::integral_constant<size_t, I>::value>...
+{
+ constexpr CompressedTupleImpl() = default;
+ explicit constexpr CompressedTupleImpl(Ts&&... args)
+ : Storage<CompressedTuple<Ts...>, I>(phmap::forward<Ts>(args))... {}
+};
+
+} // namespace internal_compressed_tuple
+
+// ---------------------------------------------------------------------------
+// Helper class to perform the Empty Base Class Optimization.
+// Ts can contain classes and non-classes, empty or not. For the ones that
+// are empty classes, we perform the CompressedTuple. If all types in Ts are
+// empty classes, then CompressedTuple<Ts...> is itself an empty class.
+//
+// To access the members, use member .get<N>() function.
+//
+// Eg:
+// phmap::priv::CompressedTuple<int, T1, T2, T3> value(7, t1, t2,
+// t3);
+// assert(value.get<0>() == 7);
+// T1& t1 = value.get<1>();
+// const T2& t2 = value.get<2>();
+// ...
+//
+// https://en.cppreference.com/w/cpp/language/ebo
+// ---------------------------------------------------------------------------
+template <typename... Ts>
+class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple
+ : private internal_compressed_tuple::CompressedTupleImpl<
+ CompressedTuple<Ts...>, phmap::index_sequence_for<Ts...>>
+{
+private:
+ template <int I>
+ using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>;
+
+public:
+ constexpr CompressedTuple() = default;
+ explicit constexpr CompressedTuple(Ts... base)
+ : CompressedTuple::CompressedTupleImpl(phmap::forward<Ts>(base)...) {}
+
+ template <int I>
+ ElemT<I>& get() & {
+ return internal_compressed_tuple::Storage<CompressedTuple, I>::get();
+ }
+
+ template <int I>
+ constexpr const ElemT<I>& get() const& {
+ return internal_compressed_tuple::Storage<CompressedTuple, I>::get();
+ }
+
+ template <int I>
+ ElemT<I>&& get() && {
+ return std::move(*this)
+ .internal_compressed_tuple::template Storage<CompressedTuple, I>::get();
+ }
+
+ template <int I>
+ constexpr const ElemT<I>&& get() const&& {
+ return phmap::move(*this)
+ .internal_compressed_tuple::template Storage<CompressedTuple, I>::get();
+ }
+};
+
+// Explicit specialization for a zero-element tuple
+// (needed to avoid ambiguous overloads for the default constructor).
+// ---------------------------------------------------------------------------
+template <>
+class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {};
+
+} // namespace priv
+} // namespace phmap
+
+
+namespace phmap {
+namespace priv {
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ // warning warning C4324: structure was padded due to alignment specifier
+ #pragma warning(disable : 4324)
+#endif
+
+
+// ----------------------------------------------------------------------------
+// Allocates at least n bytes aligned to the specified alignment.
+// Alignment must be a power of 2. It must be positive.
+//
+// Note that many allocators don't honor alignment requirements above certain
+// threshold (usually either alignof(std::max_align_t) or alignof(void*)).
+// Allocate() doesn't apply alignment corrections. If the underlying allocator
+// returns insufficiently alignment pointer, that's what you are going to get.
+// ----------------------------------------------------------------------------
+template <size_t Alignment, class Alloc>
+void* Allocate(Alloc* alloc, size_t n) {
+ static_assert(Alignment > 0, "");
+ assert(n && "n must be positive");
+ struct alignas(Alignment) M {};
+ using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>;
+ using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>;
+ A mem_alloc(*alloc);
+ void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M));
+ assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 &&
+ "allocator does not respect alignment");
+ return p;
+}
+
+// ----------------------------------------------------------------------------
+// The pointer must have been previously obtained by calling
+// Allocate<Alignment>(alloc, n).
+// ----------------------------------------------------------------------------
+template <size_t Alignment, class Alloc>
+void Deallocate(Alloc* alloc, void* p, size_t n) {
+ static_assert(Alignment > 0, "");
+ assert(n && "n must be positive");
+ struct alignas(Alignment) M {};
+ using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>;
+ using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>;
+ A mem_alloc(*alloc);
+ AT::deallocate(mem_alloc, static_cast<M*>(p),
+ (n + sizeof(M) - 1) / sizeof(M));
+}
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+// Helper functions for asan and msan.
+// ----------------------------------------------------------------------------
+inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) {
+#ifdef ADDRESS_SANITIZER
+ ASAN_POISON_MEMORY_REGION(m, s);
+#endif
+#ifdef MEMORY_SANITIZER
+ __msan_poison(m, s);
+#endif
+ (void)m;
+ (void)s;
+}
+
+inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) {
+#ifdef ADDRESS_SANITIZER
+ ASAN_UNPOISON_MEMORY_REGION(m, s);
+#endif
+#ifdef MEMORY_SANITIZER
+ __msan_unpoison(m, s);
+#endif
+ (void)m;
+ (void)s;
+}
+
+template <typename T>
+inline void SanitizerPoisonObject(const T* object) {
+ SanitizerPoisonMemoryRegion(object, sizeof(T));
+}
+
+template <typename T>
+inline void SanitizerUnpoisonObject(const T* object) {
+ SanitizerUnpoisonMemoryRegion(object, sizeof(T));
+}
+
+} // namespace priv
+} // namespace phmap
+
+
+// ---------------------------------------------------------------------------
+// thread_annotations.h
+// ---------------------------------------------------------------------------
+
+#if defined(__clang__)
+ #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
+#else
+ #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
+#endif
+
+#define PHMAP_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))
+#define PHMAP_PT_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))
+
+#define PHMAP_ACQUIRED_AFTER(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))
+
+#define PHMAP_ACQUIRED_BEFORE(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))
+
+#define PHMAP_EXCLUSIVE_LOCKS_REQUIRED(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__))
+
+#define PHMAP_SHARED_LOCKS_REQUIRED(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__))
+
+#define PHMAP_LOCKS_EXCLUDED(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__))
+
+#define PHMAP_LOCK_RETURNED(x) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
+
+#define PHMAP_LOCKABLE \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lockable)
+
+#define PHMAP_SCOPED_LOCKABLE \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
+
+#define PHMAP_EXCLUSIVE_LOCK_FUNCTION(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__))
+
+#define PHMAP_SHARED_LOCK_FUNCTION(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__))
+
+#define PHMAP_UNLOCK_FUNCTION(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__))
+
+#define PHMAP_EXCLUSIVE_TRYLOCK_FUNCTION(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__))
+
+#define PHMAP_SHARED_TRYLOCK_FUNCTION(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__))
+
+#define PHMAP_ASSERT_EXCLUSIVE_LOCK(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__))
+
+#define PHMAP_ASSERT_SHARED_LOCK(...) \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__))
+
+#define PHMAP_NO_THREAD_SAFETY_ANALYSIS \
+ PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis)
+
+//------------------------------------------------------------------------------
+// Tool-Supplied Annotations
+//------------------------------------------------------------------------------
+
+// TS_UNCHECKED should be placed around lock expressions that are not valid
+// C++ syntax, but which are present for documentation purposes. These
+// annotations will be ignored by the analysis.
+#define PHMAP_TS_UNCHECKED(x) ""
+
+// TS_FIXME is used to mark lock expressions that are not valid C++ syntax.
+// It is used by automated tools to mark and disable invalid expressions.
+// The annotation should either be fixed, or changed to TS_UNCHECKED.
+#define PHMAP_TS_FIXME(x) ""
+
+// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of
+// a particular function. However, this attribute is used to mark functions
+// that are incorrect and need to be fixed. It is used by automated tools to
+// avoid breaking the build when the analysis is updated.
+// Code owners are expected to eventually fix the routine.
+#define PHMAP_NO_THREAD_SAFETY_ANALYSIS_FIXME PHMAP_NO_THREAD_SAFETY_ANALYSIS
+
+// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY
+// annotation that needs to be fixed, because it is producing thread safety
+// warning. It disables the GUARDED_BY.
+#define PHMAP_GUARDED_BY_FIXME(x)
+
+// Disables warnings for a single read operation. This can be used to avoid
+// warnings when it is known that the read is not actually involved in a race,
+// but the compiler cannot confirm that.
+#define PHMAP_TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x)
+
+
+namespace phmap {
+namespace thread_safety_analysis {
+
+// Takes a reference to a guarded data member, and returns an unguarded
+// reference.
+template <typename T>
+inline const T& ts_unchecked_read(const T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS {
+ return v;
+}
+
+template <typename T>
+inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS {
+ return v;
+}
+
+} // namespace thread_safety_analysis
+
+namespace priv {
+
+namespace memory_internal {
+
+// ----------------------------------------------------------------------------
+// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and
+// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and
+// offsetof(Pair, second) respectively. Otherwise they are -1.
+//
+// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout
+// type, which is non-portable.
+// ----------------------------------------------------------------------------
+template <class Pair, class = std::true_type>
+struct OffsetOf {
+ static constexpr size_t kFirst = (size_t)-1;
+ static constexpr size_t kSecond = (size_t)-1;
+};
+
+template <class Pair>
+struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type>
+{
+ static constexpr size_t kFirst = offsetof(Pair, first);
+ static constexpr size_t kSecond = offsetof(Pair, second);
+};
+
+// ----------------------------------------------------------------------------
+template <class K, class V>
+struct IsLayoutCompatible
+{
+private:
+ struct Pair {
+ K first;
+ V second;
+ };
+
+ // Is P layout-compatible with Pair?
+ template <class P>
+ static constexpr bool LayoutCompatible() {
+ return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) &&
+ alignof(P) == alignof(Pair) &&
+ memory_internal::OffsetOf<P>::kFirst ==
+ memory_internal::OffsetOf<Pair>::kFirst &&
+ memory_internal::OffsetOf<P>::kSecond ==
+ memory_internal::OffsetOf<Pair>::kSecond;
+ }
+
+public:
+ // Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are,
+ // then it is safe to store them in a union and read from either.
+ static constexpr bool value = std::is_standard_layout<K>() &&
+ std::is_standard_layout<Pair>() &&
+ memory_internal::OffsetOf<Pair>::kFirst == 0 &&
+ LayoutCompatible<std::pair<K, V>>() &&
+ LayoutCompatible<std::pair<const K, V>>();
+};
+
+} // namespace memory_internal
+
+// ----------------------------------------------------------------------------
+// The internal storage type for key-value containers like flat_hash_map.
+//
+// It is convenient for the value_type of a flat_hash_map<K, V> to be
+// pair<const K, V>; the "const K" prevents accidental modification of the key
+// when dealing with the reference returned from find() and similar methods.
+// However, this creates other problems; we want to be able to emplace(K, V)
+// efficiently with move operations, and similarly be able to move a
+// pair<K, V> in insert().
+//
+// The solution is this union, which aliases the const and non-const versions
+// of the pair. This also allows flat_hash_map<const K, V> to work, even though
+// that has the same efficiency issues with move in emplace() and insert() -
+// but people do it anyway.
+//
+// If kMutableKeys is false, only the value member can be accessed.
+//
+// If kMutableKeys is true, key can be accessed through all slots while value
+// and mutable_value must be accessed only via INITIALIZED slots. Slots are
+// created and destroyed via mutable_value so that the key can be moved later.
+//
+// Accessing one of the union fields while the other is active is safe as
+// long as they are layout-compatible, which is guaranteed by the definition of
+// kMutableKeys. For C++11, the relevant section of the standard is
+// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19)
+// ----------------------------------------------------------------------------
+template <class K, class V>
+union map_slot_type
+{
+ map_slot_type() {}
+ ~map_slot_type() = delete;
+ map_slot_type(const map_slot_type&) = delete;
+ map_slot_type& operator=(const map_slot_type&) = delete;
+
+ using value_type = std::pair<const K, V>;
+ using mutable_value_type = std::pair<K, V>;
+
+ value_type value;
+ mutable_value_type mutable_value;
+ K key;
+};
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+template <class K, class V>
+struct map_slot_policy
+{
+ using slot_type = map_slot_type<K, V>;
+ using value_type = std::pair<const K, V>;
+ using mutable_value_type = std::pair<K, V>;
+
+private:
+ static void emplace(slot_type* slot) {
+ // The construction of union doesn't do anything at runtime but it allows us
+ // to access its members without violating aliasing rules.
+ new (slot) slot_type;
+ }
+ // If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one
+ // or the other via slot_type. We are also free to access the key via
+ // slot_type::key in this case.
+ using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>;
+
+public:
+ static value_type& element(slot_type* slot) { return slot->value; }
+ static const value_type& element(const slot_type* slot) {
+ return slot->value;
+ }
+
+ static const K& key(const slot_type* slot) {
+ return kMutableKeys::value ? slot->key : slot->value.first;
+ }
+
+ template <class Allocator, class... Args>
+ static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
+ emplace(slot);
+ if (kMutableKeys::value) {
+ phmap::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value,
+ std::forward<Args>(args)...);
+ } else {
+ phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value,
+ std::forward<Args>(args)...);
+ }
+ }
+
+ // Construct this slot by moving from another slot.
+ template <class Allocator>
+ static void construct(Allocator* alloc, slot_type* slot, slot_type* other) {
+ emplace(slot);
+ if (kMutableKeys::value) {
+ phmap::allocator_traits<Allocator>::construct(
+ *alloc, &slot->mutable_value, std::move(other->mutable_value));
+ } else {
+ phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value,
+ std::move(other->value));
+ }
+ }
+
+ template <class Allocator>
+ static void destroy(Allocator* alloc, slot_type* slot) {
+ if (kMutableKeys::value) {
+ phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value);
+ } else {
+ phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->value);
+ }
+ }
+
+ template <class Allocator>
+ static void transfer(Allocator* alloc, slot_type* new_slot,
+ slot_type* old_slot) {
+ emplace(new_slot);
+ if (kMutableKeys::value) {
+ phmap::allocator_traits<Allocator>::construct(
+ *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value));
+ } else {
+ phmap::allocator_traits<Allocator>::construct(*alloc, &new_slot->value,
+ std::move(old_slot->value));
+ }
+ destroy(alloc, old_slot);
+ }
+
+ template <class Allocator>
+ static void swap(Allocator* alloc, slot_type* a, slot_type* b) {
+ if (kMutableKeys::value) {
+ using std::swap;
+ swap(a->mutable_value, b->mutable_value);
+ } else {
+ value_type tmp = std::move(a->value);
+ phmap::allocator_traits<Allocator>::destroy(*alloc, &a->value);
+ phmap::allocator_traits<Allocator>::construct(*alloc, &a->value,
+ std::move(b->value));
+ phmap::allocator_traits<Allocator>::destroy(*alloc, &b->value);
+ phmap::allocator_traits<Allocator>::construct(*alloc, &b->value,
+ std::move(tmp));
+ }
+ }
+
+ template <class Allocator>
+ static void move(Allocator* alloc, slot_type* src, slot_type* dest) {
+ if (kMutableKeys::value) {
+ dest->mutable_value = std::move(src->mutable_value);
+ } else {
+ phmap::allocator_traits<Allocator>::destroy(*alloc, &dest->value);
+ phmap::allocator_traits<Allocator>::construct(*alloc, &dest->value,
+ std::move(src->value));
+ }
+ }
+
+ template <class Allocator>
+ static void move(Allocator* alloc, slot_type* first, slot_type* last,
+ slot_type* result) {
+ for (slot_type *src = first, *dest = result; src != last; ++src, ++dest)
+ move(alloc, src, dest);
+ }
+};
+
+} // namespace priv
+} // phmap
+
+
+namespace phmap {
+
+#ifdef BOOST_THREAD_LOCK_OPTIONS_HPP
+ using defer_lock_t = boost::defer_lock_t;
+ using try_to_lock_t = boost::try_to_lock_t;
+ using adopt_lock_t = boost::adopt_lock_t;
+#else
+ struct adopt_lock_t { explicit adopt_lock_t() = default; };
+ struct defer_lock_t { explicit defer_lock_t() = default; };
+ struct try_to_lock_t { explicit try_to_lock_t() = default; };
+#endif
+
+// -----------------------------------------------------------------------------
+// NullMutex
+// -----------------------------------------------------------------------------
+// A class that implements the Mutex interface, but does nothing. This is to be
+// used as a default template parameters for classes who provide optional
+// internal locking (like phmap::parallel_flat_hash_map).
+// -----------------------------------------------------------------------------
+class NullMutex {
+public:
+ NullMutex() {}
+ ~NullMutex() {}
+ void lock() {}
+ void unlock() {}
+ bool try_lock() { return true; }
+ void lock_shared() {}
+ void unlock_shared() {}
+ bool try_lock_shared() { return true; }
+};
+
+// ------------------------ lockable object used internally -------------------------
+template <class MutexType>
+class LockableBaseImpl
+{
+public:
+ // ----------------------------------------------------
+ struct DoNothing
+ {
+ using mutex_type = MutexType;
+ DoNothing() noexcept {}
+ explicit DoNothing(mutex_type& ) noexcept {}
+ explicit DoNothing(mutex_type& , mutex_type&) noexcept {}
+ DoNothing(mutex_type&, phmap::adopt_lock_t) noexcept {}
+ DoNothing(mutex_type&, phmap::defer_lock_t) noexcept {}
+ DoNothing(mutex_type&, phmap::try_to_lock_t) {}
+ template<class T> explicit DoNothing(T&&) {}
+ DoNothing& operator=(const DoNothing&) { return *this; }
+ DoNothing& operator=(DoNothing&&) { return *this; }
+ void swap(DoNothing &) {}
+ bool owns_lock() const noexcept { return true; }
+ };
+
+ // ----------------------------------------------------
+ class WriteLock
+ {
+ public:
+ using mutex_type = MutexType;
+
+ WriteLock() : m_(nullptr), locked_(false) {}
+
+ explicit WriteLock(mutex_type &m) : m_(&m) {
+ m_->lock();
+ locked_ = true;
+ }
+
+ WriteLock(mutex_type& m, adopt_lock_t) noexcept :
+ m_(&m), locked_(true)
+ {}
+
+ WriteLock(mutex_type& m, defer_lock_t) noexcept :
+ m_(&m), locked_(false)
+ {}
+
+ WriteLock(mutex_type& m, try_to_lock_t) :
+ m_(&m), locked_(false) {
+ m_->try_lock();
+ }
+
+ WriteLock(WriteLock &&o) :
+ m_(std::move(o.m_)), locked_(std::move(o.locked_)) {
+ o.locked_ = false;
+ o.m_ = nullptr;
+ }
+
+ WriteLock& operator=(WriteLock&& other) {
+ WriteLock temp(std::move(other));
+ swap(temp);
+ return *this;
+ }
+
+ ~WriteLock() {
+ if (locked_)
+ m_->unlock();
+ }
+
+ void lock() {
+ if (!locked_) {
+ m_->lock();
+ locked_ = true;
+ }
+ }
+
+ void unlock() {
+ if (locked_) {
+ m_->unlock();
+ locked_ = false;
+ }
+ }
+
+ bool try_lock() {
+ if (locked_)
+ return true;
+ locked_ = m_->try_lock();
+ return locked_;
+ }
+
+ bool owns_lock() const noexcept { return locked_; }
+
+ void swap(WriteLock &o) noexcept {
+ std::swap(m_, o.m_);
+ std::swap(locked_, o.locked_);
+ }
+
+ mutex_type *mutex() const noexcept { return m_; }
+
+ private:
+ mutex_type *m_;
+ bool locked_;
+ };
+
+ // ----------------------------------------------------
+ class ReadLock
+ {
+ public:
+ using mutex_type = MutexType;
+
+ ReadLock() : m_(nullptr), locked_(false) {}
+
+ explicit ReadLock(mutex_type &m) : m_(&m) {
+ m_->lock_shared();
+ locked_ = true;
+ }
+
+ ReadLock(mutex_type& m, adopt_lock_t) noexcept :
+ m_(&m), locked_(true)
+ {}
+
+ ReadLock(mutex_type& m, defer_lock_t) noexcept :
+ m_(&m), locked_(false)
+ {}
+
+ ReadLock(mutex_type& m, try_to_lock_t) :
+ m_(&m), locked_(false) {
+ m_->try_lock_shared();
+ }
+
+ ReadLock(ReadLock &&o) :
+ m_(std::move(o.m_)), locked_(std::move(o.locked_)) {
+ o.locked_ = false;
+ o.m_ = nullptr;
+ }
+
+ ReadLock& operator=(ReadLock&& other) {
+ ReadLock temp(std::move(other));
+ swap(temp);
+ return *this;
+ }
+
+ ~ReadLock() {
+ if (locked_)
+ m_->unlock_shared();
+ }
+
+ void lock() {
+ if (!locked_) {
+ m_->lock_shared();
+ locked_ = true;
+ }
+ }
+
+ void unlock() {
+ if (locked_) {
+ m_->unlock_shared();
+ locked_ = false;
+ }
+ }
+
+ bool try_lock() {
+ if (locked_)
+ return true;
+ locked_ = m_->try_lock_shared();
+ return locked_;
+ }
+
+ bool owns_lock() const noexcept { return locked_; }
+
+ void swap(ReadLock &o) noexcept {
+ std::swap(m_, o.m_);
+ std::swap(locked_, o.locked_);
+ }
+
+ mutex_type *mutex() const noexcept { return m_; }
+
+ private:
+ mutex_type *m_;
+ bool locked_;
+ };
+
+ // ----------------------------------------------------
+ class WriteLocks
+ {
+ public:
+ using mutex_type = MutexType;
+
+ explicit WriteLocks(mutex_type& m1, mutex_type& m2) :
+ _m1(m1), _m2(m2)
+ {
+ std::lock(m1, m2);
+ }
+
+ WriteLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) :
+ _m1(m1), _m2(m2)
+ { // adopt means we already own the mutexes
+ }
+
+ ~WriteLocks()
+ {
+ _m1.unlock();
+ _m2.unlock();
+ }
+
+ WriteLocks(WriteLocks const&) = delete;
+ WriteLocks& operator=(WriteLocks const&) = delete;
+ private:
+ mutex_type& _m1;
+ mutex_type& _m2;
+ };
+
+ // ----------------------------------------------------
+ class ReadLocks
+ {
+ public:
+ using mutex_type = MutexType;
+
+ explicit ReadLocks(mutex_type& m1, mutex_type& m2) :
+ _m1(m1), _m2(m2)
+ {
+ _m1.lock_shared();
+ _m2.lock_shared();
+ }
+
+ ReadLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) :
+ _m1(m1), _m2(m2)
+ { // adopt means we already own the mutexes
+ }
+
+ ~ReadLocks()
+ {
+ _m1.unlock_shared();
+ _m2.unlock_shared();
+ }
+
+ ReadLocks(ReadLocks const&) = delete;
+ ReadLocks& operator=(ReadLocks const&) = delete;
+ private:
+ mutex_type& _m1;
+ mutex_type& _m2;
+ };
+};
+
+// ------------------------ holds a mutex ------------------------------------
+// Default implementation for Lockable, should work fine for std::mutex
+// -----------------------------------
+// use as:
+// using Lockable = phmap::LockableImpl<mutex_type>;
+// Lockable m;
+//
+// Lockable::UpgradeLock read_lock(m); // take a upgradable lock
+//
+// {
+// Lockable::UpgradeToUnique unique_lock(read_lock);
+// // now locked for write
+// }
+//
+// ---------------------------------------------------------------------------
+// Generic mutex support (always write locks)
+// --------------------------------------------------------------------------
+template <class Mtx_>
+class LockableImpl : public Mtx_
+{
+public:
+ using mutex_type = Mtx_;
+ using Base = LockableBaseImpl<Mtx_>;
+ using SharedLock = typename Base::WriteLock;
+ using UpgradeLock = typename Base::WriteLock;
+ using UniqueLock = typename Base::WriteLock;
+ using SharedLocks = typename Base::WriteLocks;
+ using UniqueLocks = typename Base::WriteLocks;
+ using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership
+};
+
+// ---------------------------------------------------------------------------
+// Null mutex (no-op) - when we don't want internal synchronization
+// ---------------------------------------------------------------------------
+template <>
+class LockableImpl<phmap::NullMutex>: public phmap::NullMutex
+{
+public:
+ using mutex_type = phmap::NullMutex;
+ using Base = LockableBaseImpl<phmap::NullMutex>;
+ using SharedLock = typename Base::DoNothing;
+ using UpgradeLock = typename Base::DoNothing;
+ using UniqueLock = typename Base::DoNothing;
+ using UpgradeToUnique = typename Base::DoNothing;
+ using SharedLocks = typename Base::DoNothing;
+ using UniqueLocks = typename Base::DoNothing;
+};
+
+// --------------------------------------------------------------------------
+// Abseil Mutex support (read and write lock support)
+// --------------------------------------------------------------------------
+#ifdef ABSL_SYNCHRONIZATION_MUTEX_H_
+
+ struct AbslMutex : protected absl::Mutex
+ {
+ void lock() { this->Lock(); }
+ void unlock() { this->Unlock(); }
+ void try_lock() { this->TryLock(); }
+ void lock_shared() { this->ReaderLock(); }
+ void unlock_shared() { this->ReaderUnlock(); }
+ void try_lock_shared() { this->ReaderTryLock(); }
+ };
+
+ template <>
+ class LockableImpl<absl::Mutex> : public AbslMutex
+ {
+ public:
+ using mutex_type = phmap::AbslMutex;
+ using Base = LockableBaseImpl<phmap::AbslMutex>;
+ using SharedLock = typename Base::ReadLock;
+ using UpgradeLock = typename Base::WriteLock;
+ using UniqueLock = typename Base::WriteLock;
+ using SharedLocks = typename Base::ReadLocks;
+ using UniqueLocks = typename Base::WriteLocks;
+ using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership
+ };
+
+#endif
+
+// --------------------------------------------------------------------------
+// Boost shared_mutex support (read and write lock support)
+// --------------------------------------------------------------------------
+#ifdef BOOST_THREAD_SHARED_MUTEX_HPP
+
+#if 1
+ // ---------------------------------------------------------------------------
+ template <>
+ class LockableImpl<boost::shared_mutex> : public boost::shared_mutex
+ {
+ public:
+ using mutex_type = boost::shared_mutex;
+ using Base = LockableBaseImpl<boost::shared_mutex>;
+ using SharedLock = boost::shared_lock<mutex_type>;
+ using UpgradeLock = boost::unique_lock<mutex_type>; // assume can't upgrade
+ using UniqueLock = boost::unique_lock<mutex_type>;
+ using SharedLocks = typename Base::ReadLocks;
+ using UniqueLocks = typename Base::WriteLocks;
+ using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership
+ };
+#else
+ // ---------------------------------------------------------------------------
+ template <>
+ class LockableImpl<boost::upgrade_mutex> : public boost::upgrade_mutex
+ {
+ public:
+ using mutex_type = boost::upgrade_mutex;
+ using SharedLock = boost::shared_lock<mutex_type>;
+ using UpgradeLock = boost::upgrade_lock<mutex_type>;
+ using UniqueLock = boost::unique_lock<mutex_type>;
+ using SharedLocks = typename Base::ReadLocks;
+ using UniqueLocks = typename Base::WriteLocks;
+ using UpgradeToUnique = boost::upgrade_to_unique_lock<mutex_type>;
+ };
+#endif
+
+#endif // BOOST_THREAD_SHARED_MUTEX_HPP
+
+// --------------------------------------------------------------------------
+// std::shared_mutex support (read and write lock support)
+// --------------------------------------------------------------------------
+#ifdef PHMAP_HAVE_SHARED_MUTEX
+
+ // ---------------------------------------------------------------------------
+ template <>
+ class LockableImpl<std::shared_mutex> : public std::shared_mutex
+ {
+ public:
+ using mutex_type = std::shared_mutex;
+ using Base = LockableBaseImpl<std::shared_mutex>;
+ using SharedLock = std::shared_lock<mutex_type>;
+ using UpgradeLock = std::unique_lock<mutex_type>; // assume can't upgrade
+ using UniqueLock = std::unique_lock<mutex_type>;
+ using SharedLocks = typename Base::ReadLocks;
+ using UniqueLocks = typename Base::WriteLocks;
+ using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership
+ };
+#endif // PHMAP_HAVE_SHARED_MUTEX
+
+
+} // phmap
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+
+#endif // phmap_base_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_bits.h b/benchmarks/others/parallel_hashmap/phmap_bits.h
new file mode 100644
index 00000000..6b765fff
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_bits.h
@@ -0,0 +1,663 @@
+#if !defined(phmap_bits_h_guard_)
+#define phmap_bits_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
+// with modifications.
+//
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+// The following guarantees declaration of the byte swap functions
+#ifdef _MSC_VER
+ #include <stdlib.h> // NOLINT(build/include)
+#elif defined(__APPLE__)
+ // Mac OS X / Darwin features
+ #include <libkern/OSByteOrder.h>
+#elif defined(__FreeBSD__)
+ #include <sys/endian.h>
+#elif defined(__GLIBC__)
+ #include <byteswap.h> // IWYU pragma: export
+#endif
+
+#include <string.h>
+#include <cstdint>
+#include "phmap_config.h"
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+#endif
+
+// -----------------------------------------------------------------------------
+// unaligned APIs
+// -----------------------------------------------------------------------------
+// Portable handling of unaligned loads, stores, and copies.
+// On some platforms, like ARM, the copy functions can be more efficient
+// then a load and a store.
+// -----------------------------------------------------------------------------
+
+#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\
+ defined(MEMORY_SANITIZER)
+#include <stdint.h>
+
+extern "C" {
+ uint16_t __sanitizer_unaligned_load16(const void *p);
+ uint32_t __sanitizer_unaligned_load32(const void *p);
+ uint64_t __sanitizer_unaligned_load64(const void *p);
+ void __sanitizer_unaligned_store16(void *p, uint16_t v);
+ void __sanitizer_unaligned_store32(void *p, uint32_t v);
+ void __sanitizer_unaligned_store64(void *p, uint64_t v);
+} // extern "C"
+
+namespace phmap {
+namespace bits {
+
+inline uint16_t UnalignedLoad16(const void *p) {
+ return __sanitizer_unaligned_load16(p);
+}
+
+inline uint32_t UnalignedLoad32(const void *p) {
+ return __sanitizer_unaligned_load32(p);
+}
+
+inline uint64_t UnalignedLoad64(const void *p) {
+ return __sanitizer_unaligned_load64(p);
+}
+
+inline void UnalignedStore16(void *p, uint16_t v) {
+ __sanitizer_unaligned_store16(p, v);
+}
+
+inline void UnalignedStore32(void *p, uint32_t v) {
+ __sanitizer_unaligned_store32(p, v);
+}
+
+inline void UnalignedStore64(void *p, uint64_t v) {
+ __sanitizer_unaligned_store64(p, v);
+}
+
+} // namespace bits
+} // namespace phmap
+
+#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p))
+#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p))
+#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p))
+
+#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val))
+#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val))
+#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val))
+
+#else
+
+namespace phmap {
+namespace bits {
+
+inline uint16_t UnalignedLoad16(const void *p) {
+ uint16_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+}
+
+inline uint32_t UnalignedLoad32(const void *p) {
+ uint32_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+}
+
+inline uint64_t UnalignedLoad64(const void *p) {
+ uint64_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+}
+
+inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); }
+
+inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
+
+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
+
+} // namespace bits
+} // namespace phmap
+
+#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p))
+#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p))
+#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p))
+
+#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val))
+#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val))
+#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val))
+
+#endif
+
+// -----------------------------------------------------------------------------
+// File: optimization.h
+// -----------------------------------------------------------------------------
+
+#if defined(__pnacl__)
+ #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
+#elif defined(__clang__)
+ // Clang will not tail call given inline volatile assembly.
+ #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
+#elif defined(__GNUC__)
+ // GCC will not tail call given inline volatile assembly.
+ #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
+#elif defined(_MSC_VER)
+ #include <intrin.h>
+ // The __nop() intrinsic blocks the optimisation.
+ #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __nop()
+#else
+ #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
+#endif
+
+#if defined(__GNUC__)
+ #pragma GCC diagnostic push
+ #pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+
+#ifdef PHMAP_HAVE_INTRINSIC_INT128
+ __extension__ typedef unsigned __int128 phmap_uint128;
+ inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high)
+ {
+ auto result = static_cast<phmap_uint128>(a) * static_cast<phmap_uint128>(b);
+ *high = static_cast<uint64_t>(result >> 64);
+ return static_cast<uint64_t>(result);
+ }
+ #define PHMAP_HAS_UMUL128 1
+#elif (defined(_MSC_VER))
+ #if defined(_M_X64)
+ #pragma intrinsic(_umul128)
+ inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high)
+ {
+ return _umul128(a, b, high);
+ }
+ #define PHMAP_HAS_UMUL128 1
+ #endif
+#endif
+
+#if defined(__GNUC__)
+ #pragma GCC diagnostic pop
+#endif
+
+#if defined(__GNUC__)
+ // Cache line alignment
+ #if defined(__i386__) || defined(__x86_64__)
+ #define PHMAP_CACHELINE_SIZE 64
+ #elif defined(__powerpc64__)
+ #define PHMAP_CACHELINE_SIZE 128
+ #elif defined(__aarch64__)
+ // We would need to read special register ctr_el0 to find out L1 dcache size.
+ // This value is a good estimate based on a real aarch64 machine.
+ #define PHMAP_CACHELINE_SIZE 64
+ #elif defined(__arm__)
+ // Cache line sizes for ARM: These values are not strictly correct since
+ // cache line sizes depend on implementations, not architectures. There
+ // are even implementations with cache line sizes configurable at boot
+ // time.
+ #if defined(__ARM_ARCH_5T__)
+ #define PHMAP_CACHELINE_SIZE 32
+ #elif defined(__ARM_ARCH_7A__)
+ #define PHMAP_CACHELINE_SIZE 64
+ #endif
+ #endif
+
+ #ifndef PHMAP_CACHELINE_SIZE
+ // A reasonable default guess. Note that overestimates tend to waste more
+ // space, while underestimates tend to waste more time.
+ #define PHMAP_CACHELINE_SIZE 64
+ #endif
+
+ #define PHMAP_CACHELINE_ALIGNED __attribute__((aligned(PHMAP_CACHELINE_SIZE)))
+#elif defined(_MSC_VER)
+ #define PHMAP_CACHELINE_SIZE 64
+ #define PHMAP_CACHELINE_ALIGNED __declspec(align(PHMAP_CACHELINE_SIZE))
+#else
+ #define PHMAP_CACHELINE_SIZE 64
+ #define PHMAP_CACHELINE_ALIGNED
+#endif
+
+
+#if PHMAP_HAVE_BUILTIN(__builtin_expect) || \
+ (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_PREDICT_FALSE(x) (__builtin_expect(x, 0))
+ #define PHMAP_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
+#else
+ #define PHMAP_PREDICT_FALSE(x) (x)
+ #define PHMAP_PREDICT_TRUE(x) (x)
+#endif
+
+// -----------------------------------------------------------------------------
+// File: bits.h
+// -----------------------------------------------------------------------------
+
+#if defined(_MSC_VER)
+ // We can achieve something similar to attribute((always_inline)) with MSVC by
+ // using the __forceinline keyword, however this is not perfect. MSVC is
+ // much less aggressive about inlining, and even with the __forceinline keyword.
+ #define PHMAP_BASE_INTERNAL_FORCEINLINE __forceinline
+#else
+ // Use default attribute inline.
+ #define PHMAP_BASE_INTERNAL_FORCEINLINE inline PHMAP_ATTRIBUTE_ALWAYS_INLINE
+#endif
+
+
+namespace phmap {
+namespace base_internal {
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) {
+ int zeroes = 60;
+ if (n >> 32) zeroes -= 32, n >>= 32;
+ if (n >> 16) zeroes -= 16, n >>= 16;
+ if (n >> 8) zeroes -= 8, n >>= 8;
+ if (n >> 4) zeroes -= 4, n >>= 4;
+ return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) {
+#if defined(_MSC_VER) && defined(_M_X64)
+ // MSVC does not have __buitin_clzll. Use _BitScanReverse64.
+ unsigned long result = 0; // NOLINT(runtime/int)
+ if (_BitScanReverse64(&result, n)) {
+ return (int)(63 - result);
+ }
+ return 64;
+#elif defined(_MSC_VER) && !defined(__clang__)
+ // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse
+ unsigned long result = 0; // NOLINT(runtime/int)
+ if ((n >> 32) && _BitScanReverse(&result, (unsigned long)(n >> 32))) {
+ return 31 - result;
+ }
+ if (_BitScanReverse(&result, (unsigned long)n)) {
+ return 63 - result;
+ }
+ return 64;
+#elif defined(__GNUC__) || defined(__clang__)
+ // Use __builtin_clzll, which uses the following instructions:
+ // x86: bsr
+ // ARM64: clz
+ // PPC: cntlzd
+ static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
+ "__builtin_clzll does not take 64-bit arg");
+
+ // Handle 0 as a special case because __builtin_clzll(0) is undefined.
+ if (n == 0) {
+ return 64;
+ }
+ return __builtin_clzll(n);
+#else
+ return CountLeadingZeros64Slow(n);
+#endif
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) {
+ int zeroes = 28;
+ if (n >> 16) zeroes -= 16, n >>= 16;
+ if (n >> 8) zeroes -= 8, n >>= 8;
+ if (n >> 4) zeroes -= 4, n >>= 4;
+ return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) {
+#if defined(_MSC_VER) && !defined(__clang__)
+ unsigned long result = 0; // NOLINT(runtime/int)
+ if (_BitScanReverse(&result, n)) {
+ return (int)(31 - result);
+ }
+ return 32;
+#elif defined(__GNUC__) || defined(__clang__)
+ // Use __builtin_clz, which uses the following instructions:
+ // x86: bsr
+ // ARM64: clz
+ // PPC: cntlzd
+ static_assert(sizeof(int) == sizeof(n),
+ "__builtin_clz does not take 32-bit arg");
+
+ // Handle 0 as a special case because __builtin_clz(0) is undefined.
+ if (n == 0) {
+ return 32;
+ }
+ return __builtin_clz(n);
+#else
+ return CountLeadingZeros32Slow(n);
+#endif
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) {
+ int c = 63;
+ n &= ~n + 1;
+ if (n & 0x00000000FFFFFFFF) c -= 32;
+ if (n & 0x0000FFFF0000FFFF) c -= 16;
+ if (n & 0x00FF00FF00FF00FF) c -= 8;
+ if (n & 0x0F0F0F0F0F0F0F0F) c -= 4;
+ if (n & 0x3333333333333333) c -= 2;
+ if (n & 0x5555555555555555) c -= 1;
+ return c;
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) {
+#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64)
+ unsigned long result = 0; // NOLINT(runtime/int)
+ _BitScanForward64(&result, n);
+ return (int)result;
+#elif defined(_MSC_VER) && !defined(__clang__)
+ unsigned long result = 0; // NOLINT(runtime/int)
+ if (static_cast<uint32_t>(n) == 0) {
+ _BitScanForward(&result, (unsigned long)(n >> 32));
+ return result + 32;
+ }
+ _BitScanForward(&result, (unsigned long)n);
+ return result;
+#elif defined(__GNUC__) || defined(__clang__)
+ static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
+ "__builtin_ctzll does not take 64-bit arg");
+ return __builtin_ctzll(n);
+#else
+ return CountTrailingZerosNonZero64Slow(n);
+#endif
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) {
+ int c = 31;
+ n &= ~n + 1;
+ if (n & 0x0000FFFF) c -= 16;
+ if (n & 0x00FF00FF) c -= 8;
+ if (n & 0x0F0F0F0F) c -= 4;
+ if (n & 0x33333333) c -= 2;
+ if (n & 0x55555555) c -= 1;
+ return c;
+}
+
+PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) {
+#if defined(_MSC_VER) && !defined(__clang__)
+ unsigned long result = 0; // NOLINT(runtime/int)
+ _BitScanForward(&result, n);
+ return (int)result;
+#elif defined(__GNUC__) || defined(__clang__)
+ static_assert(sizeof(int) == sizeof(n),
+ "__builtin_ctz does not take 32-bit arg");
+ return __builtin_ctz(n);
+#else
+ return CountTrailingZerosNonZero32Slow(n);
+#endif
+}
+
+#undef PHMAP_BASE_INTERNAL_FORCEINLINE
+
+} // namespace base_internal
+} // namespace phmap
+
+// -----------------------------------------------------------------------------
+// File: endian.h
+// -----------------------------------------------------------------------------
+
+namespace phmap {
+
+// Use compiler byte-swapping intrinsics if they are available. 32-bit
+// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0.
+// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0.
+// For simplicity, we enable them all only for GCC 4.8.0 or later.
+#if defined(__clang__) || \
+ (defined(__GNUC__) && \
+ ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5))
+
+ inline uint64_t gbswap_64(uint64_t host_int) {
+ return __builtin_bswap64(host_int);
+ }
+ inline uint32_t gbswap_32(uint32_t host_int) {
+ return __builtin_bswap32(host_int);
+ }
+ inline uint16_t gbswap_16(uint16_t host_int) {
+ return __builtin_bswap16(host_int);
+ }
+
+#elif defined(_MSC_VER)
+
+ inline uint64_t gbswap_64(uint64_t host_int) {
+ return _byteswap_uint64(host_int);
+ }
+ inline uint32_t gbswap_32(uint32_t host_int) {
+ return _byteswap_ulong(host_int);
+ }
+ inline uint16_t gbswap_16(uint16_t host_int) {
+ return _byteswap_ushort(host_int);
+ }
+
+#elif defined(__APPLE__)
+
+ inline uint64_t gbswap_64(uint64_t host_int) { return OSSwapInt16(host_int); }
+ inline uint32_t gbswap_32(uint32_t host_int) { return OSSwapInt32(host_int); }
+ inline uint16_t gbswap_16(uint16_t host_int) { return OSSwapInt64(host_int); }
+
+#else
+
+ inline uint64_t gbswap_64(uint64_t host_int) {
+#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__)
+ // Adapted from /usr/include/byteswap.h. Not available on Mac.
+ if (__builtin_constant_p(host_int)) {
+ return __bswap_constant_64(host_int);
+ } else {
+ uint64_t result;
+ __asm__("bswap %0" : "=r"(result) : "0"(host_int));
+ return result;
+ }
+#elif defined(__GLIBC__)
+ return bswap_64(host_int);
+#else
+ return (((host_int & uint64_t{0xFF}) << 56) |
+ ((host_int & uint64_t{0xFF00}) << 40) |
+ ((host_int & uint64_t{0xFF0000}) << 24) |
+ ((host_int & uint64_t{0xFF000000}) << 8) |
+ ((host_int & uint64_t{0xFF00000000}) >> 8) |
+ ((host_int & uint64_t{0xFF0000000000}) >> 24) |
+ ((host_int & uint64_t{0xFF000000000000}) >> 40) |
+ ((host_int & uint64_t{0xFF00000000000000}) >> 56));
+#endif // bswap_64
+ }
+
+ inline uint32_t gbswap_32(uint32_t host_int) {
+#if defined(__GLIBC__)
+ return bswap_32(host_int);
+#else
+ return (((host_int & uint32_t{0xFF}) << 24) |
+ ((host_int & uint32_t{0xFF00}) << 8) |
+ ((host_int & uint32_t{0xFF0000}) >> 8) |
+ ((host_int & uint32_t{0xFF000000}) >> 24));
+#endif
+ }
+
+ inline uint16_t gbswap_16(uint16_t host_int) {
+#if defined(__GLIBC__)
+ return bswap_16(host_int);
+#else
+ return (((host_int & uint16_t{0xFF}) << 8) |
+ ((host_int & uint16_t{0xFF00}) >> 8));
+#endif
+ }
+
+#endif // intrinics available
+
+#ifdef PHMAP_IS_LITTLE_ENDIAN
+
+ // Definitions for ntohl etc. that don't require us to include
+ // netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather
+ // than just #defining them because in debug mode, gcc doesn't
+ // correctly handle the (rather involved) definitions of bswap_32.
+ // gcc guarantees that inline functions are as fast as macros, so
+ // this isn't a performance hit.
+ inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); }
+ inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); }
+ inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); }
+
+#elif defined PHMAP_IS_BIG_ENDIAN
+
+ // These definitions are simpler on big-endian machines
+ // These are functions instead of macros to avoid self-assignment warnings
+ // on calls such as "i = ghtnol(i);". This also provides type checking.
+ inline uint16_t ghtons(uint16_t x) { return x; }
+ inline uint32_t ghtonl(uint32_t x) { return x; }
+ inline uint64_t ghtonll(uint64_t x) { return x; }
+
+#else
+ #error \
+ "Unsupported byte order: Either PHMAP_IS_BIG_ENDIAN or " \
+ "PHMAP_IS_LITTLE_ENDIAN must be defined"
+#endif // byte order
+
+inline uint16_t gntohs(uint16_t x) { return ghtons(x); }
+inline uint32_t gntohl(uint32_t x) { return ghtonl(x); }
+inline uint64_t gntohll(uint64_t x) { return ghtonll(x); }
+
+// Utilities to convert numbers between the current hosts's native byte
+// order and little-endian byte order
+//
+// Load/Store methods are alignment safe
+namespace little_endian {
+// Conversion functions.
+#ifdef PHMAP_IS_LITTLE_ENDIAN
+
+ inline uint16_t FromHost16(uint16_t x) { return x; }
+ inline uint16_t ToHost16(uint16_t x) { return x; }
+
+ inline uint32_t FromHost32(uint32_t x) { return x; }
+ inline uint32_t ToHost32(uint32_t x) { return x; }
+
+ inline uint64_t FromHost64(uint64_t x) { return x; }
+ inline uint64_t ToHost64(uint64_t x) { return x; }
+
+ inline constexpr bool IsLittleEndian() { return true; }
+
+#elif defined PHMAP_IS_BIG_ENDIAN
+
+ inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); }
+ inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); }
+
+ inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); }
+ inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); }
+
+ inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); }
+ inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); }
+
+ inline constexpr bool IsLittleEndian() { return false; }
+
+#endif /* ENDIAN */
+
+// Functions to do unaligned loads and stores in little-endian order.
+inline uint16_t Load16(const void *p) {
+ return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p));
+}
+
+inline void Store16(void *p, uint16_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v));
+}
+
+inline uint32_t Load32(const void *p) {
+ return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p));
+}
+
+inline void Store32(void *p, uint32_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v));
+}
+
+inline uint64_t Load64(const void *p) {
+ return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p));
+}
+
+inline void Store64(void *p, uint64_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v));
+}
+
+} // namespace little_endian
+
+// Utilities to convert numbers between the current hosts's native byte
+// order and big-endian byte order (same as network byte order)
+//
+// Load/Store methods are alignment safe
+namespace big_endian {
+#ifdef PHMAP_IS_LITTLE_ENDIAN
+
+ inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); }
+ inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); }
+
+ inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); }
+ inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); }
+
+ inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); }
+ inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); }
+
+ inline constexpr bool IsLittleEndian() { return true; }
+
+#elif defined PHMAP_IS_BIG_ENDIAN
+
+ inline uint16_t FromHost16(uint16_t x) { return x; }
+ inline uint16_t ToHost16(uint16_t x) { return x; }
+
+ inline uint32_t FromHost32(uint32_t x) { return x; }
+ inline uint32_t ToHost32(uint32_t x) { return x; }
+
+ inline uint64_t FromHost64(uint64_t x) { return x; }
+ inline uint64_t ToHost64(uint64_t x) { return x; }
+
+ inline constexpr bool IsLittleEndian() { return false; }
+
+#endif /* ENDIAN */
+
+// Functions to do unaligned loads and stores in big-endian order.
+inline uint16_t Load16(const void *p) {
+ return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p));
+}
+
+inline void Store16(void *p, uint16_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v));
+}
+
+inline uint32_t Load32(const void *p) {
+ return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p));
+}
+
+inline void Store32(void *p, uint32_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v));
+}
+
+inline uint64_t Load64(const void *p) {
+ return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p));
+}
+
+inline void Store64(void *p, uint64_t v) {
+ PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v));
+}
+
+} // namespace big_endian
+
+} // namespace phmap
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+#endif // phmap_bits_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_config.h b/benchmarks/others/parallel_hashmap/phmap_config.h
new file mode 100644
index 00000000..fa515025
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_config.h
@@ -0,0 +1,771 @@
+#if !defined(phmap_config_h_guard_)
+#define phmap_config_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
+// with modifications.
+//
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#define PHMAP_VERSION_MAJOR 1
+#define PHMAP_VERSION_MINOR 0
+#define PHMAP_VERSION_PATCH 0
+
+// Included for the __GLIBC__ macro (or similar macros on other systems).
+#include <limits.h>
+
+#ifdef __cplusplus
+ // Included for __GLIBCXX__, _LIBCPP_VERSION
+ #include <cstddef>
+#endif // __cplusplus
+
+#if defined(__APPLE__)
+ // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED,
+ // __IPHONE_8_0.
+ #include <Availability.h>
+ #include <TargetConditionals.h>
+#endif
+
+#define PHMAP_XSTR(x) PHMAP_STR(x)
+#define PHMAP_STR(x) #x
+#define PHMAP_VAR_NAME_VALUE(var) #var "=" PHMAP_STR(var)
+
+// -----------------------------------------------------------------------------
+// Some sanity checks
+// -----------------------------------------------------------------------------
+//#if defined(__CYGWIN__)
+// #error "Cygwin is not supported."
+//#endif
+
+#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023918 && !defined(__clang__)
+ #error "phmap requires Visual Studio 2015 Update 2 or higher."
+#endif
+
+// We support gcc 4.7 and later.
+#if defined(__GNUC__) && !defined(__clang__)
+ #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7)
+ #error "phmap requires gcc 4.7 or higher."
+ #endif
+#endif
+
+// We support Apple Xcode clang 4.2.1 (version 421.11.65) and later.
+// This corresponds to Apple Xcode version 4.5.
+#if defined(__apple_build_version__) && __apple_build_version__ < 4211165
+ #error "phmap requires __apple_build_version__ of 4211165 or higher."
+#endif
+
+// Enforce C++11 as the minimum.
+#if defined(__cplusplus) && !defined(_MSC_VER)
+ #if __cplusplus < 201103L
+ #error "C++ versions less than C++11 are not supported."
+ #endif
+#endif
+
+// We have chosen glibc 2.12 as the minimum
+#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
+ #if !__GLIBC_PREREQ(2, 12)
+ #error "Minimum required version of glibc is 2.12."
+ #endif
+#endif
+
+#if defined(_STLPORT_VERSION)
+ #error "STLPort is not supported."
+#endif
+
+#if CHAR_BIT != 8
+ #error "phmap assumes CHAR_BIT == 8."
+#endif
+
+// phmap currently assumes that an int is 4 bytes.
+#if INT_MAX < 2147483647
+ #error "phmap assumes that int is at least 4 bytes. "
+#endif
+
+
+
+// -----------------------------------------------------------------------------
+// Compiler Feature Checks
+// -----------------------------------------------------------------------------
+
+#ifdef __has_builtin
+ #define PHMAP_HAVE_BUILTIN(x) __has_builtin(x)
+#else
+ #define PHMAP_HAVE_BUILTIN(x) 0
+#endif
+
+#if (defined(_MSVC_LANG) && _MSVC_LANG >= 201703) || __cplusplus >= 201703
+ #define PHMAP_HAVE_CC17 1
+#else
+ #define PHMAP_HAVE_CC17 0
+#endif
+
+#define PHMAP_BRANCHLESS 1
+
+// ----------------------------------------------------------------
+// Checks whether `std::is_trivially_destructible<T>` is supported.
+// ----------------------------------------------------------------
+#ifdef PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE
+ #error PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set
+#elif defined(_LIBCPP_VERSION) || \
+ (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \
+ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \
+ defined(_MSC_VER)
+ #define PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1
+#endif
+
+// --------------------------------------------------------------
+// Checks whether `std::is_trivially_default_constructible<T>` is
+// supported.
+// --------------------------------------------------------------
+#if defined(PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE)
+ #error PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set
+#elif defined(PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE)
+ #error PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set
+#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \
+ (!defined(__clang__) && defined(__GNUC__) && \
+ (__GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)) && \
+ (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \
+ (defined(_MSC_VER) && !defined(__NVCC__))
+ #define PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1
+ #define PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1
+#endif
+
+// -------------------------------------------------------------------
+// Checks whether C++11's `thread_local` storage duration specifier is
+// supported.
+// -------------------------------------------------------------------
+#ifdef PHMAP_HAVE_THREAD_LOCAL
+ #error PHMAP_HAVE_THREAD_LOCAL cannot be directly set
+#elif defined(__APPLE__)
+ #if __has_feature(cxx_thread_local) && \
+ !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0)
+ #define PHMAP_HAVE_THREAD_LOCAL 1
+ #endif
+#else // !defined(__APPLE__)
+ #define PHMAP_HAVE_THREAD_LOCAL 1
+#endif
+
+#if defined(__ANDROID__) && defined(__clang__)
+
+ #if __has_include(<android/ndk-version.h>)
+ #include <android/ndk-version.h>
+ #endif // __has_include(<android/ndk-version.h>)
+
+ #if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \
+ defined(__NDK_MINOR__) && \
+ ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1)))
+ #undef PHMAP_HAVE_TLS
+ #undef PHMAP_HAVE_THREAD_LOCAL
+ #endif
+#endif
+
+// ------------------------------------------------------------
+// Checks whether the __int128 compiler extension for a 128-bit
+// integral type is supported.
+// ------------------------------------------------------------
+#ifdef PHMAP_HAVE_INTRINSIC_INT128
+ #error PHMAP_HAVE_INTRINSIC_INT128 cannot be directly set
+#elif defined(__SIZEOF_INT128__)
+ #if (defined(__clang__) && !defined(_WIN32) && !defined(__aarch64__)) || \
+ (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \
+ (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__))
+ #define PHMAP_HAVE_INTRINSIC_INT128 1
+ #elif defined(__CUDACC__)
+ #if __CUDACC_VER__ >= 70000
+ #define PHMAP_HAVE_INTRINSIC_INT128 1
+ #endif // __CUDACC_VER__ >= 70000
+ #endif // defined(__CUDACC__)
+#endif
+
+// ------------------------------------------------------------------
+// Checks whether the compiler both supports and enables exceptions.
+// ------------------------------------------------------------------
+#ifdef PHMAP_HAVE_EXCEPTIONS
+ #error PHMAP_HAVE_EXCEPTIONS cannot be directly set.
+#elif defined(__clang__)
+ #if defined(__EXCEPTIONS) && __has_feature(cxx_exceptions)
+ #define PHMAP_HAVE_EXCEPTIONS 1
+ #endif // defined(__EXCEPTIONS) && __has_feature(cxx_exceptions)
+#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \
+ !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \
+ !(defined(_MSC_VER) && !defined(_CPPUNWIND))
+ #define PHMAP_HAVE_EXCEPTIONS 1
+#endif
+
+
+// -----------------------------------------------------------------------
+// Checks whether the platform has an mmap(2) implementation as defined in
+// POSIX.1-2001.
+// -----------------------------------------------------------------------
+#ifdef PHMAP_HAVE_MMAP
+ #error PHMAP_HAVE_MMAP cannot be directly set
+#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \
+ defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \
+ defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \
+ defined(__ASYLO__)
+ #define PHMAP_HAVE_MMAP 1
+#endif
+
+// -----------------------------------------------------------------------
+// Checks the endianness of the platform.
+// -----------------------------------------------------------------------
+#if defined(PHMAP_IS_BIG_ENDIAN)
+ #error "PHMAP_IS_BIG_ENDIAN cannot be directly set."
+#endif
+
+#if defined(PHMAP_IS_LITTLE_ENDIAN)
+ #error "PHMAP_IS_LITTLE_ENDIAN cannot be directly set."
+#endif
+
+#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
+ __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+ #define PHMAP_IS_LITTLE_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \
+ __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ #define PHMAP_IS_BIG_ENDIAN 1
+#elif defined(_WIN32)
+ #define PHMAP_IS_LITTLE_ENDIAN 1
+#else
+ #error "phmap endian detection needs to be set up for your compiler"
+#endif
+
+#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \
+ defined(__MAC_OS_X_VERSION_MIN_REQUIRED__) && \
+ __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400
+ #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 1
+#else
+ #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 0
+#endif
+
+// ---------------------------------------------------------------------------
+// Checks whether C++17 std::any is available by checking whether <any> exists.
+// ---------------------------------------------------------------------------
+#ifdef PHMAP_HAVE_STD_ANY
+ #error "PHMAP_HAVE_STD_ANY cannot be directly set."
+#endif
+
+#ifdef __has_include
+ #if __has_include(<any>) && __cplusplus >= 201703L && \
+ !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE
+ #define PHMAP_HAVE_STD_ANY 1
+ #endif
+#endif
+
+#ifdef PHMAP_HAVE_STD_OPTIONAL
+ #error "PHMAP_HAVE_STD_OPTIONAL cannot be directly set."
+#endif
+
+#ifdef __has_include
+ #if __has_include(<optional>) && __cplusplus >= 201703L && \
+ !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE
+ #define PHMAP_HAVE_STD_OPTIONAL 1
+ #endif
+#endif
+
+#ifdef PHMAP_HAVE_STD_VARIANT
+ #error "PHMAP_HAVE_STD_VARIANT cannot be directly set."
+#endif
+
+#ifdef __has_include
+ #if __has_include(<variant>) && __cplusplus >= 201703L && \
+ !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE
+ #define PHMAP_HAVE_STD_VARIANT 1
+ #endif
+#endif
+
+#ifdef PHMAP_HAVE_STD_STRING_VIEW
+ #error "PHMAP_HAVE_STD_STRING_VIEW cannot be directly set."
+#endif
+
+#ifdef __has_include
+ #if __has_include(<string_view>) && __cplusplus >= 201703L && \
+ (!defined(_MSC_VER) || _MSC_VER >= 1920) // vs2019
+ #define PHMAP_HAVE_STD_STRING_VIEW 1
+ #endif
+#endif
+
+// #pragma message(PHMAP_VAR_NAME_VALUE(_MSVC_LANG))
+
+#if defined(_MSC_VER) && _MSC_VER >= 1910 && PHMAP_HAVE_CC17
+ // #define PHMAP_HAVE_STD_ANY 1
+ #define PHMAP_HAVE_STD_OPTIONAL 1
+ #define PHMAP_HAVE_STD_VARIANT 1
+ #if !defined(PHMAP_HAVE_STD_STRING_VIEW) && _MSC_VER >= 1920
+ #define PHMAP_HAVE_STD_STRING_VIEW 1
+ #endif
+#endif
+
+#if PHMAP_HAVE_CC17
+ #define PHMAP_HAVE_SHARED_MUTEX 1
+#endif
+
+#ifndef PHMAP_HAVE_STD_STRING_VIEW
+ #define PHMAP_HAVE_STD_STRING_VIEW 0
+#endif
+
+// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION
+// SEH exception from emplace for variant<SomeStruct> when constructing the
+// struct can throw. This defeats some of variant_test and
+// variant_exception_safety_test.
+#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG)
+ #define PHMAP_INTERNAL_MSVC_2017_DBG_MODE
+#endif
+
+// ---------------------------------------------------------------------------
+// Checks whether wchar_t is treated as a native type
+// (MSVC: /Zc:wchar_t- treats wchar_t as unsigned short)
+// ---------------------------------------------------------------------------
+#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED)
+#define PHMAP_HAS_NATIVE_WCHAR_T
+#endif
+
+// -----------------------------------------------------------------------------
+// Sanitizer Attributes
+// -----------------------------------------------------------------------------
+//
+// Sanitizer-related attributes are not "defined" in this file (and indeed
+// are not defined as such in any file). To utilize the following
+// sanitizer-related attributes within your builds, define the following macros
+// within your build using a `-D` flag, along with the given value for
+// `-fsanitize`:
+//
+// * `ADDRESS_SANITIZER` + `-fsanitize=address` (Clang, GCC 4.8)
+// * `MEMORY_SANITIZER` + `-fsanitize=memory` (Clang-only)
+// * `THREAD_SANITIZER + `-fsanitize=thread` (Clang, GCC 4.8+)
+// * `UNDEFINED_BEHAVIOR_SANITIZER` + `-fsanitize=undefined` (Clang, GCC 4.9+)
+// * `CONTROL_FLOW_INTEGRITY` + -fsanitize=cfi (Clang-only)
+// -----------------------------------------------------------------------------
+
+// -----------------------------------------------------------------------------
+// A function-like feature checking macro that is a wrapper around
+// `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a
+// nonzero constant integer if the attribute is supported or 0 if not.
+//
+// It evaluates to zero if `__has_attribute` is not defined by the compiler.
+// -----------------------------------------------------------------------------
+#ifdef __has_attribute
+ #define PHMAP_HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+ #define PHMAP_HAVE_ATTRIBUTE(x) 0
+#endif
+
+// -----------------------------------------------------------------------------
+// A function-like feature checking macro that accepts C++11 style attributes.
+// It's a wrapper around `__has_cpp_attribute`, defined by ISO C++ SD-6
+// (https://en.cppreference.com/w/cpp/experimental/feature_test). If we don't
+// find `__has_cpp_attribute`, will evaluate to 0.
+// -----------------------------------------------------------------------------
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+ #define PHMAP_HAVE_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+ #define PHMAP_HAVE_CPP_ATTRIBUTE(x) 0
+#endif
+
+// -----------------------------------------------------------------------------
+// Function Attributes
+// -----------------------------------------------------------------------------
+#if PHMAP_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) \
+ __attribute__((__format__(__printf__, string_index, first_to_check)))
+ #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) \
+ __attribute__((__format__(__scanf__, string_index, first_to_check)))
+#else
+ #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check)
+ #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check)
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(always_inline) || \
+ (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline))
+ #define PHMAP_HAVE_ATTRIBUTE_ALWAYS_INLINE 1
+#else
+ #define PHMAP_ATTRIBUTE_ALWAYS_INLINE
+#endif
+
+#if !defined(__INTEL_COMPILER) && (PHMAP_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__)))
+ #define PHMAP_ATTRIBUTE_NOINLINE __attribute__((noinline))
+ #define PHMAP_HAVE_ATTRIBUTE_NOINLINE 1
+#else
+ #define PHMAP_ATTRIBUTE_NOINLINE
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(disable_tail_calls)
+ #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1
+ #define PHMAP_ATTRIBUTE_NO_TAIL_CALL __attribute__((disable_tail_calls))
+#elif defined(__GNUC__) && !defined(__clang__)
+ #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1
+ #define PHMAP_ATTRIBUTE_NO_TAIL_CALL \
+ __attribute__((optimize("no-optimize-sibling-calls")))
+#else
+ #define PHMAP_ATTRIBUTE_NO_TAIL_CALL
+ #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 0
+#endif
+
+#if (PHMAP_HAVE_ATTRIBUTE(weak) || \
+ (defined(__GNUC__) && !defined(__clang__))) && \
+ !(defined(__llvm__) && defined(_WIN32))
+ #undef PHMAP_ATTRIBUTE_WEAK
+ #define PHMAP_ATTRIBUTE_WEAK __attribute__((weak))
+ #define PHMAP_HAVE_ATTRIBUTE_WEAK 1
+#else
+ #define PHMAP_ATTRIBUTE_WEAK
+ #define PHMAP_HAVE_ATTRIBUTE_WEAK 0
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(nonnull) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_NONNULL(arg_index) __attribute__((nonnull(arg_index)))
+#else
+ #define PHMAP_ATTRIBUTE_NONNULL(...)
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_NORETURN __attribute__((noreturn))
+#elif defined(_MSC_VER)
+ #define PHMAP_ATTRIBUTE_NORETURN __declspec(noreturn)
+#else
+ #define PHMAP_ATTRIBUTE_NORETURN
+#endif
+
+#if defined(__GNUC__) && defined(ADDRESS_SANITIZER)
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS
+#endif
+
+#if defined(__GNUC__) && defined(MEMORY_SANITIZER)
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY
+#endif
+
+#if defined(__GNUC__) && defined(THREAD_SANITIZER)
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD
+#endif
+
+#if defined(__GNUC__) && \
+ (defined(UNDEFINED_BEHAVIOR_SANITIZER) || defined(ADDRESS_SANITIZER))
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED \
+ __attribute__((no_sanitize("undefined")))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED
+#endif
+
+#if defined(__GNUC__) && defined(CONTROL_FLOW_INTEGRITY)
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI __attribute__((no_sanitize("cfi")))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI
+#endif
+
+#if defined(__GNUC__) && defined(SAFESTACK_SANITIZER)
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK \
+ __attribute__((no_sanitize("safe-stack")))
+#else
+ #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(returns_nonnull) || \
+ (defined(__GNUC__) && \
+ (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \
+ !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull))
+#else
+ #define PHMAP_ATTRIBUTE_RETURNS_NONNULL
+#endif
+
+#ifdef PHMAP_HAVE_ATTRIBUTE_SECTION
+ #error PHMAP_HAVE_ATTRIBUTE_SECTION cannot be directly set
+#elif (PHMAP_HAVE_ATTRIBUTE(section) || \
+ (defined(__GNUC__) && !defined(__clang__))) && \
+ !defined(__APPLE__) && PHMAP_HAVE_ATTRIBUTE_WEAK
+ #define PHMAP_HAVE_ATTRIBUTE_SECTION 1
+ #ifndef PHMAP_ATTRIBUTE_SECTION
+ #define PHMAP_ATTRIBUTE_SECTION(name) \
+ __attribute__((section(#name))) __attribute__((noinline))
+ #endif
+ #ifndef PHMAP_ATTRIBUTE_SECTION_VARIABLE
+ #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name)))
+ #endif
+ #ifndef PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS
+ #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) \
+ extern char __start_##name[] PHMAP_ATTRIBUTE_WEAK; \
+ extern char __stop_##name[] PHMAP_ATTRIBUTE_WEAK
+ #endif
+ #ifndef PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS
+ #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name)
+ #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name)
+ #endif
+ #define PHMAP_ATTRIBUTE_SECTION_START(name) \
+ (reinterpret_cast<void *>(__start_##name))
+ #define PHMAP_ATTRIBUTE_SECTION_STOP(name) \
+ (reinterpret_cast<void *>(__stop_##name))
+#else // !PHMAP_HAVE_ATTRIBUTE_SECTION
+ #define PHMAP_HAVE_ATTRIBUTE_SECTION 0
+ #define PHMAP_ATTRIBUTE_SECTION(name)
+ #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name)
+ #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name)
+ #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name)
+ #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name)
+ #define PHMAP_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0))
+ #define PHMAP_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0))
+#endif // PHMAP_ATTRIBUTE_SECTION
+
+#if PHMAP_HAVE_ATTRIBUTE(force_align_arg_pointer) || \
+ (defined(__GNUC__) && !defined(__clang__))
+ #if defined(__i386__)
+ #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC \
+ __attribute__((force_align_arg_pointer))
+ #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0)
+ #elif defined(__x86_64__)
+ #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (1)
+ #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC
+ #else // !__i386__ && !__x86_64
+ #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0)
+ #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC
+ #endif // __i386__
+#else
+ #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC
+ #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0)
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(nodiscard)
+ #define PHMAP_MUST_USE_RESULT [[nodiscard]]
+#elif defined(__clang__) && PHMAP_HAVE_ATTRIBUTE(warn_unused_result)
+ #define PHMAP_MUST_USE_RESULT __attribute__((warn_unused_result))
+#else
+ #define PHMAP_MUST_USE_RESULT
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(hot) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_HOT __attribute__((hot))
+#else
+ #define PHMAP_ATTRIBUTE_HOT
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(cold) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_COLD __attribute__((cold))
+#else
+ #define PHMAP_ATTRIBUTE_COLD
+#endif
+
+#if defined(__clang__)
+ #if PHMAP_HAVE_CPP_ATTRIBUTE(clang::reinitializes)
+ #define PHMAP_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]]
+ #else
+ #define PHMAP_ATTRIBUTE_REINITIALIZES
+ #endif
+#else
+ #define PHMAP_ATTRIBUTE_REINITIALIZES
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__))
+ #undef PHMAP_ATTRIBUTE_UNUSED
+ #define PHMAP_ATTRIBUTE_UNUSED __attribute__((__unused__))
+#else
+ #define PHMAP_ATTRIBUTE_UNUSED
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(tls_model) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_INITIAL_EXEC __attribute__((tls_model("initial-exec")))
+#else
+ #define PHMAP_ATTRIBUTE_INITIAL_EXEC
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(packed) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_PACKED __attribute__((__packed__))
+#else
+ #define PHMAP_ATTRIBUTE_PACKED
+#endif
+
+#if PHMAP_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__))
+ #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes)))
+#else
+ #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes)
+#endif
+
+// ----------------------------------------------------------------------
+// Figure out SSE support
+// ----------------------------------------------------------------------
+#ifndef PHMAP_HAVE_SSE2
+ #if defined(__SSE2__) || \
+ (defined(_MSC_VER) && \
+ (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)))
+ #define PHMAP_HAVE_SSE2 1
+ #else
+ #define PHMAP_HAVE_SSE2 0
+ #endif
+#endif
+
+#ifndef PHMAP_HAVE_SSSE3
+ #if defined(__SSSE3__) || defined(__AVX2__)
+ #define PHMAP_HAVE_SSSE3 1
+ #else
+ #define PHMAP_HAVE_SSSE3 0
+ #endif
+#endif
+
+#if PHMAP_HAVE_SSSE3 && !PHMAP_HAVE_SSE2
+ #error "Bad configuration!"
+#endif
+
+#if PHMAP_HAVE_SSE2
+ #include <emmintrin.h>
+#endif
+
+#if PHMAP_HAVE_SSSE3
+ #include <tmmintrin.h>
+#endif
+
+
+// ----------------------------------------------------------------------
+// constexpr if
+// ----------------------------------------------------------------------
+#if PHMAP_HAVE_CC17
+ #define PHMAP_IF_CONSTEXPR(expr) if constexpr ((expr))
+#else
+ #define PHMAP_IF_CONSTEXPR(expr) if ((expr))
+#endif
+
+// ----------------------------------------------------------------------
+// base/macros.h
+// ----------------------------------------------------------------------
+
+// PHMAP_ARRAYSIZE()
+//
+// Returns the number of elements in an array as a compile-time constant, which
+// can be used in defining new arrays. If you use this macro on a pointer by
+// mistake, you will get a compile-time error.
+#define PHMAP_ARRAYSIZE(array) \
+ (sizeof(::phmap::macros_internal::ArraySizeHelper(array)))
+
+namespace phmap {
+namespace macros_internal {
+ // Note: this internal template function declaration is used by PHMAP_ARRAYSIZE.
+ // The function doesn't need a definition, as we only use its type.
+ template <typename T, size_t N>
+ auto ArraySizeHelper(const T (&array)[N]) -> char (&)[N];
+} // namespace macros_internal
+} // namespace phmap
+
+// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported.
+#if defined(__clang__) && defined(__has_warning)
+ #if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+ #define PHMAP_FALLTHROUGH_INTENDED [[clang::fallthrough]]
+ #endif
+#elif defined(__GNUC__) && __GNUC__ >= 7
+ #define PHMAP_FALLTHROUGH_INTENDED [[gnu::fallthrough]]
+#endif
+
+#ifndef PHMAP_FALLTHROUGH_INTENDED
+ #define PHMAP_FALLTHROUGH_INTENDED \
+ do { } while (0)
+#endif
+
+// PHMAP_DEPRECATED()
+//
+// Marks a deprecated class, struct, enum, function, method and variable
+// declarations. The macro argument is used as a custom diagnostic message (e.g.
+// suggestion of a better alternative).
+//
+// Example:
+//
+// class PHMAP_DEPRECATED("Use Bar instead") Foo {...};
+// PHMAP_DEPRECATED("Use Baz instead") void Bar() {...}
+//
+// Every usage of a deprecated entity will trigger a warning when compiled with
+// clang's `-Wdeprecated-declarations` option. This option is turned off by
+// default, but the warnings will be reported by clang-tidy.
+#if defined(__clang__) && __cplusplus >= 201103L
+ #define PHMAP_DEPRECATED(message) __attribute__((deprecated(message)))
+#endif
+
+#ifndef PHMAP_DEPRECATED
+ #define PHMAP_DEPRECATED(message)
+#endif
+
+// PHMAP_BAD_CALL_IF()
+//
+// Used on a function overload to trap bad calls: any call that matches the
+// overload will cause a compile-time error. This macro uses a clang-specific
+// "enable_if" attribute, as described at
+// http://clang.llvm.org/docs/AttributeReference.html#enable-if
+//
+// Overloads which use this macro should be bracketed by
+// `#ifdef PHMAP_BAD_CALL_IF`.
+//
+// Example:
+//
+// int isdigit(int c);
+// #ifdef PHMAP_BAD_CALL_IF
+// int isdigit(int c)
+// PHMAP_BAD_CALL_IF(c <= -1 || c > 255,
+// "'c' must have the value of an unsigned char or EOF");
+// #endif // PHMAP_BAD_CALL_IF
+
+#if defined(__clang__)
+ #if __has_attribute(enable_if)
+ #define PHMAP_BAD_CALL_IF(expr, msg) \
+ __attribute__((enable_if(expr, "Bad call trap"), unavailable(msg)))
+ #endif
+#endif
+
+// PHMAP_ASSERT()
+//
+// In C++11, `assert` can't be used portably within constexpr functions.
+// PHMAP_ASSERT functions as a runtime assert but works in C++11 constexpr
+// functions. Example:
+//
+// constexpr double Divide(double a, double b) {
+// return PHMAP_ASSERT(b != 0), a / b;
+// }
+//
+// This macro is inspired by
+// https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/
+#if defined(NDEBUG)
+ #define PHMAP_ASSERT(expr) (false ? (void)(expr) : (void)0)
+#else
+ #define PHMAP_ASSERT(expr) \
+ (PHMAP_PREDICT_TRUE((expr)) ? (void)0 \
+ : [] { assert(false && #expr); }()) // NOLINT
+#endif
+
+#ifdef PHMAP_HAVE_EXCEPTIONS
+ #define PHMAP_INTERNAL_TRY try
+ #define PHMAP_INTERNAL_CATCH_ANY catch (...)
+ #define PHMAP_INTERNAL_RETHROW do { throw; } while (false)
+#else // PHMAP_HAVE_EXCEPTIONS
+ #define PHMAP_INTERNAL_TRY if (true)
+ #define PHMAP_INTERNAL_CATCH_ANY else if (false)
+ #define PHMAP_INTERNAL_RETHROW do {} while (false)
+#endif // PHMAP_HAVE_EXCEPTIONS
+
+
+#endif // phmap_config_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_dump.h b/benchmarks/others/parallel_hashmap/phmap_dump.h
new file mode 100644
index 00000000..0f2018ef
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_dump.h
@@ -0,0 +1,227 @@
+#if !defined(phmap_dump_h_guard_)
+#define phmap_dump_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// providing dump/load/mmap_load
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include "phmap.h"
+namespace phmap
+{
+
+namespace type_traits_internal {
+
+#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801
+ template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {};
+#else
+ template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {};
+#endif
+
+template <class T1, class T2>
+struct IsTriviallyCopyable<std::pair<T1, T2>> {
+ static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value;
+};
+}
+
+namespace priv {
+
+// ------------------------------------------------------------------------
+// dump/load for raw_hash_set
+// ------------------------------------------------------------------------
+template <class Policy, class Hash, class Eq, class Alloc>
+template<typename OutputArchive>
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const {
+ static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+ "value_type should be trivially copyable");
+
+ if (!ar.dump(size_)) {
+ std::cerr << "Failed to dump size_" << std::endl;
+ return false;
+ }
+ if (size_ == 0) {
+ return true;
+ }
+ if (!ar.dump(capacity_)) {
+ std::cerr << "Failed to dump capacity_" << std::endl;
+ return false;
+ }
+ if (!ar.dump(reinterpret_cast<char*>(ctrl_),
+ sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
+
+ std::cerr << "Failed to dump ctrl_" << std::endl;
+ return false;
+ }
+ if (!ar.dump(reinterpret_cast<char*>(slots_),
+ sizeof(slot_type) * capacity_)) {
+ std::cerr << "Failed to dump slot_" << std::endl;
+ return false;
+ }
+ return true;
+}
+
+template <class Policy, class Hash, class Eq, class Alloc>
+template<typename InputArchive>
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+ static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+ "value_type should be trivially copyable");
+ raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content
+ if (!ar.load(&size_)) {
+ std::cerr << "Failed to load size_" << std::endl;
+ return false;
+ }
+ if (size_ == 0) {
+ return true;
+ }
+ if (!ar.load(&capacity_)) {
+ std::cerr << "Failed to load capacity_" << std::endl;
+ return false;
+ }
+
+ // allocate memory for ctrl_ and slots_
+ initialize_slots();
+ if (!ar.load(reinterpret_cast<char*>(ctrl_),
+ sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
+ std::cerr << "Failed to load ctrl" << std::endl;
+ return false;
+ }
+ if (!ar.load(reinterpret_cast<char*>(slots_),
+ sizeof(slot_type) * capacity_)) {
+ std::cerr << "Failed to load slot" << std::endl;
+ return false;
+ }
+ return true;
+}
+
+// ------------------------------------------------------------------------
+// dump/load for parallel_hash_set
+// ------------------------------------------------------------------------
+template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class Mtx_,
+ class Policy, class Hash, class Eq, class Alloc>
+template<typename OutputArchive>
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const {
+ static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+ "value_type should be trivially copyable");
+
+ if (! ar.dump(subcnt())) {
+ std::cerr << "Failed to dump meta!" << std::endl;
+ return false;
+ }
+ for (size_t i = 0; i < sets_.size(); ++i) {
+ auto& inner = sets_[i];
+ typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
+ if (!inner.set_.dump(ar)) {
+ std::cerr << "Failed to dump submap " << i << std::endl;
+ return false;
+ }
+ }
+ return true;
+}
+
+template <size_t N,
+ template <class, class, class, class> class RefSet,
+ class Mtx_,
+ class Policy, class Hash, class Eq, class Alloc>
+template<typename InputArchive>
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+ static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+ "value_type should be trivially copyable");
+
+ size_t submap_count = 0;
+ if (!ar.load(&submap_count)) {
+ std::cerr << "Failed to load submap count!" << std::endl;
+ return false;
+ }
+
+ if (submap_count != subcnt()) {
+ std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
+ return false;
+ }
+
+ for (size_t i = 0; i < submap_count; ++i) {
+ auto& inner = sets_[i];
+ typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
+ if (!inner.set_.load(ar)) {
+ std::cerr << "Failed to load submap " << i << std::endl;
+ return false;
+ }
+ }
+ return true;
+}
+} // namespace priv
+
+
+
+// ------------------------------------------------------------------------
+// BinaryArchive
+// File is closed when archive object is destroyed
+// ------------------------------------------------------------------------
+
+// ------------------------------------------------------------------------
+// ------------------------------------------------------------------------
+class BinaryOutputArchive {
+public:
+ BinaryOutputArchive(const char *file_path) {
+ ofs_.open(file_path, std::ios_base::binary);
+ }
+
+ bool dump(const char *p, size_t sz) {
+ ofs_.write(p, sz);
+ return true;
+ }
+
+ template<typename V>
+ typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
+ dump(const V& v) {
+ ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V));
+ return true;
+ }
+
+private:
+ std::ofstream ofs_;
+};
+
+
+class BinaryInputArchive {
+public:
+ BinaryInputArchive(const char * file_path) {
+ ifs_.open(file_path, std::ios_base::binary);
+ }
+
+ bool load(char* p, size_t sz) {
+ ifs_.read(p, sz);
+ return true;
+ }
+
+ template<typename V>
+ typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
+ load(V* v) {
+ ifs_.read(reinterpret_cast<char *>(v), sizeof(V));
+ return true;
+ }
+
+private:
+ std::ifstream ifs_;
+};
+
+} // namespace phmap
+
+#endif // phmap_dump_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h b/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h
new file mode 100644
index 00000000..a7719c49
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_fwd_decl.h
@@ -0,0 +1,154 @@
+#if !defined(phmap_fwd_decl_h_guard_)
+#define phmap_fwd_decl_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+// ---------------------------------------------------------------------------
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+ #pragma warning(disable : 4710) // function not inlined
+ #pragma warning(disable : 4711) // selected for automatic inline expansion
+#endif
+
+#include <memory>
+#include <utility>
+
+#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_)
+ namespace absl { template <class T> struct Hash; };
+#endif
+
+namespace phmap {
+
+#if defined(PHMAP_USE_ABSL_HASH)
+ template <class T> using Hash = ::absl::Hash<T>;
+#else
+ template <class T> struct Hash;
+#endif
+
+ template <class T> struct EqualTo;
+ template <class T> struct Less;
+ template <class T> using Allocator = typename std::allocator<T>;
+ template<class T1, class T2> using Pair = typename std::pair<T1, T2>;
+
+ class NullMutex;
+
+ namespace priv {
+
+ // The hash of an object of type T is computed by using phmap::Hash.
+ template <class T, class E = void>
+ struct HashEq
+ {
+ using Hash = phmap::Hash<T>;
+ using Eq = phmap::EqualTo<T>;
+ };
+
+ template <class T>
+ using hash_default_hash = typename priv::HashEq<T>::Hash;
+
+ template <class T>
+ using hash_default_eq = typename priv::HashEq<T>::Eq;
+
+ // type alias for std::allocator so we can forward declare without including other headers
+ template <class T>
+ using Allocator = typename phmap::Allocator<T>;
+
+ // type alias for std::pair so we can forward declare without including other headers
+ template<class T1, class T2>
+ using Pair = typename phmap::Pair<T1, T2>;
+
+ } // namespace priv
+
+ // ------------- forward declarations for hash containers ----------------------------------
+ template <class T,
+ class Hash = phmap::priv::hash_default_hash<T>,
+ class Eq = phmap::priv::hash_default_eq<T>,
+ class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator
+ class flat_hash_set;
+
+ template <class K, class V,
+ class Hash = phmap::priv::hash_default_hash<K>,
+ class Eq = phmap::priv::hash_default_eq<K>,
+ class Alloc = phmap::priv::Allocator<
+ phmap::priv::Pair<const K, V>>> // alias for std::allocator
+ class flat_hash_map;
+
+ template <class T,
+ class Hash = phmap::priv::hash_default_hash<T>,
+ class Eq = phmap::priv::hash_default_eq<T>,
+ class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator
+ class node_hash_set;
+
+ template <class Key, class Value,
+ class Hash = phmap::priv::hash_default_hash<Key>,
+ class Eq = phmap::priv::hash_default_eq<Key>,
+ class Alloc = phmap::priv::Allocator<
+ phmap::priv::Pair<const Key, Value>>> // alias for std::allocator
+ class node_hash_map;
+
+ template <class T,
+ class Hash = phmap::priv::hash_default_hash<T>,
+ class Eq = phmap::priv::hash_default_eq<T>,
+ class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator
+ size_t N = 4, // 2**N submaps
+ class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
+ class parallel_flat_hash_set;
+
+ template <class K, class V,
+ class Hash = phmap::priv::hash_default_hash<K>,
+ class Eq = phmap::priv::hash_default_eq<K>,
+ class Alloc = phmap::priv::Allocator<
+ phmap::priv::Pair<const K, V>>, // alias for std::allocator
+ size_t N = 4, // 2**N submaps
+ class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
+ class parallel_flat_hash_map;
+
+ template <class T,
+ class Hash = phmap::priv::hash_default_hash<T>,
+ class Eq = phmap::priv::hash_default_eq<T>,
+ class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator
+ size_t N = 4, // 2**N submaps
+ class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
+ class parallel_node_hash_set;
+
+ template <class Key, class Value,
+ class Hash = phmap::priv::hash_default_hash<Key>,
+ class Eq = phmap::priv::hash_default_eq<Key>,
+ class Alloc = phmap::priv::Allocator<
+ phmap::priv::Pair<const Key, Value>>, // alias for std::allocator
+ size_t N = 4, // 2**N submaps
+ class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
+ class parallel_node_hash_map;
+
+ // ------------- forward declarations for btree containers ----------------------------------
+ template <typename Key, typename Compare = phmap::Less<Key>,
+ typename Alloc = phmap::Allocator<Key>>
+ class btree_set;
+
+ template <typename Key, typename Compare = phmap::Less<Key>,
+ typename Alloc = phmap::Allocator<Key>>
+ class btree_multiset;
+
+ template <typename Key, typename Value, typename Compare = phmap::Less<Key>,
+ typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>>
+ class btree_map;
+
+ template <typename Key, typename Value, typename Compare = phmap::Less<Key>,
+ typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>>
+ class btree_multimap;
+
+} // namespace phmap
+
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+#endif // phmap_fwd_decl_h_guard_
diff --git a/benchmarks/others/parallel_hashmap/phmap_utils.h b/benchmarks/others/parallel_hashmap/phmap_utils.h
new file mode 100644
index 00000000..1d0c4728
--- /dev/null
+++ b/benchmarks/others/parallel_hashmap/phmap_utils.h
@@ -0,0 +1,378 @@
+#if !defined(phmap_utils_h_guard_)
+#define phmap_utils_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - [email protected]
+//
+// minimal header providing phmap::HashState
+//
+// use as: phmap::HashState().combine(0, _first_name, _last_name, _age);
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#ifdef _MSC_VER
+ #pragma warning(push)
+ #pragma warning(disable : 4514) // unreferenced inline function has been removed
+ #pragma warning(disable : 4710) // function not inlined
+ #pragma warning(disable : 4711) // selected for automatic inline expansion
+#endif
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+#include "phmap_bits.h"
+
+// ---------------------------------------------------------------
+// Absl forward declaration requires global scope.
+// ---------------------------------------------------------------
+#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) && !defined(ABSL_HASH_HASH_H_)
+ namespace absl { template <class T> struct Hash; };
+#endif
+
+namespace phmap
+{
+
+// ---------------------------------------------------------------
+// ---------------------------------------------------------------
+template<int n>
+struct phmap_mix
+{
+ inline size_t operator()(size_t) const;
+};
+
+template<>
+struct phmap_mix<4>
+{
+ inline size_t operator()(size_t a) const
+ {
+ static constexpr uint64_t kmul = 0xcc9e2d51UL;
+ // static constexpr uint64_t kmul = 0x3B9ACB93UL; // [greg] my own random prime
+ uint64_t l = a * kmul;
+ return static_cast<size_t>(l ^ (l >> 32));
+ }
+};
+
+#if defined(PHMAP_HAS_UMUL128)
+ template<>
+ struct phmap_mix<8>
+ {
+ // Very fast mixing (similar to Abseil)
+ inline size_t operator()(size_t a) const
+ {
+ static constexpr uint64_t k = 0xde5fb9d2630458e9ULL;
+ // static constexpr uint64_t k = 0x7C9D0BF0567102A5ULL; // [greg] my own random prime
+ uint64_t h;
+ uint64_t l = umul128(a, k, &h);
+ return static_cast<size_t>(h + l);
+ }
+ };
+#else
+ template<>
+ struct phmap_mix<8>
+ {
+ inline size_t operator()(size_t a) const
+ {
+ a = (~a) + (a << 21); // a = (a << 21) - a - 1;
+ a = a ^ (a >> 24);
+ a = (a + (a << 3)) + (a << 8); // a * 265
+ a = a ^ (a >> 14);
+ a = (a + (a << 2)) + (a << 4); // a * 21
+ a = a ^ (a >> 28);
+ a = a + (a << 31);
+ return static_cast<size_t>(a);
+ }
+ };
+#endif
+
+// --------------------------------------------
+template<int n>
+struct fold_if_needed
+{
+ inline size_t operator()(uint64_t) const;
+};
+
+template<>
+struct fold_if_needed<4>
+{
+ inline size_t operator()(uint64_t a) const
+ {
+ return static_cast<size_t>(a ^ (a >> 32));
+ }
+};
+
+template<>
+struct fold_if_needed<8>
+{
+ inline size_t operator()(uint64_t a) const
+ {
+ return static_cast<size_t>(a);
+ }
+};
+
+// ---------------------------------------------------------------
+// see if class T has a hash_value() friend method
+// ---------------------------------------------------------------
+template<typename T>
+struct has_hash_value
+{
+private:
+ typedef std::true_type yes;
+ typedef std::false_type no;
+
+ template<typename U> static auto test(int) -> decltype(hash_value(std::declval<const U&>()) == 1, yes());
+
+ template<typename> static no test(...);
+
+public:
+ static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value;
+};
+
+#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_)
+ template <class T> using Hash = ::absl::Hash<T>;
+#elif !defined(PHMAP_USE_ABSL_HASH)
+// ---------------------------------------------------------------
+// phmap::Hash
+// ---------------------------------------------------------------
+template <class T>
+struct Hash
+{
+ template <class U, typename std::enable_if<has_hash_value<U>::value, int>::type = 0>
+ size_t _hash(const T& val) const
+ {
+ return hash_value(val);
+ }
+
+ template <class U, typename std::enable_if<!has_hash_value<U>::value, int>::type = 0>
+ size_t _hash(const T& val) const
+ {
+ return std::hash<T>()(val);
+ }
+
+ inline size_t operator()(const T& val) const
+ {
+ return _hash<T>(val);
+ }
+};
+
+template <class T>
+struct Hash<T *>
+{
+ inline size_t operator()(const T *val) const noexcept
+ {
+ return static_cast<size_t>(reinterpret_cast<const uintptr_t>(val));
+ }
+};
+
+template<class ArgumentType, class ResultType>
+struct phmap_unary_function
+{
+ typedef ArgumentType argument_type;
+ typedef ResultType result_type;
+};
+
+template <>
+struct Hash<bool> : public phmap_unary_function<bool, size_t>
+{
+ inline size_t operator()(bool val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<char> : public phmap_unary_function<char, size_t>
+{
+ inline size_t operator()(char val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<signed char> : public phmap_unary_function<signed char, size_t>
+{
+ inline size_t operator()(signed char val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<unsigned char> : public phmap_unary_function<unsigned char, size_t>
+{
+ inline size_t operator()(unsigned char val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+#ifdef PHMAP_HAS_NATIVE_WCHAR_T
+template <>
+struct Hash<wchar_t> : public phmap_unary_function<wchar_t, size_t>
+{
+ inline size_t operator()(wchar_t val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+#endif
+
+template <>
+struct Hash<int16_t> : public phmap_unary_function<int16_t, size_t>
+{
+ inline size_t operator()(int16_t val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<uint16_t> : public phmap_unary_function<uint16_t, size_t>
+{
+ inline size_t operator()(uint16_t val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<int32_t> : public phmap_unary_function<int32_t, size_t>
+{
+ inline size_t operator()(int32_t val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<uint32_t> : public phmap_unary_function<uint32_t, size_t>
+{
+ inline size_t operator()(uint32_t val) const noexcept
+ { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<int64_t> : public phmap_unary_function<int64_t, size_t>
+{
+ inline size_t operator()(int64_t val) const noexcept
+ { return fold_if_needed<sizeof(size_t)>()(static_cast<uint64_t>(val)); }
+};
+
+template <>
+struct Hash<uint64_t> : public phmap_unary_function<uint64_t, size_t>
+{
+ inline size_t operator()(uint64_t val) const noexcept
+ { return fold_if_needed<sizeof(size_t)>()(val); }
+};
+
+template <>
+struct Hash<float> : public phmap_unary_function<float, size_t>
+{
+ inline size_t operator()(float val) const noexcept
+ {
+ // -0.0 and 0.0 should return same hash
+ uint32_t *as_int = reinterpret_cast<uint32_t *>(&val);
+ return (val == 0) ? static_cast<size_t>(0) :
+ static_cast<size_t>(*as_int);
+ }
+};
+
+template <>
+struct Hash<double> : public phmap_unary_function<double, size_t>
+{
+ inline size_t operator()(double val) const noexcept
+ {
+ // -0.0 and 0.0 should return same hash
+ uint64_t *as_int = reinterpret_cast<uint64_t *>(&val);
+ return (val == 0) ? static_cast<size_t>(0) :
+ fold_if_needed<sizeof(size_t)>()(*as_int);
+ }
+};
+
+#endif
+
+template <class H, int sz> struct Combiner
+{
+ H operator()(H seed, size_t value);
+};
+
+template <class H> struct Combiner<H, 4>
+{
+ H operator()(H seed, size_t value)
+ {
+ return seed ^ (value + 0x9e3779b9 + (seed << 6) + (seed >> 2));
+ }
+};
+
+template <class H> struct Combiner<H, 8>
+{
+ H operator()(H seed, size_t value)
+ {
+ return seed ^ (value + size_t(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2));
+ }
+};
+
+// define HashState to combine member hashes... see example below
+// -----------------------------------------------------------------------------
+template <typename H>
+class HashStateBase {
+public:
+ template <typename T, typename... Ts>
+ static H combine(H state, const T& value, const Ts&... values);
+
+ static H combine(H state) { return state; }
+};
+
+template <typename H>
+template <typename T, typename... Ts>
+H HashStateBase<H>::combine(H seed, const T& v, const Ts&... vs)
+{
+ return HashStateBase<H>::combine(Combiner<H, sizeof(H)>()(
+ seed, phmap::Hash<T>()(v)),
+ vs...);
+}
+
+using HashState = HashStateBase<size_t>;
+
+// -----------------------------------------------------------------------------
+
+#if !defined(PHMAP_USE_ABSL_HASH)
+
+// define Hash for std::pair
+// -------------------------
+template<class T1, class T2>
+struct Hash<std::pair<T1, T2>> {
+ size_t operator()(std::pair<T1, T2> const& p) const noexcept {
+ return phmap::HashState().combine(phmap::Hash<T1>()(p.first), p.second);
+ }
+};
+
+// define Hash for std::tuple
+// --------------------------
+template<class... T>
+struct Hash<std::tuple<T...>> {
+ size_t operator()(std::tuple<T...> const& t) const noexcept {
+ return _hash_helper(t);
+ }
+
+private:
+ template<size_t I = 0, class ...P>
+ typename std::enable_if<I == sizeof...(P), size_t>::type
+ _hash_helper(const std::tuple<P...> &) const noexcept { return 0; }
+
+ template<size_t I = 0, class ...P>
+ typename std::enable_if<I < sizeof...(P), size_t>::type
+ _hash_helper(const std::tuple<P...> &t) const noexcept {
+ const auto &el = std::get<I>(t);
+ using el_type = typename std::remove_cv<typename std::remove_reference<decltype(el)>::type>::type;
+ return Combiner<size_t, sizeof(size_t)>()(
+ phmap::Hash<el_type>()(el), _hash_helper<I + 1>(t));
+ }
+};
+
+
+#endif
+
+
+} // namespace phmap
+
+#ifdef _MSC_VER
+ #pragma warning(pop)
+#endif
+
+#endif // phmap_utils_h_guard_
diff --git a/benchmarks/others/robin_hood.hpp b/benchmarks/others/robin_hood.hpp
index a602dac9..ec2af8c6 100644
--- a/benchmarks/others/robin_hood.hpp
+++ b/benchmarks/others/robin_hood.hpp
@@ -1,2465 +1,2533 @@
-// ______ _____ ______ _________
-// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
-// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
-// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
-// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
-// _/_____/
-//
-// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
-// https://github.com/martinus/robin-hood-hashing
-//
-// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2018-2020 Martin Ankerl <http://martin.ankerl.com>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#ifndef ROBIN_HOOD_H_INCLUDED
-#define ROBIN_HOOD_H_INCLUDED
-
-// see https://semver.org/
-#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
-#define ROBIN_HOOD_VERSION_MINOR 10 // for adding functionality in a backwards-compatible manner
-#define ROBIN_HOOD_VERSION_PATCH 0 // for backwards-compatible bug fixes
-
-#include <algorithm>
-#include <cstdlib>
-#include <cstring>
-#include <functional>
-#include <memory> // only to support hash of smart pointers
-#include <stdexcept>
-#include <string>
-#include <type_traits>
-#include <utility>
-#if __cplusplus >= 201703L
-# include <string_view>
-#endif
-
-// #define ROBIN_HOOD_LOG_ENABLED
-#ifdef ROBIN_HOOD_LOG_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_LOG(...) \
- std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
-#else
-# define ROBIN_HOOD_LOG(x)
-#endif
-
-// #define ROBIN_HOOD_TRACE_ENABLED
-#ifdef ROBIN_HOOD_TRACE_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_TRACE(...) \
- std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
-#else
-# define ROBIN_HOOD_TRACE(x)
-#endif
-
-// #define ROBIN_HOOD_COUNT_ENABLED
-#ifdef ROBIN_HOOD_COUNT_ENABLED
-# include <iostream>
-# define ROBIN_HOOD_COUNT(x) ++counts().x;
-namespace robin_hood {
-struct Counts {
- uint64_t shiftUp{};
- uint64_t shiftDown{};
-};
-inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
- return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
-}
-
-static Counts& counts() {
- static Counts counts{};
- return counts;
-}
-} // namespace robin_hood
-#else
-# define ROBIN_HOOD_COUNT(x)
-#endif
-
-// all non-argument macros should use this facility. See
-// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
-#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
-
-// mark unused members with this macro
-#define ROBIN_HOOD_UNUSED(identifier)
-
-// bitness
-#if SIZE_MAX == UINT32_MAX
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
-#elif SIZE_MAX == UINT64_MAX
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
-#else
-# error Unsupported bitness
-#endif
-
-// endianess
-#ifdef _MSC_VER
-# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
- (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-#endif
-
-// inline
-#ifdef _MSC_VER
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
-#endif
-
-// exceptions
-#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
-#endif
-
-// count leading/trailing bits
-#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
-# ifdef _MSC_VER
-# if ROBIN_HOOD(BITNESS) == 32
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
-# endif
-# include <intrin.h>
-# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
-# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
- [](size_t mask) noexcept -> int { \
- unsigned long index; \
- return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
- : ROBIN_HOOD(BITNESS); \
- }(x)
-# else
-# if ROBIN_HOOD(BITNESS) == 32
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
-# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
-# endif
-# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
-# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
-# endif
-#endif
-
-// fallthrough
-#ifndef __has_cpp_attribute // For backwards compatibility
-# define __has_cpp_attribute(x) 0
-#endif
-#if __has_cpp_attribute(clang::fallthrough)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
-#elif __has_cpp_attribute(gnu::fallthrough)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
-#endif
-
-// likely/unlikely
-#ifdef _MSC_VER
-# define ROBIN_HOOD_LIKELY(condition) condition
-# define ROBIN_HOOD_UNLIKELY(condition) condition
-#else
-# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
-# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
-#endif
-
-// detect if native wchar_t type is availiable in MSVC
-#ifdef _MSC_VER
-# ifdef _NATIVE_WCHAR_T_DEFINED
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
-# else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
-# endif
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
-#endif
-
-// workaround missing "is_trivially_copyable" in g++ < 5.0
-// See https://stackoverflow.com/a/31798726/48181
-#if defined(__GNUC__) && __GNUC__ < 5
-# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
-#else
-# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
-#endif
-
-// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
-#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
-#else
-# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
-#endif
-
-namespace robin_hood {
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
-# define ROBIN_HOOD_STD std
-#else
-
-// c++11 compatibility layer
-namespace ROBIN_HOOD_STD {
-template <class T>
-struct alignment_of
- : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
-
-template <class T, T... Ints>
-class integer_sequence {
-public:
- using value_type = T;
- static_assert(std::is_integral<value_type>::value, "not integral type");
- static constexpr std::size_t size() noexcept {
- return sizeof...(Ints);
- }
-};
-template <std::size_t... Inds>
-using index_sequence = integer_sequence<std::size_t, Inds...>;
-
-namespace detail_ {
-template <class T, T Begin, T End, bool>
-struct IntSeqImpl {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
-
- template <class, class>
- struct IntSeqCombiner;
-
- template <TValue... Inds0, TValue... Inds1>
- struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> {
- using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
- };
-
- using TResult =
- typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
- (End - Begin) / 2 == 1>::TResult,
- typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
- (End - Begin + 1) / 2 == 1>::TResult>::TResult;
-};
-
-template <class T, T Begin>
-struct IntSeqImpl<T, Begin, Begin, false> {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0, "unexpected argument (Begin<0)");
- using TResult = integer_sequence<TValue>;
-};
-
-template <class T, T Begin, T End>
-struct IntSeqImpl<T, Begin, End, true> {
- using TValue = T;
- static_assert(std::is_integral<TValue>::value, "not integral type");
- static_assert(Begin >= 0, "unexpected argument (Begin<0)");
- using TResult = integer_sequence<TValue, Begin>;
-};
-} // namespace detail_
-
-template <class T, T N>
-using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
-
-template <std::size_t N>
-using make_index_sequence = make_integer_sequence<std::size_t, N>;
-
-template <class... T>
-using index_sequence_for = make_index_sequence<sizeof...(T)>;
-
-} // namespace ROBIN_HOOD_STD
-
-#endif
-
-namespace detail {
-
-// make sure we static_cast to the correct type for hash_int
-#if ROBIN_HOOD(BITNESS) == 64
-using SizeT = uint64_t;
-#else
-using SizeT = uint32_t;
-#endif
-
-template <typename T>
-T rotr(T x, unsigned k) {
- return (x >> k) | (x << (8U * sizeof(T) - k));
-}
-
-// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
-// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
-// care!
-template <typename T>
-inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
- return reinterpret_cast<T>(ptr);
-}
-
-template <typename T>
-inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
- return reinterpret_cast<T>(ptr);
-}
-
-// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
-// inlinings more difficult. Throws are also generally the slow path.
-template <typename E, typename... Args>
-[[noreturn]] ROBIN_HOOD(NOINLINE)
-#if ROBIN_HOOD(HAS_EXCEPTIONS)
- void doThrow(Args&&... args) {
- // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
- throw E(std::forward<Args>(args)...);
-}
-#else
- void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
- abort();
-}
-#endif
-
-template <typename E, typename T, typename... Args>
-T* assertNotNull(T* t, Args&&... args) {
- if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
- doThrow<E>(std::forward<Args>(args)...);
- }
- return t;
-}
-
-template <typename T>
-inline T unaligned_load(void const* ptr) noexcept {
- // using memcpy so we don't get into unaligned load problems.
- // compiler should optimize this very well anyways.
- T t;
- std::memcpy(&t, ptr, sizeof(T));
- return t;
-}
-
-// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
-// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
-// pointer.
-template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
-class BulkPoolAllocator {
-public:
- BulkPoolAllocator() noexcept = default;
-
- // does not copy anything, just creates a new allocator.
- BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept
- : mHead(nullptr)
- , mListForFree(nullptr) {}
-
- BulkPoolAllocator(BulkPoolAllocator&& o) noexcept
- : mHead(o.mHead)
- , mListForFree(o.mListForFree) {
- o.mListForFree = nullptr;
- o.mHead = nullptr;
- }
-
- BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept {
- reset();
- mHead = o.mHead;
- mListForFree = o.mListForFree;
- o.mListForFree = nullptr;
- o.mHead = nullptr;
- return *this;
- }
-
- BulkPoolAllocator&
- // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
- operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
- // does not do anything
- return *this;
- }
-
- ~BulkPoolAllocator() noexcept {
- reset();
- }
-
- // Deallocates all allocated memory.
- void reset() noexcept {
- while (mListForFree) {
- T* tmp = *mListForFree;
- ROBIN_HOOD_LOG("std::free")
- std::free(mListForFree);
- mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
- }
- mHead = nullptr;
- }
-
- // allocates, but does NOT initialize. Use in-place new constructor, e.g.
- // T* obj = pool.allocate();
- // ::new (static_cast<void*>(obj)) T();
- T* allocate() {
- T* tmp = mHead;
- if (!tmp) {
- tmp = performAllocation();
- }
-
- mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
- return tmp;
- }
-
- // does not actually deallocate but puts it in store.
- // make sure you have already called the destructor! e.g. with
- // obj->~T();
- // pool.deallocate(obj);
- void deallocate(T* obj) noexcept {
- *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
- mHead = obj;
- }
-
- // Adds an already allocated block of memory to the allocator. This allocator is from now on
- // responsible for freeing the data (with free()). If the provided data is not large enough to
- // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
- void addOrFree(void* ptr, const size_t numBytes) noexcept {
- // calculate number of available elements in ptr
- if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
- // not enough data for at least one element. Free and return.
- ROBIN_HOOD_LOG("std::free")
- std::free(ptr);
- } else {
- ROBIN_HOOD_LOG("add to buffer")
- add(ptr, numBytes);
- }
- }
-
- void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs>& other) noexcept {
- using std::swap;
- swap(mHead, other.mHead);
- swap(mListForFree, other.mListForFree);
- }
-
-private:
- // iterates the list of allocated memory to calculate how many to alloc next.
- // Recalculating this each time saves us a size_t member.
- // This ignores the fact that memory blocks might have been added manually with addOrFree. In
- // practice, this should not matter much.
- ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
- auto tmp = mListForFree;
- size_t numAllocs = MinNumAllocs;
-
- while (numAllocs * 2 <= MaxNumAllocs && tmp) {
- auto x = reinterpret_cast<T***>(tmp);
- tmp = *x;
- numAllocs *= 2;
- }
-
- return numAllocs;
- }
-
- // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
- void add(void* ptr, const size_t numBytes) noexcept {
- const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
-
- auto data = reinterpret_cast<T**>(ptr);
-
- // link free list
- auto x = reinterpret_cast<T***>(data);
- *x = mListForFree;
- mListForFree = data;
-
- // create linked list for newly allocated data
- auto* const headT =
- reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
-
- auto* const head = reinterpret_cast<char*>(headT);
-
- // Visual Studio compiler automatically unrolls this loop, which is pretty cool
- for (size_t i = 0; i < numElements; ++i) {
- *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
- head + (i + 1) * ALIGNED_SIZE;
- }
-
- // last one points to 0
- *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
- mHead;
- mHead = headT;
- }
-
- // Called when no memory is available (mHead == 0).
- // Don't inline this slow path.
- ROBIN_HOOD(NOINLINE) T* performAllocation() {
- size_t const numElementsToAlloc = calcNumElementsToAlloc();
-
- // alloc new memory: [prev |T, T, ... T]
- size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
- ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
- << " * " << numElementsToAlloc)
- add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
- return mHead;
- }
-
- // enforce byte alignment of the T's
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
- static constexpr size_t ALIGNMENT =
- (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
-#else
- static const size_t ALIGNMENT =
- (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value)
- ? ROBIN_HOOD_STD::alignment_of<T>::value
- : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
-#endif
-
- static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
-
- static_assert(MinNumAllocs >= 1, "MinNumAllocs");
- static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
- static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
- static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
- static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
-
- T* mHead{nullptr};
- T** mListForFree{nullptr};
-};
-
-template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
-struct NodeAllocator;
-
-// dummy allocator that does nothing
-template <typename T, size_t MinSize, size_t MaxSize>
-struct NodeAllocator<T, MinSize, MaxSize, true> {
-
- // we are not using the data, so just free it.
- void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
- ROBIN_HOOD_LOG("std::free")
- std::free(ptr);
- }
-};
-
-template <typename T, size_t MinSize, size_t MaxSize>
-struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
-
-// dummy hash, unsed as mixer when robin_hood::hash is already used
-template <typename T>
-struct identity_hash {
- constexpr size_t operator()(T const& obj) const noexcept {
- return static_cast<size_t>(obj);
- }
-};
-
-// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
-// my own here.
-namespace swappable {
-#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
-using std::swap;
-template <typename T>
-struct nothrow {
- static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
-};
-#else
-template <typename T>
-struct nothrow {
- static const bool value = std::is_nothrow_swappable<T>::value;
-};
-#endif
-} // namespace swappable
-
-} // namespace detail
-
-struct is_transparent_tag {};
-
-// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
-// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is
-// also tested.
-template <typename T1, typename T2>
-struct pair {
- using first_type = T1;
- using second_type = T2;
-
- template <typename U1 = T1, typename U2 = T2,
- typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
- std::is_default_constructible<U2>::value>::type>
- constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
- : first()
- , second() {}
-
- // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
- explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
- noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
- : first(o.first)
- , second(o.second) {}
-
- // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
- explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
- T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
- : first(std::move(o.first))
- , second(std::move(o.second)) {}
-
- constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
- T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
- : first(std::move(a))
- , second(std::move(b)) {}
-
- template <typename U1, typename U2>
- constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
- std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
- : first(std::forward<U1>(a))
- , second(std::forward<U2>(b)) {}
-
- template <typename... U1, typename... U2>
- constexpr pair(
- std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
- std::tuple<U2...> b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
- std::declval<std::tuple<U2...>&>(),
- ROBIN_HOOD_STD::index_sequence_for<U1...>(),
- ROBIN_HOOD_STD::index_sequence_for<U2...>())))
- : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(),
- ROBIN_HOOD_STD::index_sequence_for<U2...>()) {}
-
- // constructor called from the std::piecewise_construct_t ctor
- template <typename... U1, size_t... I1, typename... U2, size_t... I2>
- pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
- noexcept(T1(std::forward<U1>(std::get<I1>(
- std::declval<std::tuple<
- U1...>&>()))...)) && noexcept(T2(std::
- forward<U2>(std::get<I2>(
- std::declval<std::tuple<U2...>&>()))...)))
- : first(std::forward<U1>(std::get<I1>(a))...)
- , second(std::forward<U2>(std::get<I2>(b))...) {
- // make visual studio compiler happy about warning about unused a & b.
- // Visual studio's pair implementation disables warning 4100.
- (void)a;
- (void)b;
- }
-
- void swap(pair<T1, T2>& o) noexcept((detail::swappable::nothrow<T1>::value) &&
- (detail::swappable::nothrow<T2>::value)) {
- using std::swap;
- swap(first, o.first);
- swap(second, o.second);
- }
-
- T1 first; // NOLINT(misc-non-private-member-variables-in-classes)
- T2 second; // NOLINT(misc-non-private-member-variables-in-classes)
-};
-
-template <typename A, typename B>
-inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
- noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
- a.swap(b);
-}
-
-template <typename A, typename B>
-inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
- return (x.first == y.first) && (x.second == y.second);
-}
-template <typename A, typename B>
-inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x == y);
-}
-template <typename A, typename B>
-inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
- std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
- std::declval<B const&>())) {
- return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
-}
-template <typename A, typename B>
-inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
- return y < x;
-}
-template <typename A, typename B>
-inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x > y);
-}
-template <typename A, typename B>
-inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
- return !(x < y);
-}
-
-inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
- static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
- static constexpr uint64_t seed = UINT64_C(0xe17a1465);
- static constexpr unsigned int r = 47;
-
- auto const* const data64 = static_cast<uint64_t const*>(ptr);
- uint64_t h = seed ^ (len * m);
-
- size_t const n_blocks = len / 8;
- for (size_t i = 0; i < n_blocks; ++i) {
- auto k = detail::unaligned_load<uint64_t>(data64 + i);
-
- k *= m;
- k ^= k >> r;
- k *= m;
-
- h ^= k;
- h *= m;
- }
-
- auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
- switch (len & 7U) {
- case 7:
- h ^= static_cast<uint64_t>(data8[6]) << 48U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 6:
- h ^= static_cast<uint64_t>(data8[5]) << 40U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 5:
- h ^= static_cast<uint64_t>(data8[4]) << 32U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 4:
- h ^= static_cast<uint64_t>(data8[3]) << 24U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 3:
- h ^= static_cast<uint64_t>(data8[2]) << 16U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 2:
- h ^= static_cast<uint64_t>(data8[1]) << 8U;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- case 1:
- h ^= static_cast<uint64_t>(data8[0]);
- h *= m;
- ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
- default:
- break;
- }
-
- h ^= h >> r;
- h *= m;
- h ^= h >> r;
- return static_cast<size_t>(h);
-}
-
-inline size_t hash_int(uint64_t x) noexcept {
- // inspired by lemire's strongly universal hashing
- // https://lemire.me/blog/2018/08/15/fast-strongly-universal-64-bit-hashing-everywhere/
- //
- // Instead of shifts, we use rotations so we don't lose any bits.
- //
- // Added a final multiplcation with a constant for more mixing. It is most important that
- // the lower bits are well mixed.
- auto h1 = x * UINT64_C(0xA24BAED4963EE407);
- auto h2 = detail::rotr(x, 32U) * UINT64_C(0x9FB21C651E98DF25);
- auto h = detail::rotr(h1 + h2, 32U);
- return static_cast<size_t>(h);
-}
-
-// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
-template <typename T, typename Enable = void>
-struct hash : public std::hash<T> {
- size_t operator()(T const& obj) const
- noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
- // call base hash
- auto result = std::hash<T>::operator()(obj);
- // return mixed of that, to be save against identity has
- return hash_int(static_cast<detail::SizeT>(result));
- }
-};
-
-template <typename CharT>
-struct hash<std::basic_string<CharT>> {
- size_t operator()(std::basic_string<CharT> const& str) const noexcept {
- return hash_bytes(str.data(), sizeof(CharT) * str.size());
- }
-};
-
-#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
-template <typename CharT>
-struct hash<std::basic_string_view<CharT>> {
- size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
- return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
- }
-};
-#endif
-
-template <class T>
-struct hash<T*> {
- size_t operator()(T* ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr));
- }
-};
-
-template <class T>
-struct hash<std::unique_ptr<T>> {
- size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
- }
-};
-
-template <class T>
-struct hash<std::shared_ptr<T>> {
- size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
- return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
- }
-};
-
-template <typename Enum>
-struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
- size_t operator()(Enum e) const noexcept {
- using Underlying = typename std::underlying_type<Enum>::type;
- return hash<Underlying>{}(static_cast<Underlying>(e));
- }
-};
-
-#define ROBIN_HOOD_HASH_INT(T) \
- template <> \
- struct hash<T> { \
- size_t operator()(T const& obj) const noexcept { \
- return hash_int(static_cast<uint64_t>(obj)); \
- } \
- }
-
-#if defined(__GNUC__) && !defined(__clang__)
-# pragma GCC diagnostic push
-# pragma GCC diagnostic ignored "-Wuseless-cast"
-#endif
-// see https://en.cppreference.com/w/cpp/utility/hash
-ROBIN_HOOD_HASH_INT(bool);
-ROBIN_HOOD_HASH_INT(char);
-ROBIN_HOOD_HASH_INT(signed char);
-ROBIN_HOOD_HASH_INT(unsigned char);
-ROBIN_HOOD_HASH_INT(char16_t);
-ROBIN_HOOD_HASH_INT(char32_t);
-#if ROBIN_HOOD(HAS_NATIVE_WCHART)
-ROBIN_HOOD_HASH_INT(wchar_t);
-#endif
-ROBIN_HOOD_HASH_INT(short);
-ROBIN_HOOD_HASH_INT(unsigned short);
-ROBIN_HOOD_HASH_INT(int);
-ROBIN_HOOD_HASH_INT(unsigned int);
-ROBIN_HOOD_HASH_INT(long);
-ROBIN_HOOD_HASH_INT(long long);
-ROBIN_HOOD_HASH_INT(unsigned long);
-ROBIN_HOOD_HASH_INT(unsigned long long);
-#if defined(__GNUC__) && !defined(__clang__)
-# pragma GCC diagnostic pop
-#endif
-namespace detail {
-
-template <typename T>
-struct void_type {
- using type = void;
-};
-
-template <typename T, typename = void>
-struct has_is_transparent : public std::false_type {};
-
-template <typename T>
-struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
- : public std::true_type {};
-
-// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
-// is used. see https://stackoverflow.com/a/28771920/48181
-template <typename T>
-struct WrapHash : public T {
- WrapHash() = default;
- explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
- : T(o) {}
-};
-
-template <typename T>
-struct WrapKeyEqual : public T {
- WrapKeyEqual() = default;
- explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
- : T(o) {}
-};
-
-// A highly optimized hashmap implementation, using the Robin Hood algorithm.
-//
-// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
-// be about 2x faster in most cases and require much less allocations.
-//
-// This implementation uses the following memory layout:
-//
-// [Node, Node, ... Node | info, info, ... infoSentinel ]
-//
-// * Node: either a DataNode that directly has the std::pair<key, val> as member,
-// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
-// depends on how fast the swap() operation is. Heuristically, this is automatically choosen
-// based on sizeof(). there are always 2^n Nodes.
-//
-// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
-// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
-// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
-// actually belongs to the previous position and was pushed out because that place is already
-// taken.
-//
-// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
-// need for a idx variable.
-//
-// According to STL, order of templates has effect on throughput. That's why I've moved the
-// boolean to the front.
-// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
-template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
- typename KeyEqual>
-class Table
- : public WrapHash<Hash>,
- public WrapKeyEqual<KeyEqual>,
- detail::NodeAllocator<
- typename std::conditional<
- std::is_void<T>::value, Key,
- robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
- 4, 16384, IsFlat> {
-public:
- static constexpr bool is_flat = IsFlat;
- static constexpr bool is_map = !std::is_void<T>::value;
- static constexpr bool is_set = !is_map;
- static constexpr bool is_transparent =
- has_is_transparent<Hash>::value && has_is_transparent<KeyEqual>::value;
-
- using key_type = Key;
- using mapped_type = T;
- using value_type = typename std::conditional<
- is_set, Key,
- robin_hood::pair<typename std::conditional<is_flat, Key, Key const>::type, T>>::type;
- using size_type = size_t;
- using hasher = Hash;
- using key_equal = KeyEqual;
- using Self = Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
-
-private:
- static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
- "MaxLoadFactor100 needs to be >10 && < 100");
-
- using WHash = WrapHash<Hash>;
- using WKeyEqual = WrapKeyEqual<KeyEqual>;
-
- // configuration defaults
-
- // make sure we have 8 elements, needed to quickly rehash mInfo
- static constexpr size_t InitialNumElements = sizeof(uint64_t);
- static constexpr uint32_t InitialInfoNumBits = 5;
- static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
- static constexpr size_t InfoMask = InitialInfoInc - 1U;
- static constexpr uint8_t InitialInfoHashShift = 0;
- using DataPool = detail::NodeAllocator<value_type, 4, 16384, IsFlat>;
-
- // type needs to be wider than uint8_t.
- using InfoType = uint32_t;
-
- // DataNode ////////////////////////////////////////////////////////
-
- // Primary template for the data node. We have special implementations for small and big
- // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
- // on the heap so swap merely swaps a pointer.
- template <typename M, bool>
- class DataNode {};
-
- // Small: just allocate on the stack.
- template <typename M>
- class DataNode<M, true> final {
- public:
- template <typename... Args>
- explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
- noexcept(value_type(std::forward<Args>(args)...)))
- : mData(std::forward<Args>(args)...) {}
-
- DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
- std::is_nothrow_move_constructible<value_type>::value)
- : mData(std::move(n.mData)) {}
-
- // doesn't do anything
- void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
- void destroyDoNotDeallocate() noexcept {}
-
- value_type const* operator->() const noexcept {
- return &mData;
- }
- value_type* operator->() noexcept {
- return &mData;
- }
-
- const value_type& operator*() const noexcept {
- return mData;
- }
-
- value_type& operator*() noexcept {
- return mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
- return mData.first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
- return mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type const&>::type
- getFirst() const noexcept {
- return mData.first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
- return mData;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
- return mData.second;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
- return mData.second;
- }
-
- void swap(DataNode<M, true>& o) noexcept(
- noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
- mData.swap(o.mData);
- }
-
- private:
- value_type mData;
- };
-
- // big object: allocate on heap.
- template <typename M>
- class DataNode<M, false> {
- public:
- template <typename... Args>
- explicit DataNode(M& map, Args&&... args)
- : mData(map.allocate()) {
- ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
- }
-
- DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
- : mData(std::move(n.mData)) {}
-
- void destroy(M& map) noexcept {
- // don't deallocate, just put it into list of datapool.
- mData->~value_type();
- map.deallocate(mData);
- }
-
- void destroyDoNotDeallocate() noexcept {
- mData->~value_type();
- }
-
- value_type const* operator->() const noexcept {
- return mData;
- }
-
- value_type* operator->() noexcept {
- return mData;
- }
-
- const value_type& operator*() const {
- return *mData;
- }
-
- value_type& operator*() {
- return *mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
- return mData->first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
- return *mData;
- }
-
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, typename VT::first_type const&>::type
- getFirst() const noexcept {
- return mData->first;
- }
- template <typename VT = value_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
- return *mData;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
- return mData->second;
- }
-
- template <typename MT = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
- return mData->second;
- }
-
- void swap(DataNode<M, false>& o) noexcept {
- using std::swap;
- swap(mData, o.mData);
- }
-
- private:
- value_type* mData;
- };
-
- using Node = DataNode<Self, IsFlat>;
-
- // helpers for doInsert: extract first entry (only const required)
- ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
- return n.getFirst();
- }
-
- // in case we have void mapped_type, we are not using a pair, thus we just route k through.
- // No need to disable this because it's just not used if not applicable.
- ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
- return k;
- }
-
- // in case we have non-void mapped_type, we have a standard robin_hood::pair
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
- getFirstConst(value_type const& vt) const noexcept {
- return vt.first;
- }
-
- // Cloner //////////////////////////////////////////////////////////
-
- template <typename M, bool UseMemcpy>
- struct Cloner;
-
- // fast path: Just copy data, without allocating anything.
- template <typename M>
- struct Cloner<M, true> {
- void operator()(M const& source, M& target) const {
- auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
- auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
- auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
- std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
- }
- };
-
- template <typename M>
- struct Cloner<M, false> {
- void operator()(M const& s, M& t) const {
- auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
- std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
-
- for (size_t i = 0; i < numElementsWithBuffer; ++i) {
- if (t.mInfo[i]) {
- ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
- }
- }
- }
- };
-
- // Destroyer ///////////////////////////////////////////////////////
-
- template <typename M, bool IsFlatAndTrivial>
- struct Destroyer {};
-
- template <typename M>
- struct Destroyer<M, true> {
- void nodes(M& m) const noexcept {
- m.mNumElements = 0;
- }
-
- void nodesDoNotDeallocate(M& m) const noexcept {
- m.mNumElements = 0;
- }
- };
-
- template <typename M>
- struct Destroyer<M, false> {
- void nodes(M& m) const noexcept {
- m.mNumElements = 0;
- // clear also resets mInfo to 0, that's sometimes not necessary.
- auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
-
- for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
- if (0 != m.mInfo[idx]) {
- Node& n = m.mKeyVals[idx];
- n.destroy(m);
- n.~Node();
- }
- }
- }
-
- void nodesDoNotDeallocate(M& m) const noexcept {
- m.mNumElements = 0;
- // clear also resets mInfo to 0, that's sometimes not necessary.
- auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
- for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
- if (0 != m.mInfo[idx]) {
- Node& n = m.mKeyVals[idx];
- n.destroyDoNotDeallocate();
- n.~Node();
- }
- }
- }
- };
-
- // Iter ////////////////////////////////////////////////////////////
-
- struct fast_forward_tag {};
-
- // generic iterator for both const_iterator and iterator.
- template <bool IsConst>
- // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
- class Iter {
- private:
- using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
-
- public:
- using difference_type = std::ptrdiff_t;
- using value_type = typename Self::value_type;
- using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
- using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
- using iterator_category = std::forward_iterator_tag;
-
- // default constructed iterator can be compared to itself, but WON'T return true when
- // compared to end().
- Iter() = default;
-
- // Rule of zero: nothing specified. The conversion constructor is only enabled for
- // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
-
- // Conversion constructor from iterator to const_iterator.
- template <bool OtherIsConst,
- typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
- // NOLINTNEXTLINE(hicpp-explicit-conversions)
- Iter(Iter<OtherIsConst> const& other) noexcept
- : mKeyVals(other.mKeyVals)
- , mInfo(other.mInfo) {}
-
- Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
- : mKeyVals(valPtr)
- , mInfo(infoPtr) {}
-
- Iter(NodePtr valPtr, uint8_t const* infoPtr,
- fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
- : mKeyVals(valPtr)
- , mInfo(infoPtr) {
- fastForward();
- }
-
- template <bool OtherIsConst,
- typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
- Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
- mKeyVals = other.mKeyVals;
- mInfo = other.mInfo;
- return *this;
- }
-
- // prefix increment. Undefined behavior if we are at end()!
- Iter& operator++() noexcept {
- mInfo++;
- mKeyVals++;
- fastForward();
- return *this;
- }
-
- Iter operator++(int) noexcept {
- Iter tmp = *this;
- ++(*this);
- return tmp;
- }
-
- reference operator*() const {
- return **mKeyVals;
- }
-
- pointer operator->() const {
- return &**mKeyVals;
- }
-
- template <bool O>
- bool operator==(Iter<O> const& o) const noexcept {
- return mKeyVals == o.mKeyVals;
- }
-
- template <bool O>
- bool operator!=(Iter<O> const& o) const noexcept {
- return mKeyVals != o.mKeyVals;
- }
-
- private:
- // fast forward to the next non-free info byte
- // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
- // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
- void fastForward() noexcept {
- size_t n = 0;
- while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
- mInfo += sizeof(size_t);
- mKeyVals += sizeof(size_t);
- }
-#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
- // we know for certain that within the next 8 bytes we'll find a non-zero one.
- if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
- mInfo += 4;
- mKeyVals += 4;
- }
- if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
- mInfo += 2;
- mKeyVals += 2;
- }
- if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
- mInfo += 1;
- mKeyVals += 1;
- }
-#else
-# if ROBIN_HOOD(LITTLE_ENDIAN)
- auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
-# else
- auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
-# endif
- mInfo += inc;
- mKeyVals += inc;
-#endif
- }
-
- friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
- NodePtr mKeyVals{nullptr};
- uint8_t const* mInfo{nullptr};
- };
-
- ////////////////////////////////////////////////////////////////////
-
- // highly performance relevant code.
- // Lower bits are used for indexing into the array (2^n size)
- // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
- template <typename HashKey>
- void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
- // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as
- // an additional mixing step. This serves as a bad hash prevention, if the given data is
- // badly mixed.
- using Mix =
- typename std::conditional<std::is_same<::robin_hood::hash<key_type>, hasher>::value,
- ::robin_hood::detail::identity_hash<size_t>,
- ::robin_hood::hash<size_t>>::type;
-
- // the lower InitialInfoNumBits are reserved for info.
- auto h = Mix{}(WHash::operator()(key));
- *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
- *idx = (h >> InitialInfoNumBits) & mMask;
- }
-
- // forwards the index by one, wrapping around at the end
- void next(InfoType* info, size_t* idx) const noexcept {
- *idx = *idx + 1;
- *info += mInfoInc;
- }
-
- void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
- // unrolling this by hand did not bring any speedups.
- while (*info < mInfo[*idx]) {
- next(info, idx);
- }
- }
-
- // Shift everything up by one element. Tries to move stuff around.
- void
- shiftUp(size_t startIdx,
- size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
- auto idx = startIdx;
- ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
- while (--idx != insertion_idx) {
- mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
- }
-
- idx = startIdx;
- while (idx != insertion_idx) {
- ROBIN_HOOD_COUNT(shiftUp)
- mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
- if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
- --idx;
- }
- }
-
- void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
- // until we find one that is either empty or has zero offset.
- // TODO(martinus) we don't need to move everything, just the last one for the same
- // bucket.
- mKeyVals[idx].destroy(*this);
-
- // until we find one that is either empty or has zero offset.
- while (mInfo[idx + 1] >= 2 * mInfoInc) {
- ROBIN_HOOD_COUNT(shiftDown)
- mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
- mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
- ++idx;
- }
-
- mInfo[idx] = 0;
- // don't destroy, we've moved it
- // mKeyVals[idx].destroy(*this);
- mKeyVals[idx].~Node();
- }
-
- // copy of find(), except that it returns iterator instead of const_iterator.
- template <typename Other>
- ROBIN_HOOD(NODISCARD)
- size_t findIdx(Other const& key) const {
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
-
- do {
- // unrolling this twice gives a bit of a speedup. More unrolling did not help.
- if (info == mInfo[idx] &&
- ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
- return idx;
- }
- next(&info, &idx);
- if (info == mInfo[idx] &&
- ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
- return idx;
- }
- next(&info, &idx);
- } while (info <= mInfo[idx]);
-
- // nothing found!
- return mMask == 0 ? 0
- : static_cast<size_t>(std::distance(
- mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
- }
-
- void cloneData(const Table& o) {
- Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
- }
-
- // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
- // @return index where the element was created
- size_t insert_move(Node&& keyval) {
- // we don't retry, fail if overflowing
- // don't need to check max num elements
- if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
- throwOverflowError(); // impossible to reach LCOV_EXCL_LINE
- }
-
- size_t idx{};
- InfoType info{};
- keyToIdx(keyval.getFirst(), &idx, &info);
-
- // skip forward. Use <= because we are certain that the element is not there.
- while (info <= mInfo[idx]) {
- idx = idx + 1;
- info += mInfoInc;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = static_cast<uint8_t>(info);
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- ::new (static_cast<void*>(&l)) Node(std::move(keyval));
- } else {
- shiftUp(idx, insertion_idx);
- l = std::move(keyval);
- }
-
- // put at empty spot
- mInfo[insertion_idx] = insertion_info;
-
- ++mNumElements;
- return insertion_idx;
- }
-
-public:
- using iterator = Iter<false>;
- using const_iterator = Iter<true>;
-
- Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
- : WHash()
- , WKeyEqual() {
- ROBIN_HOOD_TRACE(this)
- }
-
- // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
- // This tremendously speeds up ctor & dtor of a map that never receives an element. The
- // penalty is payed at the first insert, and not before. Lookup of this empty map works
- // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
- // standard, but we can ignore it.
- explicit Table(
- size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
- const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- }
-
- template <typename Iter>
- Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
- const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- insert(first, last);
- }
-
- Table(std::initializer_list<value_type> initlist,
- size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
- const KeyEqual& equal = KeyEqual{})
- : WHash(h)
- , WKeyEqual(equal) {
- ROBIN_HOOD_TRACE(this)
- insert(initlist.begin(), initlist.end());
- }
-
- Table(Table&& o) noexcept
- : WHash(std::move(static_cast<WHash&>(o)))
- , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
- , DataPool(std::move(static_cast<DataPool&>(o))) {
- ROBIN_HOOD_TRACE(this)
- if (o.mMask) {
- mKeyVals = std::move(o.mKeyVals);
- mInfo = std::move(o.mInfo);
- mNumElements = std::move(o.mNumElements);
- mMask = std::move(o.mMask);
- mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
- mInfoInc = std::move(o.mInfoInc);
- mInfoHashShift = std::move(o.mInfoHashShift);
- // set other's mask to 0 so its destructor won't do anything
- o.init();
- }
- }
-
- Table& operator=(Table&& o) noexcept {
- ROBIN_HOOD_TRACE(this)
- if (&o != this) {
- if (o.mMask) {
- // only move stuff if the other map actually has some data
- destroy();
- mKeyVals = std::move(o.mKeyVals);
- mInfo = std::move(o.mInfo);
- mNumElements = std::move(o.mNumElements);
- mMask = std::move(o.mMask);
- mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
- mInfoInc = std::move(o.mInfoInc);
- mInfoHashShift = std::move(o.mInfoHashShift);
- WHash::operator=(std::move(static_cast<WHash&>(o)));
- WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
- DataPool::operator=(std::move(static_cast<DataPool&>(o)));
-
- o.init();
-
- } else {
- // nothing in the other map => just clear us.
- clear();
- }
- }
- return *this;
- }
-
- Table(const Table& o)
- : WHash(static_cast<const WHash&>(o))
- , WKeyEqual(static_cast<const WKeyEqual&>(o))
- , DataPool(static_cast<const DataPool&>(o)) {
- ROBIN_HOOD_TRACE(this)
- if (!o.empty()) {
- // not empty: create an exact copy. it is also possible to just iterate through all
- // elements and insert them, but copying is probably faster.
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
-
- ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = static_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
- // no need for calloc because clonData does memcpy
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
- mNumElements = o.mNumElements;
- mMask = o.mMask;
- mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
- mInfoInc = o.mInfoInc;
- mInfoHashShift = o.mInfoHashShift;
- cloneData(o);
- }
- }
-
- // Creates a copy of the given map. Copy constructor of each entry is used.
- // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
- // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
- Table& operator=(Table const& o) {
- ROBIN_HOOD_TRACE(this)
- if (&o == this) {
- // prevent assigning of itself
- return *this;
- }
-
- // we keep using the old allocator and not assign the new one, because we want to keep
- // the memory available. when it is the same size.
- if (o.empty()) {
- if (0 == mMask) {
- // nothing to do, we are empty too
- return *this;
- }
-
- // not empty: destroy what we have there
- // clear also resets mInfo to 0, that's sometimes not necessary.
- destroy();
- init();
- WHash::operator=(static_cast<const WHash&>(o));
- WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
- DataPool::operator=(static_cast<DataPool const&>(o));
-
- return *this;
- }
-
- // clean up old stuff
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
-
- if (mMask != o.mMask) {
- // no luck: we don't have the same array size allocated, so we need to realloc.
- if (0 != mMask) {
- // only deallocate if we actually have data!
- ROBIN_HOOD_LOG("std::free")
- std::free(mKeyVals);
- }
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
- ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = static_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
-
- // no need for calloc here because cloneData performs a memcpy.
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
- // sentinel is set in cloneData
- }
- WHash::operator=(static_cast<const WHash&>(o));
- WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
- DataPool::operator=(static_cast<DataPool const&>(o));
- mNumElements = o.mNumElements;
- mMask = o.mMask;
- mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
- mInfoInc = o.mInfoInc;
- mInfoHashShift = o.mInfoHashShift;
- cloneData(o);
-
- return *this;
- }
-
- // Swaps everything between the two maps.
- void swap(Table& o) {
- ROBIN_HOOD_TRACE(this)
- using std::swap;
- swap(o, *this);
- }
-
- // Clears all data, without resizing.
- void clear() {
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- // don't do anything! also important because we don't want to write to
- // DummyInfoByte::b, even though we would just write 0 to it.
- return;
- }
-
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
- // clear everything, then set the sentinel again
- uint8_t const z = 0;
- std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
- mInfo[numElementsWithBuffer] = 1;
-
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- // Destroys the map and all it's contents.
- ~Table() {
- ROBIN_HOOD_TRACE(this)
- destroy();
- }
-
- // Checks if both tables contain the same entries. Order is irrelevant.
- bool operator==(const Table& other) const {
- ROBIN_HOOD_TRACE(this)
- if (other.size() != size()) {
- return false;
- }
- for (auto const& otherEntry : other) {
- if (!has(otherEntry)) {
- return false;
- }
- }
-
- return true;
- }
-
- bool operator!=(const Table& other) const {
- ROBIN_HOOD_TRACE(this)
- return !operator==(other);
- }
-
- template <typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- return doCreateByKey(key);
- }
-
- template <typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
- ROBIN_HOOD_TRACE(this)
- return doCreateByKey(std::move(key));
- }
-
- template <typename Iter>
- void insert(Iter first, Iter last) {
- for (; first != last; ++first) {
- // value_type ctor needed because this might be called with std::pair's
- insert(value_type(*first));
- }
- }
-
- template <typename... Args>
- std::pair<iterator, bool> emplace(Args&&... args) {
- ROBIN_HOOD_TRACE(this)
- Node n{*this, std::forward<Args>(args)...};
- auto r = doInsert(std::move(n));
- if (!r.second) {
- // insertion not possible: destroy node
- // NOLINTNEXTLINE(bugprone-use-after-move)
- n.destroy(*this);
- }
- return r;
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
- return try_emplace_impl(key, std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
- return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const_iterator hint, const key_type& key,
- Args&&... args) {
- (void)hint;
- return try_emplace_impl(key, std::forward<Args>(args)...);
- }
-
- template <typename... Args>
- std::pair<iterator, bool> try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
- (void)hint;
- return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
- return insert_or_assign_impl(key, std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
- return insert_or_assign_impl(std::move(key), std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const_iterator hint, const key_type& key,
- Mapped&& obj) {
- (void)hint;
- return insert_or_assign_impl(key, std::forward<Mapped>(obj));
- }
-
- template <typename Mapped>
- std::pair<iterator, bool> insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) {
- (void)hint;
- return insert_or_assign_impl(std::move(key), std::forward<Mapped>(obj));
- }
-
- std::pair<iterator, bool> insert(const value_type& keyval) {
- ROBIN_HOOD_TRACE(this)
- return doInsert(keyval);
- }
-
- std::pair<iterator, bool> insert(value_type&& keyval) {
- return doInsert(std::move(keyval));
- }
-
- // Returns 1 if key is found, 0 otherwise.
- size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- return 1;
- }
- return 0;
- }
-
- template <typename OtherKey, typename Self_ = Self>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- return 1;
- }
- return 0;
- }
-
- bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- return 1U == count(key);
- }
-
- template <typename OtherKey, typename Self_ = Self>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
- return 1U == count(key);
- }
-
- // Returns a reference to the value found for key.
- // Throws std::out_of_range if element cannot be found
- template <typename Q = mapped_type>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- doThrow<std::out_of_range>("key not found");
- }
- return kv->getSecond();
- }
-
- // Returns a reference to the value found for key.
- // Throws std::out_of_range if element cannot be found
- template <typename Q = mapped_type>
- // NOLINTNEXTLINE(modernize-use-nodiscard)
- typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
- ROBIN_HOOD_TRACE(this)
- auto kv = mKeyVals + findIdx(key);
- if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
- doThrow<std::out_of_range>("key not found");
- }
- return kv->getSecond();
- }
-
- const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey>
- const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey, typename Self_ = Self>
- typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard)
- const_iterator>::type // NOLINT(modernize-use-nodiscard)
- find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return const_iterator{mKeyVals + idx, mInfo + idx};
- }
-
- iterator find(const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey>
- iterator find(const OtherKey& key, is_transparent_tag /*unused*/) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- template <typename OtherKey, typename Self_ = Self>
- typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
- ROBIN_HOOD_TRACE(this)
- const size_t idx = findIdx(key);
- return iterator{mKeyVals + idx, mInfo + idx};
- }
-
- iterator begin() {
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- return end();
- }
- return iterator(mKeyVals, mInfo, fast_forward_tag{});
- }
- const_iterator begin() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return cbegin();
- }
- const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- if (empty()) {
- return cend();
- }
- return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
- }
-
- iterator end() {
- ROBIN_HOOD_TRACE(this)
- // no need to supply valid info pointer: end() must not be dereferenced, and only node
- // pointer is compared.
- return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
- }
- const_iterator end() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return cend();
- }
- const_iterator cend() const { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
- }
-
- iterator erase(const_iterator pos) {
- ROBIN_HOOD_TRACE(this)
- // its safe to perform const cast here
- // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
- return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
- }
-
- // Erases element at pos, returns iterator to the next element.
- iterator erase(iterator pos) {
- ROBIN_HOOD_TRACE(this)
- // we assume that pos always points to a valid entry, and not end().
- auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
-
- shiftDown(idx);
- --mNumElements;
-
- if (*pos.mInfo) {
- // we've backward shifted, return this again
- return pos;
- }
-
- // no backward shift, return next element
- return ++pos;
- }
-
- size_t erase(const key_type& key) {
- ROBIN_HOOD_TRACE(this)
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
-
- // check while info matches with the source idx
- do {
- if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
- shiftDown(idx);
- --mNumElements;
- return 1;
- }
- next(&info, &idx);
- } while (info <= mInfo[idx]);
-
- // nothing found to delete
- return 0;
- }
-
- // reserves space for the specified number of elements. Makes sure the old data fits.
- // exactly the same as reserve(c).
- void rehash(size_t c) {
- // forces a reserve
- reserve(c, true);
- }
-
- // reserves space for the specified number of elements. Makes sure the old data fits.
- // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
- void reserve(size_t c) {
- // reserve, but don't force rehash
- reserve(c, false);
- }
-
- // If possible reallocates the map to a smaller one. This frees the underlying table.
- // Does not do anything if load_factor is too large for decreasing the table's size.
- void compact() {
- ROBIN_HOOD_TRACE(this)
- auto newSize = InitialNumElements;
- while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
- newSize *= 2;
- }
- if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
- throwOverflowError();
- }
-
- ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
-
- // only actually do anything when the new size is bigger than the old one. This prevents to
- // continuously allocate for each reserve() call.
- if (newSize < mMask + 1) {
- rehashPowerOfTwo(newSize, true);
- }
- }
-
- size_type size() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return mNumElements;
- }
-
- size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return static_cast<size_type>(-1);
- }
-
- ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return 0 == mNumElements;
- }
-
- float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return MaxLoadFactor100 / 100.0F;
- }
-
- // Average number of elements per bucket. Since we allow only 1 per bucket
- float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
- ROBIN_HOOD_TRACE(this)
- return static_cast<float>(size()) / static_cast<float>(mMask + 1);
- }
-
- ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return mMask;
- }
-
-#ifndef IGNORE_STANDARD_METHODS
- ROBIN_HOOD(NODISCARD) size_t bucket_count() const noexcept {
- ROBIN_HOOD_TRACE(this)
- return mMask;
- }
- void max_load_factor(float) {
- ROBIN_HOOD_TRACE(this)
- // warning, not used
- }
-#endif
-
- ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
- if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
- return maxElements * MaxLoadFactor100 / 100;
- }
-
- // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
- return (maxElements / 100) * MaxLoadFactor100;
- }
-
- ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
- // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
- // 64bit types.
- return numElements + sizeof(uint64_t);
- }
-
- ROBIN_HOOD(NODISCARD)
- size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
- auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
- return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
- }
-
- // calculation only allowed for 2^n values
- ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
-#if ROBIN_HOOD(BITNESS) == 64
- return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
-#else
- // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
- auto const ne = static_cast<uint64_t>(numElements);
- auto const s = static_cast<uint64_t>(sizeof(Node));
- auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
-
- auto const total64 = ne * s + infos;
- auto const total = static_cast<size_t>(total64);
-
- if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
- throwOverflowError();
- }
- return total;
-#endif
- }
-
-private:
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
- ROBIN_HOOD_TRACE(this)
- auto it = find(e.first);
- return it != end() && it->second == e.second;
- }
-
- template <typename Q = mapped_type>
- ROBIN_HOOD(NODISCARD)
- typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
- ROBIN_HOOD_TRACE(this)
- return find(e) != end();
- }
-
- void reserve(size_t c, bool forceRehash) {
- ROBIN_HOOD_TRACE(this)
- auto const minElementsAllowed = (std::max)(c, mNumElements);
- auto newSize = InitialNumElements;
- while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
- newSize *= 2;
- }
- if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
- throwOverflowError();
- }
-
- ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
-
- // only actually do anything when the new size is bigger than the old one. This prevents to
- // continuously allocate for each reserve() call.
- if (forceRehash || newSize > mMask + 1) {
- rehashPowerOfTwo(newSize, false);
- }
- }
-
- // reserves space for at least the specified number of elements.
- // only works if numBuckets if power of two
- void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
- ROBIN_HOOD_TRACE(this)
-
- Node* const oldKeyVals = mKeyVals;
- uint8_t const* const oldInfo = mInfo;
-
- const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
-
- // resize operation: move stuff
- init_data(numBuckets);
- if (oldMaxElementsWithBuffer > 1) {
- for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
- if (oldInfo[i] != 0) {
- insert_move(std::move(oldKeyVals[i]));
- // destroy the node but DON'T destroy the data.
- oldKeyVals[i].~Node();
- }
- }
-
- // this check is not necessary as it's guarded by the previous if, but it helps silence
- // g++'s overeager "attempt to free a non-heap object 'map'
- // [-Werror=free-nonheap-object]" warning.
- if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
- // don't destroy old data: put it into the pool instead
- if (forceFree) {
- std::free(oldKeyVals);
- } else {
- DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
- }
- }
- }
- }
-
- ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
-#if ROBIN_HOOD(HAS_EXCEPTIONS)
- throw std::overflow_error("robin_hood::map overflow");
-#else
- abort();
-#endif
- }
-
- template <typename OtherKey, typename... Args>
- std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
- ROBIN_HOOD_TRACE(this)
- auto it = find(key);
- if (it == end()) {
- return emplace(std::piecewise_construct,
- std::forward_as_tuple(std::forward<OtherKey>(key)),
- std::forward_as_tuple(std::forward<Args>(args)...));
- }
- return {it, false};
- }
-
- template <typename OtherKey, typename Mapped>
- std::pair<iterator, bool> insert_or_assign_impl(OtherKey&& key, Mapped&& obj) {
- ROBIN_HOOD_TRACE(this)
- auto it = find(key);
- if (it == end()) {
- return emplace(std::forward<OtherKey>(key), std::forward<Mapped>(obj));
- }
- it->second = std::forward<Mapped>(obj);
- return {it, false};
- }
-
- void init_data(size_t max_elements) {
- mNumElements = 0;
- mMask = max_elements - 1;
- mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
-
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
-
- // calloc also zeroes everything
- auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
- ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
- << numElementsWithBuffer << ")")
- mKeyVals = reinterpret_cast<Node*>(
- detail::assertNotNull<std::bad_alloc>(std::calloc(1, numBytesTotal)));
- mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
-
- // set sentinel
- mInfo[numElementsWithBuffer] = 1;
-
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- template <typename Arg, typename Q = mapped_type>
- typename std::enable_if<!std::is_void<Q>::value, Q&>::type doCreateByKey(Arg&& key) {
- while (true) {
- size_t idx{};
- InfoType info{};
- keyToIdx(key, &idx, &info);
- nextWhileLess(&info, &idx);
-
- // while we potentially have a match. Can't do a do-while here because when mInfo is
- // 0 we don't want to skip forward
- while (info == mInfo[idx]) {
- if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
- // key already exists, do not insert.
- return mKeyVals[idx].getSecond();
- }
- next(&info, &idx);
- }
-
- // unlikely that this evaluates to true
- if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
- increase_size();
- continue;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = info;
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- // put at empty spot. This forwards all arguments into the node where the object
- // is constructed exactly where it is needed.
- ::new (static_cast<void*>(&l))
- Node(*this, std::piecewise_construct,
- std::forward_as_tuple(std::forward<Arg>(key)), std::forward_as_tuple());
- } else {
- shiftUp(idx, insertion_idx);
- l = Node(*this, std::piecewise_construct,
- std::forward_as_tuple(std::forward<Arg>(key)), std::forward_as_tuple());
- }
-
- // mKeyVals[idx].getFirst() = std::move(key);
- mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
-
- ++mNumElements;
- return mKeyVals[insertion_idx].getSecond();
- }
- }
-
- // This is exactly the same code as operator[], except for the return values
- template <typename Arg>
- std::pair<iterator, bool> doInsert(Arg&& keyval) {
- while (true) {
- size_t idx{};
- InfoType info{};
- keyToIdx(getFirstConst(keyval), &idx, &info);
- nextWhileLess(&info, &idx);
-
- // while we potentially have a match
- while (info == mInfo[idx]) {
- if (WKeyEqual::operator()(getFirstConst(keyval), mKeyVals[idx].getFirst())) {
- // key already exists, do NOT insert.
- // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
- return std::make_pair<iterator, bool>(iterator(mKeyVals + idx, mInfo + idx),
- false);
- }
- next(&info, &idx);
- }
-
- // unlikely that this evaluates to true
- if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
- increase_size();
- continue;
- }
-
- // key not found, so we are now exactly where we want to insert it.
- auto const insertion_idx = idx;
- auto const insertion_info = info;
- if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
- mMaxNumElementsAllowed = 0;
- }
-
- // find an empty spot
- while (0 != mInfo[idx]) {
- next(&info, &idx);
- }
-
- auto& l = mKeyVals[insertion_idx];
- if (idx == insertion_idx) {
- ::new (static_cast<void*>(&l)) Node(*this, std::forward<Arg>(keyval));
- } else {
- shiftUp(idx, insertion_idx);
- l = Node(*this, std::forward<Arg>(keyval));
- }
-
- // put at empty spot
- mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
-
- ++mNumElements;
- return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true);
- }
- }
-
- bool try_increase_info() {
- ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
- << ", maxNumElementsAllowed="
- << calcMaxNumElementsAllowed(mMask + 1))
- if (mInfoInc <= 2) {
- // need to be > 2 so that shift works (otherwise undefined behavior!)
- return false;
- }
- // we got space left, try to make info smaller
- mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
-
- // remove one bit of the hash, leaving more space for the distance info.
- // This is extremely fast because we can operate on 8 bytes at once.
- ++mInfoHashShift;
- auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
-
- for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
- auto val = unaligned_load<uint64_t>(mInfo + i);
- val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
- std::memcpy(mInfo + i, &val, sizeof(val));
- }
- // update sentinel, which might have been cleared out!
- mInfo[numElementsWithBuffer] = 1;
-
- mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
- return true;
- }
-
- void increase_size() {
- // nothing allocated yet? just allocate InitialNumElements
- if (0 == mMask) {
- init_data(InitialNumElements);
- return;
- }
-
- auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
- if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
- return;
- }
-
- ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
- << maxNumElementsAllowed << ", load="
- << (static_cast<double>(mNumElements) * 100.0 /
- (static_cast<double>(mMask) + 1)))
- // it seems we have a really bad hash function! don't try to resize again
- if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
- throwOverflowError();
- }
-
- rehashPowerOfTwo((mMask + 1) * 2, false);
- }
-
- void destroy() {
- if (0 == mMask) {
- // don't deallocate!
- return;
- }
-
- Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
- .nodesDoNotDeallocate(*this);
-
- // This protection against not deleting mMask shouldn't be needed as it's sufficiently
- // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
- // reports a compile error: attempt to free a non-heap object 'fm'
- // [-Werror=free-nonheap-object]
- if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
- ROBIN_HOOD_LOG("std::free")
- std::free(mKeyVals);
- }
- }
-
- void init() noexcept {
- mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
- mInfo = reinterpret_cast<uint8_t*>(&mMask);
- mNumElements = 0;
- mMask = 0;
- mMaxNumElementsAllowed = 0;
- mInfoInc = InitialInfoInc;
- mInfoHashShift = InitialInfoHashShift;
- }
-
- // members are sorted so no padding occurs
- Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 8
- uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 16
- size_t mNumElements = 0; // 8 byte 24
- size_t mMask = 0; // 8 byte 32
- size_t mMaxNumElementsAllowed = 0; // 8 byte 40
- InfoType mInfoInc = InitialInfoInc; // 4 byte 44
- InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48
- // 16 byte 56 if NodeAllocator
-};
-
-} // namespace detail
-
-// map
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_flat_map = detail::Table<true, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_node_map = detail::Table<false, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-template <typename Key, typename T, typename Hash = hash<Key>,
- typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
-using unordered_map =
- detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
- std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
- std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
- MaxLoadFactor100, Key, T, Hash, KeyEqual>;
-
-// set
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_flat_set = detail::Table<true, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_node_set = detail::Table<false, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
- size_t MaxLoadFactor100 = 80>
-using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 &&
- std::is_nothrow_move_constructible<Key>::value &&
- std::is_nothrow_move_assignable<Key>::value,
- MaxLoadFactor100, Key, void, Hash, KeyEqual>;
-
-} // namespace robin_hood
-
-#endif \ No newline at end of file
+// ______ _____ ______ _________
+// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
+// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
+// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
+// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
+// _/_____/
+//
+// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
+// https://github.com/martinus/robin-hood-hashing
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ROBIN_HOOD_H_INCLUDED
+#define ROBIN_HOOD_H_INCLUDED
+
+// see https://semver.org/
+#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
+#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner
+#define ROBIN_HOOD_VERSION_PATCH 3 // for backwards-compatible bug fixes
+
+#include <algorithm>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <limits>
+#include <memory> // only to support hash of smart pointers
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <utility>
+#if __cplusplus >= 201703L
+# include <string_view>
+#endif
+
+// #define ROBIN_HOOD_LOG_ENABLED
+#ifdef ROBIN_HOOD_LOG_ENABLED
+# include <iostream>
+# define ROBIN_HOOD_LOG(...) \
+ std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
+#else
+# define ROBIN_HOOD_LOG(x)
+#endif
+
+// #define ROBIN_HOOD_TRACE_ENABLED
+#ifdef ROBIN_HOOD_TRACE_ENABLED
+# include <iostream>
+# define ROBIN_HOOD_TRACE(...) \
+ std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
+#else
+# define ROBIN_HOOD_TRACE(x)
+#endif
+
+// #define ROBIN_HOOD_COUNT_ENABLED
+#ifdef ROBIN_HOOD_COUNT_ENABLED
+# include <iostream>
+# define ROBIN_HOOD_COUNT(x) ++counts().x;
+namespace robin_hood {
+struct Counts {
+ uint64_t shiftUp{};
+ uint64_t shiftDown{};
+};
+inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
+ return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
+}
+
+static Counts& counts() {
+ static Counts counts{};
+ return counts;
+}
+} // namespace robin_hood
+#else
+# define ROBIN_HOOD_COUNT(x)
+#endif
+
+// all non-argument macros should use this facility. See
+// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
+#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
+
+// mark unused members with this macro
+#define ROBIN_HOOD_UNUSED(identifier)
+
+// bitness
+#if SIZE_MAX == UINT32_MAX
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
+#elif SIZE_MAX == UINT64_MAX
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
+#else
+# error Unsupported bitness
+#endif
+
+// endianess
+#ifdef _MSC_VER
+# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
+ (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#endif
+
+// inline
+#ifdef _MSC_VER
+# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
+#endif
+
+// exceptions
+#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
+# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
+#endif
+
+// count leading/trailing bits
+#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
+# ifdef _MSC_VER
+# if ROBIN_HOOD(BITNESS) == 32
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
+# else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
+# endif
+# include <intrin.h>
+# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
+# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
+ [](size_t mask) noexcept -> int { \
+ unsigned long index; \
+ return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
+ : ROBIN_HOOD(BITNESS); \
+ }(x)
+# else
+# if ROBIN_HOOD(BITNESS) == 32
+# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
+# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
+# else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
+# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
+# endif
+# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
+# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
+# endif
+#endif
+
+// fallthrough
+#ifndef __has_cpp_attribute // For backwards compatibility
+# define __has_cpp_attribute(x) 0
+#endif
+#if __has_cpp_attribute(clang::fallthrough)
+# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
+#elif __has_cpp_attribute(gnu::fallthrough)
+# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
+#endif
+
+// likely/unlikely
+#ifdef _MSC_VER
+# define ROBIN_HOOD_LIKELY(condition) condition
+# define ROBIN_HOOD_UNLIKELY(condition) condition
+#else
+# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
+# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
+#endif
+
+// detect if native wchar_t type is availiable in MSVC
+#ifdef _MSC_VER
+# ifdef _NATIVE_WCHAR_T_DEFINED
+# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
+# else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
+# endif
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
+#endif
+
+// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr
+#ifdef _MSC_VER
+# if _MSC_VER <= 1900
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1
+# else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
+# endif
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
+#endif
+
+// workaround missing "is_trivially_copyable" in g++ < 5.0
+// See https://stackoverflow.com/a/31798726/48181
+#if defined(__GNUC__) && __GNUC__ < 5
+# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
+#else
+# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
+#endif
+
+// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
+#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
+#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
+#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
+#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
+#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
+
+#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
+# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
+#else
+# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
+#endif
+
+namespace robin_hood {
+
+#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
+# define ROBIN_HOOD_STD std
+#else
+
+// c++11 compatibility layer
+namespace ROBIN_HOOD_STD {
+template <class T>
+struct alignment_of
+ : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
+
+template <class T, T... Ints>
+class integer_sequence {
+public:
+ using value_type = T;
+ static_assert(std::is_integral<value_type>::value, "not integral type");
+ static constexpr std::size_t size() noexcept {
+ return sizeof...(Ints);
+ }
+};
+template <std::size_t... Inds>
+using index_sequence = integer_sequence<std::size_t, Inds...>;
+
+namespace detail_ {
+template <class T, T Begin, T End, bool>
+struct IntSeqImpl {
+ using TValue = T;
+ static_assert(std::is_integral<TValue>::value, "not integral type");
+ static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
+
+ template <class, class>
+ struct IntSeqCombiner;
+
+ template <TValue... Inds0, TValue... Inds1>
+ struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> {
+ using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
+ };
+
+ using TResult =
+ typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
+ (End - Begin) / 2 == 1>::TResult,
+ typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
+ (End - Begin + 1) / 2 == 1>::TResult>::TResult;
+};
+
+template <class T, T Begin>
+struct IntSeqImpl<T, Begin, Begin, false> {
+ using TValue = T;
+ static_assert(std::is_integral<TValue>::value, "not integral type");
+ static_assert(Begin >= 0, "unexpected argument (Begin<0)");
+ using TResult = integer_sequence<TValue>;
+};
+
+template <class T, T Begin, T End>
+struct IntSeqImpl<T, Begin, End, true> {
+ using TValue = T;
+ static_assert(std::is_integral<TValue>::value, "not integral type");
+ static_assert(Begin >= 0, "unexpected argument (Begin<0)");
+ using TResult = integer_sequence<TValue, Begin>;
+};
+} // namespace detail_
+
+template <class T, T N>
+using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
+
+template <std::size_t N>
+using make_index_sequence = make_integer_sequence<std::size_t, N>;
+
+template <class... T>
+using index_sequence_for = make_index_sequence<sizeof...(T)>;
+
+} // namespace ROBIN_HOOD_STD
+
+#endif
+
+namespace detail {
+
+// make sure we static_cast to the correct type for hash_int
+#if ROBIN_HOOD(BITNESS) == 64
+using SizeT = uint64_t;
+#else
+using SizeT = uint32_t;
+#endif
+
+template <typename T>
+T rotr(T x, unsigned k) {
+ return (x >> k) | (x << (8U * sizeof(T) - k));
+}
+
+// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
+// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
+// care!
+template <typename T>
+inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
+ return reinterpret_cast<T>(ptr);
+}
+
+template <typename T>
+inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
+ return reinterpret_cast<T>(ptr);
+}
+
+// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
+// inlinings more difficult. Throws are also generally the slow path.
+template <typename E, typename... Args>
+[[noreturn]] ROBIN_HOOD(NOINLINE)
+#if ROBIN_HOOD(HAS_EXCEPTIONS)
+ void doThrow(Args&&... args) {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
+ throw E(std::forward<Args>(args)...);
+}
+#else
+ void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
+ abort();
+}
+#endif
+
+template <typename E, typename T, typename... Args>
+T* assertNotNull(T* t, Args&&... args) {
+ if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
+ doThrow<E>(std::forward<Args>(args)...);
+ }
+ return t;
+}
+
+template <typename T>
+inline T unaligned_load(void const* ptr) noexcept {
+ // using memcpy so we don't get into unaligned load problems.
+ // compiler should optimize this very well anyways.
+ T t;
+ std::memcpy(&t, ptr, sizeof(T));
+ return t;
+}
+
+// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
+// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
+// pointer.
+template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
+class BulkPoolAllocator {
+public:
+ BulkPoolAllocator() noexcept = default;
+
+ // does not copy anything, just creates a new allocator.
+ BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept
+ : mHead(nullptr)
+ , mListForFree(nullptr) {}
+
+ BulkPoolAllocator(BulkPoolAllocator&& o) noexcept
+ : mHead(o.mHead)
+ , mListForFree(o.mListForFree) {
+ o.mListForFree = nullptr;
+ o.mHead = nullptr;
+ }
+
+ BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept {
+ reset();
+ mHead = o.mHead;
+ mListForFree = o.mListForFree;
+ o.mListForFree = nullptr;
+ o.mHead = nullptr;
+ return *this;
+ }
+
+ BulkPoolAllocator&
+ // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
+ operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
+ // does not do anything
+ return *this;
+ }
+
+ ~BulkPoolAllocator() noexcept {
+ reset();
+ }
+
+ // Deallocates all allocated memory.
+ void reset() noexcept {
+ while (mListForFree) {
+ T* tmp = *mListForFree;
+ ROBIN_HOOD_LOG("std::free")
+ std::free(mListForFree);
+ mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
+ }
+ mHead = nullptr;
+ }
+
+ // allocates, but does NOT initialize. Use in-place new constructor, e.g.
+ // T* obj = pool.allocate();
+ // ::new (static_cast<void*>(obj)) T();
+ T* allocate() {
+ T* tmp = mHead;
+ if (!tmp) {
+ tmp = performAllocation();
+ }
+
+ mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
+ return tmp;
+ }
+
+ // does not actually deallocate but puts it in store.
+ // make sure you have already called the destructor! e.g. with
+ // obj->~T();
+ // pool.deallocate(obj);
+ void deallocate(T* obj) noexcept {
+ *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
+ mHead = obj;
+ }
+
+ // Adds an already allocated block of memory to the allocator. This allocator is from now on
+ // responsible for freeing the data (with free()). If the provided data is not large enough to
+ // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
+ void addOrFree(void* ptr, const size_t numBytes) noexcept {
+ // calculate number of available elements in ptr
+ if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
+ // not enough data for at least one element. Free and return.
+ ROBIN_HOOD_LOG("std::free")
+ std::free(ptr);
+ } else {
+ ROBIN_HOOD_LOG("add to buffer")
+ add(ptr, numBytes);
+ }
+ }
+
+ void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs>& other) noexcept {
+ using std::swap;
+ swap(mHead, other.mHead);
+ swap(mListForFree, other.mListForFree);
+ }
+
+private:
+ // iterates the list of allocated memory to calculate how many to alloc next.
+ // Recalculating this each time saves us a size_t member.
+ // This ignores the fact that memory blocks might have been added manually with addOrFree. In
+ // practice, this should not matter much.
+ ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
+ auto tmp = mListForFree;
+ size_t numAllocs = MinNumAllocs;
+
+ while (numAllocs * 2 <= MaxNumAllocs && tmp) {
+ auto x = reinterpret_cast<T***>(tmp);
+ tmp = *x;
+ numAllocs *= 2;
+ }
+
+ return numAllocs;
+ }
+
+ // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
+ void add(void* ptr, const size_t numBytes) noexcept {
+ const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
+
+ auto data = reinterpret_cast<T**>(ptr);
+
+ // link free list
+ auto x = reinterpret_cast<T***>(data);
+ *x = mListForFree;
+ mListForFree = data;
+
+ // create linked list for newly allocated data
+ auto* const headT =
+ reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
+
+ auto* const head = reinterpret_cast<char*>(headT);
+
+ // Visual Studio compiler automatically unrolls this loop, which is pretty cool
+ for (size_t i = 0; i < numElements; ++i) {
+ *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
+ head + (i + 1) * ALIGNED_SIZE;
+ }
+
+ // last one points to 0
+ *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
+ mHead;
+ mHead = headT;
+ }
+
+ // Called when no memory is available (mHead == 0).
+ // Don't inline this slow path.
+ ROBIN_HOOD(NOINLINE) T* performAllocation() {
+ size_t const numElementsToAlloc = calcNumElementsToAlloc();
+
+ // alloc new memory: [prev |T, T, ... T]
+ size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
+ ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
+ << " * " << numElementsToAlloc)
+ add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
+ return mHead;
+ }
+
+ // enforce byte alignment of the T's
+#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
+ static constexpr size_t ALIGNMENT =
+ (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
+#else
+ static const size_t ALIGNMENT =
+ (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value)
+ ? ROBIN_HOOD_STD::alignment_of<T>::value
+ : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
+#endif
+
+ static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
+
+ static_assert(MinNumAllocs >= 1, "MinNumAllocs");
+ static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
+ static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
+ static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
+ static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
+
+ T* mHead{nullptr};
+ T** mListForFree{nullptr};
+};
+
+template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
+struct NodeAllocator;
+
+// dummy allocator that does nothing
+template <typename T, size_t MinSize, size_t MaxSize>
+struct NodeAllocator<T, MinSize, MaxSize, true> {
+
+ // we are not using the data, so just free it.
+ void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
+ ROBIN_HOOD_LOG("std::free")
+ std::free(ptr);
+ }
+};
+
+template <typename T, size_t MinSize, size_t MaxSize>
+struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
+
+// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
+// my own here.
+namespace swappable {
+#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
+using std::swap;
+template <typename T>
+struct nothrow {
+ static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
+};
+#else
+template <typename T>
+struct nothrow {
+ static const bool value = std::is_nothrow_swappable<T>::value;
+};
+#endif
+} // namespace swappable
+
+} // namespace detail
+
+struct is_transparent_tag {};
+
+// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
+// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is
+// also tested.
+template <typename T1, typename T2>
+struct pair {
+ using first_type = T1;
+ using second_type = T2;
+
+ template <typename U1 = T1, typename U2 = T2,
+ typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
+ std::is_default_constructible<U2>::value>::type>
+ constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
+ : first()
+ , second() {}
+
+ // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
+ explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
+ noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
+ : first(o.first)
+ , second(o.second) {}
+
+ // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
+ explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
+ T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
+ : first(std::move(o.first))
+ , second(std::move(o.second)) {}
+
+ constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
+ T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
+ : first(std::move(a))
+ , second(std::move(b)) {}
+
+ template <typename U1, typename U2>
+ constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
+ std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
+ : first(std::forward<U1>(a))
+ , second(std::forward<U2>(b)) {}
+
+ template <typename... U1, typename... U2>
+ // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members"
+ // if this constructor is constexpr
+#if !ROBIN_HOOD(BROKEN_CONSTEXPR)
+ constexpr
+#endif
+ pair(std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
+ std::tuple<U2...>
+ b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
+ std::declval<std::tuple<U2...>&>(),
+ ROBIN_HOOD_STD::index_sequence_for<U1...>(),
+ ROBIN_HOOD_STD::index_sequence_for<U2...>())))
+ : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(),
+ ROBIN_HOOD_STD::index_sequence_for<U2...>()) {
+ }
+
+ // constructor called from the std::piecewise_construct_t ctor
+ template <typename... U1, size_t... I1, typename... U2, size_t... I2>
+ pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
+ noexcept(T1(std::forward<U1>(std::get<I1>(
+ std::declval<std::tuple<
+ U1...>&>()))...)) && noexcept(T2(std::
+ forward<U2>(std::get<I2>(
+ std::declval<std::tuple<U2...>&>()))...)))
+ : first(std::forward<U1>(std::get<I1>(a))...)
+ , second(std::forward<U2>(std::get<I2>(b))...) {
+ // make visual studio compiler happy about warning about unused a & b.
+ // Visual studio's pair implementation disables warning 4100.
+ (void)a;
+ (void)b;
+ }
+
+ void swap(pair<T1, T2>& o) noexcept((detail::swappable::nothrow<T1>::value) &&
+ (detail::swappable::nothrow<T2>::value)) {
+ using std::swap;
+ swap(first, o.first);
+ swap(second, o.second);
+ }
+
+ T1 first; // NOLINT(misc-non-private-member-variables-in-classes)
+ T2 second; // NOLINT(misc-non-private-member-variables-in-classes)
+};
+
+template <typename A, typename B>
+inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
+ noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
+ a.swap(b);
+}
+
+template <typename A, typename B>
+inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
+ return (x.first == y.first) && (x.second == y.second);
+}
+template <typename A, typename B>
+inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
+ return !(x == y);
+}
+template <typename A, typename B>
+inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
+ std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
+ std::declval<B const&>())) {
+ return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
+}
+template <typename A, typename B>
+inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
+ return y < x;
+}
+template <typename A, typename B>
+inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
+ return !(x > y);
+}
+template <typename A, typename B>
+inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
+ return !(x < y);
+}
+
+inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
+ static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
+ static constexpr uint64_t seed = UINT64_C(0xe17a1465);
+ static constexpr unsigned int r = 47;
+
+ auto const* const data64 = static_cast<uint64_t const*>(ptr);
+ uint64_t h = seed ^ (len * m);
+
+ size_t const n_blocks = len / 8;
+ for (size_t i = 0; i < n_blocks; ++i) {
+ auto k = detail::unaligned_load<uint64_t>(data64 + i);
+
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+
+ h ^= k;
+ h *= m;
+ }
+
+ auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
+ switch (len & 7U) {
+ case 7:
+ h ^= static_cast<uint64_t>(data8[6]) << 48U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 6:
+ h ^= static_cast<uint64_t>(data8[5]) << 40U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 5:
+ h ^= static_cast<uint64_t>(data8[4]) << 32U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 4:
+ h ^= static_cast<uint64_t>(data8[3]) << 24U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 3:
+ h ^= static_cast<uint64_t>(data8[2]) << 16U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 2:
+ h ^= static_cast<uint64_t>(data8[1]) << 8U;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ case 1:
+ h ^= static_cast<uint64_t>(data8[0]);
+ h *= m;
+ ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
+ default:
+ break;
+ }
+
+ h ^= h >> r;
+
+ // not doing the final step here, because this will be done by keyToIdx anyways
+ // h *= m;
+ // h ^= h >> r;
+ return static_cast<size_t>(h);
+}
+
+inline size_t hash_int(uint64_t x) noexcept {
+ // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested,
+ // and doesn't need any special 128bit operations.
+ x ^= x >> 33U;
+ x *= UINT64_C(0xff51afd7ed558ccd);
+ x ^= x >> 33U;
+
+ // not doing the final step here, because this will be done by keyToIdx anyways
+ // x *= UINT64_C(0xc4ceb9fe1a85ec53);
+ // x ^= x >> 33U;
+ return static_cast<size_t>(x);
+}
+
+// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
+template <typename T, typename Enable = void>
+struct hash : public std::hash<T> {
+ size_t operator()(T const& obj) const
+ noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
+ // call base hash
+ auto result = std::hash<T>::operator()(obj);
+ // return mixed of that, to be save against identity has
+ return hash_int(static_cast<detail::SizeT>(result));
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string<CharT>> {
+ size_t operator()(std::basic_string<CharT> const& str) const noexcept {
+ return hash_bytes(str.data(), sizeof(CharT) * str.size());
+ }
+};
+
+#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
+template <typename CharT>
+struct hash<std::basic_string_view<CharT>> {
+ size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
+ return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
+ }
+};
+#endif
+
+template <class T>
+struct hash<T*> {
+ size_t operator()(T* ptr) const noexcept {
+ return hash_int(reinterpret_cast<detail::SizeT>(ptr));
+ }
+};
+
+template <class T>
+struct hash<std::unique_ptr<T>> {
+ size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
+ return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
+ }
+};
+
+template <class T>
+struct hash<std::shared_ptr<T>> {
+ size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
+ return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
+ }
+};
+
+template <typename Enum>
+struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
+ size_t operator()(Enum e) const noexcept {
+ using Underlying = typename std::underlying_type<Enum>::type;
+ return hash<Underlying>{}(static_cast<Underlying>(e));
+ }
+};
+
+#define ROBIN_HOOD_HASH_INT(T) \
+ template <> \
+ struct hash<T> { \
+ size_t operator()(T const& obj) const noexcept { \
+ return hash_int(static_cast<uint64_t>(obj)); \
+ } \
+ }
+
+#if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wuseless-cast"
+#endif
+// see https://en.cppreference.com/w/cpp/utility/hash
+ROBIN_HOOD_HASH_INT(bool);
+ROBIN_HOOD_HASH_INT(char);
+ROBIN_HOOD_HASH_INT(signed char);
+ROBIN_HOOD_HASH_INT(unsigned char);
+ROBIN_HOOD_HASH_INT(char16_t);
+ROBIN_HOOD_HASH_INT(char32_t);
+#if ROBIN_HOOD(HAS_NATIVE_WCHART)
+ROBIN_HOOD_HASH_INT(wchar_t);
+#endif
+ROBIN_HOOD_HASH_INT(short);
+ROBIN_HOOD_HASH_INT(unsigned short);
+ROBIN_HOOD_HASH_INT(int);
+ROBIN_HOOD_HASH_INT(unsigned int);
+ROBIN_HOOD_HASH_INT(long);
+ROBIN_HOOD_HASH_INT(long long);
+ROBIN_HOOD_HASH_INT(unsigned long);
+ROBIN_HOOD_HASH_INT(unsigned long long);
+#if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic pop
+#endif
+namespace detail {
+
+template <typename T>
+struct void_type {
+ using type = void;
+};
+
+template <typename T, typename = void>
+struct has_is_transparent : public std::false_type {};
+
+template <typename T>
+struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
+ : public std::true_type {};
+
+// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
+// is used. see https://stackoverflow.com/a/28771920/48181
+template <typename T>
+struct WrapHash : public T {
+ WrapHash() = default;
+ explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
+ : T(o) {}
+};
+
+template <typename T>
+struct WrapKeyEqual : public T {
+ WrapKeyEqual() = default;
+ explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
+ : T(o) {}
+};
+
+// A highly optimized hashmap implementation, using the Robin Hood algorithm.
+//
+// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
+// be about 2x faster in most cases and require much less allocations.
+//
+// This implementation uses the following memory layout:
+//
+// [Node, Node, ... Node | info, info, ... infoSentinel ]
+//
+// * Node: either a DataNode that directly has the std::pair<key, val> as member,
+// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
+// depends on how fast the swap() operation is. Heuristically, this is automatically choosen
+// based on sizeof(). there are always 2^n Nodes.
+//
+// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
+// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
+// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
+// actually belongs to the previous position and was pushed out because that place is already
+// taken.
+//
+// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
+// need for a idx variable.
+//
+// According to STL, order of templates has effect on throughput. That's why I've moved the
+// boolean to the front.
+// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
+template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
+ typename KeyEqual>
+class Table
+ : public WrapHash<Hash>,
+ public WrapKeyEqual<KeyEqual>,
+ detail::NodeAllocator<
+ typename std::conditional<
+ std::is_void<T>::value, Key,
+ robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
+ 4, 16384, IsFlat> {
+public:
+ static constexpr bool is_flat = IsFlat;
+ static constexpr bool is_map = !std::is_void<T>::value;
+ static constexpr bool is_set = !is_map;
+ static constexpr bool is_transparent =
+ has_is_transparent<Hash>::value && has_is_transparent<KeyEqual>::value;
+
+ using key_type = Key;
+ using mapped_type = T;
+ using value_type = typename std::conditional<
+ is_set, Key,
+ robin_hood::pair<typename std::conditional<is_flat, Key, Key const>::type, T>>::type;
+ using size_type = size_t;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using Self = Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
+
+private:
+ static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
+ "MaxLoadFactor100 needs to be >10 && < 100");
+
+ using WHash = WrapHash<Hash>;
+ using WKeyEqual = WrapKeyEqual<KeyEqual>;
+
+ // configuration defaults
+
+ // make sure we have 8 elements, needed to quickly rehash mInfo
+ static constexpr size_t InitialNumElements = sizeof(uint64_t);
+ static constexpr uint32_t InitialInfoNumBits = 5;
+ static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
+ static constexpr size_t InfoMask = InitialInfoInc - 1U;
+ static constexpr uint8_t InitialInfoHashShift = 0;
+ using DataPool = detail::NodeAllocator<value_type, 4, 16384, IsFlat>;
+
+ // type needs to be wider than uint8_t.
+ using InfoType = uint32_t;
+
+ // DataNode ////////////////////////////////////////////////////////
+
+ // Primary template for the data node. We have special implementations for small and big
+ // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
+ // on the heap so swap merely swaps a pointer.
+ template <typename M, bool>
+ class DataNode {};
+
+ // Small: just allocate on the stack.
+ template <typename M>
+ class DataNode<M, true> final {
+ public:
+ template <typename... Args>
+ explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
+ noexcept(value_type(std::forward<Args>(args)...)))
+ : mData(std::forward<Args>(args)...) {}
+
+ DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
+ std::is_nothrow_move_constructible<value_type>::value)
+ : mData(std::move(n.mData)) {}
+
+ // doesn't do anything
+ void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
+ void destroyDoNotDeallocate() noexcept {}
+
+ value_type const* operator->() const noexcept {
+ return &mData;
+ }
+ value_type* operator->() noexcept {
+ return &mData;
+ }
+
+ const value_type& operator*() const noexcept {
+ return mData;
+ }
+
+ value_type& operator*() noexcept {
+ return mData;
+ }
+
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
+ return mData.first;
+ }
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
+ return mData;
+ }
+
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, typename VT::first_type const&>::type
+ getFirst() const noexcept {
+ return mData.first;
+ }
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
+ return mData;
+ }
+
+ template <typename MT = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
+ return mData.second;
+ }
+
+ template <typename MT = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
+ return mData.second;
+ }
+
+ void swap(DataNode<M, true>& o) noexcept(
+ noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
+ mData.swap(o.mData);
+ }
+
+ private:
+ value_type mData;
+ };
+
+ // big object: allocate on heap.
+ template <typename M>
+ class DataNode<M, false> {
+ public:
+ template <typename... Args>
+ explicit DataNode(M& map, Args&&... args)
+ : mData(map.allocate()) {
+ ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
+ }
+
+ DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
+ : mData(std::move(n.mData)) {}
+
+ void destroy(M& map) noexcept {
+ // don't deallocate, just put it into list of datapool.
+ mData->~value_type();
+ map.deallocate(mData);
+ }
+
+ void destroyDoNotDeallocate() noexcept {
+ mData->~value_type();
+ }
+
+ value_type const* operator->() const noexcept {
+ return mData;
+ }
+
+ value_type* operator->() noexcept {
+ return mData;
+ }
+
+ const value_type& operator*() const {
+ return *mData;
+ }
+
+ value_type& operator*() {
+ return *mData;
+ }
+
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
+ return mData->first;
+ }
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
+ return *mData;
+ }
+
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, typename VT::first_type const&>::type
+ getFirst() const noexcept {
+ return mData->first;
+ }
+ template <typename VT = value_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
+ return *mData;
+ }
+
+ template <typename MT = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
+ return mData->second;
+ }
+
+ template <typename MT = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
+ return mData->second;
+ }
+
+ void swap(DataNode<M, false>& o) noexcept {
+ using std::swap;
+ swap(mData, o.mData);
+ }
+
+ private:
+ value_type* mData;
+ };
+
+ using Node = DataNode<Self, IsFlat>;
+
+ // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required)
+ ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
+ return n.getFirst();
+ }
+
+ // in case we have void mapped_type, we are not using a pair, thus we just route k through.
+ // No need to disable this because it's just not used if not applicable.
+ ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
+ return k;
+ }
+
+ // in case we have non-void mapped_type, we have a standard robin_hood::pair
+ template <typename Q = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
+ getFirstConst(value_type const& vt) const noexcept {
+ return vt.first;
+ }
+
+ // Cloner //////////////////////////////////////////////////////////
+
+ template <typename M, bool UseMemcpy>
+ struct Cloner;
+
+ // fast path: Just copy data, without allocating anything.
+ template <typename M>
+ struct Cloner<M, true> {
+ void operator()(M const& source, M& target) const {
+ auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
+ auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
+ auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
+ std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
+ }
+ };
+
+ template <typename M>
+ struct Cloner<M, false> {
+ void operator()(M const& s, M& t) const {
+ auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
+ std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
+
+ for (size_t i = 0; i < numElementsWithBuffer; ++i) {
+ if (t.mInfo[i]) {
+ ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
+ }
+ }
+ }
+ };
+
+ // Destroyer ///////////////////////////////////////////////////////
+
+ template <typename M, bool IsFlatAndTrivial>
+ struct Destroyer {};
+
+ template <typename M>
+ struct Destroyer<M, true> {
+ void nodes(M& m) const noexcept {
+ m.mNumElements = 0;
+ }
+
+ void nodesDoNotDeallocate(M& m) const noexcept {
+ m.mNumElements = 0;
+ }
+ };
+
+ template <typename M>
+ struct Destroyer<M, false> {
+ void nodes(M& m) const noexcept {
+ m.mNumElements = 0;
+ // clear also resets mInfo to 0, that's sometimes not necessary.
+ auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
+
+ for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
+ if (0 != m.mInfo[idx]) {
+ Node& n = m.mKeyVals[idx];
+ n.destroy(m);
+ n.~Node();
+ }
+ }
+ }
+
+ void nodesDoNotDeallocate(M& m) const noexcept {
+ m.mNumElements = 0;
+ // clear also resets mInfo to 0, that's sometimes not necessary.
+ auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
+ for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
+ if (0 != m.mInfo[idx]) {
+ Node& n = m.mKeyVals[idx];
+ n.destroyDoNotDeallocate();
+ n.~Node();
+ }
+ }
+ }
+ };
+
+ // Iter ////////////////////////////////////////////////////////////
+
+ struct fast_forward_tag {};
+
+ // generic iterator for both const_iterator and iterator.
+ template <bool IsConst>
+ // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
+ class Iter {
+ private:
+ using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
+
+ public:
+ using difference_type = std::ptrdiff_t;
+ using value_type = typename Self::value_type;
+ using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
+ using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
+ using iterator_category = std::forward_iterator_tag;
+
+ // default constructed iterator can be compared to itself, but WON'T return true when
+ // compared to end().
+ Iter() = default;
+
+ // Rule of zero: nothing specified. The conversion constructor is only enabled for
+ // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
+
+ // Conversion constructor from iterator to const_iterator.
+ template <bool OtherIsConst,
+ typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ // NOLINTNEXTLINE(hicpp-explicit-conversions)
+ Iter(Iter<OtherIsConst> const& other) noexcept
+ : mKeyVals(other.mKeyVals)
+ , mInfo(other.mInfo) {}
+
+ Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
+ : mKeyVals(valPtr)
+ , mInfo(infoPtr) {}
+
+ Iter(NodePtr valPtr, uint8_t const* infoPtr,
+ fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
+ : mKeyVals(valPtr)
+ , mInfo(infoPtr) {
+ fastForward();
+ }
+
+ template <bool OtherIsConst,
+ typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
+ mKeyVals = other.mKeyVals;
+ mInfo = other.mInfo;
+ return *this;
+ }
+
+ // prefix increment. Undefined behavior if we are at end()!
+ Iter& operator++() noexcept {
+ mInfo++;
+ mKeyVals++;
+ fastForward();
+ return *this;
+ }
+
+ Iter operator++(int) noexcept {
+ Iter tmp = *this;
+ ++(*this);
+ return tmp;
+ }
+
+ reference operator*() const {
+ return **mKeyVals;
+ }
+
+ pointer operator->() const {
+ return &**mKeyVals;
+ }
+
+ template <bool O>
+ bool operator==(Iter<O> const& o) const noexcept {
+ return mKeyVals == o.mKeyVals;
+ }
+
+ template <bool O>
+ bool operator!=(Iter<O> const& o) const noexcept {
+ return mKeyVals != o.mKeyVals;
+ }
+
+ private:
+ // fast forward to the next non-free info byte
+ // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
+ // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
+ void fastForward() noexcept {
+ size_t n = 0;
+ while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
+ mInfo += sizeof(size_t);
+ mKeyVals += sizeof(size_t);
+ }
+#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
+ // we know for certain that within the next 8 bytes we'll find a non-zero one.
+ if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
+ mInfo += 4;
+ mKeyVals += 4;
+ }
+ if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
+ mInfo += 2;
+ mKeyVals += 2;
+ }
+ if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
+ mInfo += 1;
+ mKeyVals += 1;
+ }
+#else
+# if ROBIN_HOOD(LITTLE_ENDIAN)
+ auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
+# else
+ auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
+# endif
+ mInfo += inc;
+ mKeyVals += inc;
+#endif
+ }
+
+ friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
+ NodePtr mKeyVals{nullptr};
+ uint8_t const* mInfo{nullptr};
+ };
+
+ ////////////////////////////////////////////////////////////////////
+
+ // highly performance relevant code.
+ // Lower bits are used for indexing into the array (2^n size)
+ // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
+ template <typename HashKey>
+ void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
+ // In addition to whatever hash is used, add another mul & shift so we get better hashing.
+ // This serves as a bad hash prevention, if the given data is
+ // badly mixed.
+ auto h = static_cast<uint64_t>(WHash::operator()(key));
+
+ h *= mHashMultiplier;
+ h ^= h >> 33U;
+
+ // the lower InitialInfoNumBits are reserved for info.
+ *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
+ *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask;
+ }
+
+ // forwards the index by one, wrapping around at the end
+ void next(InfoType* info, size_t* idx) const noexcept {
+ *idx = *idx + 1;
+ *info += mInfoInc;
+ }
+
+ void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
+ // unrolling this by hand did not bring any speedups.
+ while (*info < mInfo[*idx]) {
+ next(info, idx);
+ }
+ }
+
+ // Shift everything up by one element. Tries to move stuff around.
+ void
+ shiftUp(size_t startIdx,
+ size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
+ auto idx = startIdx;
+ ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
+ while (--idx != insertion_idx) {
+ mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
+ }
+
+ idx = startIdx;
+ while (idx != insertion_idx) {
+ ROBIN_HOOD_COUNT(shiftUp)
+ mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
+ if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
+ mMaxNumElementsAllowed = 0;
+ }
+ --idx;
+ }
+ }
+
+ void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
+ // until we find one that is either empty or has zero offset.
+ // TODO(martinus) we don't need to move everything, just the last one for the same
+ // bucket.
+ mKeyVals[idx].destroy(*this);
+
+ // until we find one that is either empty or has zero offset.
+ while (mInfo[idx + 1] >= 2 * mInfoInc) {
+ ROBIN_HOOD_COUNT(shiftDown)
+ mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
+ mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
+ ++idx;
+ }
+
+ mInfo[idx] = 0;
+ // don't destroy, we've moved it
+ // mKeyVals[idx].destroy(*this);
+ mKeyVals[idx].~Node();
+ }
+
+ // copy of find(), except that it returns iterator instead of const_iterator.
+ template <typename Other>
+ ROBIN_HOOD(NODISCARD)
+ size_t findIdx(Other const& key) const {
+ size_t idx{};
+ InfoType info{};
+ keyToIdx(key, &idx, &info);
+
+ do {
+ // unrolling this twice gives a bit of a speedup. More unrolling did not help.
+ if (info == mInfo[idx] &&
+ ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
+ return idx;
+ }
+ next(&info, &idx);
+ if (info == mInfo[idx] &&
+ ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
+ return idx;
+ }
+ next(&info, &idx);
+ } while (info <= mInfo[idx]);
+
+ // nothing found!
+ return mMask == 0 ? 0
+ : static_cast<size_t>(std::distance(
+ mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
+ }
+
+ void cloneData(const Table& o) {
+ Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
+ }
+
+ // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
+ // @return True on success, false if something went wrong
+ void insert_move(Node&& keyval) {
+ // we don't retry, fail if overflowing
+ // don't need to check max num elements
+ if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
+ throwOverflowError();
+ }
+
+ size_t idx{};
+ InfoType info{};
+ keyToIdx(keyval.getFirst(), &idx, &info);
+
+ // skip forward. Use <= because we are certain that the element is not there.
+ while (info <= mInfo[idx]) {
+ idx = idx + 1;
+ info += mInfoInc;
+ }
+
+ // key not found, so we are now exactly where we want to insert it.
+ auto const insertion_idx = idx;
+ auto const insertion_info = static_cast<uint8_t>(info);
+ if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
+ mMaxNumElementsAllowed = 0;
+ }
+
+ // find an empty spot
+ while (0 != mInfo[idx]) {
+ next(&info, &idx);
+ }
+
+ auto& l = mKeyVals[insertion_idx];
+ if (idx == insertion_idx) {
+ ::new (static_cast<void*>(&l)) Node(std::move(keyval));
+ } else {
+ shiftUp(idx, insertion_idx);
+ l = std::move(keyval);
+ }
+
+ // put at empty spot
+ mInfo[insertion_idx] = insertion_info;
+
+ ++mNumElements;
+ }
+
+public:
+ using iterator = Iter<false>;
+ using const_iterator = Iter<true>;
+
+ Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
+ : WHash()
+ , WKeyEqual() {
+ ROBIN_HOOD_TRACE(this)
+ }
+
+ // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
+ // This tremendously speeds up ctor & dtor of a map that never receives an element. The
+ // penalty is payed at the first insert, and not before. Lookup of this empty map works
+ // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
+ // standard, but we can ignore it.
+ explicit Table(
+ size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
+ const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
+ : WHash(h)
+ , WKeyEqual(equal) {
+ ROBIN_HOOD_TRACE(this)
+ }
+
+ template <typename Iter>
+ Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
+ const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
+ : WHash(h)
+ , WKeyEqual(equal) {
+ ROBIN_HOOD_TRACE(this)
+ insert(first, last);
+ }
+
+ Table(std::initializer_list<value_type> initlist,
+ size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
+ const KeyEqual& equal = KeyEqual{})
+ : WHash(h)
+ , WKeyEqual(equal) {
+ ROBIN_HOOD_TRACE(this)
+ insert(initlist.begin(), initlist.end());
+ }
+
+ Table(Table&& o) noexcept
+ : WHash(std::move(static_cast<WHash&>(o)))
+ , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
+ , DataPool(std::move(static_cast<DataPool&>(o))) {
+ ROBIN_HOOD_TRACE(this)
+ if (o.mMask) {
+ mHashMultiplier = std::move(o.mHashMultiplier);
+ mKeyVals = std::move(o.mKeyVals);
+ mInfo = std::move(o.mInfo);
+ mNumElements = std::move(o.mNumElements);
+ mMask = std::move(o.mMask);
+ mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
+ mInfoInc = std::move(o.mInfoInc);
+ mInfoHashShift = std::move(o.mInfoHashShift);
+ // set other's mask to 0 so its destructor won't do anything
+ o.init();
+ }
+ }
+
+ Table& operator=(Table&& o) noexcept {
+ ROBIN_HOOD_TRACE(this)
+ if (&o != this) {
+ if (o.mMask) {
+ // only move stuff if the other map actually has some data
+ destroy();
+ mHashMultiplier = std::move(o.mHashMultiplier);
+ mKeyVals = std::move(o.mKeyVals);
+ mInfo = std::move(o.mInfo);
+ mNumElements = std::move(o.mNumElements);
+ mMask = std::move(o.mMask);
+ mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
+ mInfoInc = std::move(o.mInfoInc);
+ mInfoHashShift = std::move(o.mInfoHashShift);
+ WHash::operator=(std::move(static_cast<WHash&>(o)));
+ WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
+ DataPool::operator=(std::move(static_cast<DataPool&>(o)));
+
+ o.init();
+
+ } else {
+ // nothing in the other map => just clear us.
+ clear();
+ }
+ }
+ return *this;
+ }
+
+ Table(const Table& o)
+ : WHash(static_cast<const WHash&>(o))
+ , WKeyEqual(static_cast<const WKeyEqual&>(o))
+ , DataPool(static_cast<const DataPool&>(o)) {
+ ROBIN_HOOD_TRACE(this)
+ if (!o.empty()) {
+ // not empty: create an exact copy. it is also possible to just iterate through all
+ // elements and insert them, but copying is probably faster.
+
+ auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
+ auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
+
+ ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
+ << numElementsWithBuffer << ")")
+ mHashMultiplier = o.mHashMultiplier;
+ mKeyVals = static_cast<Node*>(
+ detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
+ // no need for calloc because clonData does memcpy
+ mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
+ mNumElements = o.mNumElements;
+ mMask = o.mMask;
+ mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
+ mInfoInc = o.mInfoInc;
+ mInfoHashShift = o.mInfoHashShift;
+ cloneData(o);
+ }
+ }
+
+ // Creates a copy of the given map. Copy constructor of each entry is used.
+ // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
+ // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
+ Table& operator=(Table const& o) {
+ ROBIN_HOOD_TRACE(this)
+ if (&o == this) {
+ // prevent assigning of itself
+ return *this;
+ }
+
+ // we keep using the old allocator and not assign the new one, because we want to keep
+ // the memory available. when it is the same size.
+ if (o.empty()) {
+ if (0 == mMask) {
+ // nothing to do, we are empty too
+ return *this;
+ }
+
+ // not empty: destroy what we have there
+ // clear also resets mInfo to 0, that's sometimes not necessary.
+ destroy();
+ init();
+ WHash::operator=(static_cast<const WHash&>(o));
+ WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
+ DataPool::operator=(static_cast<DataPool const&>(o));
+
+ return *this;
+ }
+
+ // clean up old stuff
+ Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
+
+ if (mMask != o.mMask) {
+ // no luck: we don't have the same array size allocated, so we need to realloc.
+ if (0 != mMask) {
+ // only deallocate if we actually have data!
+ ROBIN_HOOD_LOG("std::free")
+ std::free(mKeyVals);
+ }
+
+ auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
+ auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
+ ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
+ << numElementsWithBuffer << ")")
+ mKeyVals = static_cast<Node*>(
+ detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
+
+ // no need for calloc here because cloneData performs a memcpy.
+ mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
+ // sentinel is set in cloneData
+ }
+ WHash::operator=(static_cast<const WHash&>(o));
+ WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
+ DataPool::operator=(static_cast<DataPool const&>(o));
+ mHashMultiplier = o.mHashMultiplier;
+ mNumElements = o.mNumElements;
+ mMask = o.mMask;
+ mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
+ mInfoInc = o.mInfoInc;
+ mInfoHashShift = o.mInfoHashShift;
+ cloneData(o);
+
+ return *this;
+ }
+
+ // Swaps everything between the two maps.
+ void swap(Table& o) {
+ ROBIN_HOOD_TRACE(this)
+ using std::swap;
+ swap(o, *this);
+ }
+
+ // Clears all data, without resizing.
+ void clear() {
+ ROBIN_HOOD_TRACE(this)
+ if (empty()) {
+ // don't do anything! also important because we don't want to write to
+ // DummyInfoByte::b, even though we would just write 0 to it.
+ return;
+ }
+
+ Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
+
+ auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
+ // clear everything, then set the sentinel again
+ uint8_t const z = 0;
+ std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
+ mInfo[numElementsWithBuffer] = 1;
+
+ mInfoInc = InitialInfoInc;
+ mInfoHashShift = InitialInfoHashShift;
+ }
+
+ // Destroys the map and all it's contents.
+ ~Table() {
+ ROBIN_HOOD_TRACE(this)
+ destroy();
+ }
+
+ // Checks if both tables contain the same entries. Order is irrelevant.
+ bool operator==(const Table& other) const {
+ ROBIN_HOOD_TRACE(this)
+ if (other.size() != size()) {
+ return false;
+ }
+ for (auto const& otherEntry : other) {
+ if (!has(otherEntry)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ bool operator!=(const Table& other) const {
+ ROBIN_HOOD_TRACE(this)
+ return !operator==(other);
+ }
+
+ template <typename Q = mapped_type>
+ typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
+ ROBIN_HOOD_TRACE(this)
+ auto idxAndState = insertKeyPrepareEmptySpot(key);
+ switch (idxAndState.second) {
+ case InsertionState::key_found:
+ break;
+
+ case InsertionState::new_node:
+ ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
+ Node(*this, std::piecewise_construct, std::forward_as_tuple(key),
+ std::forward_as_tuple());
+ break;
+
+ case InsertionState::overwrite_node:
+ mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
+ std::forward_as_tuple(key), std::forward_as_tuple());
+ break;
+
+ case InsertionState::overflow_error:
+ throwOverflowError();
+ }
+
+ return mKeyVals[idxAndState.first].getSecond();
+ }
+
+ template <typename Q = mapped_type>
+ typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
+ ROBIN_HOOD_TRACE(this)
+ auto idxAndState = insertKeyPrepareEmptySpot(key);
+ switch (idxAndState.second) {
+ case InsertionState::key_found:
+ break;
+
+ case InsertionState::new_node:
+ ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
+ Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
+ std::forward_as_tuple());
+ break;
+
+ case InsertionState::overwrite_node:
+ mKeyVals[idxAndState.first] =
+ Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
+ std::forward_as_tuple());
+ break;
+
+ case InsertionState::overflow_error:
+ throwOverflowError();
+ }
+
+ return mKeyVals[idxAndState.first].getSecond();
+ }
+
+ template <typename Iter>
+ void insert(Iter first, Iter last) {
+ for (; first != last; ++first) {
+ // value_type ctor needed because this might be called with std::pair's
+ insert(value_type(*first));
+ }
+ }
+
+ void insert(std::initializer_list<value_type> ilist) {
+ for (auto&& vt : ilist) {
+ insert(std::move(vt));
+ }
+ }
+
+ template <typename... Args>
+ std::pair<iterator, bool> emplace(Args&&... args) {
+ ROBIN_HOOD_TRACE(this)
+ Node n{*this, std::forward<Args>(args)...};
+ auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n));
+ switch (idxAndState.second) {
+ case InsertionState::key_found:
+ n.destroy(*this);
+ break;
+
+ case InsertionState::new_node:
+ ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n));
+ break;
+
+ case InsertionState::overwrite_node:
+ mKeyVals[idxAndState.first] = std::move(n);
+ break;
+
+ case InsertionState::overflow_error:
+ n.destroy(*this);
+ throwOverflowError();
+ break;
+ }
+
+ return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
+ InsertionState::key_found != idxAndState.second);
+ }
+
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
+ return try_emplace_impl(key, std::forward<Args>(args)...);
+ }
+
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
+ return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
+ }
+
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(const_iterator hint, const key_type& key,
+ Args&&... args) {
+ (void)hint;
+ return try_emplace_impl(key, std::forward<Args>(args)...);
+ }
+
+ template <typename... Args>
+ std::pair<iterator, bool> try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
+ (void)hint;
+ return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
+ }
+
+ template <typename Mapped>
+ std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
+ return insertOrAssignImpl(key, std::forward<Mapped>(obj));
+ }
+
+ template <typename Mapped>
+ std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
+ return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
+ }
+
+ template <typename Mapped>
+ std::pair<iterator, bool> insert_or_assign(const_iterator hint, const key_type& key,
+ Mapped&& obj) {
+ (void)hint;
+ return insertOrAssignImpl(key, std::forward<Mapped>(obj));
+ }
+
+ template <typename Mapped>
+ std::pair<iterator, bool> insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) {
+ (void)hint;
+ return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
+ }
+
+ std::pair<iterator, bool> insert(const value_type& keyval) {
+ ROBIN_HOOD_TRACE(this)
+ return emplace(keyval);
+ }
+
+ std::pair<iterator, bool> insert(value_type&& keyval) {
+ return emplace(std::move(keyval));
+ }
+
+ // Returns 1 if key is found, 0 otherwise.
+ size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ auto kv = mKeyVals + findIdx(key);
+ if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ template <typename OtherKey, typename Self_ = Self>
+ // NOLINTNEXTLINE(modernize-use-nodiscard)
+ typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
+ ROBIN_HOOD_TRACE(this)
+ auto kv = mKeyVals + findIdx(key);
+ if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
+ return 1U == count(key);
+ }
+
+ template <typename OtherKey, typename Self_ = Self>
+ // NOLINTNEXTLINE(modernize-use-nodiscard)
+ typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
+ return 1U == count(key);
+ }
+
+ // Returns a reference to the value found for key.
+ // Throws std::out_of_range if element cannot be found
+ template <typename Q = mapped_type>
+ // NOLINTNEXTLINE(modernize-use-nodiscard)
+ typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
+ ROBIN_HOOD_TRACE(this)
+ auto kv = mKeyVals + findIdx(key);
+ if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
+ doThrow<std::out_of_range>("key not found");
+ }
+ return kv->getSecond();
+ }
+
+ // Returns a reference to the value found for key.
+ // Throws std::out_of_range if element cannot be found
+ template <typename Q = mapped_type>
+ // NOLINTNEXTLINE(modernize-use-nodiscard)
+ typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
+ ROBIN_HOOD_TRACE(this)
+ auto kv = mKeyVals + findIdx(key);
+ if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
+ doThrow<std::out_of_range>("key not found");
+ }
+ return kv->getSecond();
+ }
+
+ const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return const_iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ template <typename OtherKey>
+ const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return const_iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ template <typename OtherKey, typename Self_ = Self>
+ typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard)
+ const_iterator>::type // NOLINT(modernize-use-nodiscard)
+ find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return const_iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ iterator find(const key_type& key) {
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ template <typename OtherKey>
+ iterator find(const OtherKey& key, is_transparent_tag /*unused*/) {
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ template <typename OtherKey, typename Self_ = Self>
+ typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
+ ROBIN_HOOD_TRACE(this)
+ const size_t idx = findIdx(key);
+ return iterator{mKeyVals + idx, mInfo + idx};
+ }
+
+ iterator begin() {
+ ROBIN_HOOD_TRACE(this)
+ if (empty()) {
+ return end();
+ }
+ return iterator(mKeyVals, mInfo, fast_forward_tag{});
+ }
+ const_iterator begin() const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return cbegin();
+ }
+ const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ if (empty()) {
+ return cend();
+ }
+ return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
+ }
+
+ iterator end() {
+ ROBIN_HOOD_TRACE(this)
+ // no need to supply valid info pointer: end() must not be dereferenced, and only node
+ // pointer is compared.
+ return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
+ }
+ const_iterator end() const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return cend();
+ }
+ const_iterator cend() const { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
+ }
+
+ iterator erase(const_iterator pos) {
+ ROBIN_HOOD_TRACE(this)
+ // its safe to perform const cast here
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
+ return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
+ }
+
+ // Erases element at pos, returns iterator to the next element.
+ iterator erase(iterator pos) {
+ ROBIN_HOOD_TRACE(this)
+ // we assume that pos always points to a valid entry, and not end().
+ auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
+
+ shiftDown(idx);
+ --mNumElements;
+
+ if (*pos.mInfo) {
+ // we've backward shifted, return this again
+ return pos;
+ }
+
+ // no backward shift, return next element
+ return ++pos;
+ }
+
+ size_t erase(const key_type& key) {
+ ROBIN_HOOD_TRACE(this)
+ size_t idx{};
+ InfoType info{};
+ keyToIdx(key, &idx, &info);
+
+ // check while info matches with the source idx
+ do {
+ if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
+ shiftDown(idx);
+ --mNumElements;
+ return 1;
+ }
+ next(&info, &idx);
+ } while (info <= mInfo[idx]);
+
+ // nothing found to delete
+ return 0;
+ }
+
+ // reserves space for the specified number of elements. Makes sure the old data fits.
+ // exactly the same as reserve(c).
+ void rehash(size_t c) {
+ // forces a reserve
+ reserve(c, true);
+ }
+
+ // reserves space for the specified number of elements. Makes sure the old data fits.
+ // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
+ void reserve(size_t c) {
+ // reserve, but don't force rehash
+ reserve(c, false);
+ }
+
+ // If possible reallocates the map to a smaller one. This frees the underlying table.
+ // Does not do anything if load_factor is too large for decreasing the table's size.
+ void compact() {
+ ROBIN_HOOD_TRACE(this)
+ auto newSize = InitialNumElements;
+ while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
+ newSize *= 2;
+ }
+ if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
+ throwOverflowError();
+ }
+
+ ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
+
+ // only actually do anything when the new size is bigger than the old one. This prevents to
+ // continuously allocate for each reserve() call.
+ if (newSize < mMask + 1) {
+ rehashPowerOfTwo(newSize, true);
+ }
+ }
+
+ size_type size() const noexcept { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return mNumElements;
+ }
+
+ size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return static_cast<size_type>(-1);
+ }
+
+ ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
+ ROBIN_HOOD_TRACE(this)
+ return 0 == mNumElements;
+ }
+
+ float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return MaxLoadFactor100 / 100.0F;
+ }
+ void max_load_factor(float) const noexcept { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ }
+
+
+ // Average number of elements per bucket. Since we allow only 1 per bucket
+ float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
+ ROBIN_HOOD_TRACE(this)
+ return static_cast<float>(size()) / static_cast<float>(mMask + 1);
+ }
+
+ ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
+ ROBIN_HOOD_TRACE(this)
+ return mMask;
+ }
+
+ ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
+ if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
+ return maxElements * MaxLoadFactor100 / 100;
+ }
+
+ // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
+ return (maxElements / 100) * MaxLoadFactor100;
+ }
+
+ ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
+ // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
+ // 64bit types.
+ return numElements + sizeof(uint64_t);
+ }
+
+ ROBIN_HOOD(NODISCARD)
+ size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
+ auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
+ return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
+ }
+
+ // calculation only allowed for 2^n values
+ ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
+#if ROBIN_HOOD(BITNESS) == 64
+ return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
+#else
+ // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
+ auto const ne = static_cast<uint64_t>(numElements);
+ auto const s = static_cast<uint64_t>(sizeof(Node));
+ auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
+
+ auto const total64 = ne * s + infos;
+ auto const total = static_cast<size_t>(total64);
+
+ if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
+ throwOverflowError();
+ }
+ return total;
+#endif
+ }
+
+private:
+ template <typename Q = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
+ ROBIN_HOOD_TRACE(this)
+ auto it = find(e.first);
+ return it != end() && it->second == e.second;
+ }
+
+ template <typename Q = mapped_type>
+ ROBIN_HOOD(NODISCARD)
+ typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
+ ROBIN_HOOD_TRACE(this)
+ return find(e) != end();
+ }
+
+ void reserve(size_t c, bool forceRehash) {
+ ROBIN_HOOD_TRACE(this)
+ auto const minElementsAllowed = (std::max)(c, mNumElements);
+ auto newSize = InitialNumElements;
+ while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
+ newSize *= 2;
+ }
+ if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
+ throwOverflowError();
+ }
+
+ ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
+
+ // only actually do anything when the new size is bigger than the old one. This prevents to
+ // continuously allocate for each reserve() call.
+ if (forceRehash || newSize > mMask + 1) {
+ rehashPowerOfTwo(newSize, false);
+ }
+ }
+
+ // reserves space for at least the specified number of elements.
+ // only works if numBuckets if power of two
+ // True on success, false otherwise
+ void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
+ ROBIN_HOOD_TRACE(this)
+
+ Node* const oldKeyVals = mKeyVals;
+ uint8_t const* const oldInfo = mInfo;
+
+ const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
+
+ // resize operation: move stuff
+ initData(numBuckets);
+ if (oldMaxElementsWithBuffer > 1) {
+ for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
+ if (oldInfo[i] != 0) {
+ // might throw an exception, which is really bad since we are in the middle of
+ // moving stuff.
+ insert_move(std::move(oldKeyVals[i]));
+ // destroy the node but DON'T destroy the data.
+ oldKeyVals[i].~Node();
+ }
+ }
+
+ // this check is not necessary as it's guarded by the previous if, but it helps
+ // silence g++'s overeager "attempt to free a non-heap object 'map'
+ // [-Werror=free-nonheap-object]" warning.
+ if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
+ // don't destroy old data: put it into the pool instead
+ if (forceFree) {
+ std::free(oldKeyVals);
+ } else {
+ DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
+ }
+ }
+ }
+ }
+
+ ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
+#if ROBIN_HOOD(HAS_EXCEPTIONS)
+ throw std::overflow_error("robin_hood::map overflow");
+#else
+ abort();
+#endif
+ }
+
+ template <typename OtherKey, typename... Args>
+ std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
+ ROBIN_HOOD_TRACE(this)
+ auto idxAndState = insertKeyPrepareEmptySpot(key);
+ switch (idxAndState.second) {
+ case InsertionState::key_found:
+ break;
+
+ case InsertionState::new_node:
+ ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
+ *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ break;
+
+ case InsertionState::overwrite_node:
+ mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
+ std::forward_as_tuple(std::forward<OtherKey>(key)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ break;
+
+ case InsertionState::overflow_error:
+ throwOverflowError();
+ break;
+ }
+
+ return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
+ InsertionState::key_found != idxAndState.second);
+ }
+
+ template <typename OtherKey, typename Mapped>
+ std::pair<iterator, bool> insertOrAssignImpl(OtherKey&& key, Mapped&& obj) {
+ ROBIN_HOOD_TRACE(this)
+ auto idxAndState = insertKeyPrepareEmptySpot(key);
+ switch (idxAndState.second) {
+ case InsertionState::key_found:
+ mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj);
+ break;
+
+ case InsertionState::new_node:
+ ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
+ *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
+ std::forward_as_tuple(std::forward<Mapped>(obj)));
+ break;
+
+ case InsertionState::overwrite_node:
+ mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
+ std::forward_as_tuple(std::forward<OtherKey>(key)),
+ std::forward_as_tuple(std::forward<Mapped>(obj)));
+ break;
+
+ case InsertionState::overflow_error:
+ throwOverflowError();
+ break;
+ }
+
+ return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
+ InsertionState::key_found != idxAndState.second);
+ }
+
+ void initData(size_t max_elements) {
+ mNumElements = 0;
+ mMask = max_elements - 1;
+ mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
+
+ auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
+
+ // calloc also zeroes everything
+ auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
+ ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
+ << numElementsWithBuffer << ")")
+ mKeyVals = reinterpret_cast<Node*>(
+ detail::assertNotNull<std::bad_alloc>(std::calloc(1, numBytesTotal)));
+ mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
+
+ // set sentinel
+ mInfo[numElementsWithBuffer] = 1;
+
+ mInfoInc = InitialInfoInc;
+ mInfoHashShift = InitialInfoHashShift;
+ }
+
+ enum class InsertionState { overflow_error, key_found, new_node, overwrite_node };
+
+ // Finds key, and if not already present prepares a spot where to pot the key & value.
+ // This potentially shifts nodes out of the way, updates mInfo and number of inserted
+ // elements, so the only operation left to do is create/assign a new node at that spot.
+ template <typename OtherKey>
+ std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey&& key) {
+ for (int i = 0; i < 256; ++i) {
+ size_t idx{};
+ InfoType info{};
+ keyToIdx(key, &idx, &info);
+ nextWhileLess(&info, &idx);
+
+ // while we potentially have a match
+ while (info == mInfo[idx]) {
+ if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
+ // key already exists, do NOT insert.
+ // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
+ return std::make_pair(idx, InsertionState::key_found);
+ }
+ next(&info, &idx);
+ }
+
+ // unlikely that this evaluates to true
+ if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
+ if (!increase_size()) {
+ return std::make_pair(size_t(0), InsertionState::overflow_error);
+ }
+ continue;
+ }
+
+ // key not found, so we are now exactly where we want to insert it.
+ auto const insertion_idx = idx;
+ auto const insertion_info = info;
+ if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
+ mMaxNumElementsAllowed = 0;
+ }
+
+ // find an empty spot
+ while (0 != mInfo[idx]) {
+ next(&info, &idx);
+ }
+
+ if (idx != insertion_idx) {
+ shiftUp(idx, insertion_idx);
+ }
+ // put at empty spot
+ mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
+ ++mNumElements;
+ return std::make_pair(insertion_idx, idx == insertion_idx
+ ? InsertionState::new_node
+ : InsertionState::overwrite_node);
+ }
+
+ // enough attempts failed, so finally give up.
+ return std::make_pair(size_t(0), InsertionState::overflow_error);
+ }
+
+ bool try_increase_info() {
+ ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
+ << ", maxNumElementsAllowed="
+ << calcMaxNumElementsAllowed(mMask + 1))
+ if (mInfoInc <= 2) {
+ // need to be > 2 so that shift works (otherwise undefined behavior!)
+ return false;
+ }
+ // we got space left, try to make info smaller
+ mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
+
+ // remove one bit of the hash, leaving more space for the distance info.
+ // This is extremely fast because we can operate on 8 bytes at once.
+ ++mInfoHashShift;
+ auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
+
+ for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
+ auto val = unaligned_load<uint64_t>(mInfo + i);
+ val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
+ std::memcpy(mInfo + i, &val, sizeof(val));
+ }
+ // update sentinel, which might have been cleared out!
+ mInfo[numElementsWithBuffer] = 1;
+
+ mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
+ return true;
+ }
+
+ // True if resize was possible, false otherwise
+ bool increase_size() {
+ // nothing allocated yet? just allocate InitialNumElements
+ if (0 == mMask) {
+ initData(InitialNumElements);
+ return true;
+ }
+
+ auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
+ if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
+ return true;
+ }
+
+ ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
+ << maxNumElementsAllowed << ", load="
+ << (static_cast<double>(mNumElements) * 100.0 /
+ (static_cast<double>(mMask) + 1)))
+
+ if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
+ // we have to resize, even though there would still be plenty of space left!
+ // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case
+ // we have to rehash a few times
+ nextHashMultiplier();
+ rehashPowerOfTwo(mMask + 1, true);
+ } else {
+ // we've reached the capacity of the map, so the hash seems to work nice. Keep using it.
+ rehashPowerOfTwo((mMask + 1) * 2, false);
+ }
+ return true;
+ }
+
+ void nextHashMultiplier() {
+ // adding an *even* number, so that the multiplier will always stay odd. This is necessary
+ // so that the hash stays a mixing function (and thus doesn't have any information loss).
+ mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54);
+ }
+
+ void destroy() {
+ if (0 == mMask) {
+ // don't deallocate!
+ return;
+ }
+
+ Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
+ .nodesDoNotDeallocate(*this);
+
+ // This protection against not deleting mMask shouldn't be needed as it's sufficiently
+ // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
+ // reports a compile error: attempt to free a non-heap object 'fm'
+ // [-Werror=free-nonheap-object]
+ if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
+ ROBIN_HOOD_LOG("std::free")
+ std::free(mKeyVals);
+ }
+ }
+
+ void init() noexcept {
+ mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
+ mInfo = reinterpret_cast<uint8_t*>(&mMask);
+ mNumElements = 0;
+ mMask = 0;
+ mMaxNumElementsAllowed = 0;
+ mInfoInc = InitialInfoInc;
+ mInfoHashShift = InitialInfoHashShift;
+ }
+
+ // members are sorted so no padding occurs
+ uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8
+ Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 16
+ uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 24
+ size_t mNumElements = 0; // 8 byte 32
+ size_t mMask = 0; // 8 byte 40
+ size_t mMaxNumElementsAllowed = 0; // 8 byte 48
+ InfoType mInfoInc = InitialInfoInc; // 4 byte 52
+ InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56
+ // 16 byte 56 if NodeAllocator
+};
+
+} // namespace detail
+
+// map
+
+template <typename Key, typename T, typename Hash = hash<Key>,
+ typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
+using unordered_flat_map = detail::Table<true, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
+
+template <typename Key, typename T, typename Hash = hash<Key>,
+ typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
+using unordered_node_map = detail::Table<false, MaxLoadFactor100, Key, T, Hash, KeyEqual>;
+
+template <typename Key, typename T, typename Hash = hash<Key>,
+ typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
+using unordered_map =
+ detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
+ std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
+ std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
+ MaxLoadFactor100, Key, T, Hash, KeyEqual>;
+
+// set
+
+template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
+ size_t MaxLoadFactor100 = 80>
+using unordered_flat_set = detail::Table<true, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
+
+template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
+ size_t MaxLoadFactor100 = 80>
+using unordered_node_set = detail::Table<false, MaxLoadFactor100, Key, void, Hash, KeyEqual>;
+
+template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
+ size_t MaxLoadFactor100 = 80>
+using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 &&
+ std::is_nothrow_move_constructible<Key>::value &&
+ std::is_nothrow_move_assignable<Key>::value,
+ MaxLoadFactor100, Key, void, Hash, KeyEqual>;
+
+} // namespace robin_hood
+
+#endif
diff --git a/benchmarks/others/sparsepp/spp.h b/benchmarks/others/sparsepp/spp.h
deleted file mode 100644
index 26aa2001..00000000
--- a/benchmarks/others/sparsepp/spp.h
+++ /dev/null
@@ -1,4361 +0,0 @@
-#if !defined(sparsepp_h_guard_)
-#define sparsepp_h_guard_
-
-
-// ----------------------------------------------------------------------
-// Copyright (c) 2016, Gregory Popovitch - [email protected]
-// All rights reserved.
-//
-// This work is derived from Google's sparsehash library
-//
-// Copyright (c) 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// ----------------------------------------------------------------------
-
-
-// some macros for portability
-// ---------------------------
-// includes
-// --------
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <limits> // for numeric_limits
-#include <algorithm> // For swap(), eg
-#include <iterator> // for iterator tags
-#include <functional> // for equal_to<>, select1st<>, std::unary_function, etc
-#include <memory> // for alloc, uninitialized_copy, uninitialized_fill
-#include <cstdlib> // for malloc/realloc/free
-#include <cstddef> // for ptrdiff_t
-#include <new> // for placement new
-#include <stdexcept> // For length_error
-#include <utility> // for pair<>
-#include <cstdio>
-#include <iosfwd>
-#include <ios>
-
-#include "spp_stdint.h" // includes spp_config.h
-#include "spp_traits.h"
-#include "spp_utils.h"
-
-#ifdef SPP_INCLUDE_SPP_ALLOC
- #include "spp_dlalloc.h"
-#endif
-
-#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST)
- #include <initializer_list>
-#endif
-
-#if (SPP_GROUP_SIZE == 32)
- #define SPP_SHIFT_ 5
- #define SPP_MASK_ 0x1F
- typedef uint32_t group_bm_type;
-#elif (SPP_GROUP_SIZE == 64)
- #define SPP_SHIFT_ 6
- #define SPP_MASK_ 0x3F
- typedef uint64_t group_bm_type;
-#else
- #error "SPP_GROUP_SIZE must be either 32 or 64"
-#endif
-
-namespace spp_ {
-
-// ----------------------------------------------------------------------
-// U T I L F U N C T I O N S
-// ----------------------------------------------------------------------
-template <class E>
-inline void throw_exception(const E& exception)
-{
-#if !defined(SPP_NO_EXCEPTIONS)
- throw exception;
-#else
- assert(0);
- abort();
-#endif
-}
-
-// ----------------------------------------------------------------------
-// M U T A B L E P A I R H A C K
-// turn std::pair<const K, V> into mutable std::pair<K, V>
-// ----------------------------------------------------------------------
-template <class T>
-struct cvt
-{
- typedef T type;
-};
-
-template <class K, class V>
-struct cvt<std::pair<const K, V> >
-{
- typedef std::pair<K, V> type;
-};
-
-template <class K, class V>
-struct cvt<const std::pair<const K, V> >
-{
- typedef const std::pair<K, V> type;
-};
-
-// ----------------------------------------------------------------------
-// M O V E I T E R A T O R
-// ----------------------------------------------------------------------
-#ifdef SPP_NO_CXX11_RVALUE_REFERENCES
- #define MK_MOVE_IT(p) (p)
-#else
- #define MK_MOVE_IT(p) std::make_move_iterator(p)
-#endif
-
-
-// ----------------------------------------------------------------------
-// I N T E R N A L S T U F F
-// ----------------------------------------------------------------------
-#ifdef SPP_NO_CXX11_STATIC_ASSERT
- template <bool> struct SppCompileAssert { };
- #define SPP_COMPILE_ASSERT(expr, msg) \
- SPP_ATTRIBUTE_UNUSED typedef SppCompileAssert<(bool(expr))> spp_bogus_[bool(expr) ? 1 : -1]
-#else
- #define SPP_COMPILE_ASSERT static_assert
-#endif
-
-namespace sparsehash_internal
-{
-
-// Adaptor methods for reading/writing data from an INPUT or OUPTUT
-// variable passed to serialize() or unserialize(). For now we
-// have implemented INPUT/OUTPUT for FILE*, istream*/ostream* (note
-// they are pointers, unlike typical use), or else a pointer to
-// something that supports a Read()/Write() method.
-//
-// For technical reasons, we implement read_data/write_data in two
-// stages. The actual work is done in *_data_internal, which takes
-// the stream argument twice: once as a template type, and once with
-// normal type information. (We only use the second version.) We do
-// this because of how C++ picks what function overload to use. If we
-// implemented this the naive way:
-// bool read_data(istream* is, const void* data, size_t length);
-// template<typename T> read_data(T* fp, const void* data, size_t length);
-// C++ would prefer the second version for every stream type except
-// istream. However, we want C++ to prefer the first version for
-// streams that are *subclasses* of istream, such as istringstream.
-// This is not possible given the way template types are resolved. So
-// we split the stream argument in two, one of which is templated and
-// one of which is not. The specialized functions (like the istream
-// version above) ignore the template arg and use the second, 'type'
-// arg, getting subclass matching as normal. The 'catch-all'
-// functions (the second version above) use the template arg to deduce
-// the type, and use a second, void* arg to achieve the desired
-// 'catch-all' semantics.
-
- // ----- low-level I/O for FILE* ----
-
- template<typename Ignored>
- inline bool read_data_internal(Ignored* /*unused*/, FILE* fp,
- void* data, size_t length)
- {
- return fread(data, length, 1, fp) == 1;
- }
-
- template<typename Ignored>
- inline bool write_data_internal(Ignored* /*unused*/, FILE* fp,
- const void* data, size_t length)
- {
- return fwrite(data, length, 1, fp) == 1;
- }
-
- // ----- low-level I/O for iostream ----
-
- // We want the caller to be responsible for #including <iostream>, not
- // us, because iostream is a big header! According to the standard,
- // it's only legal to delay the instantiation the way we want to if
- // the istream/ostream is a template type. So we jump through hoops.
- template<typename ISTREAM>
- inline bool read_data_internal_for_istream(ISTREAM* fp,
- void* data, size_t length)
- {
- return fp->read(reinterpret_cast<char*>(data),
- static_cast<std::streamsize>(length)).good();
- }
- template<typename Ignored>
- inline bool read_data_internal(Ignored* /*unused*/, std::istream* fp,
- void* data, size_t length)
- {
- return read_data_internal_for_istream(fp, data, length);
- }
-
- template<typename OSTREAM>
- inline bool write_data_internal_for_ostream(OSTREAM* fp,
- const void* data, size_t length)
- {
- return fp->write(reinterpret_cast<const char*>(data),
- static_cast<std::streamsize>(length)).good();
- }
- template<typename Ignored>
- inline bool write_data_internal(Ignored* /*unused*/, std::ostream* fp,
- const void* data, size_t length)
- {
- return write_data_internal_for_ostream(fp, data, length);
- }
-
- // ----- low-level I/O for custom streams ----
-
- // The INPUT type needs to support a Read() method that takes a
- // buffer and a length and returns the number of bytes read.
- template <typename INPUT>
- inline bool read_data_internal(INPUT* fp, void* /*unused*/,
- void* data, size_t length)
- {
- return static_cast<size_t>(fp->Read(data, length)) == length;
- }
-
- // The OUTPUT type needs to support a Write() operation that takes
- // a buffer and a length and returns the number of bytes written.
- template <typename OUTPUT>
- inline bool write_data_internal(OUTPUT* fp, void* /*unused*/,
- const void* data, size_t length)
- {
- return static_cast<size_t>(fp->Write(data, length)) == length;
- }
-
- // ----- low-level I/O: the public API ----
-
- template <typename INPUT>
- inline bool read_data(INPUT* fp, void* data, size_t length)
- {
- return read_data_internal(fp, fp, data, length);
- }
-
- template <typename OUTPUT>
- inline bool write_data(OUTPUT* fp, const void* data, size_t length)
- {
- return write_data_internal(fp, fp, data, length);
- }
-
- // Uses read_data() and write_data() to read/write an integer.
- // length is the number of bytes to read/write (which may differ
- // from sizeof(IntType), allowing us to save on a 32-bit system
- // and load on a 64-bit system). Excess bytes are taken to be 0.
- // INPUT and OUTPUT must match legal inputs to read/write_data (above).
- // --------------------------------------------------------------------
- template <typename INPUT, typename IntType>
- bool read_bigendian_number(INPUT* fp, IntType* value, size_t length)
- {
- *value = 0;
- unsigned char byte;
- // We require IntType to be unsigned or else the shifting gets all screwy.
- SPP_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), "serializing_int_requires_an_unsigned_type");
- for (size_t i = 0; i < length; ++i)
- {
- if (!read_data(fp, &byte, sizeof(byte)))
- return false;
- *value |= static_cast<IntType>(byte) << ((length - 1 - i) * 8);
- }
- return true;
- }
-
- template <typename OUTPUT, typename IntType>
- bool write_bigendian_number(OUTPUT* fp, IntType value, size_t length)
- {
- unsigned char byte;
- // We require IntType to be unsigned or else the shifting gets all screwy.
- SPP_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), "serializing_int_requires_an_unsigned_type");
- for (size_t i = 0; i < length; ++i)
- {
- byte = (sizeof(value) <= length-1 - i)
- ? static_cast<unsigned char>(0) : static_cast<unsigned char>((value >> ((length-1 - i) * 8)) & 255);
- if (!write_data(fp, &byte, sizeof(byte))) return false;
- }
- return true;
- }
-
- // If your keys and values are simple enough, you can pass this
- // serializer to serialize()/unserialize(). "Simple enough" means
- // value_type is a POD type that contains no pointers. Note,
- // however, we don't try to normalize endianness.
- // This is the type used for NopointerSerializer.
- // ---------------------------------------------------------------
- template <typename value_type> struct pod_serializer
- {
- template <typename INPUT>
- bool operator()(INPUT* fp, value_type* value) const
- {
- return read_data(fp, value, sizeof(*value));
- }
-
- template <typename OUTPUT>
- bool operator()(OUTPUT* fp, const value_type& value) const
- {
- return write_data(fp, &value, sizeof(value));
- }
- };
-
-
- // Settings contains parameters for growing and shrinking the table.
- // It also packages zero-size functor (ie. hasher).
- //
- // It does some munging of the hash value for the cases where
- // the original hash function is not be very good.
- // ---------------------------------------------------------------
- template<typename Key, typename HashFunc, typename SizeType, int HT_MIN_BUCKETS>
- class sh_hashtable_settings : public HashFunc
- {
- private:
-#ifndef SPP_MIX_HASH
- template <class T, int sz> struct Mixer
- {
- inline T operator()(T h) const { return h; }
- };
-#else
- template <class T, int sz> struct Mixer
- {
- inline T operator()(T h) const;
- };
-
- template <class T> struct Mixer<T, 4>
- {
- inline T operator()(T h) const
- {
- // from Thomas Wang - https://gist.github.com/badboy/6267743
- // ---------------------------------------------------------
- h = (h ^ 61) ^ (h >> 16);
- h = h + (h << 3);
- h = h ^ (h >> 4);
- h = h * 0x27d4eb2d;
- h = h ^ (h >> 15);
- return h;
- }
- };
-
- template <class T> struct Mixer<T, 8>
- {
- inline T operator()(T h) const
- {
- // from Thomas Wang - https://gist.github.com/badboy/6267743
- // ---------------------------------------------------------
- h = (~h) + (h << 21); // h = (h << 21) - h - 1;
- h = h ^ (h >> 24);
- h = (h + (h << 3)) + (h << 8); // h * 265
- h = h ^ (h >> 14);
- h = (h + (h << 2)) + (h << 4); // h * 21
- h = h ^ (h >> 28);
- h = h + (h << 31);
- return h;
- }
- };
-#endif
-
- public:
- typedef Key key_type;
- typedef HashFunc hasher;
- typedef SizeType size_type;
-
- public:
- sh_hashtable_settings(const hasher& hf,
- const float ht_occupancy_flt,
- const float ht_empty_flt)
- : hasher(hf),
- enlarge_threshold_(0),
- shrink_threshold_(0),
- consider_shrink_(false),
- num_ht_copies_(0)
- {
- set_enlarge_factor(ht_occupancy_flt);
- set_shrink_factor(ht_empty_flt);
- }
-
- size_t hash(const key_type& v) const
- {
- size_t h = hasher::operator()(v);
- Mixer<size_t, sizeof(size_t)> mixer;
-
- return mixer(h);
- }
-
- float enlarge_factor() const { return enlarge_factor_; }
- void set_enlarge_factor(float f) { enlarge_factor_ = f; }
- float shrink_factor() const { return shrink_factor_; }
- void set_shrink_factor(float f) { shrink_factor_ = f; }
-
- size_type enlarge_threshold() const { return enlarge_threshold_; }
- void set_enlarge_threshold(size_type t) { enlarge_threshold_ = t; }
- size_type shrink_threshold() const { return shrink_threshold_; }
- void set_shrink_threshold(size_type t) { shrink_threshold_ = t; }
-
- size_type enlarge_size(size_type x) const { return static_cast<size_type>(x * enlarge_factor_); }
- size_type shrink_size(size_type x) const { return static_cast<size_type>(x * shrink_factor_); }
-
- bool consider_shrink() const { return consider_shrink_; }
- void set_consider_shrink(bool t) { consider_shrink_ = t; }
-
- unsigned int num_ht_copies() const { return num_ht_copies_; }
- void inc_num_ht_copies() { ++num_ht_copies_; }
-
- // Reset the enlarge and shrink thresholds
- void reset_thresholds(size_type num_buckets)
- {
- set_enlarge_threshold(enlarge_size(num_buckets));
- set_shrink_threshold(shrink_size(num_buckets));
- // whatever caused us to reset already considered
- set_consider_shrink(false);
- }
-
- // Caller is resposible for calling reset_threshold right after
- // set_resizing_parameters.
- // ------------------------------------------------------------
- void set_resizing_parameters(float shrink, float grow)
- {
- assert(shrink >= 0);
- assert(grow <= 1);
- if (shrink > grow/2.0f)
- shrink = grow / 2.0f; // otherwise we thrash hashtable size
- set_shrink_factor(shrink);
- set_enlarge_factor(grow);
- }
-
- // This is the smallest size a hashtable can be without being too crowded
- // If you like, you can give a min #buckets as well as a min #elts
- // ----------------------------------------------------------------------
- size_type min_buckets(size_type num_elts, size_type min_buckets_wanted)
- {
- float enlarge = enlarge_factor();
- size_type sz = HT_MIN_BUCKETS; // min buckets allowed
- while (sz < min_buckets_wanted ||
- num_elts >= static_cast<size_type>(sz * enlarge))
- {
- // This just prevents overflowing size_type, since sz can exceed
- // max_size() here.
- // -------------------------------------------------------------
- if (static_cast<size_type>(sz * 2) < sz)
- throw_exception(std::length_error("resize overflow")); // protect against overflow
- sz *= 2;
- }
- return sz;
- }
-
- private:
- size_type enlarge_threshold_; // table.size() * enlarge_factor
- size_type shrink_threshold_; // table.size() * shrink_factor
- float enlarge_factor_; // how full before resize
- float shrink_factor_; // how empty before resize
- bool consider_shrink_; // if we should try to shrink before next insert
-
- unsigned int num_ht_copies_; // num_ht_copies is a counter incremented every Copy/Move
- };
-
-} // namespace sparsehash_internal
-
-#undef SPP_COMPILE_ASSERT
-
-// ----------------------------------------------------------------------
-// S P A R S E T A B L E
-// ----------------------------------------------------------------------
-//
-// A sparsetable is a random container that implements a sparse array,
-// that is, an array that uses very little memory to store unassigned
-// indices (in this case, between 1-2 bits per unassigned index). For
-// instance, if you allocate an array of size 5 and assign a[2] = <big
-// struct>, then a[2] will take up a lot of memory but a[0], a[1],
-// a[3], and a[4] will not. Array elements that have a value are
-// called "assigned". Array elements that have no value yet, or have
-// had their value cleared using erase() or clear(), are called
-// "unassigned".
-//
-// Unassigned values seem to have the default value of T (see below).
-// Nevertheless, there is a difference between an unassigned index and
-// one explicitly assigned the value of T(). The latter is considered
-// assigned.
-//
-// Access to an array element is constant time, as is insertion and
-// deletion. Insertion and deletion may be fairly slow, however:
-// because of this container's memory economy, each insert and delete
-// causes a memory reallocation.
-//
-// NOTE: You should not test(), get(), or set() any index that is
-// greater than sparsetable.size(). If you need to do that, call
-// resize() first.
-//
-// --- Template parameters
-// PARAMETER DESCRIPTION DEFAULT
-// T The value of the array: the type of --
-// object that is stored in the array.
-//
-// Alloc: Allocator to use to allocate memory.
-//
-// --- Model of
-// Random Access Container
-//
-// --- Type requirements
-// T must be Copy Constructible. It need not be Assignable.
-//
-// --- Public base classes
-// None.
-//
-// --- Members
-//
-// [*] All iterators are const in a sparsetable (though nonempty_iterators
-// may not be). Use get() and set() to assign values, not iterators.
-//
-// [+] iterators are random-access iterators. nonempty_iterators are
-// bidirectional iterators.
-
-// [*] If you shrink a sparsetable using resize(), assigned elements
-// past the end of the table are removed using erase(). If you grow
-// a sparsetable, new unassigned indices are created.
-//
-// [+] Note that operator[] returns a const reference. You must use
-// set() to change the value of a table element.
-//
-// [!] Unassignment also calls the destructor.
-//
-// Iterators are invalidated whenever an item is inserted or
-// deleted (ie set() or erase() is used) or when the size of
-// the table changes (ie resize() or clear() is used).
-
-
-
-// ---------------------------------------------------------------------------
-// Our iterator as simple as iterators can be: basically it's just
-// the index into our table. Dereference, the only complicated
-// thing, we punt to the table class. This just goes to show how
-// much machinery STL requires to do even the most trivial tasks.
-//
-// A NOTE ON ASSIGNING:
-// A sparse table does not actually allocate memory for entries
-// that are not filled. Because of this, it becomes complicated
-// to have a non-const iterator: we don't know, if the iterator points
-// to a not-filled bucket, whether you plan to fill it with something
-// or whether you plan to read its value (in which case you'll get
-// the default bucket value). Therefore, while we can define const
-// operations in a pretty 'normal' way, for non-const operations, we
-// define something that returns a helper object with operator= and
-// operator& that allocate a bucket lazily. We use this for table[]
-// and also for regular table iterators.
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-// Our iterator as simple as iterators can be: basically it's just
-// the index into our table. Dereference, the only complicated
-// thing, we punt to the table class. This just goes to show how
-// much machinery STL requires to do even the most trivial tasks.
-//
-// By templatizing over tabletype, we have one iterator type which
-// we can use for both sparsetables and sparsebins. In fact it
-// works on any class that allows size() and operator[] (eg vector),
-// as long as it does the standard STL typedefs too (eg value_type).
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-template <class tabletype>
-class table_iterator
-{
-public:
- typedef table_iterator iterator;
-
- typedef std::random_access_iterator_tag iterator_category;
- typedef typename tabletype::value_type value_type;
- typedef typename tabletype::difference_type difference_type;
- typedef typename tabletype::size_type size_type;
-
- explicit table_iterator(tabletype *tbl = 0, size_type p = 0) :
- table(tbl), pos(p)
- { }
-
- // Helper function to assert things are ok; eg pos is still in range
- void check() const
- {
- assert(table);
- assert(pos <= table->size());
- }
-
- // Arithmetic: we just do arithmetic on pos. We don't even need to
- // do bounds checking, since STL doesn't consider that its job. :-)
- iterator& operator+=(size_type t) { pos += t; check(); return *this; }
- iterator& operator-=(size_type t) { pos -= t; check(); return *this; }
- iterator& operator++() { ++pos; check(); return *this; }
- iterator& operator--() { --pos; check(); return *this; }
- iterator operator++(int)
- {
- iterator tmp(*this); // for x++
- ++pos; check(); return tmp;
- }
-
- iterator operator--(int)
- {
- iterator tmp(*this); // for x--
- --pos; check(); return tmp;
- }
-
- iterator operator+(difference_type i) const
- {
- iterator tmp(*this);
- tmp += i; return tmp;
- }
-
- iterator operator-(difference_type i) const
- {
- iterator tmp(*this);
- tmp -= i; return tmp;
- }
-
- difference_type operator-(iterator it) const
- {
- // for "x = it2 - it"
- assert(table == it.table);
- return pos - it.pos;
- }
-
- // Comparisons.
- bool operator==(const iterator& it) const
- {
- return table == it.table && pos == it.pos;
- }
-
- bool operator<(const iterator& it) const
- {
- assert(table == it.table); // life is bad bad bad otherwise
- return pos < it.pos;
- }
-
- bool operator!=(const iterator& it) const { return !(*this == it); }
- bool operator<=(const iterator& it) const { return !(it < *this); }
- bool operator>(const iterator& it) const { return it < *this; }
- bool operator>=(const iterator& it) const { return !(*this < it); }
-
- // Here's the info we actually need to be an iterator
- tabletype *table; // so we can dereference and bounds-check
- size_type pos; // index into the table
-};
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-template <class tabletype>
-class const_table_iterator
-{
-public:
- typedef table_iterator<tabletype> iterator;
- typedef const_table_iterator const_iterator;
-
- typedef std::random_access_iterator_tag iterator_category;
- typedef typename tabletype::value_type value_type;
- typedef typename tabletype::difference_type difference_type;
- typedef typename tabletype::size_type size_type;
- typedef typename tabletype::const_reference reference; // we're const-only
- typedef typename tabletype::const_pointer pointer;
-
- // The "real" constructor
- const_table_iterator(const tabletype *tbl, size_type p)
- : table(tbl), pos(p) { }
-
- // The default constructor, used when I define vars of type table::iterator
- const_table_iterator() : table(NULL), pos(0) { }
-
- // The copy constructor, for when I say table::iterator foo = tbl.begin()
- // Also converts normal iterators to const iterators // not explicit on purpose
- const_table_iterator(const iterator &from)
- : table(from.table), pos(from.pos) { }
-
- // The default destructor is fine; we don't define one
- // The default operator= is fine; we don't define one
-
- // The main thing our iterator does is dereference. If the table entry
- // we point to is empty, we return the default value type.
- reference operator*() const { return (*table)[pos]; }
- pointer operator->() const { return &(operator*()); }
-
- // Helper function to assert things are ok; eg pos is still in range
- void check() const
- {
- assert(table);
- assert(pos <= table->size());
- }
-
- // Arithmetic: we just do arithmetic on pos. We don't even need to
- // do bounds checking, since STL doesn't consider that its job. :-)
- const_iterator& operator+=(size_type t) { pos += t; check(); return *this; }
- const_iterator& operator-=(size_type t) { pos -= t; check(); return *this; }
- const_iterator& operator++() { ++pos; check(); return *this; }
- const_iterator& operator--() { --pos; check(); return *this; }
- const_iterator operator++(int)
- {
- const_iterator tmp(*this); // for x++
- ++pos; check();
- return tmp;
- }
- const_iterator operator--(int)
- {
- const_iterator tmp(*this); // for x--
- --pos; check();
- return tmp;
- }
- const_iterator operator+(difference_type i) const
- {
- const_iterator tmp(*this);
- tmp += i;
- return tmp;
- }
- const_iterator operator-(difference_type i) const
- {
- const_iterator tmp(*this);
- tmp -= i;
- return tmp;
- }
- difference_type operator-(const_iterator it) const
- {
- // for "x = it2 - it"
- assert(table == it.table);
- return pos - it.pos;
- }
- reference operator[](difference_type n) const
- {
- return *(*this + n); // simple though not totally efficient
- }
-
- // Comparisons.
- bool operator==(const const_iterator& it) const
- {
- return table == it.table && pos == it.pos;
- }
-
- bool operator<(const const_iterator& it) const
- {
- assert(table == it.table); // life is bad bad bad otherwise
- return pos < it.pos;
- }
- bool operator!=(const const_iterator& it) const { return !(*this == it); }
- bool operator<=(const const_iterator& it) const { return !(it < *this); }
- bool operator>(const const_iterator& it) const { return it < *this; }
- bool operator>=(const const_iterator& it) const { return !(*this < it); }
-
- // Here's the info we actually need to be an iterator
- const tabletype *table; // so we can dereference and bounds-check
- size_type pos; // index into the table
-};
-
-// ---------------------------------------------------------------------------
-// This is a 2-D iterator. You specify a begin and end over a list
-// of *containers*. We iterate over each container by iterating over
-// it. It's actually simple:
-// VECTOR.begin() VECTOR[0].begin() --------> VECTOR[0].end() ---,
-// | ________________________________________________/
-// | \_> VECTOR[1].begin() --------> VECTOR[1].end() -,
-// | ___________________________________________________/
-// v \_> ......
-// VECTOR.end()
-//
-// It's impossible to do random access on one of these things in constant
-// time, so it's just a bidirectional iterator.
-//
-// Unfortunately, because we need to use this for a non-empty iterator,
-// we use ne_begin() and ne_end() instead of begin() and end()
-// (though only going across, not down).
-// ---------------------------------------------------------------------------
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-template <class T, class row_it, class col_it, class iter_type>
-class Two_d_iterator
-{
-public:
- typedef Two_d_iterator iterator;
- typedef iter_type iterator_category;
- typedef T value_type;
- typedef std::ptrdiff_t difference_type;
- typedef T* pointer;
- typedef T& reference;
-
- explicit Two_d_iterator(row_it curr) : row_current(curr), col_current(0)
- {
- if (row_current && !row_current->is_marked())
- {
- col_current = row_current->ne_begin();
- advance_past_end(); // in case cur->begin() == cur->end()
- }
- }
-
- explicit Two_d_iterator(row_it curr, col_it col) : row_current(curr), col_current(col)
- {
- assert(col);
- }
-
- // The default constructor
- Two_d_iterator() : row_current(0), col_current(0) { }
-
- // Need this explicitly so we can convert normal iterators <=> const iterators
- // not explicit on purpose
- // ---------------------------------------------------------------------------
- template <class T2, class row_it2, class col_it2, class iter_type2>
- Two_d_iterator(const Two_d_iterator<T2, row_it2, col_it2, iter_type2>& it) :
- row_current (*(row_it *)&it.row_current),
- col_current (*(col_it *)&it.col_current)
- { }
-
- // The default destructor is fine; we don't define one
- // The default operator= is fine; we don't define one
-
- value_type& operator*() const { return *(col_current); }
- value_type* operator->() const { return &(operator*()); }
-
- // Arithmetic: we just do arithmetic on pos. We don't even need to
- // do bounds checking, since STL doesn't consider that its job. :-)
- // NOTE: this is not amortized constant time! What do we do about it?
- // ------------------------------------------------------------------
- void advance_past_end()
- {
- // used when col_current points to end()
- while (col_current == row_current->ne_end())
- {
- // end of current row
- // ------------------
- ++row_current; // go to beginning of next
- if (!row_current->is_marked()) // col is irrelevant at end
- col_current = row_current->ne_begin();
- else
- break; // don't go past row_end
- }
- }
-
- friend size_t operator-(iterator l, iterator f)
- {
- if (f.row_current->is_marked())
- return 0;
-
- size_t diff(0);
- while (f != l)
- {
- ++diff;
- ++f;
- }
- return diff;
- }
-
- iterator& operator++()
- {
- // assert(!row_current->is_marked()); // how to ++ from there?
- ++col_current;
- advance_past_end(); // in case col_current is at end()
- return *this;
- }
-
- iterator& operator--()
- {
- while (row_current->is_marked() ||
- col_current == row_current->ne_begin())
- {
- --row_current;
- col_current = row_current->ne_end(); // this is 1 too far
- }
- --col_current;
- return *this;
- }
- iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; }
- iterator operator--(int) { iterator tmp(*this); --*this; return tmp; }
-
-
- // Comparisons.
- bool operator==(const iterator& it) const
- {
- return (row_current == it.row_current &&
- (!row_current || row_current->is_marked() || col_current == it.col_current));
- }
-
- bool operator!=(const iterator& it) const { return !(*this == it); }
-
- // Here's the info we actually need to be an iterator
- // These need to be public so we convert from iterator to const_iterator
- // ---------------------------------------------------------------------
- row_it row_current;
- col_it col_current;
-};
-
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-template <class T, class row_it, class col_it, class iter_type, class Alloc>
-class Two_d_destructive_iterator : public Two_d_iterator<T, row_it, col_it, iter_type>
-{
-public:
- typedef Two_d_destructive_iterator iterator;
-
- Two_d_destructive_iterator(Alloc &alloc, row_it curr) :
- _alloc(alloc)
- {
- this->row_current = curr;
- this->col_current = 0;
- if (this->row_current && !this->row_current->is_marked())
- {
- this->col_current = this->row_current->ne_begin();
- advance_past_end(); // in case cur->begin() == cur->end()
- }
- }
-
- // Arithmetic: we just do arithmetic on pos. We don't even need to
- // do bounds checking, since STL doesn't consider that its job. :-)
- // NOTE: this is not amortized constant time! What do we do about it?
- // ------------------------------------------------------------------
- void advance_past_end()
- {
- // used when col_current points to end()
- while (this->col_current == this->row_current->ne_end())
- {
- this->row_current->clear(_alloc, true); // This is what differs from non-destructive iterators above
-
- // end of current row
- // ------------------
- ++this->row_current; // go to beginning of next
- if (!this->row_current->is_marked()) // col is irrelevant at end
- this->col_current = this->row_current->ne_begin();
- else
- break; // don't go past row_end
- }
- }
-
- iterator& operator++()
- {
- // assert(!this->row_current->is_marked()); // how to ++ from there?
- ++this->col_current;
- advance_past_end(); // in case col_current is at end()
- return *this;
- }
-
-private:
- Two_d_destructive_iterator& operator=(const Two_d_destructive_iterator &o);
-
- Alloc &_alloc;
-};
-
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-#if defined(SPP_POPCNT_CHECK)
-static inline bool spp_popcount_check()
-{
- int cpuInfo[4] = { -1 };
- spp_cpuid(cpuInfo, 1);
- if (cpuInfo[2] & (1 << 23))
- return true; // means SPP_POPCNT supported
- return false;
-}
-#endif
-
-#if defined(SPP_POPCNT_CHECK) && defined(SPP_POPCNT)
-
-static inline uint32_t spp_popcount(uint32_t i)
-{
- static const bool s_ok = spp_popcount_check();
- return s_ok ? SPP_POPCNT(i) : s_spp_popcount_default(i);
-}
-
-#else
-
-static inline uint32_t spp_popcount(uint32_t i)
-{
-#if defined(SPP_POPCNT)
- return static_cast<uint32_t>(SPP_POPCNT(i));
-#else
- return s_spp_popcount_default(i);
-#endif
-}
-
-#endif
-
-#if defined(SPP_POPCNT_CHECK) && defined(SPP_POPCNT64)
-
-static inline uint32_t spp_popcount(uint64_t i)
-{
- static const bool s_ok = spp_popcount_check();
- return s_ok ? (uint32_t)SPP_POPCNT64(i) : s_spp_popcount_default(i);
-}
-
-#else
-
-static inline uint32_t spp_popcount(uint64_t i)
-{
-#if defined(SPP_POPCNT64)
- return static_cast<uint32_t>(SPP_POPCNT64(i));
-#elif 1
- return s_spp_popcount_default(i);
-#endif
-}
-
-#endif
-
-// ---------------------------------------------------------------------------
-// SPARSE-TABLE
-// ------------
-// The idea is that a table with (logically) t buckets is divided
-// into t/M *groups* of M buckets each. (M is a constant, typically
-// 32) Each group is stored sparsely.
-// Thus, inserting into the table causes some array to grow, which is
-// slow but still constant time. Lookup involves doing a
-// logical-position-to-sparse-position lookup, which is also slow but
-// constant time. The larger M is, the slower these operations are
-// but the less overhead (slightly).
-//
-// To store the sparse array, we store a bitmap B, where B[i] = 1 iff
-// bucket i is non-empty. Then to look up bucket i we really look up
-// array[# of 1s before i in B]. This is constant time for fixed M.
-//
-// Terminology: the position of an item in the overall table (from
-// 1 .. t) is called its "location." The logical position in a group
-// (from 1 .. M) is called its "position." The actual location in
-// the array (from 1 .. # of non-empty buckets in the group) is
-// called its "offset."
-// ---------------------------------------------------------------------------
-
-template <class T, class Alloc>
-class sparsegroup
-{
-public:
- // Basic types
- typedef T value_type;
- typedef Alloc allocator_type;
- typedef value_type& reference;
- typedef const value_type& const_reference;
- typedef value_type* pointer;
- typedef const value_type* const_pointer;
-
- typedef uint8_t size_type; // max # of buckets
-
- // These are our special iterators, that go over non-empty buckets in a
- // group. These aren't const-only because you can change non-empty bcks.
- // ---------------------------------------------------------------------
- typedef pointer ne_iterator;
- typedef const_pointer const_ne_iterator;
- typedef std::reverse_iterator<ne_iterator> reverse_ne_iterator;
- typedef std::reverse_iterator<const_ne_iterator> const_reverse_ne_iterator;
-
- // We'll have versions for our special non-empty iterator too
- // ----------------------------------------------------------
- ne_iterator ne_begin() { return reinterpret_cast<pointer>(_group); }
- const_ne_iterator ne_begin() const { return reinterpret_cast<pointer>(_group); }
- const_ne_iterator ne_cbegin() const { return reinterpret_cast<pointer>(_group); }
- ne_iterator ne_end() { return reinterpret_cast<pointer>(_group + _num_items()); }
- const_ne_iterator ne_end() const { return reinterpret_cast<pointer>(_group + _num_items()); }
- const_ne_iterator ne_cend() const { return reinterpret_cast<pointer>(_group + _num_items()); }
- reverse_ne_iterator ne_rbegin() { return reverse_ne_iterator(ne_end()); }
- const_reverse_ne_iterator ne_rbegin() const { return const_reverse_ne_iterator(ne_cend()); }
- const_reverse_ne_iterator ne_crbegin() const { return const_reverse_ne_iterator(ne_cend()); }
- reverse_ne_iterator ne_rend() { return reverse_ne_iterator(ne_begin()); }
- const_reverse_ne_iterator ne_rend() const { return const_reverse_ne_iterator(ne_cbegin()); }
- const_reverse_ne_iterator ne_crend() const { return const_reverse_ne_iterator(ne_cbegin()); }
-
-private:
- // T can be std::pair<const K, V>, but sometime we need to cast to a mutable type
- // ------------------------------------------------------------------------------
- typedef typename spp_::cvt<T>::type mutable_value_type;
- typedef mutable_value_type & mutable_reference;
- typedef mutable_value_type * mutable_pointer;
- typedef const mutable_value_type * const_mutable_pointer;
-
- bool _bmtest(size_type i) const { return !!(_bitmap & (static_cast<group_bm_type>(1) << i)); }
- void _bmset(size_type i) { _bitmap |= static_cast<group_bm_type>(1) << i; }
- void _bmclear(size_type i) { _bitmap &= ~(static_cast<group_bm_type>(1) << i); }
-
- bool _bme_test(size_type i) const { return !!(_bm_erased & (static_cast<group_bm_type>(1) << i)); }
- void _bme_set(size_type i) { _bm_erased |= static_cast<group_bm_type>(1) << i; }
- void _bme_clear(size_type i) { _bm_erased &= ~(static_cast<group_bm_type>(1) << i); }
-
- bool _bmtest_strict(size_type i) const
- { return !!((_bitmap | _bm_erased) & (static_cast<group_bm_type>(1) << i)); }
-
-
- static uint32_t _sizing(uint32_t n)
- {
-#if !defined(SPP_ALLOC_SZ) || (SPP_ALLOC_SZ == 0)
- // aggressive allocation first, then decreasing as sparsegroups fill up
- // --------------------------------------------------------------------
- struct alloc_batch_size
- {
- // 32 bit bitmap
- // ........ .... .... .. .. .. .. . . . . . . . .
- // 8 12 16 18 20 22 24 25 26 ... 32
- // ------------------------------------------------------
- SPP_CXX14_CONSTEXPR alloc_batch_size()
- : data()
- {
- uint8_t group_sz = SPP_GROUP_SIZE / 4;
- uint8_t group_start_alloc = SPP_GROUP_SIZE / 8; //4;
- uint8_t alloc_sz = group_start_alloc;
- for (int i=0; i<4; ++i)
- {
- for (int j=0; j<group_sz; ++j)
- {
- if (j && j % group_start_alloc == 0)
- alloc_sz += group_start_alloc;
- data[i * group_sz + j] = alloc_sz;
- }
- if (group_start_alloc > 2)
- group_start_alloc /= 2;
- alloc_sz += group_start_alloc;
- }
- }
- uint8_t data[SPP_GROUP_SIZE];
- };
-
- static alloc_batch_size s_alloc_batch_sz;
- return n ? static_cast<uint32_t>(s_alloc_batch_sz.data[n-1]) : 0; // more aggressive alloc at the beginning
-
-#elif (SPP_ALLOC_SZ == 1)
- // use as little memory as possible - slowest insert/delete in table
- // -----------------------------------------------------------------
- return n;
-#else
- // decent compromise when SPP_ALLOC_SZ == 2
- // ----------------------------------------
- static size_type sz_minus_1 = SPP_ALLOC_SZ - 1;
- return (n + sz_minus_1) & ~sz_minus_1;
-#endif
- }
-
- pointer _allocate_group(allocator_type &alloc, uint32_t n /* , bool tight = false */)
- {
- // ignore tight since we don't store num_alloc
- // num_alloc = (uint8_t)(tight ? n : _sizing(n));
-
- uint32_t num_alloc = (uint8_t)_sizing(n);
- _set_num_alloc(num_alloc);
- pointer retval = alloc.allocate(static_cast<size_type>(num_alloc));
- if (retval == NULL)
- {
- // the allocator is supposed to throw an exception if the allocation fails.
- throw_exception(std::bad_alloc());
- }
- return retval;
- }
-
- void _free_group(allocator_type &alloc, uint32_t num_alloc)
- {
- if (_group)
- {
- uint32_t num_buckets = _num_items();
- if (num_buckets)
- {
- mutable_pointer end_it = (mutable_pointer)(_group + num_buckets);
- for (mutable_pointer p = (mutable_pointer)_group; p != end_it; ++p)
- p->~mutable_value_type();
- }
- alloc.deallocate(_group, (typename allocator_type::size_type)num_alloc);
- _group = NULL;
- }
- }
-
- // private because should not be called - no allocator!
- sparsegroup &operator=(const sparsegroup& x);
-
- static size_type _pos_to_offset(group_bm_type bm, size_type pos)
- {
- //return (size_type)((uint32_t)~((int32_t(-1) + pos) >> 31) & spp_popcount(bm << (SPP_GROUP_SIZE - pos)));
- //return (size_type)(pos ? spp_popcount(bm << (SPP_GROUP_SIZE - pos)) : 0);
- return static_cast<size_type>(spp_popcount(bm & ((static_cast<group_bm_type>(1) << pos) - 1)));
- }
-
-public:
-
- // get_iter() in sparsetable needs it
- size_type pos_to_offset(size_type pos) const
- {
- return _pos_to_offset(_bitmap, pos);
- }
-
-#ifdef _MSC_VER
-#pragma warning(push)
-#pragma warning(disable : 4146)
-#endif
-
- // Returns the (logical) position in the bm[] array, i, such that
- // bm[i] is the offset-th set bit in the array. It is the inverse
- // of pos_to_offset. get_pos() uses this function to find the index
- // of an ne_iterator in the table. Bit-twiddling from
- // http://hackersdelight.org/basics.pdf
- // -----------------------------------------------------------------
- static size_type offset_to_pos(group_bm_type bm, size_type offset)
- {
- for (; offset > 0; offset--)
- bm &= (bm-1); // remove right-most set bit
-
- // Clear all bits to the left of the rightmost bit (the &),
- // and then clear the rightmost bit but set all bits to the
- // right of it (the -1).
- // --------------------------------------------------------
- bm = (bm & -bm) - 1;
- return static_cast<size_type>(spp_popcount(bm));
- }
-
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
-
- size_type offset_to_pos(size_type offset) const
- {
- return offset_to_pos(_bitmap, offset);
- }
-
-public:
- // Constructors -- default and copy -- and destructor
- explicit sparsegroup() :
- _group(0), _bitmap(0), _bm_erased(0)
- {
- _set_num_items(0);
- _set_num_alloc(0);
- }
-
- sparsegroup(const sparsegroup& x) :
- _group(0), _bitmap(x._bitmap), _bm_erased(x._bm_erased)
- {
- _set_num_items(0);
- _set_num_alloc(0);
- assert(_group == 0);
- }
-
- sparsegroup(const sparsegroup& x, allocator_type& a) :
- _group(0), _bitmap(x._bitmap), _bm_erased(x._bm_erased)
- {
- _set_num_items(0);
- _set_num_alloc(0);
-
- uint32_t num_items = x._num_items();
- if (num_items)
- {
- _group = _allocate_group(a, num_items /* , true */);
- _set_num_items(num_items);
- std::uninitialized_copy(x._group, x._group + num_items, _group);
- }
- }
-
- ~sparsegroup() { assert(_group == 0); }
-
- void destruct(allocator_type& a) { _free_group(a, _num_alloc()); }
-
- // Many STL algorithms use swap instead of copy constructors
- void swap(sparsegroup& x)
- {
- using std::swap;
-
- swap(_group, x._group);
- swap(_bitmap, x._bitmap);
- swap(_bm_erased, x._bm_erased);
-#ifdef SPP_STORE_NUM_ITEMS
- swap(_num_buckets, x._num_buckets);
- swap(_num_allocated, x._num_allocated);
-#endif
- }
-
- // It's always nice to be able to clear a table without deallocating it
- void clear(allocator_type &alloc, bool erased)
- {
- _free_group(alloc, _num_alloc());
- _bitmap = 0;
- if (erased)
- _bm_erased = 0;
- _set_num_items(0);
- _set_num_alloc(0);
- }
-
- // Functions that tell you about size. Alas, these aren't so useful
- // because our table is always fixed size.
- size_type size() const { return static_cast<size_type>(SPP_GROUP_SIZE); }
- size_type max_size() const { return static_cast<size_type>(SPP_GROUP_SIZE); }
-
- bool empty() const { return false; }
-
- // We also may want to know how many *used* buckets there are
- size_type num_nonempty() const { return (size_type)_num_items(); }
-
- // TODO(csilvers): make protected + friend
- // This is used by sparse_hashtable to get an element from the table
- // when we know it exists.
- reference unsafe_get(size_type i) const
- {
- // assert(_bmtest(i));
- return (reference)_group[pos_to_offset(i)];
- }
-
- typedef std::pair<pointer, bool> SetResult;
-
-private:
- //typedef spp_::integral_constant<bool, spp_::is_relocatable<value_type>::value> check_relocatable;
- typedef spp_::true_type realloc_ok_type;
- typedef spp_::false_type realloc_not_ok_type;
-
- //typedef spp_::zero_type libc_reloc_type;
- //typedef spp_::one_type spp_reloc_type;
- //typedef spp_::two_type spp_not_reloc_type;
- //typedef spp_::three_type generic_alloc_type;
-
-#if 1
- typedef typename if_<((spp_::is_same<allocator_type, libc_allocator<value_type> >::value ||
- spp_::is_same<allocator_type, spp_allocator<value_type> >::value) &&
- spp_::is_relocatable<value_type>::value), realloc_ok_type, realloc_not_ok_type>::type
- check_alloc_type;
-#else
- typedef typename if_<spp_::is_same<allocator_type, spp_allocator<value_type> >::value,
- typename if_<spp_::is_relocatable<value_type>::value, spp_reloc_type, spp_not_reloc_type>::type,
- typename if_<(spp_::is_same<allocator_type, libc_allocator<value_type> >::value &&
- spp_::is_relocatable<value_type>::value), libc_reloc_type, generic_alloc_type>::type >::type
- check_alloc_type;
-#endif
-
-
- //typedef if_<spp_::is_same<allocator_type, libc_allocator<value_type> >::value,
- // libc_alloc_type,
- // if_<spp_::is_same<allocator_type, spp_allocator<value_type> >::value,
- // spp_alloc_type, user_alloc_type> > check_alloc_type;
-
- //typedef spp_::integral_constant<bool,
- // (spp_::is_relocatable<value_type>::value &&
- // (spp_::is_same<allocator_type, spp_allocator<value_type> >::value ||
- // spp_::is_same<allocator_type, libc_allocator<value_type> >::value)) >
- // realloc_and_memmove_ok;
-
- // ------------------------- memory at *p is uninitialized => need to construct
- void _init_val(mutable_value_type *p, reference val)
- {
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- ::new (p) value_type(std::move((mutable_reference)val));
-#else
- ::new (p) value_type((mutable_reference)val);
-#endif
- }
-
- // ------------------------- memory at *p is uninitialized => need to construct
- void _init_val(mutable_value_type *p, const_reference val)
- {
- ::new (p) value_type(val);
- }
-
- // ------------------------------------------------ memory at *p is initialized
- void _set_val(value_type *p, reference val)
- {
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- *(mutable_pointer)p = std::move((mutable_reference)val);
-#else
- using std::swap;
- swap(*(mutable_pointer)p, *(mutable_pointer)&val);
-#endif
- }
-
- // ------------------------------------------------ memory at *p is initialized
- void _set_val(value_type *p, const_reference val)
- {
- *(mutable_pointer)p = *(const_mutable_pointer)&val;
- }
-
- // Create space at _group[offset], assuming value_type is relocatable, and the
- // allocator_type is the spp allocator.
- // return true if the slot was constructed (i.e. contains a valid value_type
- // ---------------------------------------------------------------------------------
- template <class Val>
- void _set_aux(allocator_type &alloc, size_type offset, Val &val, realloc_ok_type)
- {
- //static int x=0; if (++x < 10) printf("x\n"); // check we are getting here
-
- uint32_t num_items = _num_items();
- uint32_t num_alloc = _sizing(num_items);
-
- if (num_items == num_alloc)
- {
- num_alloc = _sizing(num_items + 1);
- _group = alloc.reallocate(_group, num_alloc);
- _set_num_alloc(num_alloc);
- }
-
- for (uint32_t i = num_items; i > offset; --i)
- memcpy(static_cast<void *>(_group + i), _group + i-1, sizeof(*_group));
-
- _init_val((mutable_pointer)(_group + offset), val);
- }
-
- // Create space at _group[offset], assuming value_type is *not* relocatable, and the
- // allocator_type is the spp allocator.
- // return true if the slot was constructed (i.e. contains a valid value_type
- // ---------------------------------------------------------------------------------
- template <class Val>
- void _set_aux(allocator_type &alloc, size_type offset, Val &val, realloc_not_ok_type)
- {
- uint32_t num_items = _num_items();
- uint32_t num_alloc = _sizing(num_items);
-
- //assert(num_alloc == (uint32_t)_num_allocated);
- if (num_items < num_alloc)
- {
- // create new object at end and rotate it to position
- _init_val((mutable_pointer)&_group[num_items], val);
- std::rotate((mutable_pointer)(_group + offset),
- (mutable_pointer)(_group + num_items),
- (mutable_pointer)(_group + num_items + 1));
- return;
- }
-
- // This is valid because 0 <= offset <= num_items
- pointer p = _allocate_group(alloc, _sizing(num_items + 1));
- if (offset)
- std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)_group),
- MK_MOVE_IT((mutable_pointer)(_group + offset)),
- (mutable_pointer)p);
- if (num_items > offset)
- std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group + offset)),
- MK_MOVE_IT((mutable_pointer)(_group + num_items)),
- (mutable_pointer)(p + offset + 1));
- _init_val((mutable_pointer)(p + offset), val);
- _free_group(alloc, num_alloc);
- _group = p;
- }
-
- // ----------------------------------------------------------------------------------
- template <class Val>
- void _set(allocator_type &alloc, size_type i, size_type offset, Val &val)
- {
- if (!_bmtest(i))
- {
- _set_aux(alloc, offset, val, check_alloc_type());
- _incr_num_items();
- _bmset(i);
- }
- else
- _set_val(&_group[offset], val);
- }
-
-public:
-
- // This returns the pointer to the inserted item
- // ---------------------------------------------
- template <class Val>
- pointer set(allocator_type &alloc, size_type i, Val &val)
- {
- _bme_clear(i); // in case this was an "erased" location
-
- size_type offset = pos_to_offset(i);
- _set(alloc, i, offset, val); // may change _group pointer
- return (pointer)(_group + offset);
- }
-
- // We let you see if a bucket is non-empty without retrieving it
- // -------------------------------------------------------------
- bool test(size_type i) const
- {
- return _bmtest(i);
- }
-
- // also tests for erased values
- // ----------------------------
- bool test_strict(size_type i) const
- {
- return _bmtest_strict(i);
- }
-
-private:
- // Shrink the array, assuming value_type is relocatable, and the
- // allocator_type is the libc allocator (supporting reallocate).
- // -------------------------------------------------------------
- void _group_erase_aux(allocator_type &alloc, size_type offset, realloc_ok_type)
- {
- // static int x=0; if (++x < 10) printf("Y\n"); // check we are getting here
- uint32_t num_items = _num_items();
- uint32_t num_alloc = _sizing(num_items);
-
- if (num_items == 1)
- {
- assert(offset == 0);
- _free_group(alloc, num_alloc);
- _set_num_alloc(0);
- return;
- }
-
- _group[offset].~value_type();
-
- for (size_type i = offset; i < num_items - 1; ++i)
- memcpy(static_cast<void *>(_group + i), _group + i + 1, sizeof(*_group));
-
- if (_sizing(num_items - 1) != num_alloc)
- {
- num_alloc = _sizing(num_items - 1);
- assert(num_alloc); // because we have at least 1 item left
- _set_num_alloc(num_alloc);
- _group = alloc.reallocate(_group, num_alloc);
- }
- }
-
- // Shrink the array, without any special assumptions about value_type and
- // allocator_type.
- // --------------------------------------------------------------------------
- void _group_erase_aux(allocator_type &alloc, size_type offset, realloc_not_ok_type)
- {
- uint32_t num_items = _num_items();
- uint32_t num_alloc = _sizing(num_items);
-
- if (_sizing(num_items - 1) != num_alloc)
- {
- pointer p = 0;
- if (num_items > 1)
- {
- p = _allocate_group(alloc, num_items - 1);
- if (offset)
- std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group)),
- MK_MOVE_IT((mutable_pointer)(_group + offset)),
- (mutable_pointer)(p));
- if (static_cast<uint32_t>(offset + 1) < num_items)
- std::uninitialized_copy(MK_MOVE_IT((mutable_pointer)(_group + offset + 1)),
- MK_MOVE_IT((mutable_pointer)(_group + num_items)),
- (mutable_pointer)(p + offset));
- }
- else
- {
- assert(offset == 0);
- _set_num_alloc(0);
- }
- _free_group(alloc, num_alloc);
- _group = p;
- }
- else
- {
- std::rotate((mutable_pointer)(_group + offset),
- (mutable_pointer)(_group + offset + 1),
- (mutable_pointer)(_group + num_items));
- ((mutable_pointer)(_group + num_items - 1))->~mutable_value_type();
- }
- }
-
- void _group_erase(allocator_type &alloc, size_type offset)
- {
- _group_erase_aux(alloc, offset, check_alloc_type());
- }
-
-public:
- template <class twod_iter>
- bool erase_ne(allocator_type &alloc, twod_iter &it)
- {
- assert(_group && it.col_current != ne_end());
- size_type offset = (size_type)(it.col_current - ne_begin());
- size_type pos = offset_to_pos(offset);
-
- if (_num_items() <= 1)
- {
- clear(alloc, false);
- it.col_current = 0;
- }
- else
- {
- _group_erase(alloc, offset);
- _decr_num_items();
- _bmclear(pos);
-
- // in case _group_erase reallocated the buffer
- it.col_current = reinterpret_cast<pointer>(_group) + offset;
- }
- _bme_set(pos); // remember that this position has been erased
- it.advance_past_end();
- return true;
- }
-
-
- // This takes the specified elements out of the group. This is
- // "undefining", rather than "clearing".
- // TODO(austern): Make this exception safe: handle exceptions from
- // value_type's copy constructor.
- // ---------------------------------------------------------------
- void erase(allocator_type &alloc, size_type i)
- {
- if (_bmtest(i))
- {
- // trivial to erase empty bucket
- if (_num_items() == 1)
- clear(alloc, false);
- else
- {
- _group_erase(alloc, pos_to_offset(i));
- _decr_num_items();
- _bmclear(i);
- }
- _bme_set(i); // remember that this position has been erased
- }
- }
-
- // I/O
- // We support reading and writing groups to disk. We don't store
- // the actual array contents (which we don't know how to store),
- // just the bitmap and size. Meant to be used with table I/O.
- // --------------------------------------------------------------
- template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const
- {
- // warning: we write 4 or 8 bytes for the bitmap, instead of 6 in the
- // original google sparsehash
- // ------------------------------------------------------------------
- if (!sparsehash_internal::write_data(fp, &_bitmap, sizeof(_bitmap)))
- return false;
-
- return true;
- }
-
- // Reading destroys the old group contents! Returns true if all was ok.
- template <typename INPUT> bool read_metadata(allocator_type &alloc, INPUT *fp)
- {
- clear(alloc, true);
-
- if (!sparsehash_internal::read_data(fp, &_bitmap, sizeof(_bitmap)))
- return false;
-
- // We'll allocate the space, but we won't fill it: it will be
- // left as uninitialized raw memory.
- uint32_t num_items = spp_popcount(_bitmap); // yes, _num_buckets not set
- _set_num_items(num_items);
- _group = num_items ? _allocate_group(alloc, num_items/* , true */) : 0;
- return true;
- }
-
- // Again, only meaningful if value_type is a POD.
- template <typename INPUT> bool read_nopointer_data(INPUT *fp)
- {
- for (ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!sparsehash_internal::read_data(fp, &(*it), sizeof(*it)))
- return false;
- return true;
- }
-
- // If your keys and values are simple enough, we can write them
- // to disk for you. "simple enough" means POD and no pointers.
- // However, we don't try to normalize endianness.
- // ------------------------------------------------------------
- template <typename OUTPUT> bool write_nopointer_data(OUTPUT *fp) const
- {
- for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!sparsehash_internal::write_data(fp, &(*it), sizeof(*it)))
- return false;
- return true;
- }
-
-
- // Comparisons. We only need to define == and < -- we get
- // != > <= >= via relops.h (which we happily included above).
- // Note the comparisons are pretty arbitrary: we compare
- // values of the first index that isn't equal (using default
- // value for empty buckets).
- // ---------------------------------------------------------
- bool operator==(const sparsegroup& x) const
- {
- return (_bitmap == x._bitmap &&
- _bm_erased == x._bm_erased &&
- std::equal(_group, _group + _num_items(), x._group));
- }
-
- bool operator<(const sparsegroup& x) const
- {
- // also from <algorithm>
- return std::lexicographical_compare(_group, _group + _num_items(),
- x._group, x._group + x._num_items());
- }
-
- bool operator!=(const sparsegroup& x) const { return !(*this == x); }
- bool operator<=(const sparsegroup& x) const { return !(x < *this); }
- bool operator> (const sparsegroup& x) const { return x < *this; }
- bool operator>=(const sparsegroup& x) const { return !(*this < x); }
-
- void mark() { _group = (value_type *)static_cast<uintptr_t>(-1); }
- bool is_marked() const { return _group == (value_type *)static_cast<uintptr_t>(-1); }
-
-private:
- // ---------------------------------------------------------------------------
- template <class A>
- class alloc_impl : public A
- {
- public:
- typedef typename A::pointer pointer;
- typedef typename A::size_type size_type;
-
- // Convert a normal allocator to one that has realloc_or_die()
- explicit alloc_impl(const A& a) : A(a) { }
-
- // realloc_or_die should only be used when using the default
- // allocator (spp::spp_allocator).
- pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/)
- {
- throw_exception(std::runtime_error("realloc_or_die is only supported for spp::spp_allocator\n"));
- return NULL;
- }
- };
-
- // A template specialization of alloc_impl for
- // spp::libc_allocator that can handle realloc_or_die.
- // -----------------------------------------------------------
- template <class A>
- class alloc_impl<spp_::libc_allocator<A> > : public spp_::libc_allocator<A>
- {
- public:
- typedef typename spp_::libc_allocator<A>::pointer pointer;
- typedef typename spp_::libc_allocator<A>::size_type size_type;
-
- explicit alloc_impl(const spp_::libc_allocator<A>& a)
- : spp_::libc_allocator<A>(a)
- { }
-
- pointer realloc_or_die(pointer ptr, size_type n)
- {
- pointer retval = this->reallocate(ptr, n);
- if (retval == NULL)
- {
- // the allocator is supposed to throw an exception if the allocation fails.
- throw_exception(std::bad_alloc());
- }
- return retval;
- }
- };
-
- // A template specialization of alloc_impl for
- // spp::spp_allocator that can handle realloc_or_die.
- // -----------------------------------------------------------
- template <class A>
- class alloc_impl<spp_::spp_allocator<A> > : public spp_::spp_allocator<A>
- {
- public:
- typedef typename spp_::spp_allocator<A>::pointer pointer;
- typedef typename spp_::spp_allocator<A>::size_type size_type;
-
- explicit alloc_impl(const spp_::spp_allocator<A>& a)
- : spp_::spp_allocator<A>(a)
- { }
-
- pointer realloc_or_die(pointer ptr, size_type n)
- {
- pointer retval = this->reallocate(ptr, n);
- if (retval == NULL)
- {
- // the allocator is supposed to throw an exception if the allocation fails.
- throw_exception(std::bad_alloc());
- }
- return retval;
- }
- };
-
-
-#ifdef SPP_STORE_NUM_ITEMS
- uint32_t _num_items() const { return (uint32_t)_num_buckets; }
- void _set_num_items(uint32_t val) { _num_buckets = static_cast<size_type>(val); }
- void _incr_num_items() { ++_num_buckets; }
- void _decr_num_items() { --_num_buckets; }
- uint32_t _num_alloc() const { return (uint32_t)_num_allocated; }
- void _set_num_alloc(uint32_t val) { _num_allocated = static_cast<size_type>(val); }
-#else
- uint32_t _num_items() const { return spp_popcount(_bitmap); }
- void _set_num_items(uint32_t ) { }
- void _incr_num_items() { }
- void _decr_num_items() { }
- uint32_t _num_alloc() const { return _sizing(_num_items()); }
- void _set_num_alloc(uint32_t val) { }
-#endif
-
- // The actual data
- // ---------------
- value_type * _group; // (small) array of T's
- group_bm_type _bitmap;
- group_bm_type _bm_erased; // ones where items have been erased
-
-#ifdef SPP_STORE_NUM_ITEMS
- size_type _num_buckets;
- size_type _num_allocated;
-#endif
-};
-
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-template <class T, class Alloc>
-class sparsetable
-{
-public:
- typedef T value_type;
- typedef Alloc allocator_type;
- typedef sparsegroup<value_type, allocator_type> group_type;
-
-private:
- typedef typename Alloc::template rebind<group_type>::other group_alloc_type;
- typedef typename group_alloc_type::size_type group_size_type;
-
-public:
- // Basic types
- // -----------
- typedef typename allocator_type::size_type size_type;
- typedef typename allocator_type::difference_type difference_type;
- typedef value_type& reference;
- typedef const value_type& const_reference;
- typedef value_type* pointer;
- typedef const value_type* const_pointer;
-
- typedef group_type& GroupsReference;
- typedef const group_type& GroupsConstReference;
-
- typedef typename group_type::ne_iterator ColIterator;
- typedef typename group_type::const_ne_iterator ColConstIterator;
-
- typedef table_iterator<sparsetable<T, allocator_type> > iterator; // defined with index
- typedef const_table_iterator<sparsetable<T, allocator_type> > const_iterator; // defined with index
- typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
- typedef std::reverse_iterator<iterator> reverse_iterator;
-
- // These are our special iterators, that go over non-empty buckets in a
- // table. These aren't const only because you can change non-empty bcks.
- // ----------------------------------------------------------------------
- typedef Two_d_iterator<T,
- group_type *,
- ColIterator,
- std::bidirectional_iterator_tag> ne_iterator;
-
- typedef Two_d_iterator<const T,
- const group_type *,
- ColConstIterator,
- std::bidirectional_iterator_tag> const_ne_iterator;
-
- // Another special iterator: it frees memory as it iterates (used to resize).
- // Obviously, you can only iterate over it once, which is why it's an input iterator
- // ---------------------------------------------------------------------------------
- typedef Two_d_destructive_iterator<T,
- group_type *,
- ColIterator,
- std::input_iterator_tag,
- allocator_type> destructive_iterator;
-
- typedef std::reverse_iterator<ne_iterator> reverse_ne_iterator;
- typedef std::reverse_iterator<const_ne_iterator> const_reverse_ne_iterator;
-
-
- // Iterator functions
- // ------------------
- iterator begin() { return iterator(this, 0); }
- const_iterator begin() const { return const_iterator(this, 0); }
- const_iterator cbegin() const { return const_iterator(this, 0); }
- iterator end() { return iterator(this, size()); }
- const_iterator end() const { return const_iterator(this, size()); }
- const_iterator cend() const { return const_iterator(this, size()); }
- reverse_iterator rbegin() { return reverse_iterator(end()); }
- const_reverse_iterator rbegin() const { return const_reverse_iterator(cend()); }
- const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); }
- reverse_iterator rend() { return reverse_iterator(begin()); }
- const_reverse_iterator rend() const { return const_reverse_iterator(cbegin()); }
- const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); }
-
- // Versions for our special non-empty iterator
- // ------------------------------------------
- ne_iterator ne_begin() { return ne_iterator (_first_group); }
- const_ne_iterator ne_begin() const { return const_ne_iterator(_first_group); }
- const_ne_iterator ne_cbegin() const { return const_ne_iterator(_first_group); }
- ne_iterator ne_end() { return ne_iterator (_last_group); }
- const_ne_iterator ne_end() const { return const_ne_iterator(_last_group); }
- const_ne_iterator ne_cend() const { return const_ne_iterator(_last_group); }
-
- reverse_ne_iterator ne_rbegin() { return reverse_ne_iterator(ne_end()); }
- const_reverse_ne_iterator ne_rbegin() const { return const_reverse_ne_iterator(ne_end()); }
- const_reverse_ne_iterator ne_crbegin() const { return const_reverse_ne_iterator(ne_end()); }
- reverse_ne_iterator ne_rend() { return reverse_ne_iterator(ne_begin()); }
- const_reverse_ne_iterator ne_rend() const { return const_reverse_ne_iterator(ne_begin()); }
- const_reverse_ne_iterator ne_crend() const { return const_reverse_ne_iterator(ne_begin()); }
-
- destructive_iterator destructive_begin()
- {
- return destructive_iterator(_alloc, _first_group);
- }
-
- destructive_iterator destructive_end()
- {
- return destructive_iterator(_alloc, _last_group);
- }
-
- // How to deal with the proper group
- static group_size_type num_groups(size_type num)
- {
- // how many to hold num buckets
- return num == 0 ? (group_size_type)0 :
- (group_size_type)(((num-1) / SPP_GROUP_SIZE) + 1);
- }
-
- typename group_type::size_type pos_in_group(size_type i) const
- {
- return static_cast<typename group_type::size_type>(i & SPP_MASK_);
- }
-
- size_type group_num(size_type i) const
- {
- return (size_type)(i >> SPP_SHIFT_);
- }
-
- GroupsReference which_group(size_type i)
- {
- return _first_group[group_num(i)];
- }
-
- GroupsConstReference which_group(size_type i) const
- {
- return _first_group[group_num(i)];
- }
-
- void _alloc_group_array(group_size_type sz, group_type *&first, group_type *&last)
- {
- if (sz)
- {
- first = _group_alloc.allocate((size_type)(sz + 1)); // + 1 for end marker
- first[sz].mark(); // for the ne_iterator
- last = first + sz;
- }
- }
-
- void _free_group_array(group_type *&first, group_type *&last)
- {
- if (first)
- {
- _group_alloc.deallocate(first, (group_size_type)(last - first + 1)); // + 1 for end marker
- first = last = 0;
- }
- }
-
- void _allocate_groups(size_type sz)
- {
- if (sz)
- {
- _alloc_group_array(sz, _first_group, _last_group);
- std::uninitialized_fill(_first_group, _last_group, group_type());
- }
- }
-
- void _free_groups()
- {
- if (_first_group)
- {
- for (group_type *g = _first_group; g != _last_group; ++g)
- g->destruct(_alloc);
- _free_group_array(_first_group, _last_group);
- }
- }
-
- void _cleanup()
- {
- _free_groups(); // sets _first_group = _last_group = 0
- _table_size = 0;
- _num_buckets = 0;
- }
-
- void _init()
- {
- _first_group = 0;
- _last_group = 0;
- _table_size = 0;
- _num_buckets = 0;
- }
-
- void _copy(const sparsetable &o)
- {
- _table_size = o._table_size;
- _num_buckets = o._num_buckets;
- _alloc = o._alloc; // todo - copy or move allocator according to...
- _group_alloc = o._group_alloc; // http://en.cppreference.com/w/cpp/container/unordered_map/unordered_map
-
- group_size_type sz = (group_size_type)(o._last_group - o._first_group);
- if (sz)
- {
- _alloc_group_array(sz, _first_group, _last_group);
- for (group_size_type i=0; i<sz; ++i)
- new (_first_group + i) group_type(o._first_group[i], _alloc);
- }
- }
-
-public:
- // Constructors -- default, normal (when you specify size), and copy
- explicit sparsetable(size_type sz = 0, const allocator_type &alloc = allocator_type()) :
- _first_group(0),
- _last_group(0),
- _table_size(sz),
- _num_buckets(0),
- _group_alloc(alloc),
- _alloc(alloc)
- // todo - copy or move allocator according to
- // http://en.cppreference.com/w/cpp/container/unordered_map/unordered_map
- {
- _allocate_groups(num_groups(sz));
- }
-
- ~sparsetable()
- {
- _free_groups();
- }
-
- sparsetable(const sparsetable &o)
- {
- _init();
- _copy(o);
- }
-
- sparsetable& operator=(const sparsetable &o)
- {
- _cleanup();
- _copy(o);
- return *this;
- }
-
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- sparsetable(sparsetable&& o)
- {
- _init();
- this->swap(o);
- }
-
- sparsetable(sparsetable&& o, const allocator_type &alloc)
- {
- _init();
- this->swap(o);
- _alloc = alloc; // [gp todo] is this correct?
- }
-
- sparsetable& operator=(sparsetable&& o)
- {
- _cleanup();
- this->swap(o);
- return *this;
- }
-#endif
-
- // Many STL algorithms use swap instead of copy constructors
- void swap(sparsetable& o)
- {
- using std::swap;
-
- swap(_first_group, o._first_group);
- swap(_last_group, o._last_group);
- swap(_table_size, o._table_size);
- swap(_num_buckets, o._num_buckets);
- if (_alloc != o._alloc)
- swap(_alloc, o._alloc);
- if (_group_alloc != o._group_alloc)
- swap(_group_alloc, o._group_alloc);
- }
-
- // It's always nice to be able to clear a table without deallocating it
- void clear()
- {
- _free_groups();
- _num_buckets = 0;
- _table_size = 0;
- }
-
- inline allocator_type get_allocator() const
- {
- return _alloc;
- }
-
-
- // Functions that tell you about size.
- // NOTE: empty() is non-intuitive! It does not tell you the number
- // of not-empty buckets (use num_nonempty() for that). Instead
- // it says whether you've allocated any buckets or not.
- // ----------------------------------------------------------------
- size_type size() const { return _table_size; }
- size_type max_size() const { return _alloc.max_size(); }
- bool empty() const { return _table_size == 0; }
- size_type num_nonempty() const { return _num_buckets; }
-
- // OK, we'll let you resize one of these puppies
- void resize(size_type new_size)
- {
- group_size_type sz = num_groups(new_size);
- group_size_type old_sz = (group_size_type)(_last_group - _first_group);
-
- if (sz != old_sz)
- {
- // resize group array
- // ------------------
- group_type *first = 0, *last = 0;
- if (sz)
- {
- _alloc_group_array(sz, first, last);
- if (old_sz)
- memcpy(static_cast<void *>(first), _first_group, sizeof(*first) * (std::min)(sz, old_sz));
- }
-
- if (sz < old_sz)
- {
- for (group_type *g = _first_group + sz; g != _last_group; ++g)
- g->destruct(_alloc);
- }
- else
- std::uninitialized_fill(first + old_sz, last, group_type());
-
- _free_group_array(_first_group, _last_group);
- _first_group = first;
- _last_group = last;
- }
-#if 0
- // used only in test program
- // todo: fix if sparsetable to be used directly
- // --------------------------------------------
- if (new_size < _table_size)
- {
- // lower num_buckets, clear last group
- if (pos_in_group(new_size) > 0) // need to clear inside last group
- groups.back().erase(_alloc, groups.back().begin() + pos_in_group(new_size),
- groups.back().end());
- _num_buckets = 0; // refigure # of used buckets
- for (const group_type *group = _first_group; group != _last_group; ++group)
- _num_buckets += group->num_nonempty();
- }
-#endif
- _table_size = new_size;
- }
-
- // We let you see if a bucket is non-empty without retrieving it
- // -------------------------------------------------------------
- bool test(size_type i) const
- {
- // assert(i < _table_size);
- return which_group(i).test(pos_in_group(i));
- }
-
- // also tests for erased values
- // ----------------------------
- bool test_strict(size_type i) const
- {
- // assert(i < _table_size);
- return which_group(i).test_strict(pos_in_group(i));
- }
-
- friend struct GrpPos;
-
- struct GrpPos
- {
- typedef typename sparsetable::ne_iterator ne_iter;
- GrpPos(const sparsetable &table, size_type i) :
- grp(table.which_group(i)), pos(table.pos_in_group(i)) {}
-
- bool test_strict() const { return grp.test_strict(pos); }
- bool test() const { return grp.test(pos); }
- typename sparsetable::reference unsafe_get() const { return grp.unsafe_get(pos); }
- ne_iter get_iter(typename sparsetable::reference ref)
- {
- return ne_iter((group_type *)&grp, &ref);
- }
-
- void erase(sparsetable &table) // item *must* be present
- {
- assert(table._num_buckets);
- ((group_type &)grp).erase(table._alloc, pos);
- --table._num_buckets;
- }
-
- private:
- GrpPos* operator=(const GrpPos&);
-
- const group_type &grp;
- typename group_type::size_type pos;
- };
-
- bool test(iterator pos) const
- {
- return which_group(pos.pos).test(pos_in_group(pos.pos));
- }
-
- bool test(const_iterator pos) const
- {
- return which_group(pos.pos).test(pos_in_group(pos.pos));
- }
-
- // TODO(csilvers): make protected + friend
- // This is used by sparse_hashtable to get an element from the table
- // when we know it exists (because the caller has called test(i)).
- // -----------------------------------------------------------------
- reference unsafe_get(size_type i) const
- {
- assert(i < _table_size);
- // assert(test(i));
- return which_group(i).unsafe_get(pos_in_group(i));
- }
-
- // Needed for hashtables, gets as a ne_iterator. Crashes for empty bcks
- const_ne_iterator get_iter(size_type i) const
- {
- //assert(test(i)); // how can a ne_iterator point to an empty bucket?
-
- size_type grp_idx = group_num(i);
-
- return const_ne_iterator(_first_group + grp_idx,
- (_first_group[grp_idx].ne_begin() +
- _first_group[grp_idx].pos_to_offset(pos_in_group(i))));
- }
-
- const_ne_iterator get_iter(size_type i, ColIterator col_it) const
- {
- return const_ne_iterator(_first_group + group_num(i), col_it);
- }
-
- // For nonempty we can return a non-const version
- ne_iterator get_iter(size_type i)
- {
- //assert(test(i)); // how can a nonempty_iterator point to an empty bucket?
-
- size_type grp_idx = group_num(i);
-
- return ne_iterator(_first_group + grp_idx,
- (_first_group[grp_idx].ne_begin() +
- _first_group[grp_idx].pos_to_offset(pos_in_group(i))));
- }
-
- ne_iterator get_iter(size_type i, ColIterator col_it)
- {
- return ne_iterator(_first_group + group_num(i), col_it);
- }
-
- // And the reverse transformation.
- size_type get_pos(const const_ne_iterator& it) const
- {
- difference_type current_row = it.row_current - _first_group;
- difference_type current_col = (it.col_current - _first_group[current_row].ne_begin());
- return ((current_row * SPP_GROUP_SIZE) +
- _first_group[current_row].offset_to_pos(current_col));
- }
-
- // Val can be reference or const_reference
- // ---------------------------------------
- template <class Val>
- reference set(size_type i, Val &val)
- {
- assert(i < _table_size);
- group_type &group = which_group(i);
- typename group_type::size_type old_numbuckets = group.num_nonempty();
- pointer p(group.set(_alloc, pos_in_group(i), val));
- _num_buckets += group.num_nonempty() - old_numbuckets;
- return *p;
- }
-
- // used in _move_from (where we can move the old value instead of copying it
- void move(size_type i, reference val)
- {
- assert(i < _table_size);
- which_group(i).set(_alloc, pos_in_group(i), val);
- ++_num_buckets;
- }
-
- // This takes the specified elements out of the table.
- // --------------------------------------------------
- void erase(size_type i)
- {
- assert(i < _table_size);
-
- GroupsReference grp(which_group(i));
- typename group_type::size_type old_numbuckets = grp.num_nonempty();
- grp.erase(_alloc, pos_in_group(i));
- _num_buckets += grp.num_nonempty() - old_numbuckets;
- }
-
- void erase(iterator pos)
- {
- erase(pos.pos);
- }
-
- void erase(iterator start_it, iterator end_it)
- {
- // This could be more efficient, but then we'd need to figure
- // out if we spanned groups or not. Doesn't seem worth it.
- for (; start_it != end_it; ++start_it)
- erase(start_it);
- }
-
- const_ne_iterator erase(const_ne_iterator it)
- {
- ne_iterator res(it);
- if (res.row_current->erase_ne(_alloc, res))
- _num_buckets--;
- return res;
- }
-
- const_ne_iterator erase(const_ne_iterator f, const_ne_iterator l)
- {
- size_t diff = l - f;
- while (diff--)
- f = erase(f);
- return f;
- }
-
- // We support reading and writing tables to disk. We don't store
- // the actual array contents (which we don't know how to store),
- // just the groups and sizes. Returns true if all went ok.
-
-private:
- // Every time the disk format changes, this should probably change too
- typedef unsigned long MagicNumberType;
- static const MagicNumberType MAGIC_NUMBER = 0x24687531;
-
- // Old versions of this code write all data in 32 bits. We need to
- // support these files as well as having support for 64-bit systems.
- // So we use the following encoding scheme: for values < 2^32-1, we
- // store in 4 bytes in big-endian order. For values > 2^32, we
- // store 0xFFFFFFF followed by 8 bytes in big-endian order. This
- // causes us to mis-read old-version code that stores exactly
- // 0xFFFFFFF, but I don't think that is likely to have happened for
- // these particular values.
- template <typename OUTPUT, typename IntType>
- static bool write_32_or_64(OUTPUT* fp, IntType value)
- {
- if (value < 0xFFFFFFFFULL) // fits in 4 bytes
- {
- if (!sparsehash_internal::write_bigendian_number(fp, value, 4))
- return false;
- }
- else
- {
- if (!sparsehash_internal::write_bigendian_number(fp, 0xFFFFFFFFUL, 4))
- return false;
- if (!sparsehash_internal::write_bigendian_number(fp, value, 8))
- return false;
- }
- return true;
- }
-
- template <typename INPUT, typename IntType>
- static bool read_32_or_64(INPUT* fp, IntType *value)
- {
- // reads into value
- MagicNumberType first4 = 0; // a convenient 32-bit unsigned type
- if (!sparsehash_internal::read_bigendian_number(fp, &first4, 4))
- return false;
-
- if (first4 < 0xFFFFFFFFULL)
- {
- *value = first4;
- }
- else
- {
- if (!sparsehash_internal::read_bigendian_number(fp, value, 8))
- return false;
- }
- return true;
- }
-
-public:
- // read/write_metadata() and read_write/nopointer_data() are DEPRECATED.
- // Use serialize() and unserialize(), below, for new code.
-
- template <typename OUTPUT>
- bool write_metadata(OUTPUT *fp) const
- {
- if (!write_32_or_64(fp, MAGIC_NUMBER)) return false;
- if (!write_32_or_64(fp, _table_size)) return false;
- if (!write_32_or_64(fp, _num_buckets)) return false;
-
- for (const group_type *group = _first_group; group != _last_group; ++group)
- if (group->write_metadata(fp) == false)
- return false;
- return true;
- }
-
- // Reading destroys the old table contents! Returns true if read ok.
- template <typename INPUT>
- bool read_metadata(INPUT *fp)
- {
- size_type magic_read = 0;
- if (!read_32_or_64(fp, &magic_read)) return false;
- if (magic_read != MAGIC_NUMBER)
- {
- clear(); // just to be consistent
- return false;
- }
-
- if (!read_32_or_64(fp, &_table_size)) return false;
- if (!read_32_or_64(fp, &_num_buckets)) return false;
-
- resize(_table_size); // so the vector's sized ok
- for (group_type *group = _first_group; group != _last_group; ++group)
- if (group->read_metadata(_alloc, fp) == false)
- return false;
- return true;
- }
-
- // This code is identical to that for SparseGroup
- // If your keys and values are simple enough, we can write them
- // to disk for you. "simple enough" means no pointers.
- // However, we don't try to normalize endianness
- bool write_nopointer_data(FILE *fp) const
- {
- for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!fwrite(&*it, sizeof(*it), 1, fp))
- return false;
- return true;
- }
-
- // When reading, we have to override the potential const-ness of *it
- bool read_nopointer_data(FILE *fp)
- {
- for (ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!fread(reinterpret_cast<void*>(&(*it)), sizeof(*it), 1, fp))
- return false;
- return true;
- }
-
- // INPUT and OUTPUT must be either a FILE, *or* a C++ stream
- // (istream, ostream, etc) *or* a class providing
- // Read(void*, size_t) and Write(const void*, size_t)
- // (respectively), which writes a buffer into a stream
- // (which the INPUT/OUTPUT instance presumably owns).
-
- typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer;
-
- // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&)
- template <typename ValueSerializer, typename OUTPUT>
- bool serialize(ValueSerializer serializer, OUTPUT *fp)
- {
- if (!write_metadata(fp))
- return false;
- for (const_ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!serializer(fp, *it))
- return false;
- return true;
- }
-
- // ValueSerializer: a functor. operator()(INPUT*, value_type*)
- template <typename ValueSerializer, typename INPUT>
- bool unserialize(ValueSerializer serializer, INPUT *fp)
- {
- clear();
- if (!read_metadata(fp))
- return false;
- for (ne_iterator it = ne_begin(); it != ne_end(); ++it)
- if (!serializer(fp, &*it))
- return false;
- return true;
- }
-
- // Comparisons. Note the comparisons are pretty arbitrary: we
- // compare values of the first index that isn't equal (using default
- // value for empty buckets).
- bool operator==(const sparsetable& x) const
- {
- return (_table_size == x._table_size &&
- _num_buckets == x._num_buckets &&
- _first_group == x._first_group);
- }
-
- bool operator<(const sparsetable& x) const
- {
- return std::lexicographical_compare(begin(), end(), x.begin(), x.end());
- }
- bool operator!=(const sparsetable& x) const { return !(*this == x); }
- bool operator<=(const sparsetable& x) const { return !(x < *this); }
- bool operator>(const sparsetable& x) const { return x < *this; }
- bool operator>=(const sparsetable& x) const { return !(*this < x); }
-
-
-private:
- // The actual data
- // ---------------
- group_type * _first_group;
- group_type * _last_group;
- size_type _table_size; // how many buckets they want
- size_type _num_buckets; // number of non-empty buckets
- group_alloc_type _group_alloc;
- allocator_type _alloc;
-};
-
-// ----------------------------------------------------------------------
-// S P A R S E _ H A S H T A B L E
-// ----------------------------------------------------------------------
-// Hashtable class, used to implement the hashed associative containers
-// hash_set and hash_map.
-//
-// Value: what is stored in the table (each bucket is a Value).
-// Key: something in a 1-to-1 correspondence to a Value, that can be used
-// to search for a Value in the table (find() takes a Key).
-// HashFcn: Takes a Key and returns an integer, the more unique the better.
-// ExtractKey: given a Value, returns the unique Key associated with it.
-// Must inherit from unary_function, or at least have a
-// result_type enum indicating the return type of operator().
-// EqualKey: Given two Keys, says whether they are the same (that is,
-// if they are both associated with the same Value).
-// Alloc: STL allocator to use to allocate memory.
-//
-// ----------------------------------------------------------------------
-
-// The probing method
-// ------------------
-// Linear probing
-// #define JUMP_(key, num_probes) ( 1 )
-// Quadratic probing
-#define JUMP_(key, num_probes) ( num_probes )
-
-
-// -------------------------------------------------------------------
-// -------------------------------------------------------------------
-template <class Value, class Key, class HashFcn,
- class ExtractKey, class SetKey, class EqualKey, class Alloc>
-class sparse_hashtable
-{
-public:
- typedef Key key_type;
- typedef Value value_type;
- typedef HashFcn hasher; // user provided or spp_hash<Key>
- typedef EqualKey key_equal;
- typedef Alloc allocator_type;
-
- typedef typename allocator_type::size_type size_type;
- typedef typename allocator_type::difference_type difference_type;
- typedef value_type& reference;
- typedef const value_type& const_reference;
- typedef value_type* pointer;
- typedef const value_type* const_pointer;
-
- // Table is the main storage class.
- typedef sparsetable<value_type, allocator_type> Table;
- typedef typename Table::ne_iterator ne_it;
- typedef typename Table::const_ne_iterator cne_it;
- typedef typename Table::destructive_iterator dest_it;
- typedef typename Table::ColIterator ColIterator;
-
- typedef ne_it iterator;
- typedef cne_it const_iterator;
- typedef dest_it destructive_iterator;
-
- // These come from tr1. For us they're the same as regular iterators.
- // -------------------------------------------------------------------
- typedef iterator local_iterator;
- typedef const_iterator const_local_iterator;
-
- // How full we let the table get before we resize
- // ----------------------------------------------
- static const int HT_OCCUPANCY_PCT; // = 80 (out of 100);
-
- // How empty we let the table get before we resize lower, by default.
- // (0.0 means never resize lower.)
- // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing
- // ------------------------------------------------------------------
- static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT;
-
- // Minimum size we're willing to let hashtables be.
- // Must be a power of two, and at least 4.
- // Note, however, that for a given hashtable, the initial size is a
- // function of the first constructor arg, and may be >HT_MIN_BUCKETS.
- // ------------------------------------------------------------------
- static const size_type HT_MIN_BUCKETS = 4;
-
- // By default, if you don't specify a hashtable size at
- // construction-time, we use this size. Must be a power of two, and
- // at least HT_MIN_BUCKETS.
- // -----------------------------------------------------------------
- static const size_type HT_DEFAULT_STARTING_BUCKETS = 32;
-
- // iterators
- // ---------
- iterator begin() { return _mk_iterator(table.ne_begin()); }
- iterator end() { return _mk_iterator(table.ne_end()); }
- const_iterator begin() const { return _mk_const_iterator(table.ne_cbegin()); }
- const_iterator end() const { return _mk_const_iterator(table.ne_cend()); }
- const_iterator cbegin() const { return _mk_const_iterator(table.ne_cbegin()); }
- const_iterator cend() const { return _mk_const_iterator(table.ne_cend()); }
-
- // These come from tr1 unordered_map. They iterate over 'bucket' n.
- // For sparsehashtable, we could consider each 'group' to be a bucket,
- // I guess, but I don't really see the point. We'll just consider
- // bucket n to be the n-th element of the sparsetable, if it's occupied,
- // or some empty element, otherwise.
- // ---------------------------------------------------------------------
- local_iterator begin(size_type i)
- {
- return _mk_iterator(table.test(i) ? table.get_iter(i) : table.ne_end());
- }
-
- local_iterator end(size_type i)
- {
- local_iterator it = begin(i);
- if (table.test(i))
- ++it;
- return _mk_iterator(it);
- }
-
- const_local_iterator begin(size_type i) const
- {
- return _mk_const_iterator(table.test(i) ? table.get_iter(i) : table.ne_cend());
- }
-
- const_local_iterator end(size_type i) const
- {
- const_local_iterator it = begin(i);
- if (table.test(i))
- ++it;
- return _mk_const_iterator(it);
- }
-
- const_local_iterator cbegin(size_type i) const { return begin(i); }
- const_local_iterator cend(size_type i) const { return end(i); }
-
- // This is used when resizing
- // --------------------------
- destructive_iterator destructive_begin() { return _mk_destructive_iterator(table.destructive_begin()); }
- destructive_iterator destructive_end() { return _mk_destructive_iterator(table.destructive_end()); }
-
-
- // accessor functions for the things we templatize on, basically
- // -------------------------------------------------------------
- hasher hash_funct() const { return settings; }
- key_equal key_eq() const { return key_info; }
- allocator_type get_allocator() const { return table.get_allocator(); }
-
- // Accessor function for statistics gathering.
- unsigned int num_table_copies() const { return settings.num_ht_copies(); }
-
-private:
- // This is used as a tag for the copy constructor, saying to destroy its
- // arg We have two ways of destructively copying: with potentially growing
- // the hashtable as we copy, and without. To make sure the outside world
- // can't do a destructive copy, we make the typename private.
- // -----------------------------------------------------------------------
- enum MoveDontCopyT {MoveDontCopy, MoveDontGrow};
-
- // creating iterators from sparsetable::ne_iterators
- // -------------------------------------------------
- iterator _mk_iterator(ne_it it) const { return it; }
- const_iterator _mk_const_iterator(cne_it it) const { return it; }
- destructive_iterator _mk_destructive_iterator(dest_it it) const { return it; }
-
-public:
- size_type size() const { return table.num_nonempty(); }
- size_type max_size() const { return table.max_size(); }
- bool empty() const { return size() == 0; }
- size_type bucket_count() const { return table.size(); }
- size_type max_bucket_count() const { return max_size(); }
- // These are tr1 methods. Their idea of 'bucket' doesn't map well to
- // what we do. We just say every bucket has 0 or 1 items in it.
- size_type bucket_size(size_type i) const
- {
- return (size_type)(begin(i) == end(i) ? 0 : 1);
- }
-
-private:
- // Because of the above, size_type(-1) is never legal; use it for errors
- // ---------------------------------------------------------------------
- static const size_type ILLEGAL_BUCKET = size_type(-1);
-
- // Used after a string of deletes. Returns true if we actually shrunk.
- // TODO(csilvers): take a delta so we can take into account inserts
- // done after shrinking. Maybe make part of the Settings class?
- // --------------------------------------------------------------------
- bool _maybe_shrink()
- {
- assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two
- assert(bucket_count() >= HT_MIN_BUCKETS);
- bool retval = false;
-
- // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS,
- // we'll never shrink until you get relatively big, and we'll never
- // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something
- // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will
- // shrink us down to HT_MIN_BUCKETS buckets, which is too small.
- // ---------------------------------------------------------------
- const size_type num_remain = table.num_nonempty();
- const size_type shrink_threshold = settings.shrink_threshold();
- if (shrink_threshold > 0 && num_remain < shrink_threshold &&
- bucket_count() > HT_DEFAULT_STARTING_BUCKETS)
- {
- const float shrink_factor = settings.shrink_factor();
- size_type sz = (size_type)(bucket_count() / 2); // find how much we should shrink
- while (sz > HT_DEFAULT_STARTING_BUCKETS &&
- num_remain < static_cast<size_type>(sz * shrink_factor))
- {
- sz /= 2; // stay a power of 2
- }
- sparse_hashtable tmp(MoveDontCopy, *this, sz);
- swap(tmp); // now we are tmp
- retval = true;
- }
- settings.set_consider_shrink(false); // because we just considered it
- return retval;
- }
-
- // We'll let you resize a hashtable -- though this makes us copy all!
- // When you resize, you say, "make it big enough for this many more elements"
- // Returns true if we actually resized, false if size was already ok.
- // --------------------------------------------------------------------------
- bool _resize_delta(size_type delta)
- {
- bool did_resize = false;
- if (settings.consider_shrink())
- {
- // see if lots of deletes happened
- if (_maybe_shrink())
- did_resize = true;
- }
- if (table.num_nonempty() >=
- (std::numeric_limits<size_type>::max)() - delta)
- {
- throw_exception(std::length_error("resize overflow"));
- }
-
- size_type num_occupied = (size_type)(table.num_nonempty() + num_deleted);
-
- if (bucket_count() >= HT_MIN_BUCKETS &&
- (num_occupied + delta) <= settings.enlarge_threshold())
- return did_resize; // we're ok as we are
-
- // Sometimes, we need to resize just to get rid of all the
- // "deleted" buckets that are clogging up the hashtable. So when
- // deciding whether to resize, count the deleted buckets (which
- // are currently taking up room).
- // -------------------------------------------------------------
- const size_type needed_size =
- settings.min_buckets((size_type)(num_occupied + delta), (size_type)0);
-
- if (needed_size <= bucket_count()) // we have enough buckets
- return did_resize;
-
- size_type resize_to = settings.min_buckets((size_type)(num_occupied + delta), bucket_count());
-
- if (resize_to < needed_size && // may double resize_to
- resize_to < (std::numeric_limits<size_type>::max)() / 2)
- {
- // This situation means that we have enough deleted elements,
- // that once we purge them, we won't actually have needed to
- // grow. But we may want to grow anyway: if we just purge one
- // element, say, we'll have to grow anyway next time we
- // insert. Might as well grow now, since we're already going
- // through the trouble of copying (in order to purge the
- // deleted elements).
- const size_type target =
- static_cast<size_type>(settings.shrink_size((size_type)(resize_to*2)));
- if (table.num_nonempty() + delta >= target)
- {
- // Good, we won't be below the shrink threshhold even if we double.
- resize_to *= 2;
- }
- }
-
- sparse_hashtable tmp(MoveDontCopy, *this, resize_to);
- swap(tmp); // now we are tmp
- return true;
- }
-
- // Used to actually do the rehashing when we grow/shrink a hashtable
- // -----------------------------------------------------------------
- void _copy_from(const sparse_hashtable &ht, size_type min_buckets_wanted)
- {
- clear(); // clear table, set num_deleted to 0
-
- // If we need to change the size of our table, do it now
- const size_type resize_to = settings.min_buckets(ht.size(), min_buckets_wanted);
-
- if (resize_to > bucket_count())
- {
- // we don't have enough buckets
- table.resize(resize_to); // sets the number of buckets
- settings.reset_thresholds(bucket_count());
- }
-
- // We use a normal iterator to get bcks from ht
- // We could use insert() here, but since we know there are
- // no duplicates, we can be more efficient
- assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two
- for (const_iterator it = ht.begin(); it != ht.end(); ++it)
- {
- size_type num_probes = 0; // how many times we've probed
- size_type bucknum;
- const size_type bucket_count_minus_one = bucket_count() - 1;
- for (bucknum = hash(get_key(*it)) & bucket_count_minus_one;
- table.test(bucknum); // table.test() OK since no erase()
- bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one)
- {
- ++num_probes;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- table.set(bucknum, *it); // copies the value to here
- }
- settings.inc_num_ht_copies();
- }
-
- // Implementation is like _copy_from, but it destroys the table of the
- // "from" guy by freeing sparsetable memory as we iterate. This is
- // useful in resizing, since we're throwing away the "from" guy anyway.
- // --------------------------------------------------------------------
- void _move_from(MoveDontCopyT mover, sparse_hashtable &ht,
- size_type min_buckets_wanted)
- {
- clear();
-
- // If we need to change the size of our table, do it now
- size_type resize_to;
- if (mover == MoveDontGrow)
- resize_to = ht.bucket_count(); // keep same size as old ht
- else // MoveDontCopy
- resize_to = settings.min_buckets(ht.size(), min_buckets_wanted);
- if (resize_to > bucket_count())
- {
- // we don't have enough buckets
- table.resize(resize_to); // sets the number of buckets
- settings.reset_thresholds(bucket_count());
- }
-
- // We use a normal iterator to get bcks from ht
- // We could use insert() here, but since we know there are
- // no duplicates, we can be more efficient
- assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two
- const size_type bucket_count_minus_one = (const size_type)(bucket_count() - 1);
-
- // THIS IS THE MAJOR LINE THAT DIFFERS FROM COPY_FROM():
- for (destructive_iterator it = ht.destructive_begin();
- it != ht.destructive_end(); ++it)
- {
- size_type num_probes = 0;
- size_type bucknum;
- for (bucknum = hash(get_key(*it)) & bucket_count_minus_one;
- table.test(bucknum); // table.test() OK since no erase()
- bucknum = (size_type)((bucknum + JUMP_(key, num_probes)) & (bucket_count()-1)))
- {
- ++num_probes;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- table.move(bucknum, *it); // moves the value to here
- }
- settings.inc_num_ht_copies();
- }
-
-
- // Required by the spec for hashed associative container
-public:
- // Though the docs say this should be num_buckets, I think it's much
- // more useful as num_elements. As a special feature, calling with
- // req_elements==0 will cause us to shrink if we can, saving space.
- // -----------------------------------------------------------------
- void resize(size_type req_elements)
- {
- // resize to this or larger
- if (settings.consider_shrink() || req_elements == 0)
- _maybe_shrink();
- if (req_elements > table.num_nonempty()) // we only grow
- _resize_delta((size_type)(req_elements - table.num_nonempty()));
- }
-
- // Get and change the value of shrink_factor and enlarge_factor. The
- // description at the beginning of this file explains how to choose
- // the values. Setting the shrink parameter to 0.0 ensures that the
- // table never shrinks.
- // ------------------------------------------------------------------
- void get_resizing_parameters(float* shrink, float* grow) const
- {
- *shrink = settings.shrink_factor();
- *grow = settings.enlarge_factor();
- }
-
- float get_shrink_factor() const { return settings.shrink_factor(); }
- float get_enlarge_factor() const { return settings.enlarge_factor(); }
-
- void set_resizing_parameters(float shrink, float grow)
- {
- settings.set_resizing_parameters(shrink, grow);
- settings.reset_thresholds(bucket_count());
- }
-
- void set_shrink_factor(float shrink)
- {
- set_resizing_parameters(shrink, get_enlarge_factor());
- }
-
- void set_enlarge_factor(float grow)
- {
- set_resizing_parameters(get_shrink_factor(), grow);
- }
-
- // CONSTRUCTORS -- as required by the specs, we take a size,
- // but also let you specify a hashfunction, key comparator,
- // and key extractor. We also define a copy constructor and =.
- // DESTRUCTOR -- the default is fine, surprisingly.
- // ------------------------------------------------------------
- explicit sparse_hashtable(size_type expected_max_items_in_table = 0,
- const HashFcn& hf = HashFcn(),
- const EqualKey& eql = EqualKey(),
- const ExtractKey& ext = ExtractKey(),
- const SetKey& set = SetKey(),
- const allocator_type& alloc = allocator_type())
- : settings(hf),
- key_info(ext, set, eql),
- num_deleted(0),
- table((expected_max_items_in_table == 0
- ? HT_DEFAULT_STARTING_BUCKETS
- : settings.min_buckets(expected_max_items_in_table, 0)),
- alloc)
- {
- settings.reset_thresholds(bucket_count());
- }
-
- // As a convenience for resize(), we allow an optional second argument
- // which lets you make this new hashtable a different size than ht.
- // We also provide a mechanism of saying you want to "move" the ht argument
- // into us instead of copying.
- // ------------------------------------------------------------------------
- sparse_hashtable(const sparse_hashtable& ht,
- size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS)
- : settings(ht.settings),
- key_info(ht.key_info),
- num_deleted(0),
- table(0)
- {
- settings.reset_thresholds(bucket_count());
- _copy_from(ht, min_buckets_wanted);
- }
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
-
- sparse_hashtable(sparse_hashtable&& o, const allocator_type& alloc = allocator_type()) :
- settings(o.settings),
- key_info(o.key_info),
- num_deleted(0),
- table(HT_DEFAULT_STARTING_BUCKETS, alloc)
- {
- settings.reset_thresholds(bucket_count());
- this->swap(o);
- }
-
- sparse_hashtable& operator=(sparse_hashtable&& o)
- {
- this->swap(o);
- return *this;
- }
-#endif
-
- sparse_hashtable(MoveDontCopyT mover,
- sparse_hashtable& ht,
- size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS)
- : settings(ht.settings),
- key_info(ht.key_info),
- num_deleted(0),
- table(min_buckets_wanted, ht.table.get_allocator())
- //table(min_buckets_wanted)
- {
- settings.reset_thresholds(bucket_count());
- _move_from(mover, ht, min_buckets_wanted);
- }
-
- sparse_hashtable& operator=(const sparse_hashtable& ht)
- {
- if (&ht == this)
- return *this; // don't copy onto ourselves
- settings = ht.settings;
- key_info = ht.key_info;
- num_deleted = ht.num_deleted;
-
- // _copy_from() calls clear and sets num_deleted to 0 too
- _copy_from(ht, HT_MIN_BUCKETS);
-
- // we purposefully don't copy the allocator, which may not be copyable
- return *this;
- }
-
- // Many STL algorithms use swap instead of copy constructors
- void swap(sparse_hashtable& ht)
- {
- using std::swap;
-
- swap(settings, ht.settings);
- swap(key_info, ht.key_info);
- swap(num_deleted, ht.num_deleted);
- table.swap(ht.table);
- settings.reset_thresholds(bucket_count()); // also resets consider_shrink
- ht.settings.reset_thresholds(ht.bucket_count());
- // we purposefully don't swap the allocator, which may not be swap-able
- }
-
- // It's always nice to be able to clear a table without deallocating it
- void clear()
- {
- if (!empty() || num_deleted != 0)
- {
- table.clear();
- table = Table(HT_DEFAULT_STARTING_BUCKETS, table.get_allocator());
- }
- settings.reset_thresholds(bucket_count());
- num_deleted = 0;
- }
-
- // LOOKUP ROUTINES
-private:
-
- enum pos_type { pt_empty = 0, pt_erased, pt_full };
- // -------------------------------------------------------------------
- class Position
- {
- public:
-
- Position() : _t(pt_empty) {}
- Position(pos_type t, size_type idx) : _t(t), _idx(idx) {}
-
- pos_type _t;
- size_type _idx;
- };
-
- // Returns a pair:
- // - 'first' is a code, 2 if key already present, 0 or 1 otherwise.
- // - 'second' is a position, where the key should go
- // Note: because of deletions where-to-insert is not trivial: it's the
- // first deleted bucket we see, as long as we don't find the key later
- // -------------------------------------------------------------------
- Position _find_position(const key_type &key) const
- {
- size_type num_probes = 0; // how many times we've probed
- const size_type bucket_count_minus_one = (const size_type)(bucket_count() - 1);
- size_type bucknum = hash(key) & bucket_count_minus_one;
- Position pos;
-
- while (1)
- {
- // probe until something happens
- // -----------------------------
- typename Table::GrpPos grp_pos(table, bucknum);
-
- if (!grp_pos.test_strict())
- {
- // bucket is empty => key not present
- return pos._t ? pos : Position(pt_empty, bucknum);
- }
- else if (grp_pos.test())
- {
- reference ref(grp_pos.unsafe_get());
-
- if (equals(key, get_key(ref)))
- return Position(pt_full, bucknum);
- }
- else if (pos._t == pt_empty)
- {
- // first erased position
- pos._t = pt_erased;
- pos._idx = bucknum;
- }
-
- ++num_probes; // we're doing another probe
- bucknum = (size_type)((bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one);
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- }
-
-public:
- // I hate to duplicate find() like that, but it is
- // significantly faster to not have the intermediate pair
- // ------------------------------------------------------------------
- iterator find(const key_type& key)
- {
- size_type num_probes = 0; // how many times we've probed
- const size_type bucket_count_minus_one = bucket_count() - 1;
- size_type bucknum = hash(key) & bucket_count_minus_one;
-
- while (1) // probe until something happens
- {
- typename Table::GrpPos grp_pos(table, bucknum);
-
- if (!grp_pos.test_strict())
- return end(); // bucket is empty
- if (grp_pos.test())
- {
- reference ref(grp_pos.unsafe_get());
-
- if (equals(key, get_key(ref)))
- return grp_pos.get_iter(ref);
- }
- ++num_probes; // we're doing another probe
- bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- }
-
- // Wish I could avoid the duplicate find() const and non-const.
- // ------------------------------------------------------------
- const_iterator find(const key_type& key) const
- {
- size_type num_probes = 0; // how many times we've probed
- const size_type bucket_count_minus_one = bucket_count() - 1;
- size_type bucknum = hash(key) & bucket_count_minus_one;
-
- while (1) // probe until something happens
- {
- typename Table::GrpPos grp_pos(table, bucknum);
-
- if (!grp_pos.test_strict())
- return end(); // bucket is empty
- else if (grp_pos.test())
- {
- reference ref(grp_pos.unsafe_get());
-
- if (equals(key, get_key(ref)))
- return _mk_const_iterator(table.get_iter(bucknum, &ref));
- }
- ++num_probes; // we're doing another probe
- bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- }
-
- // This is a tr1 method: the bucket a given key is in, or what bucket
- // it would be put in, if it were to be inserted. Shrug.
- // ------------------------------------------------------------------
- size_type bucket(const key_type& key) const
- {
- Position pos = _find_position(key);
- return pos._idx;
- }
-
- // Counts how many elements have key key. For maps, it's either 0 or 1.
- // ---------------------------------------------------------------------
- size_type count(const key_type &key) const
- {
- Position pos = _find_position(key);
- return (size_type)(pos._t == pt_full ? 1 : 0);
- }
-
- // Likewise, equal_range doesn't really make sense for us. Oh well.
- // -----------------------------------------------------------------
- std::pair<iterator,iterator> equal_range(const key_type& key)
- {
- iterator pos = find(key); // either an iterator or end
- if (pos == end())
- return std::pair<iterator,iterator>(pos, pos);
- else
- {
- const iterator startpos = pos++;
- return std::pair<iterator,iterator>(startpos, pos);
- }
- }
-
- std::pair<const_iterator,const_iterator> equal_range(const key_type& key) const
- {
- const_iterator pos = find(key); // either an iterator or end
- if (pos == end())
- return std::pair<const_iterator,const_iterator>(pos, pos);
- else
- {
- const const_iterator startpos = pos++;
- return std::pair<const_iterator,const_iterator>(startpos, pos);
- }
- }
-
-
- // INSERTION ROUTINES
-private:
- // Private method used by insert_noresize and find_or_insert.
- template <class T>
- reference _insert_at(T& obj, size_type pos, bool erased)
- {
- if (size() >= max_size())
- {
- throw_exception(std::length_error("insert overflow"));
- }
- if (erased)
- {
- assert(num_deleted);
- --num_deleted;
- }
- return table.set(pos, obj);
- }
-
- // If you know *this is big enough to hold obj, use this routine
- template <class T>
- std::pair<iterator, bool> _insert_noresize(T& obj)
- {
- Position pos = _find_position(get_key(obj));
- bool already_there = (pos._t == pt_full);
-
- if (!already_there)
- {
- reference ref(_insert_at(obj, pos._idx, pos._t == pt_erased));
- return std::pair<iterator, bool>(_mk_iterator(table.get_iter(pos._idx, &ref)), true);
- }
- return std::pair<iterator,bool>(_mk_iterator(table.get_iter(pos._idx)), false);
- }
-
- // Specializations of insert(it, it) depending on the power of the iterator:
- // (1) Iterator supports operator-, resize before inserting
- template <class ForwardIterator>
- void _insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag /*unused*/)
- {
- int64_t dist = std::distance(f, l);
- if (dist < 0 || static_cast<size_t>(dist) >= (std::numeric_limits<size_type>::max)())
- throw_exception(std::length_error("insert-range overflow"));
-
- _resize_delta(static_cast<size_type>(dist));
-
- for (; dist > 0; --dist, ++f)
- _insert_noresize(*f);
- }
-
- // (2) Arbitrary iterator, can't tell how much to resize
- template <class InputIterator>
- void _insert(InputIterator f, InputIterator l, std::input_iterator_tag /*unused*/)
- {
- for (; f != l; ++f)
- _insert(*f);
- }
-
-public:
-
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class... Args>
- std::pair<iterator, bool> emplace(Args&&... args)
- {
- _resize_delta(1);
- value_type obj(std::forward<Args>(args)...);
- return _insert_noresize(obj);
- }
-#endif
-
- // This is the normal insert routine, used by the outside world
- std::pair<iterator, bool> insert(const_reference obj)
- {
- _resize_delta(1); // adding an object, grow if need be
- return _insert_noresize(obj);
- }
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- template< class P >
- std::pair<iterator, bool> insert(P &&obj)
- {
- _resize_delta(1); // adding an object, grow if need be
- value_type val(std::forward<P>(obj));
- return _insert_noresize(val);
- }
-#endif
-
- // When inserting a lot at a time, we specialize on the type of iterator
- template <class InputIterator>
- void insert(InputIterator f, InputIterator l)
- {
- // specializes on iterator type
- _insert(f, l,
- typename std::iterator_traits<InputIterator>::iterator_category());
- }
-
- // DefaultValue is a functor that takes a key and returns a value_type
- // representing the default value to be inserted if none is found.
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class DefaultValue, class KT>
- value_type& find_or_insert(KT&& key)
-#else
- template <class DefaultValue>
- value_type& find_or_insert(const key_type& key)
-#endif
- {
- size_type num_probes = 0; // how many times we've probed
- const size_type bucket_count_minus_one = bucket_count() - 1;
- size_type bucknum = hash(key) & bucket_count_minus_one;
- DefaultValue default_value;
- size_type erased_pos = 0;
- bool erased = false;
-
- while (1) // probe until something happens
- {
- typename Table::GrpPos grp_pos(table, bucknum);
-
- if (!grp_pos.test_strict())
- {
- // not found
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- auto&& def(default_value(std::forward<KT>(key)));
-#else
- value_type def(default_value(key));
-#endif
- if (_resize_delta(1))
- {
- // needed to rehash to make room
- // Since we resized, we can't use pos, so recalculate where to insert.
- return *(_insert_noresize(def).first);
- }
- else
- {
- // no need to rehash, insert right here
- return _insert_at(def, erased ? erased_pos : bucknum, erased);
- }
- }
- if (grp_pos.test())
- {
- reference ref(grp_pos.unsafe_get());
-
- if (equals(key, get_key(ref)))
- return ref;
- }
- else if (!erased)
- {
- // first erased position
- erased_pos = bucknum;
- erased = true;
- }
-
- ++num_probes; // we're doing another probe
- bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- }
-
- size_type erase(const key_type& key)
- {
- size_type num_probes = 0; // how many times we've probed
- const size_type bucket_count_minus_one = bucket_count() - 1;
- size_type bucknum = hash(key) & bucket_count_minus_one;
-
- while (1) // probe until something happens
- {
- typename Table::GrpPos grp_pos(table, bucknum);
-
- if (!grp_pos.test_strict())
- return 0; // bucket is empty, we deleted nothing
- if (grp_pos.test())
- {
- reference ref(grp_pos.unsafe_get());
-
- if (equals(key, get_key(ref)))
- {
- grp_pos.erase(table);
- ++num_deleted;
- settings.set_consider_shrink(true); // will think about shrink after next insert
- return 1; // because we deleted one thing
- }
- }
- ++num_probes; // we're doing another probe
- bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one;
- assert(num_probes < bucket_count()
- && "Hashtable is full: an error in key_equal<> or hash<>");
- }
- }
-
- const_iterator erase(const_iterator pos)
- {
- if (pos == cend())
- return cend(); // sanity check
-
- const_iterator nextpos = table.erase(pos);
- ++num_deleted;
- settings.set_consider_shrink(true);
- return nextpos;
- }
-
- const_iterator erase(const_iterator f, const_iterator l)
- {
- if (f == cend())
- return cend(); // sanity check
-
- size_type num_before = table.num_nonempty();
- const_iterator nextpos = table.erase(f, l);
- num_deleted += num_before - table.num_nonempty();
- settings.set_consider_shrink(true);
- return nextpos;
- }
-
- // Deleted key routines - just to keep google test framework happy
- // we don't actually use the deleted key
- // ---------------------------------------------------------------
- void set_deleted_key(const key_type&)
- {
- }
-
- void clear_deleted_key()
- {
- }
-
- bool operator==(const sparse_hashtable& ht) const
- {
- if (this == &ht)
- return true;
-
- if (size() != ht.size())
- return false;
-
- for (const_iterator it = begin(); it != end(); ++it)
- {
- const_iterator it2 = ht.find(get_key(*it));
- if ((it2 == ht.end()) || (*it != *it2))
- return false;
- }
-
- return true;
- }
-
- bool operator!=(const sparse_hashtable& ht) const
- {
- return !(*this == ht);
- }
-
-
- // I/O
- // We support reading and writing hashtables to disk. NOTE that
- // this only stores the hashtable metadata, not the stuff you've
- // actually put in the hashtable! Alas, since I don't know how to
- // write a hasher or key_equal, you have to make sure everything
- // but the table is the same. We compact before writing.
- //
- // The OUTPUT type needs to support a Write() operation. File and
- // OutputBuffer are appropriate types to pass in.
- //
- // The INPUT type needs to support a Read() operation. File and
- // InputBuffer are appropriate types to pass in.
- // -------------------------------------------------------------
- template <typename OUTPUT>
- bool write_metadata(OUTPUT *fp)
- {
- return table.write_metadata(fp);
- }
-
- template <typename INPUT>
- bool read_metadata(INPUT *fp)
- {
- num_deleted = 0; // since we got rid before writing
- const bool result = table.read_metadata(fp);
- settings.reset_thresholds(bucket_count());
- return result;
- }
-
- // Only meaningful if value_type is a POD.
- template <typename OUTPUT>
- bool write_nopointer_data(OUTPUT *fp)
- {
- return table.write_nopointer_data(fp);
- }
-
- // Only meaningful if value_type is a POD.
- template <typename INPUT>
- bool read_nopointer_data(INPUT *fp)
- {
- return table.read_nopointer_data(fp);
- }
-
- // INPUT and OUTPUT must be either a FILE, *or* a C++ stream
- // (istream, ostream, etc) *or* a class providing
- // Read(void*, size_t) and Write(const void*, size_t)
- // (respectively), which writes a buffer into a stream
- // (which the INPUT/OUTPUT instance presumably owns).
-
- typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer;
-
- // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&)
- template <typename ValueSerializer, typename OUTPUT>
- bool serialize(ValueSerializer serializer, OUTPUT *fp)
- {
- return table.serialize(serializer, fp);
- }
-
- // ValueSerializer: a functor. operator()(INPUT*, value_type*)
- template <typename ValueSerializer, typename INPUT>
- bool unserialize(ValueSerializer serializer, INPUT *fp)
- {
- num_deleted = 0; // since we got rid before writing
- const bool result = table.unserialize(serializer, fp);
- settings.reset_thresholds(bucket_count());
- return result;
- }
-
-private:
-
- // Package templated functors with the other types to eliminate memory
- // needed for storing these zero-size operators. Since ExtractKey and
- // hasher's operator() might have the same function signature, they
- // must be packaged in different classes.
- // -------------------------------------------------------------------------
- struct Settings :
- sparsehash_internal::sh_hashtable_settings<key_type, hasher,
- size_type, HT_MIN_BUCKETS>
- {
- explicit Settings(const hasher& hf)
- : sparsehash_internal::sh_hashtable_settings<key_type, hasher, size_type,
- HT_MIN_BUCKETS>
- (hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {}
- };
-
- // KeyInfo stores delete key and packages zero-size functors:
- // ExtractKey and SetKey.
- // ---------------------------------------------------------
- class KeyInfo : public ExtractKey, public SetKey, public EqualKey
- {
- public:
- KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq)
- : ExtractKey(ek), SetKey(sk), EqualKey(eq)
- {
- }
-
- // We want to return the exact same type as ExtractKey: Key or const Key&
- typename ExtractKey::result_type get_key(const_reference v) const
- {
- return ExtractKey::operator()(v);
- }
-
- bool equals(const key_type& a, const key_type& b) const
- {
- return EqualKey::operator()(a, b);
- }
- };
-
- // Utility functions to access the templated operators
- size_t hash(const key_type& v) const
- {
- return settings.hash(v);
- }
-
- bool equals(const key_type& a, const key_type& b) const
- {
- return key_info.equals(a, b);
- }
-
- typename ExtractKey::result_type get_key(const_reference v) const
- {
- return key_info.get_key(v);
- }
-
-private:
- // Actual data
- // -----------
- Settings settings;
- KeyInfo key_info;
- size_type num_deleted;
- Table table; // holds num_buckets and num_elements too
-};
-
-#undef JUMP_
-
-// -----------------------------------------------------------------------------
-template <class V, class K, class HF, class ExK, class SetK, class EqK, class A>
-const typename sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type
-sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET;
-
-// How full we let the table get before we resize. Knuth says .8 is
-// good -- higher causes us to probe too much, though saves memory
-// -----------------------------------------------------------------------------
-template <class V, class K, class HF, class ExK, class SetK, class EqK, class A>
-const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 50;
-
-// How empty we let the table get before we resize lower.
-// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing
-// -----------------------------------------------------------------------------
-template <class V, class K, class HF, class ExK, class SetK, class EqK, class A>
-const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT
-= static_cast<int>(0.4 *
- sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT);
-
-
-// ----------------------------------------------------------------------
-// S P A R S E _ H A S H _ M A P
-// ----------------------------------------------------------------------
-template <class Key, class T,
- class HashFcn = spp_hash<Key>,
- class EqualKey = std::equal_to<Key>,
- class Alloc = SPP_DEFAULT_ALLOCATOR<std::pair<const Key, T> > >
-class sparse_hash_map
-{
-public:
- typedef typename std::pair<const Key, T> value_type;
-
-private:
- // Apparently select1st is not stl-standard, so we define our own
- struct SelectKey
- {
- typedef const Key& result_type;
-
- inline const Key& operator()(const value_type& p) const
- {
- return p.first;
- }
- };
-
- struct SetKey
- {
- inline void operator()(value_type* value, const Key& new_key) const
- {
- *const_cast<Key*>(&value->first) = new_key;
- }
- };
-
- // For operator[].
- struct DefaultValue
- {
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class KT>
- inline value_type operator()(KT&& key) const
- {
- return { std::forward<KT>(key), T() };
- }
-#else
- inline value_type operator()(const Key& key) const
- {
- return std::make_pair(key, T());
- }
-#endif
- };
-
- // The actual data
- typedef sparse_hashtable<value_type, Key, HashFcn, SelectKey,
- SetKey, EqualKey, Alloc> ht;
-
-public:
- typedef typename ht::key_type key_type;
- typedef T data_type;
- typedef T mapped_type;
- typedef typename ht::hasher hasher;
- typedef typename ht::key_equal key_equal;
- typedef Alloc allocator_type;
-
- typedef typename ht::size_type size_type;
- typedef typename ht::difference_type difference_type;
- typedef typename ht::pointer pointer;
- typedef typename ht::const_pointer const_pointer;
- typedef typename ht::reference reference;
- typedef typename ht::const_reference const_reference;
-
- typedef typename ht::iterator iterator;
- typedef typename ht::const_iterator const_iterator;
- typedef typename ht::local_iterator local_iterator;
- typedef typename ht::const_local_iterator const_local_iterator;
-
- // Iterator functions
- iterator begin() { return rep.begin(); }
- iterator end() { return rep.end(); }
- const_iterator begin() const { return rep.cbegin(); }
- const_iterator end() const { return rep.cend(); }
- const_iterator cbegin() const { return rep.cbegin(); }
- const_iterator cend() const { return rep.cend(); }
-
- // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements.
- local_iterator begin(size_type i) { return rep.begin(i); }
- local_iterator end(size_type i) { return rep.end(i); }
- const_local_iterator begin(size_type i) const { return rep.begin(i); }
- const_local_iterator end(size_type i) const { return rep.end(i); }
- const_local_iterator cbegin(size_type i) const { return rep.cbegin(i); }
- const_local_iterator cend(size_type i) const { return rep.cend(i); }
-
- // Accessor functions
- // ------------------
- allocator_type get_allocator() const { return rep.get_allocator(); }
- hasher hash_funct() const { return rep.hash_funct(); }
- hasher hash_function() const { return hash_funct(); }
- key_equal key_eq() const { return rep.key_eq(); }
-
-
- // Constructors
- // ------------
- explicit sparse_hash_map(size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type())
- : rep(n, hf, eql, SelectKey(), SetKey(), alloc)
- {
- }
-
- explicit sparse_hash_map(const allocator_type& alloc) :
- rep(0, hasher(), key_equal(), SelectKey(), SetKey(), alloc)
- {
- }
-
- sparse_hash_map(size_type n, const allocator_type& alloc) :
- rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc)
- {
- }
-
- sparse_hash_map(size_type n, const hasher& hf, const allocator_type& alloc) :
- rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc)
- {
- }
-
- template <class InputIterator>
- sparse_hash_map(InputIterator f, InputIterator l,
- size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type())
- : rep(n, hf, eql, SelectKey(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- template <class InputIterator>
- sparse_hash_map(InputIterator f, InputIterator l,
- size_type n, const allocator_type& alloc)
- : rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- template <class InputIterator>
- sparse_hash_map(InputIterator f, InputIterator l,
- size_type n, const hasher& hf, const allocator_type& alloc)
- : rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- sparse_hash_map(const sparse_hash_map &o) :
- rep(o.rep)
- {}
-
- sparse_hash_map(const sparse_hash_map &o,
- const allocator_type& alloc) :
- rep(o.rep, alloc)
- {}
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- sparse_hash_map(sparse_hash_map &&o) :
- rep(std::move(o.rep))
- {}
-
- sparse_hash_map(sparse_hash_map &&o,
- const allocator_type& alloc) :
- rep(std::move(o.rep), alloc)
- {}
-
- sparse_hash_map& operator=(sparse_hash_map &&o)
- {
- rep = std::move(o.rep);
- }
-#endif
-
-#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST)
- sparse_hash_map(std::initializer_list<value_type> init,
- size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type())
- : rep(n, hf, eql, SelectKey(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_map(std::initializer_list<value_type> init,
- size_type n, const allocator_type& alloc) :
- rep(n, hasher(), key_equal(), SelectKey(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_map(std::initializer_list<value_type> init,
- size_type n, const hasher& hf, const allocator_type& alloc) :
- rep(n, hf, key_equal(), SelectKey(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_map& operator=(std::initializer_list<value_type> init)
- {
- rep.clear();
- rep.insert(init.begin(), init.end());
- return *this;
- }
-
- void insert(std::initializer_list<value_type> init)
- {
- rep.insert(init.begin(), init.end());
- }
-#endif
-
- sparse_hash_map& operator=(const sparse_hash_map &o)
- {
- rep = o.rep;
- return *this;
- }
-
- void clear() { rep.clear(); }
- void swap(sparse_hash_map& hs) { rep.swap(hs.rep); }
-
- // Functions concerning size
- // -------------------------
- size_type size() const { return rep.size(); }
- size_type max_size() const { return rep.max_size(); }
- bool empty() const { return rep.empty(); }
- size_type bucket_count() const { return rep.bucket_count(); }
- size_type max_bucket_count() const { return rep.max_bucket_count(); }
-
- size_type bucket_size(size_type i) const { return rep.bucket_size(i); }
- size_type bucket(const key_type& key) const { return rep.bucket(key); }
- float load_factor() const { return size() * 1.0f / bucket_count(); }
-
- float max_load_factor() const { return rep.get_enlarge_factor(); }
- void max_load_factor(float grow) { rep.set_enlarge_factor(grow); }
-
- float min_load_factor() const { return rep.get_shrink_factor(); }
- void min_load_factor(float shrink){ rep.set_shrink_factor(shrink); }
-
- void set_resizing_parameters(float shrink, float grow)
- {
- rep.set_resizing_parameters(shrink, grow);
- }
-
- void resize(size_type cnt) { rep.resize(cnt); }
- void rehash(size_type cnt) { resize(cnt); } // c++11 name
- void reserve(size_type cnt) { resize(cnt); } // c++11
-
- // Lookup
- // ------
- iterator find(const key_type& key) { return rep.find(key); }
- const_iterator find(const key_type& key) const { return rep.find(key); }
- bool contains(const key_type& key) const { return rep.find(key) != rep.end(); }
-
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class KT>
- mapped_type& operator[](KT&& key)
- {
- return rep.template find_or_insert<DefaultValue>(std::forward<KT>(key)).second;
- }
-#else
- mapped_type& operator[](const key_type& key)
- {
- return rep.template find_or_insert<DefaultValue>(key).second;
- }
-#endif
-
- size_type count(const key_type& key) const { return rep.count(key); }
-
- std::pair<iterator, iterator>
- equal_range(const key_type& key) { return rep.equal_range(key); }
-
- std::pair<const_iterator, const_iterator>
- equal_range(const key_type& key) const { return rep.equal_range(key); }
-
- mapped_type& at(const key_type& key)
- {
- iterator it = rep.find(key);
- if (it == rep.end())
- throw_exception(std::out_of_range("at: key not present"));
- return it->second;
- }
-
- const mapped_type& at(const key_type& key) const
- {
- const_iterator it = rep.find(key);
- if (it == rep.cend())
- throw_exception(std::out_of_range("at: key not present"));
- return it->second;
- }
-
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class... Args>
- std::pair<iterator, bool> emplace(Args&&... args)
- {
- return rep.emplace(std::forward<Args>(args)...);
- }
-
- template <class... Args>
- iterator emplace_hint(const_iterator , Args&&... args)
- {
- return rep.emplace(std::forward<Args>(args)...).first;
- }
-#endif
-
- // Insert
- // ------
- std::pair<iterator, bool>
- insert(const value_type& obj) { return rep.insert(obj); }
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- template< class P >
- std::pair<iterator, bool> insert(P&& obj) { return rep.insert(std::forward<P>(obj)); }
-#endif
-
- template <class InputIterator>
- void insert(InputIterator f, InputIterator l) { rep.insert(f, l); }
-
- void insert(const_iterator f, const_iterator l) { rep.insert(f, l); }
-
- iterator insert(iterator /*unused*/, const value_type& obj) { return insert(obj).first; }
- iterator insert(const_iterator /*unused*/, const value_type& obj) { return insert(obj).first; }
-
- // Deleted key routines - just to keep google test framework happy
- // we don't actually use the deleted key
- // ---------------------------------------------------------------
- void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); }
- void clear_deleted_key() { rep.clear_deleted_key(); }
- key_type deleted_key() const { return rep.deleted_key(); }
-
- // Erase
- // -----
- size_type erase(const key_type& key) { return rep.erase(key); }
- iterator erase(iterator it) { return rep.erase(it); }
- iterator erase(iterator f, iterator l) { return rep.erase(f, l); }
- iterator erase(const_iterator it) { return rep.erase(it); }
- iterator erase(const_iterator f, const_iterator l){ return rep.erase(f, l); }
-
- // Comparison
- // ----------
- bool operator==(const sparse_hash_map& hs) const { return rep == hs.rep; }
- bool operator!=(const sparse_hash_map& hs) const { return rep != hs.rep; }
-
-
- // I/O -- this is an add-on for writing metainformation to disk
- //
- // For maximum flexibility, this does not assume a particular
- // file type (though it will probably be a FILE *). We just pass
- // the fp through to rep.
-
- // If your keys and values are simple enough, you can pass this
- // serializer to serialize()/unserialize(). "Simple enough" means
- // value_type is a POD type that contains no pointers. Note,
- // however, we don't try to normalize endianness.
- // ---------------------------------------------------------------
- typedef typename ht::NopointerSerializer NopointerSerializer;
-
- // serializer: a class providing operator()(OUTPUT*, const value_type&)
- // (writing value_type to OUTPUT). You can specify a
- // NopointerSerializer object if appropriate (see above).
- // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a
- // pointer to a class providing size_t Write(const void*, size_t),
- // which writes a buffer into a stream (which fp presumably
- // owns) and returns the number of bytes successfully written.
- // Note basic_ostream<not_char> is not currently supported.
- // ---------------------------------------------------------------
- template <typename ValueSerializer, typename OUTPUT>
- bool serialize(ValueSerializer serializer, OUTPUT* fp)
- {
- return rep.serialize(serializer, fp);
- }
-
- // serializer: a functor providing operator()(INPUT*, value_type*)
- // (reading from INPUT and into value_type). You can specify a
- // NopointerSerializer object if appropriate (see above).
- // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a
- // pointer to a class providing size_t Read(void*, size_t),
- // which reads into a buffer from a stream (which fp presumably
- // owns) and returns the number of bytes successfully read.
- // Note basic_istream<not_char> is not currently supported.
- // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer
- // may need to do a const cast in order to fill in the key.
- // NOTE: if Key or T are not POD types, the serializer MUST use
- // placement-new to initialize their values, rather than a normal
- // equals-assignment or similar. (The value_type* passed into the
- // serializer points to garbage memory.)
- // ---------------------------------------------------------------
- template <typename ValueSerializer, typename INPUT>
- bool unserialize(ValueSerializer serializer, INPUT* fp)
- {
- return rep.unserialize(serializer, fp);
- }
-
- // The four methods below are DEPRECATED.
- // Use serialize() and unserialize() for new code.
- // -----------------------------------------------
- template <typename OUTPUT>
- bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); }
-
- template <typename INPUT>
- bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); }
-
- template <typename OUTPUT>
- bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); }
-
- template <typename INPUT>
- bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); }
-
-
-private:
- // The actual data
- // ---------------
- ht rep;
-};
-
-// ----------------------------------------------------------------------
-// S P A R S E _ H A S H _ S E T
-// ----------------------------------------------------------------------
-
-template <class Value,
- class HashFcn = spp_hash<Value>,
- class EqualKey = std::equal_to<Value>,
- class Alloc = SPP_DEFAULT_ALLOCATOR<Value> >
-class sparse_hash_set
-{
-private:
- // Apparently identity is not stl-standard, so we define our own
- struct Identity
- {
- typedef const Value& result_type;
- inline const Value& operator()(const Value& v) const { return v; }
- };
-
- struct SetKey
- {
- inline void operator()(Value* value, const Value& new_key) const
- {
- *value = new_key;
- }
- };
-
- typedef sparse_hashtable<Value, Value, HashFcn, Identity, SetKey,
- EqualKey, Alloc> ht;
-
-public:
- typedef typename ht::key_type key_type;
- typedef typename ht::value_type value_type;
- typedef typename ht::hasher hasher;
- typedef typename ht::key_equal key_equal;
- typedef Alloc allocator_type;
-
- typedef typename ht::size_type size_type;
- typedef typename ht::difference_type difference_type;
- typedef typename ht::const_pointer pointer;
- typedef typename ht::const_pointer const_pointer;
- typedef typename ht::const_reference reference;
- typedef typename ht::const_reference const_reference;
-
- typedef typename ht::const_iterator iterator;
- typedef typename ht::const_iterator const_iterator;
- typedef typename ht::const_local_iterator local_iterator;
- typedef typename ht::const_local_iterator const_local_iterator;
-
-
- // Iterator functions -- recall all iterators are const
- iterator begin() const { return rep.begin(); }
- iterator end() const { return rep.end(); }
- const_iterator cbegin() const { return rep.cbegin(); }
- const_iterator cend() const { return rep.cend(); }
-
- // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements.
- local_iterator begin(size_type i) const { return rep.begin(i); }
- local_iterator end(size_type i) const { return rep.end(i); }
- local_iterator cbegin(size_type i) const { return rep.cbegin(i); }
- local_iterator cend(size_type i) const { return rep.cend(i); }
-
-
- // Accessor functions
- // ------------------
- allocator_type get_allocator() const { return rep.get_allocator(); }
- hasher hash_funct() const { return rep.hash_funct(); }
- hasher hash_function() const { return hash_funct(); } // tr1 name
- key_equal key_eq() const { return rep.key_eq(); }
-
-
- // Constructors
- // ------------
- explicit sparse_hash_set(size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type()) :
- rep(n, hf, eql, Identity(), SetKey(), alloc)
- {
- }
-
- explicit sparse_hash_set(const allocator_type& alloc) :
- rep(0, hasher(), key_equal(), Identity(), SetKey(), alloc)
- {
- }
-
- sparse_hash_set(size_type n, const allocator_type& alloc) :
- rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc)
- {
- }
-
- sparse_hash_set(size_type n, const hasher& hf,
- const allocator_type& alloc) :
- rep(n, hf, key_equal(), Identity(), SetKey(), alloc)
- {
- }
-
- template <class InputIterator>
- sparse_hash_set(InputIterator f, InputIterator l,
- size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type())
- : rep(n, hf, eql, Identity(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- template <class InputIterator>
- sparse_hash_set(InputIterator f, InputIterator l,
- size_type n, const allocator_type& alloc)
- : rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- template <class InputIterator>
- sparse_hash_set(InputIterator f, InputIterator l,
- size_type n, const hasher& hf, const allocator_type& alloc)
- : rep(n, hf, key_equal(), Identity(), SetKey(), alloc)
- {
- rep.insert(f, l);
- }
-
- sparse_hash_set(const sparse_hash_set &o) :
- rep(o.rep)
- {}
-
- sparse_hash_set(const sparse_hash_set &o,
- const allocator_type& alloc) :
- rep(o.rep, alloc)
- {}
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- sparse_hash_set(sparse_hash_set &&o) :
- rep(std::move(o.rep))
- {}
-
- sparse_hash_set(sparse_hash_set &&o,
- const allocator_type& alloc) :
- rep(std::move(o.rep), alloc)
- {}
-#endif
-
-#if !defined(SPP_NO_CXX11_HDR_INITIALIZER_LIST)
- sparse_hash_set(std::initializer_list<value_type> init,
- size_type n = 0,
- const hasher& hf = hasher(),
- const key_equal& eql = key_equal(),
- const allocator_type& alloc = allocator_type()) :
- rep(n, hf, eql, Identity(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_set(std::initializer_list<value_type> init,
- size_type n, const allocator_type& alloc) :
- rep(n, hasher(), key_equal(), Identity(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_set(std::initializer_list<value_type> init,
- size_type n, const hasher& hf,
- const allocator_type& alloc) :
- rep(n, hf, key_equal(), Identity(), SetKey(), alloc)
- {
- rep.insert(init.begin(), init.end());
- }
-
- sparse_hash_set& operator=(std::initializer_list<value_type> init)
- {
- rep.clear();
- rep.insert(init.begin(), init.end());
- return *this;
- }
-
- void insert(std::initializer_list<value_type> init)
- {
- rep.insert(init.begin(), init.end());
- }
-
-#endif
-
- sparse_hash_set& operator=(const sparse_hash_set &o)
- {
- rep = o.rep;
- return *this;
- }
-
- void clear() { rep.clear(); }
- void swap(sparse_hash_set& hs) { rep.swap(hs.rep); }
-
-
- // Functions concerning size
- // -------------------------
- size_type size() const { return rep.size(); }
- size_type max_size() const { return rep.max_size(); }
- bool empty() const { return rep.empty(); }
- size_type bucket_count() const { return rep.bucket_count(); }
- size_type max_bucket_count() const { return rep.max_bucket_count(); }
-
- size_type bucket_size(size_type i) const { return rep.bucket_size(i); }
- size_type bucket(const key_type& key) const { return rep.bucket(key); }
-
- float load_factor() const { return size() * 1.0f / bucket_count(); }
-
- float max_load_factor() const { return rep.get_enlarge_factor(); }
- void max_load_factor(float grow) { rep.set_enlarge_factor(grow); }
-
- float min_load_factor() const { return rep.get_shrink_factor(); }
- void min_load_factor(float shrink){ rep.set_shrink_factor(shrink); }
-
- void set_resizing_parameters(float shrink, float grow)
- {
- rep.set_resizing_parameters(shrink, grow);
- }
-
- void resize(size_type cnt) { rep.resize(cnt); }
- void rehash(size_type cnt) { resize(cnt); } // c++11 name
- void reserve(size_type cnt) { resize(cnt); } // c++11
-
- // Lookup
- // ------
- iterator find(const key_type& key) const { return rep.find(key); }
- bool contains(const key_type& key) const { return rep.find(key) != rep.end(); }
-
- size_type count(const key_type& key) const { return rep.count(key); }
-
- std::pair<iterator, iterator>
- equal_range(const key_type& key) const { return rep.equal_range(key); }
-
-#if !defined(SPP_NO_CXX11_VARIADIC_TEMPLATES)
- template <class... Args>
- std::pair<iterator, bool> emplace(Args&&... args)
- {
- return rep.emplace(std::forward<Args>(args)...);
- }
-
- template <class... Args>
- iterator emplace_hint(const_iterator , Args&&... args)
- {
- return rep.emplace(std::forward<Args>(args)...).first;
- }
-#endif
-
- // Insert
- // ------
- std::pair<iterator, bool> insert(const value_type& obj)
- {
- std::pair<typename ht::iterator, bool> p = rep.insert(obj);
- return std::pair<iterator, bool>(p.first, p.second); // const to non-const
- }
-
-#if !defined(SPP_NO_CXX11_RVALUE_REFERENCES)
- template<class P>
- std::pair<iterator, bool> insert(P&& obj) { return rep.insert(std::forward<P>(obj)); }
-#endif
-
- template <class InputIterator>
- void insert(InputIterator f, InputIterator l) { rep.insert(f, l); }
-
- void insert(const_iterator f, const_iterator l) { rep.insert(f, l); }
-
- iterator insert(iterator /*unused*/, const value_type& obj) { return insert(obj).first; }
-
- // Deleted key - do nothing - just to keep google test framework happy
- // -------------------------------------------------------------------
- void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); }
- void clear_deleted_key() { rep.clear_deleted_key(); }
- key_type deleted_key() const { return rep.deleted_key(); }
-
- // Erase
- // -----
- size_type erase(const key_type& key) { return rep.erase(key); }
- iterator erase(iterator it) { return rep.erase(it); }
- iterator erase(iterator f, iterator l) { return rep.erase(f, l); }
-
- // Comparison
- // ----------
- bool operator==(const sparse_hash_set& hs) const { return rep == hs.rep; }
- bool operator!=(const sparse_hash_set& hs) const { return rep != hs.rep; }
-
-
- // I/O -- this is an add-on for writing metainformation to disk
- //
- // For maximum flexibility, this does not assume a particular
- // file type (though it will probably be a FILE *). We just pass
- // the fp through to rep.
-
- // If your keys and values are simple enough, you can pass this
- // serializer to serialize()/unserialize(). "Simple enough" means
- // value_type is a POD type that contains no pointers. Note,
- // however, we don't try to normalize endianness.
- // ---------------------------------------------------------------
- typedef typename ht::NopointerSerializer NopointerSerializer;
-
- // serializer: a class providing operator()(OUTPUT*, const value_type&)
- // (writing value_type to OUTPUT). You can specify a
- // NopointerSerializer object if appropriate (see above).
- // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a
- // pointer to a class providing size_t Write(const void*, size_t),
- // which writes a buffer into a stream (which fp presumably
- // owns) and returns the number of bytes successfully written.
- // Note basic_ostream<not_char> is not currently supported.
- // ---------------------------------------------------------------
- template <typename ValueSerializer, typename OUTPUT>
- bool serialize(ValueSerializer serializer, OUTPUT* fp)
- {
- return rep.serialize(serializer, fp);
- }
-
- // serializer: a functor providing operator()(INPUT*, value_type*)
- // (reading from INPUT and into value_type). You can specify a
- // NopointerSerializer object if appropriate (see above).
- // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a
- // pointer to a class providing size_t Read(void*, size_t),
- // which reads into a buffer from a stream (which fp presumably
- // owns) and returns the number of bytes successfully read.
- // Note basic_istream<not_char> is not currently supported.
- // NOTE: Since value_type is const Key, ValueSerializer
- // may need to do a const cast in order to fill in the key.
- // NOTE: if Key is not a POD type, the serializer MUST use
- // placement-new to initialize its value, rather than a normal
- // equals-assignment or similar. (The value_type* passed into
- // the serializer points to garbage memory.)
- // ---------------------------------------------------------------
- template <typename ValueSerializer, typename INPUT>
- bool unserialize(ValueSerializer serializer, INPUT* fp)
- {
- return rep.unserialize(serializer, fp);
- }
-
- // The four methods below are DEPRECATED.
- // Use serialize() and unserialize() for new code.
- // -----------------------------------------------
- template <typename OUTPUT>
- bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); }
-
- template <typename INPUT>
- bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); }
-
- template <typename OUTPUT>
- bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); }
-
- template <typename INPUT>
- bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); }
-
-private:
- // The actual data
- // ---------------
- ht rep;
-};
-
-} // spp_ namespace
-
-
-// We need a global swap for all our classes as well
-// -------------------------------------------------
-
-template <class T, class Alloc>
-inline void swap(spp_::sparsegroup<T,Alloc> &x, spp_::sparsegroup<T,Alloc> &y)
-{
- x.swap(y);
-}
-
-template <class T, class Alloc>
-inline void swap(spp_::sparsetable<T,Alloc> &x, spp_::sparsetable<T,Alloc> &y)
-{
- x.swap(y);
-}
-
-template <class V, class K, class HF, class ExK, class SetK, class EqK, class A>
-inline void swap(spp_::sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &x,
- spp_::sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &y)
-{
- x.swap(y);
-}
-
-template <class Key, class T, class HashFcn, class EqualKey, class Alloc>
-inline void swap(spp_::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1,
- spp_::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2)
-{
- hm1.swap(hm2);
-}
-
-template <class Val, class HashFcn, class EqualKey, class Alloc>
-inline void swap(spp_::sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1,
- spp_::sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2)
-{
- hs1.swap(hs2);
-}
-
-#endif // sparsepp_h_guard_
diff --git a/benchmarks/others/sparsepp/spp_config.h b/benchmarks/others/sparsepp/spp_config.h
deleted file mode 100644
index 46eeee5c..00000000
--- a/benchmarks/others/sparsepp/spp_config.h
+++ /dev/null
@@ -1,781 +0,0 @@
-#if !defined(spp_config_h_guard)
-#define spp_config_h_guard
-
-// --------------------------------------------------
-// Sparsepp config macros
-// some can be overriden on the command line
-// --------------------------------------------------
-#ifndef SPP_NAMESPACE
- #define SPP_NAMESPACE spp
-#endif
-
-#ifndef spp_
- #define spp_ SPP_NAMESPACE
-#endif
-
-#ifndef SPP_DEFAULT_ALLOCATOR
- #if (defined(SPP_USE_SPP_ALLOC) && SPP_USE_SPP_ALLOC) && defined(_MSC_VER)
- // -----------------------------------------------------------------------------
- // When building with the Microsoft compiler, we use a custom allocator because
- // the default one fragments memory when reallocating. This is desirable only
- // when creating large sparsepp hash maps. If you create lots of small hash_maps,
- // define the following before including spp.h:
- // #define SPP_DEFAULT_ALLOCATOR spp::libc_allocator
- // -----------------------------------------------------------------------------
- #define SPP_DEFAULT_ALLOCATOR spp_::spp_allocator
- #define SPP_INCLUDE_SPP_ALLOC
- #else
- #define SPP_DEFAULT_ALLOCATOR spp_::libc_allocator
- #endif
-#endif
-
-#ifndef SPP_GROUP_SIZE
- // must be 32 or 64
- #define SPP_GROUP_SIZE 32
-#endif
-
-#ifndef SPP_ALLOC_SZ
- // must be power of 2 (0 = agressive alloc, 1 = smallest memory usage, 2 = good compromise)
- #define SPP_ALLOC_SZ 0
-#endif
-
-#ifndef SPP_STORE_NUM_ITEMS
- // 1 uses a little bit more memory, but faster!!
- #define SPP_STORE_NUM_ITEMS 1
-#endif
-
-
-// ---------------------------------------------------------------------------
-// Compiler detection code (SPP_ proprocessor macros) derived from Boost
-// libraries. Therefore Boost software licence reproduced below.
-// ---------------------------------------------------------------------------
-// Boost Software License - Version 1.0 - August 17th, 2003
-//
-// Permission is hereby granted, free of charge, to any person or organization
-// obtaining a copy of the software and accompanying documentation covered by
-// this license (the "Software") to use, reproduce, display, distribute,
-// execute, and transmit the Software, and to prepare derivative works of the
-// Software, and to permit third-parties to whom the Software is furnished to
-// do so, all subject to the following:
-//
-// The copyright notices in the Software and this entire statement, including
-// the above license grant, this restriction and the following disclaimer,
-// must be included in all copies of the Software, in whole or in part, and
-// all derivative works of the Software, unless such copies or derivative
-// works are solely in the form of machine-executable object code generated by
-// a source language processor.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-// DEALINGS IN THE SOFTWARE.
-// ---------------------------------------------------------------------------
-
-// Boost like configuration
-// ------------------------
-#if defined __clang__
-
- #if defined(i386)
- #include <cpuid.h>
- inline void spp_cpuid(int info[4], int InfoType) {
- __cpuid_count(InfoType, 0, info[0], info[1], info[2], info[3]);
- }
- #endif
-
- #define SPP_POPCNT __builtin_popcount
- #define SPP_POPCNT64 __builtin_popcountll
-
- #define SPP_HAS_CSTDINT
-
- #ifndef __has_extension
- #define __has_extension __has_feature
- #endif
-
- #if !__has_feature(cxx_exceptions) && !defined(SPP_NO_EXCEPTIONS)
- #define SPP_NO_EXCEPTIONS
- #endif
-
- #if !__has_feature(cxx_rtti) && !defined(SPP_NO_RTTI)
- #define SPP_NO_RTTI
- #endif
-
- #if !__has_feature(cxx_rtti) && !defined(SPP_NO_TYPEID)
- #define SPP_NO_TYPEID
- #endif
-
- #if defined(__int64) && !defined(__GNUC__)
- #define SPP_HAS_MS_INT64
- #endif
-
- #define SPP_HAS_NRVO
-
- // Branch prediction hints
- #if defined(__has_builtin)
- #if __has_builtin(__builtin_expect)
- #define SPP_LIKELY(x) __builtin_expect(x, 1)
- #define SPP_UNLIKELY(x) __builtin_expect(x, 0)
- #endif
- #endif
-
- // Clang supports "long long" in all compilation modes.
- #define SPP_HAS_LONG_LONG
-
- #if !__has_feature(cxx_constexpr)
- #define SPP_NO_CXX11_CONSTEXPR
- #endif
-
- #if !__has_feature(cxx_decltype)
- #define SPP_NO_CXX11_DECLTYPE
- #endif
-
- #if !__has_feature(cxx_decltype_incomplete_return_types)
- #define SPP_NO_CXX11_DECLTYPE_N3276
- #endif
-
- #if !__has_feature(cxx_defaulted_functions)
- #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS
- #endif
-
- #if !__has_feature(cxx_deleted_functions)
- #define SPP_NO_CXX11_DELETED_FUNCTIONS
- #endif
-
- #if !__has_feature(cxx_explicit_conversions)
- #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS
- #endif
-
- #if !__has_feature(cxx_default_function_template_args)
- #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS
- #endif
-
- #if !__has_feature(cxx_generalized_initializers)
- #define SPP_NO_CXX11_HDR_INITIALIZER_LIST
- #endif
-
- #if !__has_feature(cxx_lambdas)
- #define SPP_NO_CXX11_LAMBDAS
- #endif
-
- #if !__has_feature(cxx_local_type_template_args)
- #define SPP_NO_CXX11_LOCAL_CLASS_TEMPLATE_PARAMETERS
- #endif
-
- #if !__has_feature(cxx_raw_string_literals)
- #define SPP_NO_CXX11_RAW_LITERALS
- #endif
-
- #if !__has_feature(cxx_reference_qualified_functions)
- #define SPP_NO_CXX11_REF_QUALIFIERS
- #endif
-
- #if !__has_feature(cxx_generalized_initializers)
- #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX
- #endif
-
- #if !__has_feature(cxx_rvalue_references)
- #define SPP_NO_CXX11_RVALUE_REFERENCES
- #endif
-
- #if !__has_feature(cxx_static_assert)
- #define SPP_NO_CXX11_STATIC_ASSERT
- #endif
-
- #if !__has_feature(cxx_alias_templates)
- #define SPP_NO_CXX11_TEMPLATE_ALIASES
- #endif
-
- #if !__has_feature(cxx_variadic_templates)
- #define SPP_NO_CXX11_VARIADIC_TEMPLATES
- #endif
-
- #if !__has_feature(cxx_user_literals)
- #define SPP_NO_CXX11_USER_DEFINED_LITERALS
- #endif
-
- #if !__has_feature(cxx_alignas)
- #define SPP_NO_CXX11_ALIGNAS
- #endif
-
- #if !__has_feature(cxx_trailing_return)
- #define SPP_NO_CXX11_TRAILING_RESULT_TYPES
- #endif
-
- #if !__has_feature(cxx_inline_namespaces)
- #define SPP_NO_CXX11_INLINE_NAMESPACES
- #endif
-
- #if !__has_feature(cxx_override_control)
- #define SPP_NO_CXX11_FINAL
- #endif
-
- #if !(__has_feature(__cxx_binary_literals__) || __has_extension(__cxx_binary_literals__))
- #define SPP_NO_CXX14_BINARY_LITERALS
- #endif
-
- #if !__has_feature(__cxx_decltype_auto__)
- #define SPP_NO_CXX14_DECLTYPE_AUTO
- #endif
-
- #if !__has_feature(__cxx_init_captures__)
- #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES
- #endif
-
- #if !__has_feature(__cxx_generic_lambdas__)
- #define SPP_NO_CXX14_GENERIC_LAMBDAS
- #endif
-
-
- #if !__has_feature(__cxx_generic_lambdas__) || !__has_feature(__cxx_relaxed_constexpr__)
- #define SPP_NO_CXX14_CONSTEXPR
- #endif
-
- #if !__has_feature(__cxx_return_type_deduction__)
- #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION
- #endif
-
- #if !__has_feature(__cxx_variable_templates__)
- #define SPP_NO_CXX14_VARIABLE_TEMPLATES
- #endif
-
- #if __cplusplus < 201400
- #define SPP_NO_CXX14_DIGIT_SEPARATORS
- #endif
-
- #if defined(__has_builtin) && __has_builtin(__builtin_unreachable)
- #define SPP_UNREACHABLE_RETURN(x) __builtin_unreachable();
- #endif
-
- #define SPP_ATTRIBUTE_UNUSED __attribute__((__unused__))
-
- #ifndef SPP_COMPILER
- #define SPP_COMPILER "Clang version " __clang_version__
- #endif
-
- #define SPP_CLANG 1
-
-
-#elif defined __GNUC__
-
- #define SPP_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
-
- // definition to expand macro then apply to pragma message
- // #define VALUE_TO_STRING(x) #x
- // #define VALUE(x) VALUE_TO_STRING(x)
- // #define VAR_NAME_VALUE(var) #var "=" VALUE(var)
- // #pragma message(VAR_NAME_VALUE(SPP_GCC_VERSION))
-
- #if defined(i386)
- #include <cpuid.h>
- inline void spp_cpuid(int info[4], int InfoType) {
- __cpuid_count(InfoType, 0, info[0], info[1], info[2], info[3]);
- }
- #endif
-
- // __POPCNT__ defined when the compiled with popcount support
- // (-mpopcnt compiler option is given for example)
- #ifdef __POPCNT__
- // slower unless compiled iwith -mpopcnt
- #define SPP_POPCNT __builtin_popcount
- #define SPP_POPCNT64 __builtin_popcountll
- #endif
-
- #if defined(__GXX_EXPERIMENTAL_CXX0X__) || (__cplusplus >= 201103L)
- #define SPP_GCC_CXX11
- #endif
-
- #if __GNUC__ == 3
- #if defined (__PATHSCALE__)
- #define SPP_NO_TWO_PHASE_NAME_LOOKUP
- #define SPP_NO_IS_ABSTRACT
- #endif
-
- #if __GNUC_MINOR__ < 4
- #define SPP_NO_IS_ABSTRACT
- #endif
-
- #define SPP_NO_CXX11_EXTERN_TEMPLATE
- #endif
-
- #if __GNUC__ < 4
- //
- // All problems to gcc-3.x and earlier here:
- //
- #define SPP_NO_TWO_PHASE_NAME_LOOKUP
- #ifdef __OPEN64__
- #define SPP_NO_IS_ABSTRACT
- #endif
- #endif
-
- // GCC prior to 3.4 had #pragma once too but it didn't work well with filesystem links
- #if SPP_GCC_VERSION >= 30400
- #define SPP_HAS_PRAGMA_ONCE
- #endif
-
- #if SPP_GCC_VERSION < 40400
- // Previous versions of GCC did not completely implement value-initialization:
- // GCC Bug 30111, "Value-initialization of POD base class doesn't initialize
- // members", reported by Jonathan Wakely in 2006,
- // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=30111 (fixed for GCC 4.4)
- // GCC Bug 33916, "Default constructor fails to initialize array members",
- // reported by Michael Elizabeth Chastain in 2007,
- // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=33916 (fixed for GCC 4.2.4)
- // See also: http://www.boost.org/libs/utility/value_init.htm #compiler_issues
- #define SPP_NO_COMPLETE_VALUE_INITIALIZATION
- #endif
-
- #if !defined(__EXCEPTIONS) && !defined(SPP_NO_EXCEPTIONS)
- #define SPP_NO_EXCEPTIONS
- #endif
-
- //
- // Threading support: Turn this on unconditionally here (except for
- // those platforms where we can know for sure). It will get turned off again
- // later if no threading API is detected.
- //
- #if !defined(__MINGW32__) && !defined(linux) && !defined(__linux) && !defined(__linux__)
- #define SPP_HAS_THREADS
- #endif
-
- //
- // gcc has "long long"
- // Except on Darwin with standard compliance enabled (-pedantic)
- // Apple gcc helpfully defines this macro we can query
- //
- #if !defined(__DARWIN_NO_LONG_LONG)
- #define SPP_HAS_LONG_LONG
- #endif
-
- //
- // gcc implements the named return value optimization since version 3.1
- //
- #define SPP_HAS_NRVO
-
- // Branch prediction hints
- #define SPP_LIKELY(x) __builtin_expect(x, 1)
- #define SPP_UNLIKELY(x) __builtin_expect(x, 0)
-
- //
- // Dynamic shared object (DSO) and dynamic-link library (DLL) support
- //
- #if __GNUC__ >= 4
- #if (defined(_WIN32) || defined(__WIN32__) || defined(WIN32)) && !defined(__CYGWIN__)
- // All Win32 development environments, including 64-bit Windows and MinGW, define
- // _WIN32 or one of its variant spellings. Note that Cygwin is a POSIX environment,
- // so does not define _WIN32 or its variants.
- #define SPP_HAS_DECLSPEC
- #define SPP_SYMBOL_EXPORT __attribute__((__dllexport__))
- #define SPP_SYMBOL_IMPORT __attribute__((__dllimport__))
- #else
- #define SPP_SYMBOL_EXPORT __attribute__((__visibility__("default")))
- #define SPP_SYMBOL_IMPORT
- #endif
-
- #define SPP_SYMBOL_VISIBLE __attribute__((__visibility__("default")))
- #else
- // config/platform/win32.hpp will define SPP_SYMBOL_EXPORT, etc., unless already defined
- #define SPP_SYMBOL_EXPORT
- #endif
-
- //
- // RTTI and typeinfo detection is possible post gcc-4.3:
- //
- #if SPP_GCC_VERSION > 40300
- #ifndef __GXX_RTTI
- #ifndef SPP_NO_TYPEID
- #define SPP_NO_TYPEID
- #endif
- #ifndef SPP_NO_RTTI
- #define SPP_NO_RTTI
- #endif
- #endif
- #endif
-
- //
- // Recent GCC versions have __int128 when in 64-bit mode.
- //
- // We disable this if the compiler is really nvcc with C++03 as it
- // doesn't actually support __int128 as of CUDA_VERSION=7500
- // even though it defines __SIZEOF_INT128__.
- // See https://svn.boost.org/trac/boost/ticket/8048
- // https://svn.boost.org/trac/boost/ticket/11852
- // Only re-enable this for nvcc if you're absolutely sure
- // of the circumstances under which it's supported:
- //
- #if defined(__CUDACC__)
- #if defined(SPP_GCC_CXX11)
- #define SPP_NVCC_CXX11
- #else
- #define SPP_NVCC_CXX03
- #endif
- #endif
-
- #if defined(__SIZEOF_INT128__) && !defined(SPP_NVCC_CXX03)
- #define SPP_HAS_INT128
- #endif
- //
- // Recent GCC versions have a __float128 native type, we need to
- // include a std lib header to detect this - not ideal, but we'll
- // be including <cstddef> later anyway when we select the std lib.
- //
- // Nevertheless, as of CUDA 7.5, using __float128 with the host
- // compiler in pre-C++11 mode is still not supported.
- // See https://svn.boost.org/trac/boost/ticket/11852
- //
- #ifdef __cplusplus
- #include <cstddef>
- #else
- #include <stddef.h>
- #endif
-
- #if defined(_GLIBCXX_USE_FLOAT128) && !defined(__STRICT_ANSI__) && !defined(SPP_NVCC_CXX03)
- #define SPP_HAS_FLOAT128
- #endif
-
- // C++0x features in 4.3.n and later
- //
- #if (SPP_GCC_VERSION >= 40300) && defined(SPP_GCC_CXX11)
- // C++0x features are only enabled when -std=c++0x or -std=gnu++0x are
- // passed on the command line, which in turn defines
- // __GXX_EXPERIMENTAL_CXX0X__.
- #define SPP_HAS_DECLTYPE
- #define SPP_HAS_RVALUE_REFS
- #define SPP_HAS_STATIC_ASSERT
- #define SPP_HAS_VARIADIC_TMPL
- #define SPP_HAS_CSTDINT
- #else
- #define SPP_NO_CXX11_DECLTYPE
- #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS
- #define SPP_NO_CXX11_RVALUE_REFERENCES
- #define SPP_NO_CXX11_STATIC_ASSERT
- #endif
-
- // C++0x features in 4.4.n and later
- //
- #if (SPP_GCC_VERSION < 40400) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_AUTO_DECLARATIONS
- #define SPP_NO_CXX11_AUTO_MULTIDECLARATIONS
- #define SPP_NO_CXX11_CHAR16_T
- #define SPP_NO_CXX11_CHAR32_T
- #define SPP_NO_CXX11_HDR_INITIALIZER_LIST
- #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS
- #define SPP_NO_CXX11_DELETED_FUNCTIONS
- #define SPP_NO_CXX11_TRAILING_RESULT_TYPES
- #define SPP_NO_CXX11_INLINE_NAMESPACES
- #define SPP_NO_CXX11_VARIADIC_TEMPLATES
- #endif
-
- #if SPP_GCC_VERSION < 40500
- #define SPP_NO_SFINAE_EXPR
- #endif
-
- // GCC 4.5 forbids declaration of defaulted functions in private or protected sections
- #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ == 5) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_NON_PUBLIC_DEFAULTED_FUNCTIONS
- #endif
-
- // C++0x features in 4.5.0 and later
- //
- #if (SPP_GCC_VERSION < 40500) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS
- #define SPP_NO_CXX11_LAMBDAS
- #define SPP_NO_CXX11_LOCAL_CLASS_TEMPLATE_PARAMETERS
- #define SPP_NO_CXX11_RAW_LITERALS
- #endif
-
- // C++0x features in 4.6.n and later
- //
- #if (SPP_GCC_VERSION < 40600) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_CONSTEXPR
- #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX
- #endif
-
- // C++0x features in 4.7.n and later
- //
- #if (SPP_GCC_VERSION < 40700) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_FINAL
- #define SPP_NO_CXX11_TEMPLATE_ALIASES
- #define SPP_NO_CXX11_USER_DEFINED_LITERALS
- #define SPP_NO_CXX11_FIXED_LENGTH_VARIADIC_TEMPLATE_EXPANSION_PACKS
- #endif
-
- // C++0x features in 4.8.n and later
- //
- #if (SPP_GCC_VERSION < 40800) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_ALIGNAS
- #endif
-
- // C++0x features in 4.8.1 and later
- //
- #if (SPP_GCC_VERSION < 40801) || !defined(SPP_GCC_CXX11)
- #define SPP_NO_CXX11_DECLTYPE_N3276
- #define SPP_NO_CXX11_REF_QUALIFIERS
- #define SPP_NO_CXX14_BINARY_LITERALS
- #endif
-
- // C++14 features in 4.9.0 and later
- //
- #if (SPP_GCC_VERSION < 40900) || (__cplusplus < 201300)
- #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION
- #define SPP_NO_CXX14_GENERIC_LAMBDAS
- #define SPP_NO_CXX14_DIGIT_SEPARATORS
- #define SPP_NO_CXX14_DECLTYPE_AUTO
- #if !((SPP_GCC_VERSION >= 40801) && (SPP_GCC_VERSION < 40900) && defined(SPP_GCC_CXX11))
- #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES
- #endif
- #endif
-
-
- // C++ 14:
- #if !defined(__cpp_constexpr) || (__cpp_constexpr < 201304)
- #define SPP_NO_CXX14_CONSTEXPR
- #endif
- #if !defined(__cpp_variable_templates) || (__cpp_variable_templates < 201304)
- #define SPP_NO_CXX14_VARIABLE_TEMPLATES
- #endif
-
- //
- // Unused attribute:
- #if __GNUC__ >= 4
- #define SPP_ATTRIBUTE_UNUSED __attribute__((__unused__))
- #endif
- //
- // __builtin_unreachable:
- #if SPP_GCC_VERSION >= 40800
- #define SPP_UNREACHABLE_RETURN(x) __builtin_unreachable();
- #endif
-
- #ifndef SPP_COMPILER
- #define SPP_COMPILER "GNU C++ version " __VERSION__
- #endif
-
- // ConceptGCC compiler:
- // http://www.generic-programming.org/software/ConceptGCC/
- #ifdef __GXX_CONCEPTS__
- #define SPP_HAS_CONCEPTS
- #define SPP_COMPILER "ConceptGCC version " __VERSION__
- #endif
-
-#elif defined _MSC_VER
-
- #include <intrin.h> // for __popcnt()
-
- #define SPP_POPCNT_CHECK // slower when defined, but we have to check!
- #define spp_cpuid(info, x) __cpuid(info, x)
-
- #define SPP_POPCNT __popcnt
- #if (SPP_GROUP_SIZE == 64 && INTPTR_MAX == INT64_MAX)
- #define SPP_POPCNT64 __popcnt64
- #endif
-
- // Attempt to suppress VC6 warnings about the length of decorated names (obsolete):
- #pragma warning( disable : 4503 ) // warning: decorated name length exceeded
-
- #define SPP_HAS_PRAGMA_ONCE
- #define SPP_HAS_CSTDINT
-
- //
- // versions check:
- // we don't support Visual C++ prior to version 7.1:
- #if _MSC_VER < 1310
- #error "Antique compiler not supported"
- #endif
-
- #if _MSC_FULL_VER < 180020827
- #define SPP_NO_FENV_H
- #endif
-
- #if _MSC_VER < 1400
- // although a conforming signature for swprint exists in VC7.1
- // it appears not to actually work:
- #define SPP_NO_SWPRINTF
-
- // Our extern template tests also fail for this compiler:
- #define SPP_NO_CXX11_EXTERN_TEMPLATE
-
- // Variadic macros do not exist for VC7.1 and lower
- #define SPP_NO_CXX11_VARIADIC_MACROS
- #endif
-
- #if _MSC_VER < 1500 // 140X == VC++ 8.0
- #undef SPP_HAS_CSTDINT
- #define SPP_NO_MEMBER_TEMPLATE_FRIENDS
- #endif
-
- #if _MSC_VER < 1600 // 150X == VC++ 9.0
- // A bug in VC9:
- #define SPP_NO_ADL_BARRIER
- #endif
-
-
- // MSVC (including the latest checked version) has not yet completely
- // implemented value-initialization, as is reported:
- // "VC++ does not value-initialize members of derived classes without
- // user-declared constructor", reported in 2009 by Sylvester Hesp:
- // https: //connect.microsoft.com/VisualStudio/feedback/details/484295
- // "Presence of copy constructor breaks member class initialization",
- // reported in 2009 by Alex Vakulenko:
- // https: //connect.microsoft.com/VisualStudio/feedback/details/499606
- // "Value-initialization in new-expression", reported in 2005 by
- // Pavel Kuznetsov (MetaCommunications Engineering):
- // https: //connect.microsoft.com/VisualStudio/feedback/details/100744
- // See also: http: //www.boost.org/libs/utility/value_init.htm #compiler_issues
- // (Niels Dekker, LKEB, May 2010)
- #define SPP_NO_COMPLETE_VALUE_INITIALIZATION
-
- #ifndef _NATIVE_WCHAR_T_DEFINED
- #define SPP_NO_INTRINSIC_WCHAR_T
- #endif
-
- //
- // check for exception handling support:
- #if !defined(_CPPUNWIND) && !defined(SPP_NO_EXCEPTIONS)
- #define SPP_NO_EXCEPTIONS
- #endif
-
- //
- // __int64 support:
- //
- #define SPP_HAS_MS_INT64
- #if defined(_MSC_EXTENSIONS) || (_MSC_VER >= 1400)
- #define SPP_HAS_LONG_LONG
- #else
- #define SPP_NO_LONG_LONG
- #endif
-
- #if (_MSC_VER >= 1400) && !defined(_DEBUG)
- #define SPP_HAS_NRVO
- #endif
-
- #if _MSC_VER >= 1500 // 150X == VC++ 9.0
- #define SPP_HAS_PRAGMA_DETECT_MISMATCH
- #endif
-
- //
- // disable Win32 API's if compiler extensions are
- // turned off:
- //
- #if !defined(_MSC_EXTENSIONS) && !defined(SPP_DISABLE_WIN32)
- #define SPP_DISABLE_WIN32
- #endif
-
- #if !defined(_CPPRTTI) && !defined(SPP_NO_RTTI)
- #define SPP_NO_RTTI
- #endif
-
- //
- // TR1 features:
- //
- #if _MSC_VER >= 1700
- // #define SPP_HAS_TR1_HASH // don't know if this is true yet.
- // #define SPP_HAS_TR1_TYPE_TRAITS // don't know if this is true yet.
- #define SPP_HAS_TR1_UNORDERED_MAP
- #define SPP_HAS_TR1_UNORDERED_SET
- #endif
-
- //
- // C++0x features
- //
- // See above for SPP_NO_LONG_LONG
-
- // C++ features supported by VC++ 10 (aka 2010)
- //
- #if _MSC_VER < 1600
- #define SPP_NO_CXX11_AUTO_DECLARATIONS
- #define SPP_NO_CXX11_AUTO_MULTIDECLARATIONS
- #define SPP_NO_CXX11_LAMBDAS
- #define SPP_NO_CXX11_RVALUE_REFERENCES
- #define SPP_NO_CXX11_STATIC_ASSERT
- #define SPP_NO_CXX11_DECLTYPE
- #endif // _MSC_VER < 1600
-
- #if _MSC_VER >= 1600
- #define SPP_HAS_STDINT_H
- #endif
-
- // C++11 features supported by VC++ 11 (aka 2012)
- //
- #if _MSC_VER < 1700
- #define SPP_NO_CXX11_FINAL
- #endif // _MSC_VER < 1700
-
- // C++11 features supported by VC++ 12 (aka 2013).
- //
- #if _MSC_FULL_VER < 180020827
- #define SPP_NO_CXX11_DEFAULTED_FUNCTIONS
- #define SPP_NO_CXX11_DELETED_FUNCTIONS
- #define SPP_NO_CXX11_EXPLICIT_CONVERSION_OPERATORS
- #define SPP_NO_CXX11_FUNCTION_TEMPLATE_DEFAULT_ARGS
- #define SPP_NO_CXX11_RAW_LITERALS
- #define SPP_NO_CXX11_TEMPLATE_ALIASES
- #define SPP_NO_CXX11_TRAILING_RESULT_TYPES
- #define SPP_NO_CXX11_VARIADIC_TEMPLATES
- #define SPP_NO_CXX11_UNIFIED_INITIALIZATION_SYNTAX
- #define SPP_NO_CXX11_DECLTYPE_N3276
- #endif
-
- // C++11 features supported by VC++ 14 (aka 2014) CTP1
- #if (_MSC_FULL_VER < 190021730)
- #define SPP_NO_CXX11_REF_QUALIFIERS
- #define SPP_NO_CXX11_USER_DEFINED_LITERALS
- #define SPP_NO_CXX11_ALIGNAS
- #define SPP_NO_CXX11_INLINE_NAMESPACES
- #define SPP_NO_CXX14_DECLTYPE_AUTO
- #define SPP_NO_CXX14_INITIALIZED_LAMBDA_CAPTURES
- #define SPP_NO_CXX14_RETURN_TYPE_DEDUCTION
- #define SPP_NO_CXX11_HDR_INITIALIZER_LIST
- #endif
-
- // C++11 features not supported by any versions
- #define SPP_NO_CXX11_CHAR16_T
- #define SPP_NO_CXX11_CHAR32_T
- #define SPP_NO_CXX11_CONSTEXPR
- #define SPP_NO_SFINAE_EXPR
- #define SPP_NO_TWO_PHASE_NAME_LOOKUP
-
- // C++ 14:
- #if !defined(__cpp_binary_literals) || (__cpp_binary_literals < 201304)
- #define SPP_NO_CXX14_BINARY_LITERALS
- #endif
-
- #if !defined(__cpp_constexpr) || (__cpp_constexpr < 201304)
- #define SPP_NO_CXX14_CONSTEXPR
- #endif
-
- #if (__cplusplus < 201304) // There's no SD6 check for this....
- #define SPP_NO_CXX14_DIGIT_SEPARATORS
- #endif
-
- #if !defined(__cpp_generic_lambdas) || (__cpp_generic_lambdas < 201304)
- #define SPP_NO_CXX14_GENERIC_LAMBDAS
- #endif
-
- #if !defined(__cpp_variable_templates) || (__cpp_variable_templates < 201304)
- #define SPP_NO_CXX14_VARIABLE_TEMPLATES
- #endif
-
-#endif
-
-// from boost/config/suffix.hpp
-// ----------------------------
-#ifndef SPP_ATTRIBUTE_UNUSED
- #define SPP_ATTRIBUTE_UNUSED
-#endif
-
-/*
- Try to persuade compilers to inline.
-*/
-#ifndef SPP_FORCEINLINE
- #if defined(__GNUC__)
- #define SPP_FORCEINLINE __inline __attribute__ ((always_inline))
- #elif defined(_MSC_VER)
- #define SPP_FORCEINLINE __forceinline
- #else
- #define SPP_FORCEINLINE inline
- #endif
-#endif
-
-
-#endif // spp_config_h_guard
diff --git a/benchmarks/others/sparsepp/spp_dlalloc.h b/benchmarks/others/sparsepp/spp_dlalloc.h
deleted file mode 100644
index f88aab7c..00000000
--- a/benchmarks/others/sparsepp/spp_dlalloc.h
+++ /dev/null
@@ -1,4044 +0,0 @@
-#ifndef spp_dlalloc__h_
-#define spp_dlalloc__h_
-
-/* This is a C++ allocator created from Doug Lea's dlmalloc
- (Version 2.8.6 Wed Aug 29 06:57:58 2012)
- see: http://g.oswego.edu/dl/html/malloc.html
-*/
-
-#include "spp_utils.h"
-#include "spp_smartptr.h"
-
-
-#ifndef SPP_FORCEINLINE
- #if defined(__GNUC__)
- #define SPP_FORCEINLINE __inline __attribute__ ((always_inline))
- #elif defined(_MSC_VER)
- #define SPP_FORCEINLINE __forceinline
- #else
- #define SPP_FORCEINLINE inline
- #endif
-#endif
-
-
-#ifndef SPP_IMPL
- #define SPP_IMPL SPP_FORCEINLINE
-#endif
-
-#ifndef SPP_API
- #define SPP_API static
-#endif
-
-
-namespace spp
-{
- // ---------------------- allocator internal API -----------------------
- typedef void* mspace;
-
- /*
- create_mspace creates and returns a new independent space with the
- given initial capacity, or, if 0, the default granularity size. It
- returns null if there is no system memory available to create the
- space. If argument locked is non-zero, the space uses a separate
- lock to control access. The capacity of the space will grow
- dynamically as needed to service mspace_malloc requests. You can
- control the sizes of incremental increases of this space by
- compiling with a different SPP_DEFAULT_GRANULARITY or dynamically
- setting with mallopt(M_GRANULARITY, value).
- */
- SPP_API mspace create_mspace(size_t capacity, int locked);
- SPP_API size_t destroy_mspace(mspace msp);
- SPP_API void* mspace_malloc(mspace msp, size_t bytes);
- SPP_API void mspace_free(mspace msp, void* mem);
- SPP_API void* mspace_realloc(mspace msp, void* mem, size_t newsize);
-
-#if 0
- SPP_API mspace create_mspace_with_base(void* base, size_t capacity, int locked);
- SPP_API int mspace_track_large_chunks(mspace msp, int enable);
- SPP_API void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
- SPP_API void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
- SPP_API void** mspace_independent_calloc(mspace msp, size_t n_elements,
- size_t elem_size, void* chunks[]);
- SPP_API void** mspace_independent_comalloc(mspace msp, size_t n_elements,
- size_t sizes[], void* chunks[]);
- SPP_API size_t mspace_footprint(mspace msp);
- SPP_API size_t mspace_max_footprint(mspace msp);
- SPP_API size_t mspace_usable_size(const void* mem);
- SPP_API int mspace_trim(mspace msp, size_t pad);
- SPP_API int mspace_mallopt(int, int);
-#endif
-
- // -----------------------------------------------------------
- // -----------------------------------------------------------
- struct MSpace : public spp_rc
- {
- MSpace() :
- _sp(create_mspace(0, 0))
- {}
-
- ~MSpace()
- {
- destroy_mspace(_sp);
- }
-
- mspace _sp;
- };
-
- // -----------------------------------------------------------
- // -----------------------------------------------------------
- template<class T>
- class spp_allocator
- {
- public:
- typedef T value_type;
- typedef T* pointer;
- typedef ptrdiff_t difference_type;
- typedef const T* const_pointer;
- typedef size_t size_type;
-
- MSpace *getSpace() const { return _space.get(); }
-
- spp_allocator() : _space(new MSpace) {}
-
- template<class U>
- spp_allocator(const spp_allocator<U> &o) : _space(o.getSpace()) {}
-
- template<class U>
- spp_allocator& operator=(const spp_allocator<U> &o)
- {
- if (&o != this)
- _space = o.getSpace();
- return *this;
- }
-
- void swap(spp_allocator &o)
- {
- std::swap(_space, o._space);
- }
-
- pointer allocate(size_t n, const_pointer /* unused */ = 0)
- {
- pointer res = static_cast<pointer>(mspace_malloc(_space->_sp, n * sizeof(T)));
- if (!res)
- throw std::bad_alloc();
- return res;
- }
-
- void deallocate(pointer p, size_t /* unused */)
- {
- mspace_free(_space->_sp, p);
- }
-
- pointer reallocate(pointer p, size_t new_size)
- {
- pointer res = static_cast<pointer>(mspace_realloc(_space->_sp, p, new_size * sizeof(T)));
- if (!res)
- throw std::bad_alloc();
- return res;
- }
-
- pointer reallocate(pointer p, size_type /* old_size */, size_t new_size)
- {
- return reallocate(p, new_size);
- }
-
- size_type max_size() const
- {
- return static_cast<size_type>(-1) / sizeof(value_type);
- }
-
- void construct(pointer p, const value_type& val)
- {
- new (p) value_type(val);
- }
-
- void destroy(pointer p) { p->~value_type(); }
-
- template<class U>
- struct rebind
- {
- // rebind to libc_allocator because we want to use malloc_inspect_all in destructive_iterator
- // to reduce peak memory usage (we don't want <group_items> mixed with value_type when
- // we traverse the allocated memory).
- typedef spp::spp_allocator<U> other;
- };
-
- mspace space() const { return _space->_sp; }
-
- // check if we can clear the whole allocator memory at once => works only if the allocator
- // is not be shared. If can_clear() returns true, we expect that the next allocator call
- // will be clear() - not allocate() or deallocate()
- bool can_clear()
- {
- assert(!_space_to_clear);
- _space_to_clear.reset();
- _space_to_clear.swap(_space);
- if (_space_to_clear->count() == 1)
- return true;
- else
- _space_to_clear.swap(_space);
- return false;
- }
-
- void clear()
- {
- assert(!_space && !!_space_to_clear);
- _space_to_clear.reset();
- _space = new MSpace;
- }
-
- private:
- spp_sptr<MSpace> _space;
- spp_sptr<MSpace> _space_to_clear;
- };
-}
-
-
-// allocators are "equal" whenever memory allocated with one can be deallocated with the other
-template<class T>
-inline bool operator==(const spp_::spp_allocator<T> &a, const spp_::spp_allocator<T> &b)
-{
- return a.space() == b.space();
-}
-
-template<class T>
-inline bool operator!=(const spp_::spp_allocator<T> &a, const spp_::spp_allocator<T> &b)
-{
- return !(a == b);
-}
-
-namespace std
-{
- template <class T>
- inline void swap(spp_::spp_allocator<T> &a, spp_::spp_allocator<T> &b)
- {
- a.swap(b);
- }
-}
-
-#if !defined(SPP_EXCLUDE_IMPLEMENTATION)
-
-#ifndef WIN32
- #ifdef _WIN32
- #define WIN32 1
- #endif
- #ifdef _WIN32_WCE
- #define SPP_LACKS_FCNTL_H
- #define WIN32 1
- #endif
-#endif
-
-#ifdef WIN32
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- #include <tchar.h>
- #define SPP_HAVE_MMAP 1
- #define SPP_LACKS_UNISTD_H
- #define SPP_LACKS_SYS_PARAM_H
- #define SPP_LACKS_SYS_MMAN_H
- #define SPP_LACKS_STRING_H
- #define SPP_LACKS_STRINGS_H
- #define SPP_LACKS_SYS_TYPES_H
- #define SPP_LACKS_ERRNO_H
- #define SPP_LACKS_SCHED_H
- #ifndef SPP_MALLOC_FAILURE_ACTION
- #define SPP_MALLOC_FAILURE_ACTION
- #endif
- #ifndef SPP_MMAP_CLEARS
- #ifdef _WIN32_WCE /* WINCE reportedly does not clear */
- #define SPP_MMAP_CLEARS 0
- #else
- #define SPP_MMAP_CLEARS 1
- #endif
- #endif
-#endif
-
-#if defined(DARWIN) || defined(_DARWIN)
- #define SPP_HAVE_MMAP 1
- /* OSX allocators provide 16 byte alignment */
- #ifndef SPP_MALLOC_ALIGNMENT
- #define SPP_MALLOC_ALIGNMENT ((size_t)16U)
- #endif
-#endif
-
-#ifndef SPP_LACKS_SYS_TYPES_H
- #include <sys/types.h> /* For size_t */
-#endif
-
-#ifndef SPP_MALLOC_ALIGNMENT
- #define SPP_MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
-#endif
-
-/* ------------------- size_t and alignment properties -------------------- */
-static const size_t spp_max_size_t = ~(size_t)0;
-static const size_t spp_size_t_bitsize = sizeof(size_t) << 3;
-static const size_t spp_half_max_size_t = spp_max_size_t / 2U;
-static const size_t spp_chunk_align_mask = SPP_MALLOC_ALIGNMENT - 1;
-
-#if defined(SPP_DEBUG) || !defined(NDEBUG)
-static bool spp_is_aligned(void *p) { return ((size_t)p & spp_chunk_align_mask) == 0; }
-#endif
-
-// the number of bytes to offset an address to align it
-static size_t align_offset(void *p)
-{
- return (((size_t)p & spp_chunk_align_mask) == 0) ? 0 :
- ((SPP_MALLOC_ALIGNMENT - ((size_t)p & spp_chunk_align_mask)) & spp_chunk_align_mask);
-}
-
-
-#ifndef SPP_FOOTERS
- #define SPP_FOOTERS 0
-#endif
-
-#ifndef SPP_ABORT
- #define SPP_ABORT abort()
-#endif
-
-#ifndef SPP_ABORT_ON_ASSERT_FAILURE
- #define SPP_ABORT_ON_ASSERT_FAILURE 1
-#endif
-
-#ifndef SPP_PROCEED_ON_ERROR
- #define SPP_PROCEED_ON_ERROR 0
-#endif
-
-#ifndef SPP_INSECURE
- #define SPP_INSECURE 0
-#endif
-
-#ifndef SPP_MALLOC_INSPECT_ALL
- #define SPP_MALLOC_INSPECT_ALL 0
-#endif
-
-#ifndef SPP_HAVE_MMAP
- #define SPP_HAVE_MMAP 1
-#endif
-
-#ifndef SPP_MMAP_CLEARS
- #define SPP_MMAP_CLEARS 1
-#endif
-
-#ifndef SPP_HAVE_MREMAP
- #ifdef linux
- #define SPP_HAVE_MREMAP 1
- #ifndef _GNU_SOURCE
- #define _GNU_SOURCE /* Turns on mremap() definition */
- #endif
- #else
- #define SPP_HAVE_MREMAP 0
- #endif
-#endif
-
-#ifndef SPP_MALLOC_FAILURE_ACTION
- // ENOMEM = 12
- #define SPP_MALLOC_FAILURE_ACTION errno = 12
-#endif
-
-
-#ifndef SPP_DEFAULT_GRANULARITY
- #if defined(WIN32)
- #define SPP_DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */
- #else
- #define SPP_DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
- #endif
-#endif
-
-#ifndef SPP_DEFAULT_TRIM_THRESHOLD
- #define SPP_DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
-#endif
-
-#ifndef SPP_DEFAULT_MMAP_THRESHOLD
- #if SPP_HAVE_MMAP
- #define SPP_DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
- #else
- #define SPP_DEFAULT_MMAP_THRESHOLD spp_max_size_t
- #endif
-#endif
-
-#ifndef SPP_MAX_RELEASE_CHECK_RATE
- #if SPP_HAVE_MMAP
- #define SPP_MAX_RELEASE_CHECK_RATE 4095
- #else
- #define SPP_MAX_RELEASE_CHECK_RATE spp_max_size_t
- #endif
-#endif
-
-#ifndef SPP_USE_BUILTIN_FFS
- #define SPP_USE_BUILTIN_FFS 0
-#endif
-
-#ifndef SPP_USE_DEV_RANDOM
- #define SPP_USE_DEV_RANDOM 0
-#endif
-
-#ifndef SPP_NO_SEGMENT_TRAVERSAL
- #define SPP_NO_SEGMENT_TRAVERSAL 0
-#endif
-
-
-
-/*------------------------------ internal #includes ---------------------- */
-
-#ifdef _MSC_VER
- #pragma warning( disable : 4146 ) /* no "unsigned" warnings */
-#endif
-#ifndef SPP_LACKS_ERRNO_H
- #include <errno.h> /* for SPP_MALLOC_FAILURE_ACTION */
-#endif
-
-#ifdef SPP_DEBUG
- #if SPP_ABORT_ON_ASSERT_FAILURE
- #undef assert
- #define assert(x) if(!(x)) SPP_ABORT
- #else
- #include <assert.h>
- #endif
-#else
- #ifndef assert
- #define assert(x)
- #endif
- #define SPP_DEBUG 0
-#endif
-
-#if !defined(WIN32) && !defined(SPP_LACKS_TIME_H)
- #include <time.h> /* for magic initialization */
-#endif
-
-#ifndef SPP_LACKS_STDLIB_H
- #include <stdlib.h> /* for abort() */
-#endif
-
-#ifndef SPP_LACKS_STRING_H
- #include <string.h> /* for memset etc */
-#endif
-
-#if SPP_USE_BUILTIN_FFS
- #ifndef SPP_LACKS_STRINGS_H
- #include <strings.h> /* for ffs */
- #endif
-#endif
-
-#if SPP_HAVE_MMAP
- #ifndef SPP_LACKS_SYS_MMAN_H
- /* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
- #if (defined(linux) && !defined(__USE_GNU))
- #define __USE_GNU 1
- #include <sys/mman.h> /* for mmap */
- #undef __USE_GNU
- #else
- #include <sys/mman.h> /* for mmap */
- #endif
- #endif
- #ifndef SPP_LACKS_FCNTL_H
- #include <fcntl.h>
- #endif
-#endif
-
-#ifndef SPP_LACKS_UNISTD_H
- #include <unistd.h> /* for sbrk, sysconf */
-#else
- #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
- extern void* sbrk(ptrdiff_t);
- #endif
-#endif
-
-#include <new>
-
-namespace spp
-{
-
-/* Declarations for bit scanning on win32 */
-#if defined(_MSC_VER) && _MSC_VER>=1300
- #ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
- extern "C" {
- unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
- unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
- }
-
- #define BitScanForward _BitScanForward
- #define BitScanReverse _BitScanReverse
- #pragma intrinsic(_BitScanForward)
- #pragma intrinsic(_BitScanReverse)
- #endif /* BitScanForward */
-#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
-
-#ifndef WIN32
- #ifndef malloc_getpagesize
- #ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */
- #ifndef _SC_PAGE_SIZE
- #define _SC_PAGE_SIZE _SC_PAGESIZE
- #endif
- #endif
- #ifdef _SC_PAGE_SIZE
- #define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
- #else
- #if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
- extern size_t getpagesize();
- #define malloc_getpagesize getpagesize()
- #else
- #ifdef WIN32 /* use supplied emulation of getpagesize */
- #define malloc_getpagesize getpagesize()
- #else
- #ifndef SPP_LACKS_SYS_PARAM_H
- #include <sys/param.h>
- #endif
- #ifdef EXEC_PAGESIZE
- #define malloc_getpagesize EXEC_PAGESIZE
- #else
- #ifdef NBPG
- #ifndef CLSIZE
- #define malloc_getpagesize NBPG
- #else
- #define malloc_getpagesize (NBPG * CLSIZE)
- #endif
- #else
- #ifdef NBPC
- #define malloc_getpagesize NBPC
- #else
- #ifdef PAGESIZE
- #define malloc_getpagesize PAGESIZE
- #else /* just guess */
- #define malloc_getpagesize ((size_t)4096U)
- #endif
- #endif
- #endif
- #endif
- #endif
- #endif
- #endif
- #endif
-#endif
-
-/* -------------------------- MMAP preliminaries ------------------------- */
-
-/*
- If SPP_HAVE_MORECORE or SPP_HAVE_MMAP are false, we just define calls and
- checks to fail so compiler optimizer can delete code rather than
- using so many "#if"s.
-*/
-
-
-/* MMAP must return mfail on failure */
-static void *mfail = (void*)spp_max_size_t;
-static char *cmfail = (char*)mfail;
-
-#if SPP_HAVE_MMAP
-
-#ifndef WIN32
- #define SPP_MUNMAP_DEFAULT(a, s) munmap((a), (s))
- #define SPP_MMAP_PROT (PROT_READ | PROT_WRITE)
- #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
- #define MAP_ANONYMOUS MAP_ANON
- #endif
-
- #ifdef MAP_ANONYMOUS
- #define SPP_MMAP_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS)
- #define SPP_MMAP_DEFAULT(s) mmap(0, (s), SPP_MMAP_PROT, SPP_MMAP_FLAGS, -1, 0)
- #else /* MAP_ANONYMOUS */
- /*
- Nearly all versions of mmap support MAP_ANONYMOUS, so the following
- is unlikely to be needed, but is supplied just in case.
- */
- #define SPP_MMAP_FLAGS (MAP_PRIVATE)
- static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
- void SPP_MMAP_DEFAULT(size_t s)
- {
- if (dev_zero_fd < 0)
- dev_zero_fd = open("/dev/zero", O_RDWR);
- mmap(0, s, SPP_MMAP_PROT, SPP_MMAP_FLAGS, dev_zero_fd, 0);
- }
- #endif /* MAP_ANONYMOUS */
-
- #define SPP_DIRECT_MMAP_DEFAULT(s) SPP_MMAP_DEFAULT(s)
-
-#else /* WIN32 */
-
- /* Win32 MMAP via VirtualAlloc */
- static SPP_FORCEINLINE void* win32mmap(size_t size)
- {
- void* ptr = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
- return (ptr != 0) ? ptr : mfail;
- }
-
- /* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
- static SPP_FORCEINLINE void* win32direct_mmap(size_t size)
- {
- void* ptr = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
- PAGE_READWRITE);
- return (ptr != 0) ? ptr : mfail;
- }
-
- /* This function supports releasing coalesed segments */
- static SPP_FORCEINLINE int win32munmap(void* ptr, size_t size)
- {
- MEMORY_BASIC_INFORMATION minfo;
- char* cptr = (char*)ptr;
- while (size)
- {
- if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
- return -1;
- if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
- minfo.State != MEM_COMMIT || minfo.RegionSize > size)
- return -1;
- if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
- return -1;
- cptr += minfo.RegionSize;
- size -= minfo.RegionSize;
- }
- return 0;
- }
-
- #define SPP_MMAP_DEFAULT(s) win32mmap(s)
- #define SPP_MUNMAP_DEFAULT(a, s) win32munmap((a), (s))
- #define SPP_DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s)
-#endif /* WIN32 */
-#endif /* SPP_HAVE_MMAP */
-
-#if SPP_HAVE_MREMAP
- #ifndef WIN32
- #define SPP_MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
- #endif
-#endif
-
-/**
- * Define SPP_CALL_MMAP/SPP_CALL_MUNMAP/SPP_CALL_DIRECT_MMAP
- */
-#if SPP_HAVE_MMAP
- #define USE_MMAP_BIT 1
-
- #ifdef SPP_MMAP
- #define SPP_CALL_MMAP(s) SPP_MMAP(s)
- #else
- #define SPP_CALL_MMAP(s) SPP_MMAP_DEFAULT(s)
- #endif
-
- #ifdef SPP_MUNMAP
- #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP((a), (s))
- #else
- #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP_DEFAULT((a), (s))
- #endif
-
- #ifdef SPP_DIRECT_MMAP
- #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP(s)
- #else
- #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP_DEFAULT(s)
- #endif
-
-#else /* SPP_HAVE_MMAP */
- #define USE_MMAP_BIT 0
-
- #define SPP_MMAP(s) mfail
- #define SPP_MUNMAP(a, s) (-1)
- #define SPP_DIRECT_MMAP(s) mfail
- #define SPP_CALL_DIRECT_MMAP(s) SPP_DIRECT_MMAP(s)
- #define SPP_CALL_MMAP(s) SPP_MMAP(s)
- #define SPP_CALL_MUNMAP(a, s) SPP_MUNMAP((a), (s))
-#endif
-
-/**
- * Define SPP_CALL_MREMAP
- */
-#if SPP_HAVE_MMAP && SPP_HAVE_MREMAP
- #ifdef MREMAP
- #define SPP_CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
- #else
- #define SPP_CALL_MREMAP(addr, osz, nsz, mv) SPP_MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
- #endif
-#else
- #define SPP_CALL_MREMAP(addr, osz, nsz, mv) mfail
-#endif
-
-/* mstate bit set if continguous morecore disabled or failed */
-static const unsigned USE_NONCONTIGUOUS_BIT = 4U;
-
-/* segment bit set in create_mspace_with_base */
-static const unsigned EXTERN_BIT = 8U;
-
-
-/* --------------------------- flags ------------------------ */
-
-static const unsigned PINUSE_BIT = 1;
-static const unsigned CINUSE_BIT = 2;
-static const unsigned FLAG4_BIT = 4;
-static const unsigned INUSE_BITS = (PINUSE_BIT | CINUSE_BIT);
-static const unsigned FLAG_BITS = (PINUSE_BIT | CINUSE_BIT | FLAG4_BIT);
-
-/* ------------------- Chunks sizes and alignments ----------------------- */
-
-#if SPP_FOOTERS
- static const unsigned CHUNK_OVERHEAD = 2 * sizeof(size_t);
-#else
- static const unsigned CHUNK_OVERHEAD = sizeof(size_t);
-#endif
-
-/* MMapped chunks need a second word of overhead ... */
-static const unsigned SPP_MMAP_CHUNK_OVERHEAD = 2 * sizeof(size_t);
-
-/* ... and additional padding for fake next-chunk at foot */
-static const unsigned SPP_MMAP_FOOT_PAD = 4 * sizeof(size_t);
-
-// ===============================================================================
-struct malloc_chunk_header
-{
- void set_size_and_pinuse_of_free_chunk(size_t s)
- {
- _head = s | PINUSE_BIT;
- set_foot(s);
- }
-
- void set_foot(size_t s)
- {
- ((malloc_chunk_header *)((char*)this + s))->_prev_foot = s;
- }
-
- // extraction of fields from head words
- bool cinuse() const { return !!(_head & CINUSE_BIT); }
- bool pinuse() const { return !!(_head & PINUSE_BIT); }
- bool flag4inuse() const { return !!(_head & FLAG4_BIT); }
- bool is_inuse() const { return (_head & INUSE_BITS) != PINUSE_BIT; }
- bool is_mmapped() const { return (_head & INUSE_BITS) == 0; }
-
- size_t chunksize() const { return _head & ~(FLAG_BITS); }
-
- void clear_pinuse() { _head &= ~PINUSE_BIT; }
- void set_flag4() { _head |= FLAG4_BIT; }
- void clear_flag4() { _head &= ~FLAG4_BIT; }
-
- // Treat space at ptr +/- offset as a chunk
- malloc_chunk_header * chunk_plus_offset(size_t s)
- {
- return (malloc_chunk_header *)((char*)this + s);
- }
- malloc_chunk_header * chunk_minus_offset(size_t s)
- {
- return (malloc_chunk_header *)((char*)this - s);
- }
-
- // Ptr to next or previous physical malloc_chunk.
- malloc_chunk_header * next_chunk()
- {
- return (malloc_chunk_header *)((char*)this + (_head & ~FLAG_BITS));
- }
- malloc_chunk_header * prev_chunk()
- {
- return (malloc_chunk_header *)((char*)this - (_prev_foot));
- }
-
- // extract next chunk's pinuse bit
- size_t next_pinuse() { return next_chunk()->_head & PINUSE_BIT; }
-
- size_t _prev_foot; // Size of previous chunk (if free).
- size_t _head; // Size and inuse bits.
-};
-
-// ===============================================================================
-struct malloc_chunk : public malloc_chunk_header
-{
- // Set size, pinuse bit, foot, and clear next pinuse
- void set_free_with_pinuse(size_t s, malloc_chunk* n)
- {
- n->clear_pinuse();
- set_size_and_pinuse_of_free_chunk(s);
- }
-
- // Get the internal overhead associated with chunk p
- size_t overhead_for() { return is_mmapped() ? SPP_MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD; }
-
- // Return true if malloced space is not necessarily cleared
- bool calloc_must_clear()
- {
-#if SPP_MMAP_CLEARS
- return !is_mmapped();
-#else
- return true;
-#endif
- }
-
- struct malloc_chunk* _fd; // double links -- used only if free.
- struct malloc_chunk* _bk;
-};
-
-static const unsigned MCHUNK_SIZE = sizeof(malloc_chunk);
-
-/* The smallest size we can malloc is an aligned minimal chunk */
-static const unsigned MIN_CHUNK_SIZE = (MCHUNK_SIZE + spp_chunk_align_mask) & ~spp_chunk_align_mask;
-
-typedef malloc_chunk mchunk;
-typedef malloc_chunk* mchunkptr;
-typedef malloc_chunk_header *hchunkptr;
-typedef malloc_chunk* sbinptr; // The type of bins of chunks
-typedef unsigned int bindex_t; // Described below
-typedef unsigned int binmap_t; // Described below
-typedef unsigned int flag_t; // The type of various bit flag sets
-
-// conversion from malloc headers to user pointers, and back
-static SPP_FORCEINLINE void *chunk2mem(const void *p) { return (void *)((char *)p + 2 * sizeof(size_t)); }
-static SPP_FORCEINLINE mchunkptr mem2chunk(const void *mem) { return (mchunkptr)((char *)mem - 2 * sizeof(size_t)); }
-
-// chunk associated with aligned address A
-static SPP_FORCEINLINE mchunkptr align_as_chunk(char *A) { return (mchunkptr)(A + align_offset(chunk2mem(A))); }
-
-// Bounds on request (not chunk) sizes.
-static const unsigned MAX_REQUEST = (-MIN_CHUNK_SIZE) << 2;
-static const unsigned MIN_REQUEST = MIN_CHUNK_SIZE - CHUNK_OVERHEAD - 1;
-
-// pad request bytes into a usable size
-static SPP_FORCEINLINE size_t pad_request(size_t req)
-{
- return (req + CHUNK_OVERHEAD + spp_chunk_align_mask) & ~spp_chunk_align_mask;
-}
-
-// pad request, checking for minimum (but not maximum)
-static SPP_FORCEINLINE size_t request2size(size_t req)
-{
- return req < MIN_REQUEST ? MIN_CHUNK_SIZE : pad_request(req);
-}
-
-
-/* ------------------ Operations on head and foot fields ----------------- */
-
-/*
- The head field of a chunk is or'ed with PINUSE_BIT when previous
- adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in
- use, unless mmapped, in which case both bits are cleared.
-
- FLAG4_BIT is not used by this malloc, but might be useful in extensions.
-*/
-
-// Head value for fenceposts
-static const unsigned FENCEPOST_HEAD = INUSE_BITS | sizeof(size_t);
-
-
-/* ---------------------- Overlaid data structures ----------------------- */
-
-/*
- When chunks are not in use, they are treated as nodes of either
- lists or trees.
-
- "Small" chunks are stored in circular doubly-linked lists, and look
- like this:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `head:' | Size of chunk, in bytes |P|
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Forward pointer to next chunk in list |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Back pointer to previous chunk in list |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Unused space (may be 0 bytes long) .
- . .
- . |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `foot:' | Size of chunk, in bytes |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- Larger chunks are kept in a form of bitwise digital trees (aka
- tries) keyed on chunksizes. Because malloc_tree_chunks are only for
- free chunks greater than 256 bytes, their size doesn't impose any
- constraints on user chunk sizes. Each node looks like:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `head:' | Size of chunk, in bytes |P|
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Forward pointer to next chunk of same size |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Back pointer to previous chunk of same size |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to left child (child[0]) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to right child (child[1]) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to parent |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | bin index of this chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Unused space .
- . |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `foot:' | Size of chunk, in bytes |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- Each tree holding treenodes is a tree of unique chunk sizes. Chunks
- of the same size are arranged in a circularly-linked list, with only
- the oldest chunk (the next to be used, in our FIFO ordering)
- actually in the tree. (Tree members are distinguished by a non-null
- parent pointer.) If a chunk with the same size an an existing node
- is inserted, it is linked off the existing node using pointers that
- work in the same way as fd/bk pointers of small chunks.
-
- Each tree contains a power of 2 sized range of chunk sizes (the
- smallest is 0x100 <= x < 0x180), which is is divided in half at each
- tree level, with the chunks in the smaller half of the range (0x100
- <= x < 0x140 for the top nose) in the left subtree and the larger
- half (0x140 <= x < 0x180) in the right subtree. This is, of course,
- done by inspecting individual bits.
-
- Using these rules, each node's left subtree contains all smaller
- sizes than its right subtree. However, the node at the root of each
- subtree has no particular ordering relationship to either. (The
- dividing line between the subtree sizes is based on trie relation.)
- If we remove the last chunk of a given size from the interior of the
- tree, we need to replace it with a leaf node. The tree ordering
- rules permit a node to be replaced by any leaf below it.
-
- The smallest chunk in a tree (a common operation in a best-fit
- allocator) can be found by walking a path to the leftmost leaf in
- the tree. Unlike a usual binary tree, where we follow left child
- pointers until we reach a null, here we follow the right child
- pointer any time the left one is null, until we reach a leaf with
- both child pointers null. The smallest chunk in the tree will be
- somewhere along that path.
-
- The worst case number of steps to add, find, or remove a node is
- bounded by the number of bits differentiating chunks within
- bins. Under current bin calculations, this ranges from 6 up to 21
- (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case
- is of course much better.
-*/
-
-// ===============================================================================
-struct malloc_tree_chunk : public malloc_chunk_header
-{
- malloc_tree_chunk *leftmost_child()
- {
- return _child[0] ? _child[0] : _child[1];
- }
-
-
- malloc_tree_chunk* _fd;
- malloc_tree_chunk* _bk;
-
- malloc_tree_chunk* _child[2];
- malloc_tree_chunk* _parent;
- bindex_t _index;
-};
-
-typedef malloc_tree_chunk tchunk;
-typedef malloc_tree_chunk* tchunkptr;
-typedef malloc_tree_chunk* tbinptr; // The type of bins of trees
-
-/* ----------------------------- Segments -------------------------------- */
-
-/*
- Each malloc space may include non-contiguous segments, held in a
- list headed by an embedded malloc_segment record representing the
- top-most space. Segments also include flags holding properties of
- the space. Large chunks that are directly allocated by mmap are not
- included in this list. They are instead independently created and
- destroyed without otherwise keeping track of them.
-
- Segment management mainly comes into play for spaces allocated by
- MMAP. Any call to MMAP might or might not return memory that is
- adjacent to an existing segment. MORECORE normally contiguously
- extends the current space, so this space is almost always adjacent,
- which is simpler and faster to deal with. (This is why MORECORE is
- used preferentially to MMAP when both are available -- see
- sys_alloc.) When allocating using MMAP, we don't use any of the
- hinting mechanisms (inconsistently) supported in various
- implementations of unix mmap, or distinguish reserving from
- committing memory. Instead, we just ask for space, and exploit
- contiguity when we get it. It is probably possible to do
- better than this on some systems, but no general scheme seems
- to be significantly better.
-
- Management entails a simpler variant of the consolidation scheme
- used for chunks to reduce fragmentation -- new adjacent memory is
- normally prepended or appended to an existing segment. However,
- there are limitations compared to chunk consolidation that mostly
- reflect the fact that segment processing is relatively infrequent
- (occurring only when getting memory from system) and that we
- don't expect to have huge numbers of segments:
-
- * Segments are not indexed, so traversal requires linear scans. (It
- would be possible to index these, but is not worth the extra
- overhead and complexity for most programs on most platforms.)
- * New segments are only appended to old ones when holding top-most
- memory; if they cannot be prepended to others, they are held in
- different segments.
-
- Except for the top-most segment of an mstate, each segment record
- is kept at the tail of its segment. Segments are added by pushing
- segment records onto the list headed by &mstate.seg for the
- containing mstate.
-
- Segment flags control allocation/merge/deallocation policies:
- * If EXTERN_BIT set, then we did not allocate this segment,
- and so should not try to deallocate or merge with others.
- (This currently holds only for the initial segment passed
- into create_mspace_with_base.)
- * If USE_MMAP_BIT set, the segment may be merged with
- other surrounding mmapped segments and trimmed/de-allocated
- using munmap.
- * If neither bit is set, then the segment was obtained using
- MORECORE so can be merged with surrounding MORECORE'd segments
- and deallocated/trimmed using MORECORE with negative arguments.
-*/
-
-// ===============================================================================
-struct malloc_segment
-{
- bool is_mmapped_segment() { return !!(_sflags & USE_MMAP_BIT); }
- bool is_extern_segment() { return !!(_sflags & EXTERN_BIT); }
-
- char* _base; // base address
- size_t _size; // allocated size
- malloc_segment* _next; // ptr to next segment
- flag_t _sflags; // mmap and extern flag
-};
-
-typedef malloc_segment msegment;
-typedef malloc_segment* msegmentptr;
-
-/* ------------- Malloc_params ------------------- */
-
-/*
- malloc_params holds global properties, including those that can be
- dynamically set using mallopt. There is a single instance, mparams,
- initialized in init_mparams. Note that the non-zeroness of "magic"
- also serves as an initialization flag.
-*/
-
-// ===============================================================================
-struct malloc_params
-{
- malloc_params() : _magic(0) {}
-
- void ensure_initialization()
- {
- if (!_magic)
- _init();
- }
-
- SPP_IMPL int change(int param_number, int value);
-
- size_t page_align(size_t sz)
- {
- return (sz + (_page_size - 1)) & ~(_page_size - 1);
- }
-
- size_t granularity_align(size_t sz)
- {
- return (sz + (_granularity - 1)) & ~(_granularity - 1);
- }
-
- bool is_page_aligned(char *S)
- {
- return ((size_t)S & (_page_size - 1)) == 0;
- }
-
- SPP_IMPL int _init();
-
- size_t _magic;
- size_t _page_size;
- size_t _granularity;
- size_t _mmap_threshold;
- size_t _trim_threshold;
- flag_t _default_mflags;
-};
-
-static malloc_params mparams;
-
-/* ---------------------------- malloc_state ----------------------------- */
-
-/*
- A malloc_state holds all of the bookkeeping for a space.
- The main fields are:
-
- Top
- The topmost chunk of the currently active segment. Its size is
- cached in topsize. The actual size of topmost space is
- topsize+TOP_FOOT_SIZE, which includes space reserved for adding
- fenceposts and segment records if necessary when getting more
- space from the system. The size at which to autotrim top is
- cached from mparams in trim_check, except that it is disabled if
- an autotrim fails.
-
- Designated victim (dv)
- This is the preferred chunk for servicing small requests that
- don't have exact fits. It is normally the chunk split off most
- recently to service another small request. Its size is cached in
- dvsize. The link fields of this chunk are not maintained since it
- is not kept in a bin.
-
- SmallBins
- An array of bin headers for free chunks. These bins hold chunks
- with sizes less than MIN_LARGE_SIZE bytes. Each bin contains
- chunks of all the same size, spaced 8 bytes apart. To simplify
- use in double-linked lists, each bin header acts as a malloc_chunk
- pointing to the real first node, if it exists (else pointing to
- itself). This avoids special-casing for headers. But to avoid
- waste, we allocate only the fd/bk pointers of bins, and then use
- repositioning tricks to treat these as the fields of a chunk.
-
- TreeBins
- Treebins are pointers to the roots of trees holding a range of
- sizes. There are 2 equally spaced treebins for each power of two
- from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything
- larger.
-
- Bin maps
- There is one bit map for small bins ("smallmap") and one for
- treebins ("treemap). Each bin sets its bit when non-empty, and
- clears the bit when empty. Bit operations are then used to avoid
- bin-by-bin searching -- nearly all "search" is done without ever
- looking at bins that won't be selected. The bit maps
- conservatively use 32 bits per map word, even if on 64bit system.
- For a good description of some of the bit-based techniques used
- here, see Henry S. Warren Jr's book "Hacker's Delight" (and
- supplement at http://hackersdelight.org/). Many of these are
- intended to reduce the branchiness of paths through malloc etc, as
- well as to reduce the number of memory locations read or written.
-
- Segments
- A list of segments headed by an embedded malloc_segment record
- representing the initial space.
-
- Address check support
- The least_addr field is the least address ever obtained from
- MORECORE or MMAP. Attempted frees and reallocs of any address less
- than this are trapped (unless SPP_INSECURE is defined).
-
- Magic tag
- A cross-check field that should always hold same value as mparams._magic.
-
- Max allowed footprint
- The maximum allowed bytes to allocate from system (zero means no limit)
-
- Flags
- Bits recording whether to use MMAP, locks, or contiguous MORECORE
-
- Statistics
- Each space keeps track of current and maximum system memory
- obtained via MORECORE or MMAP.
-
- Trim support
- Fields holding the amount of unused topmost memory that should trigger
- trimming, and a counter to force periodic scanning to release unused
- non-topmost segments.
-
- Extension support
- A void* pointer and a size_t field that can be used to help implement
- extensions to this malloc.
-*/
-
-
-// ================================================================================
-class malloc_state
-{
-public:
- /* ----------------------- _malloc, _free, etc... --- */
- SPP_FORCEINLINE void* _malloc(size_t bytes);
- SPP_FORCEINLINE void _free(mchunkptr p);
-
-
- /* ------------------------ Relays to internal calls to malloc/free from realloc, memalign etc */
- void *internal_malloc(size_t b) { return mspace_malloc(this, b); }
- void internal_free(void *mem) { mspace_free(this, mem); }
-
- /* ------------------------ ----------------------- */
-
- SPP_IMPL void init_top(mchunkptr p, size_t psize);
- SPP_IMPL void init_bins();
- SPP_IMPL void init(char* tbase, size_t tsize);
-
- /* ------------------------ System alloc/dealloc -------------------------- */
- SPP_IMPL void* sys_alloc(size_t nb);
- SPP_IMPL size_t release_unused_segments();
- SPP_IMPL int sys_trim(size_t pad);
- SPP_IMPL void dispose_chunk(mchunkptr p, size_t psize);
-
- /* ----------------------- Internal support for realloc, memalign, etc --- */
- SPP_IMPL mchunkptr try_realloc_chunk(mchunkptr p, size_t nb, int can_move);
- SPP_IMPL void* internal_memalign(size_t alignment, size_t bytes);
- SPP_IMPL void** ialloc(size_t n_elements, size_t* sizes, int opts, void* chunks[]);
- SPP_IMPL size_t internal_bulk_free(void* array[], size_t nelem);
- SPP_IMPL void internal_inspect_all(void(*handler)(void *start, void *end,
- size_t used_bytes, void* callback_arg),
- void* arg);
-
- /* -------------------------- system alloc setup (Operations on mflags) ----- */
- bool use_lock() const { return false; }
- void enable_lock() {}
- void set_lock(int) {}
- void disable_lock() {}
-
- bool use_mmap() const { return !!(_mflags & USE_MMAP_BIT); }
- void enable_mmap() { _mflags |= USE_MMAP_BIT; }
-
-#if SPP_HAVE_MMAP
- void disable_mmap() { _mflags &= ~USE_MMAP_BIT; }
-#else
- void disable_mmap() {}
-#endif
-
- /* ----------------------- Runtime Check Support ------------------------- */
-
- /*
- For security, the main invariant is that malloc/free/etc never
- writes to a static address other than malloc_state, unless static
- malloc_state itself has been corrupted, which cannot occur via
- malloc (because of these checks). In essence this means that we
- believe all pointers, sizes, maps etc held in malloc_state, but
- check all of those linked or offsetted from other embedded data
- structures. These checks are interspersed with main code in a way
- that tends to minimize their run-time cost.
-
- When SPP_FOOTERS is defined, in addition to range checking, we also
- verify footer fields of inuse chunks, which can be used guarantee
- that the mstate controlling malloc/free is intact. This is a
- streamlined version of the approach described by William Robertson
- et al in "Run-time Detection of Heap-based Overflows" LISA'03
- http://www.usenix.org/events/lisa03/tech/robertson.html The footer
- of an inuse chunk holds the xor of its mstate and a random seed,
- that is checked upon calls to free() and realloc(). This is
- (probabalistically) unguessable from outside the program, but can be
- computed by any code successfully malloc'ing any chunk, so does not
- itself provide protection against code that has already broken
- security through some other means. Unlike Robertson et al, we
- always dynamically check addresses of all offset chunks (previous,
- next, etc). This turns out to be cheaper than relying on hashes.
- */
-
-
-#if !SPP_INSECURE
- // Check if address a is at least as high as any from MORECORE or MMAP
- bool ok_address(void *a) const { return (char *)a >= _least_addr; }
-
- // Check if address of next chunk n is higher than base chunk p
- static bool ok_next(void *p, void *n) { return p < n; }
-
- // Check if p has inuse status
- static bool ok_inuse(mchunkptr p) { return p->is_inuse(); }
-
- // Check if p has its pinuse bit on
- static bool ok_pinuse(mchunkptr p) { return p->pinuse(); }
-
- // Check if (alleged) mstate m has expected magic field
- bool ok_magic() const { return _magic == mparams._magic; }
-
- // In gcc, use __builtin_expect to minimize impact of checks
- #if defined(__GNUC__) && __GNUC__ >= 3
- static bool rtcheck(bool e) { return __builtin_expect(e, 1); }
- #else
- static bool rtcheck(bool e) { return e; }
- #endif
-#else
- static bool ok_address(void *) { return true; }
- static bool ok_next(void *, void *) { return true; }
- static bool ok_inuse(mchunkptr) { return true; }
- static bool ok_pinuse(mchunkptr) { return true; }
- static bool ok_magic() { return true; }
- static bool rtcheck(bool) { return true; }
-#endif
-
- bool is_initialized() const { return _top != 0; }
-
- bool use_noncontiguous() const { return !!(_mflags & USE_NONCONTIGUOUS_BIT); }
- void disable_contiguous() { _mflags |= USE_NONCONTIGUOUS_BIT; }
-
- // Return segment holding given address
- msegmentptr segment_holding(char* addr) const
- {
- msegmentptr sp = (msegmentptr)&_seg;
- for (;;)
- {
- if (addr >= sp->_base && addr < sp->_base + sp->_size)
- return sp;
- if ((sp = sp->_next) == 0)
- return 0;
- }
- }
-
- // Return true if segment contains a segment link
- int has_segment_link(msegmentptr ss) const
- {
- msegmentptr sp = (msegmentptr)&_seg;
- for (;;)
- {
- if ((char*)sp >= ss->_base && (char*)sp < ss->_base + ss->_size)
- return 1;
- if ((sp = sp->_next) == 0)
- return 0;
- }
- }
-
- bool should_trim(size_t s) const { return s > _trim_check; }
-
- /* -------------------------- Debugging setup ---------------------------- */
-
-#if ! SPP_DEBUG
- void check_free_chunk(mchunkptr) {}
- void check_inuse_chunk(mchunkptr) {}
- void check_malloced_chunk(void*, size_t) {}
- void check_mmapped_chunk(mchunkptr) {}
- void check_malloc_state() {}
- void check_top_chunk(mchunkptr) {}
-#else /* SPP_DEBUG */
- void check_free_chunk(mchunkptr p) { do_check_free_chunk(p); }
- void check_inuse_chunk(mchunkptr p) { do_check_inuse_chunk(p); }
- void check_malloced_chunk(void* p, size_t s) { do_check_malloced_chunk(p, s); }
- void check_mmapped_chunk(mchunkptr p) { do_check_mmapped_chunk(p); }
- void check_malloc_state() { do_check_malloc_state(); }
- void check_top_chunk(mchunkptr p) { do_check_top_chunk(p); }
-
- void do_check_any_chunk(mchunkptr p) const;
- void do_check_top_chunk(mchunkptr p) const;
- void do_check_mmapped_chunk(mchunkptr p) const;
- void do_check_inuse_chunk(mchunkptr p) const;
- void do_check_free_chunk(mchunkptr p) const;
- void do_check_malloced_chunk(void* mem, size_t s) const;
- void do_check_tree(tchunkptr t);
- void do_check_treebin(bindex_t i);
- void do_check_smallbin(bindex_t i);
- void do_check_malloc_state();
- int bin_find(mchunkptr x);
- size_t traverse_and_check();
-#endif
-
-private:
-
- /* ---------------------------- Indexing Bins ---------------------------- */
-
- static bool is_small(size_t s) { return (s >> SMALLBIN_SHIFT) < NSMALLBINS; }
- static bindex_t small_index(size_t s) { return (bindex_t)(s >> SMALLBIN_SHIFT); }
- static size_t small_index2size(size_t i) { return i << SMALLBIN_SHIFT; }
- static bindex_t MIN_SMALL_INDEX() { return small_index(MIN_CHUNK_SIZE); }
-
- // assign tree index for size S to variable I. Use x86 asm if possible
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
- SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S)
- {
- unsigned int X = S >> TREEBIN_SHIFT;
- if (X == 0)
- return 0;
- else if (X > 0xFFFF)
- return NTREEBINS - 1;
-
- unsigned int K = (unsigned) sizeof(X) * __CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X);
- return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1)));
- }
-
-#elif defined (__INTEL_COMPILER)
- SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S)
- {
- size_t X = S >> TREEBIN_SHIFT;
- if (X == 0)
- return 0;
- else if (X > 0xFFFF)
- return NTREEBINS - 1;
-
- unsigned int K = _bit_scan_reverse(X);
- return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1)));
- }
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
- SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S)
- {
- size_t X = S >> TREEBIN_SHIFT;
- if (X == 0)
- return 0;
- else if (X > 0xFFFF)
- return NTREEBINS - 1;
-
- unsigned int K;
- _BitScanReverse((DWORD *) &K, (DWORD) X);
- return (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1)));
- }
-
-#else // GNUC
- SPP_FORCEINLINE static bindex_t compute_tree_index(size_t S)
- {
- size_t X = S >> TREEBIN_SHIFT;
- if (X == 0)
- return 0;
- else if (X > 0xFFFF)
- return NTREEBINS - 1;
-
- unsigned int Y = (unsigned int)X;
- unsigned int N = ((Y - 0x100) >> 16) & 8;
- unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;
- N += K;
- N += K = (((Y <<= K) - 0x4000) >> 16) & 2;
- K = 14 - N + ((Y <<= K) >> 15);
- return (K << 1) + ((S >> (K + (TREEBIN_SHIFT - 1)) & 1));
- }
-#endif
-
- // Shift placing maximum resolved bit in a treebin at i as sign bit
- static bindex_t leftshift_for_tree_index(bindex_t i)
- {
- return (i == NTREEBINS - 1) ? 0 :
- ((spp_size_t_bitsize - 1) - ((i >> 1) + TREEBIN_SHIFT - 2));
- }
-
- // The size of the smallest chunk held in bin with index i
- static bindex_t minsize_for_tree_index(bindex_t i)
- {
- return ((size_t)1 << ((i >> 1) + TREEBIN_SHIFT)) |
- (((size_t)(i & 1)) << ((i >> 1) + TREEBIN_SHIFT - 1));
- }
-
-
- // ----------- isolate the least set bit of a bitmap
- static binmap_t least_bit(binmap_t x) { return x & -x; }
-
- // ----------- mask with all bits to left of least bit of x on
- static binmap_t left_bits(binmap_t x) { return (x << 1) | -(x << 1); }
-
- // index corresponding to given bit. Use x86 asm if possible
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
- static bindex_t compute_bit2idx(binmap_t X)
- {
- unsigned int J;
- J = __builtin_ctz(X);
- return (bindex_t)J;
- }
-
-#elif defined (__INTEL_COMPILER)
- static bindex_t compute_bit2idx(binmap_t X)
- {
- unsigned int J;
- J = _bit_scan_forward(X);
- return (bindex_t)J;
- }
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
- static bindex_t compute_bit2idx(binmap_t X)
- {
- unsigned int J;
- _BitScanForward((DWORD *) &J, X);
- return (bindex_t)J;
- }
-
-#elif SPP_USE_BUILTIN_FFS
- static bindex_t compute_bit2idx(binmap_t X) { return ffs(X) - 1; }
-
-#else
- static bindex_t compute_bit2idx(binmap_t X)
- {
- unsigned int Y = X - 1;
- unsigned int K = Y >> (16 - 4) & 16;
- unsigned int N = K; Y >>= K;
- N += K = Y >> (8 - 3) & 8; Y >>= K;
- N += K = Y >> (4 - 2) & 4; Y >>= K;
- N += K = Y >> (2 - 1) & 2; Y >>= K;
- N += K = Y >> (1 - 0) & 1; Y >>= K;
- return (bindex_t)(N + Y);
- }
-#endif
-
- /* ------------------------ Set up inuse chunks with or without footers ---*/
-#if !SPP_FOOTERS
- void mark_inuse_foot(malloc_chunk_header *, size_t) {}
-#else
- //Set foot of inuse chunk to be xor of mstate and seed
- void mark_inuse_foot(malloc_chunk_header *p, size_t s)
- {
- (((mchunkptr)((char*)p + s))->prev_foot = (size_t)this ^ mparams._magic);
- }
-#endif
-
- void set_inuse(malloc_chunk_header *p, size_t s)
- {
- p->_head = (p->_head & PINUSE_BIT) | s | CINUSE_BIT;
- ((mchunkptr)(((char*)p) + s))->_head |= PINUSE_BIT;
- mark_inuse_foot(p, s);
- }
-
- void set_inuse_and_pinuse(malloc_chunk_header *p, size_t s)
- {
- p->_head = s | PINUSE_BIT | CINUSE_BIT;
- ((mchunkptr)(((char*)p) + s))->_head |= PINUSE_BIT;
- mark_inuse_foot(p, s);
- }
-
- void set_size_and_pinuse_of_inuse_chunk(malloc_chunk_header *p, size_t s)
- {
- p->_head = s | PINUSE_BIT | CINUSE_BIT;
- mark_inuse_foot(p, s);
- }
-
- /* ------------------------ Addressing by index. See about smallbin repositioning --- */
- sbinptr smallbin_at(bindex_t i) const { return (sbinptr)((char*)&_smallbins[i << 1]); }
- tbinptr* treebin_at(bindex_t i) { return &_treebins[i]; }
-
- /* ----------------------- bit corresponding to given index ---------*/
- static binmap_t idx2bit(bindex_t i) { return ((binmap_t)1 << i); }
-
- // --------------- Mark/Clear bits with given index
- void mark_smallmap(bindex_t i) { _smallmap |= idx2bit(i); }
- void clear_smallmap(bindex_t i) { _smallmap &= ~idx2bit(i); }
- binmap_t smallmap_is_marked(bindex_t i) const { return _smallmap & idx2bit(i); }
-
- void mark_treemap(bindex_t i) { _treemap |= idx2bit(i); }
- void clear_treemap(bindex_t i) { _treemap &= ~idx2bit(i); }
- binmap_t treemap_is_marked(bindex_t i) const { return _treemap & idx2bit(i); }
-
- /* ------------------------ ----------------------- */
- SPP_FORCEINLINE void insert_small_chunk(mchunkptr P, size_t S);
- SPP_FORCEINLINE void unlink_small_chunk(mchunkptr P, size_t S);
- SPP_FORCEINLINE void unlink_first_small_chunk(mchunkptr B, mchunkptr P, bindex_t I);
- SPP_FORCEINLINE void replace_dv(mchunkptr P, size_t S);
-
- /* ------------------------- Operations on trees ------------------------- */
- SPP_FORCEINLINE void insert_large_chunk(tchunkptr X, size_t S);
- SPP_FORCEINLINE void unlink_large_chunk(tchunkptr X);
-
- /* ------------------------ Relays to large vs small bin operations */
- SPP_FORCEINLINE void insert_chunk(mchunkptr P, size_t S);
- SPP_FORCEINLINE void unlink_chunk(mchunkptr P, size_t S);
-
- /* ----------------------- Direct-mmapping chunks ----------------------- */
- SPP_IMPL void* mmap_alloc(size_t nb);
- SPP_IMPL mchunkptr mmap_resize(mchunkptr oldp, size_t nb, int flags);
-
- SPP_IMPL void reset_on_error();
- SPP_IMPL void* prepend_alloc(char* newbase, char* oldbase, size_t nb);
- SPP_IMPL void add_segment(char* tbase, size_t tsize, flag_t mmapped);
-
- /* ------------------------ malloc --------------------------- */
- SPP_IMPL void* tmalloc_large(size_t nb);
- SPP_IMPL void* tmalloc_small(size_t nb);
-
- /* ------------------------Bin types, widths and sizes -------- */
- static const size_t NSMALLBINS = 32;
- static const size_t NTREEBINS = 32;
- static const size_t SMALLBIN_SHIFT = 3;
- static const size_t SMALLBIN_WIDTH = 1 << SMALLBIN_SHIFT;
- static const size_t TREEBIN_SHIFT = 8;
- static const size_t MIN_LARGE_SIZE = 1 << TREEBIN_SHIFT;
- static const size_t MAX_SMALL_SIZE = (MIN_LARGE_SIZE - 1);
- static const size_t MAX_SMALL_REQUEST = (MAX_SMALL_SIZE - spp_chunk_align_mask - CHUNK_OVERHEAD);
-
- /* ------------------------ data members --------------------------- */
- binmap_t _smallmap;
- binmap_t _treemap;
- size_t _dvsize;
- size_t _topsize;
- char* _least_addr;
- mchunkptr _dv;
- mchunkptr _top;
- size_t _trim_check;
- size_t _release_checks;
- size_t _magic;
- mchunkptr _smallbins[(NSMALLBINS + 1) * 2];
- tbinptr _treebins[NTREEBINS];
-public:
- size_t _footprint;
- size_t _max_footprint;
- size_t _footprint_limit; // zero means no limit
- flag_t _mflags;
-
- msegment _seg;
-
-private:
- void* _extp; // Unused but available for extensions
- size_t _exts;
-};
-
-typedef malloc_state* mstate;
-
-/* ------------- end malloc_state ------------------- */
-
-#if SPP_FOOTERS
-static malloc_state* get_mstate_for(malloc_chunk_header *p)
-{
- return (malloc_state*)(((mchunkptr)((char*)(p) +
- (p->chunksize())))->prev_foot ^ mparams._magic);
-}
-#endif
-
-/* -------------------------- system alloc setup ------------------------- */
-
-
-
-// For mmap, use granularity alignment on windows, else page-align
-#ifdef WIN32
- #define mmap_align(S) mparams.granularity_align(S)
-#else
- #define mmap_align(S) mparams.page_align(S)
-#endif
-
-// True if segment S holds address A
-static bool segment_holds(msegmentptr S, mchunkptr A)
-{
- return (char*)A >= S->_base && (char*)A < S->_base + S->_size;
-}
-
-/*
- top_foot_size is padding at the end of a segment, including space
- that may be needed to place segment records and fenceposts when new
- noncontiguous segments are added.
-*/
-static SPP_FORCEINLINE size_t top_foot_size()
-{
- return align_offset(chunk2mem((void *)0)) +
- pad_request(sizeof(struct malloc_segment)) +
- MIN_CHUNK_SIZE;
-}
-
-
-// For sys_alloc, enough padding to ensure can malloc request on success
-static SPP_FORCEINLINE size_t sys_alloc_padding()
-{
- return top_foot_size() + SPP_MALLOC_ALIGNMENT;
-}
-
-
-#define SPP_USAGE_ERROR_ACTION(m,p) SPP_ABORT
-
-/* ---------------------------- setting mparams -------------------------- */
-
-// Initialize mparams
-int malloc_params::_init()
-{
-#ifdef NEED_GLOBAL_LOCK_INIT
- if (malloc_global_mutex_status <= 0)
- init_malloc_global_mutex();
-#endif
-
- if (_magic == 0)
- {
- size_t magic;
- size_t psize;
- size_t gsize;
-
-#ifndef WIN32
- psize = malloc_getpagesize;
- gsize = ((SPP_DEFAULT_GRANULARITY != 0) ? SPP_DEFAULT_GRANULARITY : psize);
-#else
- {
- SYSTEM_INFO system_info;
- GetSystemInfo(&system_info);
- psize = system_info.dwPageSize;
- gsize = ((SPP_DEFAULT_GRANULARITY != 0) ?
- SPP_DEFAULT_GRANULARITY : system_info.dwAllocationGranularity);
- }
-#endif
-
- /* Sanity-check configuration:
- size_t must be unsigned and as wide as pointer type.
- ints must be at least 4 bytes.
- alignment must be at least 8.
- Alignment, min chunk size, and page size must all be powers of 2.
- */
- if ((sizeof(size_t) != sizeof(char*)) ||
- (spp_max_size_t < MIN_CHUNK_SIZE) ||
- (sizeof(int) < 4) ||
- (SPP_MALLOC_ALIGNMENT < (size_t)8U) ||
- ((SPP_MALLOC_ALIGNMENT & (SPP_MALLOC_ALIGNMENT - 1)) != 0) ||
- ((MCHUNK_SIZE & (MCHUNK_SIZE - 1)) != 0) ||
- ((gsize & (gsize - 1)) != 0) ||
- ((psize & (psize - 1)) != 0))
- SPP_ABORT;
- _granularity = gsize;
- _page_size = psize;
- _mmap_threshold = SPP_DEFAULT_MMAP_THRESHOLD;
- _trim_threshold = SPP_DEFAULT_TRIM_THRESHOLD;
- _default_mflags = USE_MMAP_BIT | USE_NONCONTIGUOUS_BIT;
-
- {
-#if SPP_USE_DEV_RANDOM
- int fd;
- unsigned char buf[sizeof(size_t)];
- // Try to use /dev/urandom, else fall back on using time
- if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 &&
- read(fd, buf, sizeof(buf)) == sizeof(buf))
- {
- magic = *((size_t *) buf);
- close(fd);
- }
- else
-#endif
- {
-#ifdef WIN32
- magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U);
-#elif defined(SPP_LACKS_TIME_H)
- magic = (size_t)&magic ^ (size_t)0x55555555U;
-#else
- magic = (size_t)(time(0) ^ (size_t)0x55555555U);
-#endif
- }
- magic |= (size_t)8U; // ensure nonzero
- magic &= ~(size_t)7U; // improve chances of fault for bad values
- // Until memory modes commonly available, use volatile-write
- (*(volatile size_t *)(&(_magic))) = magic;
- }
- }
-
- return 1;
-}
-
-/*
- mallopt tuning options. SVID/XPG defines four standard parameter
- numbers for mallopt, normally defined in malloc.h. None of these
- are used in this malloc, so setting them has no effect. But this
- malloc does support the following options.
-*/
-static const int m_trim_threshold = -1;
-static const int m_granularity = -2;
-static const int m_mmap_threshold = -3;
-
-// support for mallopt
-int malloc_params::change(int param_number, int value)
-{
- size_t val;
- ensure_initialization();
- val = (value == -1) ? spp_max_size_t : (size_t)value;
-
- switch (param_number)
- {
- case m_trim_threshold:
- _trim_threshold = val;
- return 1;
-
- case m_granularity:
- if (val >= _page_size && ((val & (val - 1)) == 0))
- {
- _granularity = val;
- return 1;
- }
- else
- return 0;
-
- case m_mmap_threshold:
- _mmap_threshold = val;
- return 1;
-
- default:
- return 0;
- }
-}
-
-#if SPP_DEBUG
-/* ------------------------- Debugging Support --------------------------- */
-
-// Check properties of any chunk, whether free, inuse, mmapped etc
-void malloc_state::do_check_any_chunk(mchunkptr p) const
-{
- assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD));
- assert(ok_address(p));
-}
-
-// Check properties of top chunk
-void malloc_state::do_check_top_chunk(mchunkptr p) const
-{
- msegmentptr sp = segment_holding((char*)p);
- size_t sz = p->_head & ~INUSE_BITS; // third-lowest bit can be set!
- assert(sp != 0);
- assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD));
- assert(ok_address(p));
- assert(sz == _topsize);
- assert(sz > 0);
- assert(sz == ((sp->_base + sp->_size) - (char*)p) - top_foot_size());
- assert(p->pinuse());
- assert(!p->chunk_plus_offset(sz)->pinuse());
-}
-
-// Check properties of (inuse) mmapped chunks
-void malloc_state::do_check_mmapped_chunk(mchunkptr p) const
-{
- size_t sz = p->chunksize();
- size_t len = (sz + (p->_prev_foot) + SPP_MMAP_FOOT_PAD);
- assert(p->is_mmapped());
- assert(use_mmap());
- assert((spp_is_aligned(chunk2mem(p))) || (p->_head == FENCEPOST_HEAD));
- assert(ok_address(p));
- assert(!is_small(sz));
- assert((len & (mparams._page_size - 1)) == 0);
- assert(p->chunk_plus_offset(sz)->_head == FENCEPOST_HEAD);
- assert(p->chunk_plus_offset(sz + sizeof(size_t))->_head == 0);
-}
-
-// Check properties of inuse chunks
-void malloc_state::do_check_inuse_chunk(mchunkptr p) const
-{
- do_check_any_chunk(p);
- assert(p->is_inuse());
- assert(p->next_pinuse());
- // If not pinuse and not mmapped, previous chunk has OK offset
- assert(p->is_mmapped() || p->pinuse() || (mchunkptr)p->prev_chunk()->next_chunk() == p);
- if (p->is_mmapped())
- do_check_mmapped_chunk(p);
-}
-
-// Check properties of free chunks
-void malloc_state::do_check_free_chunk(mchunkptr p) const
-{
- size_t sz = p->chunksize();
- mchunkptr next = (mchunkptr)p->chunk_plus_offset(sz);
- do_check_any_chunk(p);
- assert(!p->is_inuse());
- assert(!p->next_pinuse());
- assert(!p->is_mmapped());
- if (p != _dv && p != _top)
- {
- if (sz >= MIN_CHUNK_SIZE)
- {
- assert((sz & spp_chunk_align_mask) == 0);
- assert(spp_is_aligned(chunk2mem(p)));
- assert(next->_prev_foot == sz);
- assert(p->pinuse());
- assert(next == _top || next->is_inuse());
- assert(p->_fd->_bk == p);
- assert(p->_bk->_fd == p);
- }
- else // markers are always of size sizeof(size_t)
- assert(sz == sizeof(size_t));
- }
-}
-
-// Check properties of malloced chunks at the point they are malloced
-void malloc_state::do_check_malloced_chunk(void* mem, size_t s) const
-{
- if (mem != 0)
- {
- mchunkptr p = mem2chunk(mem);
- size_t sz = p->_head & ~INUSE_BITS;
- do_check_inuse_chunk(p);
- assert((sz & spp_chunk_align_mask) == 0);
- assert(sz >= MIN_CHUNK_SIZE);
- assert(sz >= s);
- // unless mmapped, size is less than MIN_CHUNK_SIZE more than request
- assert(p->is_mmapped() || sz < (s + MIN_CHUNK_SIZE));
- }
-}
-
-// Check a tree and its subtrees.
-void malloc_state::do_check_tree(tchunkptr t)
-{
- tchunkptr head = 0;
- tchunkptr u = t;
- bindex_t tindex = t->_index;
- size_t tsize = t->chunksize();
- bindex_t idx = compute_tree_index(tsize);
- assert(tindex == idx);
- assert(tsize >= MIN_LARGE_SIZE);
- assert(tsize >= minsize_for_tree_index(idx));
- assert((idx == NTREEBINS - 1) || (tsize < minsize_for_tree_index((idx + 1))));
-
- do
- {
- // traverse through chain of same-sized nodes
- do_check_any_chunk((mchunkptr)u);
- assert(u->_index == tindex);
- assert(u->chunksize() == tsize);
- assert(!u->is_inuse());
- assert(!u->next_pinuse());
- assert(u->_fd->_bk == u);
- assert(u->_bk->_fd == u);
- if (u->_parent == 0)
- {
- assert(u->_child[0] == 0);
- assert(u->_child[1] == 0);
- }
- else
- {
- assert(head == 0); // only one node on chain has parent
- head = u;
- assert(u->_parent != u);
- assert(u->_parent->_child[0] == u ||
- u->_parent->_child[1] == u ||
- *((tbinptr*)(u->_parent)) == u);
- if (u->_child[0] != 0)
- {
- assert(u->_child[0]->_parent == u);
- assert(u->_child[0] != u);
- do_check_tree(u->_child[0]);
- }
- if (u->_child[1] != 0)
- {
- assert(u->_child[1]->_parent == u);
- assert(u->_child[1] != u);
- do_check_tree(u->_child[1]);
- }
- if (u->_child[0] != 0 && u->_child[1] != 0)
- assert(u->_child[0]->chunksize() < u->_child[1]->chunksize());
- }
- u = u->_fd;
- }
- while (u != t);
- assert(head != 0);
-}
-
-// Check all the chunks in a treebin.
-void malloc_state::do_check_treebin(bindex_t i)
-{
- tbinptr* tb = (tbinptr*)treebin_at(i);
- tchunkptr t = *tb;
- int empty = (_treemap & (1U << i)) == 0;
- if (t == 0)
- assert(empty);
- if (!empty)
- do_check_tree(t);
-}
-
-// Check all the chunks in a smallbin.
-void malloc_state::do_check_smallbin(bindex_t i)
-{
- sbinptr b = smallbin_at(i);
- mchunkptr p = b->_bk;
- unsigned int empty = (_smallmap & (1U << i)) == 0;
- if (p == b)
- assert(empty);
- if (!empty)
- {
- for (; p != b; p = p->_bk)
- {
- size_t size = p->chunksize();
- mchunkptr q;
- // each chunk claims to be free
- do_check_free_chunk(p);
- // chunk belongs in bin
- assert(small_index(size) == i);
- assert(p->_bk == b || p->_bk->chunksize() == p->chunksize());
- // chunk is followed by an inuse chunk
- q = (mchunkptr)p->next_chunk();
- if (q->_head != FENCEPOST_HEAD)
- do_check_inuse_chunk(q);
- }
- }
-}
-
-// Find x in a bin. Used in other check functions.
-int malloc_state::bin_find(mchunkptr x)
-{
- size_t size = x->chunksize();
- if (is_small(size))
- {
- bindex_t sidx = small_index(size);
- sbinptr b = smallbin_at(sidx);
- if (smallmap_is_marked(sidx))
- {
- mchunkptr p = b;
- do
- {
- if (p == x)
- return 1;
- }
- while ((p = p->_fd) != b);
- }
- }
- else
- {
- bindex_t tidx = compute_tree_index(size);
- if (treemap_is_marked(tidx))
- {
- tchunkptr t = *treebin_at(tidx);
- size_t sizebits = size << leftshift_for_tree_index(tidx);
- while (t != 0 && t->chunksize() != size)
- {
- t = t->_child[(sizebits >> (spp_size_t_bitsize - 1)) & 1];
- sizebits <<= 1;
- }
- if (t != 0)
- {
- tchunkptr u = t;
- do
- {
- if (u == (tchunkptr)x)
- return 1;
- }
- while ((u = u->_fd) != t);
- }
- }
- }
- return 0;
-}
-
-// Traverse each chunk and check it; return total
-size_t malloc_state::traverse_and_check()
-{
- size_t sum = 0;
- if (is_initialized())
- {
- msegmentptr s = (msegmentptr)&_seg;
- sum += _topsize + top_foot_size();
- while (s != 0)
- {
- mchunkptr q = align_as_chunk(s->_base);
- mchunkptr lastq = 0;
- assert(q->pinuse());
- while (segment_holds(s, q) &&
- q != _top && q->_head != FENCEPOST_HEAD)
- {
- sum += q->chunksize();
- if (q->is_inuse())
- {
- assert(!bin_find(q));
- do_check_inuse_chunk(q);
- }
- else
- {
- assert(q == _dv || bin_find(q));
- assert(lastq == 0 || lastq->is_inuse()); // Not 2 consecutive free
- do_check_free_chunk(q);
- }
- lastq = q;
- q = (mchunkptr)q->next_chunk();
- }
- s = s->_next;
- }
- }
- return sum;
-}
-
-
-// Check all properties of malloc_state.
-void malloc_state::do_check_malloc_state()
-{
- bindex_t i;
- size_t total;
- // check bins
- for (i = 0; i < NSMALLBINS; ++i)
- do_check_smallbin(i);
- for (i = 0; i < NTREEBINS; ++i)
- do_check_treebin(i);
-
- if (_dvsize != 0)
- {
- // check dv chunk
- do_check_any_chunk(_dv);
- assert(_dvsize == _dv->chunksize());
- assert(_dvsize >= MIN_CHUNK_SIZE);
- assert(bin_find(_dv) == 0);
- }
-
- if (_top != 0)
- {
- // check top chunk
- do_check_top_chunk(_top);
- //assert(topsize == top->chunksize()); redundant
- assert(_topsize > 0);
- assert(bin_find(_top) == 0);
- }
-
- total = traverse_and_check();
- assert(total <= _footprint);
- assert(_footprint <= _max_footprint);
-}
-#endif // SPP_DEBUG
-
-/* ----------------------- Operations on smallbins ----------------------- */
-
-/*
- Various forms of linking and unlinking are defined as macros. Even
- the ones for trees, which are very long but have very short typical
- paths. This is ugly but reduces reliance on inlining support of
- compilers.
-*/
-
-// Link a free chunk into a smallbin
-void malloc_state::insert_small_chunk(mchunkptr p, size_t s)
-{
- bindex_t I = small_index(s);
- mchunkptr B = smallbin_at(I);
- mchunkptr F = B;
- assert(s >= MIN_CHUNK_SIZE);
- if (!smallmap_is_marked(I))
- mark_smallmap(I);
- else if (rtcheck(ok_address(B->_fd)))
- F = B->_fd;
- else
- SPP_ABORT;
- B->_fd = p;
- F->_bk = p;
- p->_fd = F;
- p->_bk = B;
-}
-
-// Unlink a chunk from a smallbin
-void malloc_state::unlink_small_chunk(mchunkptr p, size_t s)
-{
- mchunkptr F = p->_fd;
- mchunkptr B = p->_bk;
- bindex_t I = small_index(s);
- assert(p != B);
- assert(p != F);
- assert(p->chunksize() == small_index2size(I));
- if (rtcheck(F == smallbin_at(I) || (ok_address(F) && F->_bk == p)))
- {
- if (B == F)
- clear_smallmap(I);
- else if (rtcheck(B == smallbin_at(I) ||
- (ok_address(B) && B->_fd == p)))
- {
- F->_bk = B;
- B->_fd = F;
- }
- else
- SPP_ABORT;
- }
- else
- SPP_ABORT;
-}
-
-// Unlink the first chunk from a smallbin
-void malloc_state::unlink_first_small_chunk(mchunkptr B, mchunkptr p, bindex_t I)
-{
- mchunkptr F = p->_fd;
- assert(p != B);
- assert(p != F);
- assert(p->chunksize() == small_index2size(I));
- if (B == F)
- clear_smallmap(I);
- else if (rtcheck(ok_address(F) && F->_bk == p))
- {
- F->_bk = B;
- B->_fd = F;
- }
- else
- SPP_ABORT;
-}
-
-// Replace dv node, binning the old one
-// Used only when dvsize known to be small
-void malloc_state::replace_dv(mchunkptr p, size_t s)
-{
- size_t DVS = _dvsize;
- assert(is_small(DVS));
- if (DVS != 0)
- {
- mchunkptr DV = _dv;
- insert_small_chunk(DV, DVS);
- }
- _dvsize = s;
- _dv = p;
-}
-
-/* ------------------------- Operations on trees ------------------------- */
-
-// Insert chunk into tree
-void malloc_state::insert_large_chunk(tchunkptr X, size_t s)
-{
- tbinptr* H;
- bindex_t I = compute_tree_index(s);
- H = treebin_at(I);
- X->_index = I;
- X->_child[0] = X->_child[1] = 0;
- if (!treemap_is_marked(I))
- {
- mark_treemap(I);
- *H = X;
- X->_parent = (tchunkptr)H;
- X->_fd = X->_bk = X;
- }
- else
- {
- tchunkptr T = *H;
- size_t K = s << leftshift_for_tree_index(I);
- for (;;)
- {
- if (T->chunksize() != s)
- {
- tchunkptr* C = &(T->_child[(K >> (spp_size_t_bitsize - 1)) & 1]);
- K <<= 1;
- if (*C != 0)
- T = *C;
- else if (rtcheck(ok_address(C)))
- {
- *C = X;
- X->_parent = T;
- X->_fd = X->_bk = X;
- break;
- }
- else
- {
- SPP_ABORT;
- break;
- }
- }
- else
- {
- tchunkptr F = T->_fd;
- if (rtcheck(ok_address(T) && ok_address(F)))
- {
- T->_fd = F->_bk = X;
- X->_fd = F;
- X->_bk = T;
- X->_parent = 0;
- break;
- }
- else
- {
- SPP_ABORT;
- break;
- }
- }
- }
- }
-}
-
-/*
- Unlink steps:
-
- 1. If x is a chained node, unlink it from its same-sized fd/bk links
- and choose its bk node as its replacement.
- 2. If x was the last node of its size, but not a leaf node, it must
- be replaced with a leaf node (not merely one with an open left or
- right), to make sure that lefts and rights of descendents
- correspond properly to bit masks. We use the rightmost descendent
- of x. We could use any other leaf, but this is easy to locate and
- tends to counteract removal of leftmosts elsewhere, and so keeps
- paths shorter than minimally guaranteed. This doesn't loop much
- because on average a node in a tree is near the bottom.
- 3. If x is the base of a chain (i.e., has parent links) relink
- x's parent and children to x's replacement (or null if none).
-*/
-
-void malloc_state::unlink_large_chunk(tchunkptr X)
-{
- tchunkptr XP = X->_parent;
- tchunkptr R;
- if (X->_bk != X)
- {
- tchunkptr F = X->_fd;
- R = X->_bk;
- if (rtcheck(ok_address(F) && F->_bk == X && R->_fd == X))
- {
- F->_bk = R;
- R->_fd = F;
- }
- else
- SPP_ABORT;
- }
- else
- {
- tchunkptr* RP;
- if (((R = *(RP = &(X->_child[1]))) != 0) ||
- ((R = *(RP = &(X->_child[0]))) != 0))
- {
- tchunkptr* CP;
- while ((*(CP = &(R->_child[1])) != 0) ||
- (*(CP = &(R->_child[0])) != 0))
- R = *(RP = CP);
- if (rtcheck(ok_address(RP)))
- *RP = 0;
- else
- SPP_ABORT;
- }
- }
- if (XP != 0)
- {
- tbinptr* H = treebin_at(X->_index);
- if (X == *H)
- {
- if ((*H = R) == 0)
- clear_treemap(X->_index);
- }
- else if (rtcheck(ok_address(XP)))
- {
- if (XP->_child[0] == X)
- XP->_child[0] = R;
- else
- XP->_child[1] = R;
- }
- else
- SPP_ABORT;
- if (R != 0)
- {
- if (rtcheck(ok_address(R)))
- {
- tchunkptr C0, C1;
- R->_parent = XP;
- if ((C0 = X->_child[0]) != 0)
- {
- if (rtcheck(ok_address(C0)))
- {
- R->_child[0] = C0;
- C0->_parent = R;
- }
- else
- SPP_ABORT;
- }
- if ((C1 = X->_child[1]) != 0)
- {
- if (rtcheck(ok_address(C1)))
- {
- R->_child[1] = C1;
- C1->_parent = R;
- }
- else
- SPP_ABORT;
- }
- }
- else
- SPP_ABORT;
- }
- }
-}
-
-// Relays to large vs small bin operations
-
-void malloc_state::insert_chunk(mchunkptr p, size_t s)
-{
- if (is_small(s))
- insert_small_chunk(p, s);
- else
- {
- tchunkptr tp = (tchunkptr)(p);
- insert_large_chunk(tp, s);
- }
-}
-
-void malloc_state::unlink_chunk(mchunkptr p, size_t s)
-{
- if (is_small(s))
- unlink_small_chunk(p, s);
- else
- {
- tchunkptr tp = (tchunkptr)(p);
- unlink_large_chunk(tp);
- }
-}
-
-
-/* ----------------------- Direct-mmapping chunks ----------------------- */
-
-/*
- Directly mmapped chunks are set up with an offset to the start of
- the mmapped region stored in the prev_foot field of the chunk. This
- allows reconstruction of the required argument to MUNMAP when freed,
- and also allows adjustment of the returned chunk to meet alignment
- requirements (especially in memalign).
-*/
-
-// Malloc using mmap
-void* malloc_state::mmap_alloc(size_t nb)
-{
- size_t mmsize = mmap_align(nb + 6 * sizeof(size_t) + spp_chunk_align_mask);
- if (_footprint_limit != 0)
- {
- size_t fp = _footprint + mmsize;
- if (fp <= _footprint || fp > _footprint_limit)
- return 0;
- }
- if (mmsize > nb)
- {
- // Check for wrap around 0
- char* mm = (char*)(SPP_CALL_DIRECT_MMAP(mmsize));
- if (mm != cmfail)
- {
- size_t offset = align_offset(chunk2mem(mm));
- size_t psize = mmsize - offset - SPP_MMAP_FOOT_PAD;
- mchunkptr p = (mchunkptr)(mm + offset);
- p->_prev_foot = offset;
- p->_head = psize;
- mark_inuse_foot(p, psize);
- p->chunk_plus_offset(psize)->_head = FENCEPOST_HEAD;
- p->chunk_plus_offset(psize + sizeof(size_t))->_head = 0;
-
- if (_least_addr == 0 || mm < _least_addr)
- _least_addr = mm;
- if ((_footprint += mmsize) > _max_footprint)
- _max_footprint = _footprint;
- assert(spp_is_aligned(chunk2mem(p)));
- check_mmapped_chunk(p);
- return chunk2mem(p);
- }
- }
- return 0;
-}
-
-// Realloc using mmap
-mchunkptr malloc_state::mmap_resize(mchunkptr oldp, size_t nb, int flags)
-{
- size_t oldsize = oldp->chunksize();
- (void)flags; // placate people compiling -Wunused
- if (is_small(nb)) // Can't shrink mmap regions below small size
- return 0;
-
- // Keep old chunk if big enough but not too big
- if (oldsize >= nb + sizeof(size_t) &&
- (oldsize - nb) <= (mparams._granularity << 1))
- return oldp;
- else
- {
- size_t offset = oldp->_prev_foot;
- size_t oldmmsize = oldsize + offset + SPP_MMAP_FOOT_PAD;
- size_t newmmsize = mmap_align(nb + 6 * sizeof(size_t) + spp_chunk_align_mask);
- char* cp = (char*)SPP_CALL_MREMAP((char*)oldp - offset,
- oldmmsize, newmmsize, flags);
- if (cp != cmfail)
- {
- mchunkptr newp = (mchunkptr)(cp + offset);
- size_t psize = newmmsize - offset - SPP_MMAP_FOOT_PAD;
- newp->_head = psize;
- mark_inuse_foot(newp, psize);
- newp->chunk_plus_offset(psize)->_head = FENCEPOST_HEAD;
- newp->chunk_plus_offset(psize + sizeof(size_t))->_head = 0;
-
- if (cp < _least_addr)
- _least_addr = cp;
- if ((_footprint += newmmsize - oldmmsize) > _max_footprint)
- _max_footprint = _footprint;
- check_mmapped_chunk(newp);
- return newp;
- }
- }
- return 0;
-}
-
-
-/* -------------------------- mspace management -------------------------- */
-
-// Initialize top chunk and its size
-void malloc_state::init_top(mchunkptr p, size_t psize)
-{
- // Ensure alignment
- size_t offset = align_offset(chunk2mem(p));
- p = (mchunkptr)((char*)p + offset);
- psize -= offset;
-
- _top = p;
- _topsize = psize;
- p->_head = psize | PINUSE_BIT;
- // set size of fake trailing chunk holding overhead space only once
- p->chunk_plus_offset(psize)->_head = top_foot_size();
- _trim_check = mparams._trim_threshold; // reset on each update
-}
-
-// Initialize bins for a new mstate that is otherwise zeroed out
-void malloc_state::init_bins()
-{
- // Establish circular links for smallbins
- bindex_t i;
- for (i = 0; i < NSMALLBINS; ++i)
- {
- sbinptr bin = smallbin_at(i);
- bin->_fd = bin->_bk = bin;
- }
-}
-
-#if SPP_PROCEED_ON_ERROR
-
-// default corruption action
-void malloc_state::reset_on_error()
-{
- int i;
- ++malloc_corruption_error_count;
- // Reinitialize fields to forget about all memory
- _smallmap = _treemap = 0;
- _dvsize = _topsize = 0;
- _seg._base = 0;
- _seg._size = 0;
- _seg._next = 0;
- _top = _dv = 0;
- for (i = 0; i < NTREEBINS; ++i)
- *treebin_at(i) = 0;
- init_bins();
-}
-#endif
-
-/* Allocate chunk and prepend remainder with chunk in successor base. */
-void* malloc_state::prepend_alloc(char* newbase, char* oldbase, size_t nb)
-{
- mchunkptr p = align_as_chunk(newbase);
- mchunkptr oldfirst = align_as_chunk(oldbase);
- size_t psize = (char*)oldfirst - (char*)p;
- mchunkptr q = (mchunkptr)p->chunk_plus_offset(nb);
- size_t qsize = psize - nb;
- set_size_and_pinuse_of_inuse_chunk(p, nb);
-
- assert((char*)oldfirst > (char*)q);
- assert(oldfirst->pinuse());
- assert(qsize >= MIN_CHUNK_SIZE);
-
- // consolidate remainder with first chunk of old base
- if (oldfirst == _top)
- {
- size_t tsize = _topsize += qsize;
- _top = q;
- q->_head = tsize | PINUSE_BIT;
- check_top_chunk(q);
- }
- else if (oldfirst == _dv)
- {
- size_t dsize = _dvsize += qsize;
- _dv = q;
- q->set_size_and_pinuse_of_free_chunk(dsize);
- }
- else
- {
- if (!oldfirst->is_inuse())
- {
- size_t nsize = oldfirst->chunksize();
- unlink_chunk(oldfirst, nsize);
- oldfirst = (mchunkptr)oldfirst->chunk_plus_offset(nsize);
- qsize += nsize;
- }
- q->set_free_with_pinuse(qsize, oldfirst);
- insert_chunk(q, qsize);
- check_free_chunk(q);
- }
-
- check_malloced_chunk(chunk2mem(p), nb);
- return chunk2mem(p);
-}
-
-// Add a segment to hold a new noncontiguous region
-void malloc_state::add_segment(char* tbase, size_t tsize, flag_t mmapped)
-{
- // Determine locations and sizes of segment, fenceposts, old top
- char* old_top = (char*)_top;
- msegmentptr oldsp = segment_holding(old_top);
- char* old_end = oldsp->_base + oldsp->_size;
- size_t ssize = pad_request(sizeof(struct malloc_segment));
- char* rawsp = old_end - (ssize + 4 * sizeof(size_t) + spp_chunk_align_mask);
- size_t offset = align_offset(chunk2mem(rawsp));
- char* asp = rawsp + offset;
- char* csp = (asp < (old_top + MIN_CHUNK_SIZE)) ? old_top : asp;
- mchunkptr sp = (mchunkptr)csp;
- msegmentptr ss = (msegmentptr)(chunk2mem(sp));
- mchunkptr tnext = (mchunkptr)sp->chunk_plus_offset(ssize);
- mchunkptr p = tnext;
- int nfences = 0;
-
- // reset top to new space
- init_top((mchunkptr)tbase, tsize - top_foot_size());
-
- // Set up segment record
- assert(spp_is_aligned(ss));
- set_size_and_pinuse_of_inuse_chunk(sp, ssize);
- *ss = _seg; // Push current record
- _seg._base = tbase;
- _seg._size = tsize;
- _seg._sflags = mmapped;
- _seg._next = ss;
-
- // Insert trailing fenceposts
- for (;;)
- {
- mchunkptr nextp = (mchunkptr)p->chunk_plus_offset(sizeof(size_t));
- p->_head = FENCEPOST_HEAD;
- ++nfences;
- if ((char*)(&(nextp->_head)) < old_end)
- p = nextp;
- else
- break;
- }
- assert(nfences >= 2);
-
- // Insert the rest of old top into a bin as an ordinary free chunk
- if (csp != old_top)
- {
- mchunkptr q = (mchunkptr)old_top;
- size_t psize = csp - old_top;
- mchunkptr tn = (mchunkptr)q->chunk_plus_offset(psize);
- q->set_free_with_pinuse(psize, tn);
- insert_chunk(q, psize);
- }
-
- check_top_chunk(_top);
-}
-
-/* -------------------------- System allocation -------------------------- */
-
-// Get memory from system using MMAP
-void* malloc_state::sys_alloc(size_t nb)
-{
- char* tbase = cmfail;
- size_t tsize = 0;
- flag_t mmap_flag = 0;
- size_t asize; // allocation size
-
- mparams.ensure_initialization();
-
- // Directly map large chunks, but only if already initialized
- if (use_mmap() && nb >= mparams._mmap_threshold && _topsize != 0)
- {
- void* mem = mmap_alloc(nb);
- if (mem != 0)
- return mem;
- }
-
- asize = mparams.granularity_align(nb + sys_alloc_padding());
- if (asize <= nb)
- return 0; // wraparound
- if (_footprint_limit != 0)
- {
- size_t fp = _footprint + asize;
- if (fp <= _footprint || fp > _footprint_limit)
- return 0;
- }
-
- /*
- Try getting memory with a call to MMAP new space (disabled if not SPP_HAVE_MMAP).
- We need to request enough bytes from system to ensure
- we can malloc nb bytes upon success, so pad with enough space for
- top_foot, plus alignment-pad to make sure we don't lose bytes if
- not on boundary, and round this up to a granularity unit.
- */
-
- if (SPP_HAVE_MMAP && tbase == cmfail)
- {
- // Try MMAP
- char* mp = (char*)(SPP_CALL_MMAP(asize));
- if (mp != cmfail)
- {
- tbase = mp;
- tsize = asize;
- mmap_flag = USE_MMAP_BIT;
- }
- }
-
- if (tbase != cmfail)
- {
-
- if ((_footprint += tsize) > _max_footprint)
- _max_footprint = _footprint;
-
- if (!is_initialized())
- {
- // first-time initialization
- if (_least_addr == 0 || tbase < _least_addr)
- _least_addr = tbase;
- _seg._base = tbase;
- _seg._size = tsize;
- _seg._sflags = mmap_flag;
- _magic = mparams._magic;
- _release_checks = SPP_MAX_RELEASE_CHECK_RATE;
- init_bins();
-
- // Offset top by embedded malloc_state
- mchunkptr mn = (mchunkptr)mem2chunk(this)->next_chunk();
- init_top(mn, (size_t)((tbase + tsize) - (char*)mn) - top_foot_size());
- }
-
- else
- {
- // Try to merge with an existing segment
- msegmentptr sp = &_seg;
- // Only consider most recent segment if traversal suppressed
- while (sp != 0 && tbase != sp->_base + sp->_size)
- sp = (SPP_NO_SEGMENT_TRAVERSAL) ? 0 : sp->_next;
- if (sp != 0 &&
- !sp->is_extern_segment() &&
- (sp->_sflags & USE_MMAP_BIT) == mmap_flag &&
- segment_holds(sp, _top))
- {
- // append
- sp->_size += tsize;
- init_top(_top, _topsize + tsize);
- }
- else
- {
- if (tbase < _least_addr)
- _least_addr = tbase;
- sp = &_seg;
- while (sp != 0 && sp->_base != tbase + tsize)
- sp = (SPP_NO_SEGMENT_TRAVERSAL) ? 0 : sp->_next;
- if (sp != 0 &&
- !sp->is_extern_segment() &&
- (sp->_sflags & USE_MMAP_BIT) == mmap_flag)
- {
- char* oldbase = sp->_base;
- sp->_base = tbase;
- sp->_size += tsize;
- return prepend_alloc(tbase, oldbase, nb);
- }
- else
- add_segment(tbase, tsize, mmap_flag);
- }
- }
-
- if (nb < _topsize)
- {
- // Allocate from new or extended top space
- size_t rsize = _topsize -= nb;
- mchunkptr p = _top;
- mchunkptr r = _top = (mchunkptr)p->chunk_plus_offset(nb);
- r->_head = rsize | PINUSE_BIT;
- set_size_and_pinuse_of_inuse_chunk(p, nb);
- check_top_chunk(_top);
- check_malloced_chunk(chunk2mem(p), nb);
- return chunk2mem(p);
- }
- }
-
- SPP_MALLOC_FAILURE_ACTION;
- return 0;
-}
-
-/* ----------------------- system deallocation -------------------------- */
-
-// Unmap and unlink any mmapped segments that don't contain used chunks
-size_t malloc_state::release_unused_segments()
-{
- size_t released = 0;
- int nsegs = 0;
- msegmentptr pred = &_seg;
- msegmentptr sp = pred->_next;
- while (sp != 0)
- {
- char* base = sp->_base;
- size_t size = sp->_size;
- msegmentptr next = sp->_next;
- ++nsegs;
- if (sp->is_mmapped_segment() && !sp->is_extern_segment())
- {
- mchunkptr p = align_as_chunk(base);
- size_t psize = p->chunksize();
- // Can unmap if first chunk holds entire segment and not pinned
- if (!p->is_inuse() && (char*)p + psize >= base + size - top_foot_size())
- {
- tchunkptr tp = (tchunkptr)p;
- assert(segment_holds(sp, p));
- if (p == _dv)
- {
- _dv = 0;
- _dvsize = 0;
- }
- else
- unlink_large_chunk(tp);
- if (SPP_CALL_MUNMAP(base, size) == 0)
- {
- released += size;
- _footprint -= size;
- // unlink obsoleted record
- sp = pred;
- sp->_next = next;
- }
- else
- {
- // back out if cannot unmap
- insert_large_chunk(tp, psize);
- }
- }
- }
- if (SPP_NO_SEGMENT_TRAVERSAL) // scan only first segment
- break;
- pred = sp;
- sp = next;
- }
- // Reset check counter
- _release_checks = (((size_t) nsegs > (size_t) SPP_MAX_RELEASE_CHECK_RATE) ?
- (size_t) nsegs : (size_t) SPP_MAX_RELEASE_CHECK_RATE);
- return released;
-}
-
-int malloc_state::sys_trim(size_t pad)
-{
- size_t released = 0;
- mparams.ensure_initialization();
- if (pad < MAX_REQUEST && is_initialized())
- {
- pad += top_foot_size(); // ensure enough room for segment overhead
-
- if (_topsize > pad)
- {
- // Shrink top space in _granularity - size units, keeping at least one
- size_t unit = mparams._granularity;
- size_t extra = ((_topsize - pad + (unit - 1)) / unit -
- 1) * unit;
- msegmentptr sp = segment_holding((char*)_top);
-
- if (!sp->is_extern_segment())
- {
- if (sp->is_mmapped_segment())
- {
- if (SPP_HAVE_MMAP &&
- sp->_size >= extra &&
- !has_segment_link(sp))
- {
- // can't shrink if pinned
- size_t newsize = sp->_size - extra;
- (void)newsize; // placate people compiling -Wunused-variable
- // Prefer mremap, fall back to munmap
- if ((SPP_CALL_MREMAP(sp->_base, sp->_size, newsize, 0) != mfail) ||
- (SPP_CALL_MUNMAP(sp->_base + newsize, extra) == 0))
- released = extra;
- }
- }
- }
-
- if (released != 0)
- {
- sp->_size -= released;
- _footprint -= released;
- init_top(_top, _topsize - released);
- check_top_chunk(_top);
- }
- }
-
- // Unmap any unused mmapped segments
- if (SPP_HAVE_MMAP)
- released += release_unused_segments();
-
- // On failure, disable autotrim to avoid repeated failed future calls
- if (released == 0 && _topsize > _trim_check)
- _trim_check = spp_max_size_t;
- }
-
- return (released != 0) ? 1 : 0;
-}
-
-/* Consolidate and bin a chunk. Differs from exported versions
- of free mainly in that the chunk need not be marked as inuse.
-*/
-void malloc_state::dispose_chunk(mchunkptr p, size_t psize)
-{
- mchunkptr next = (mchunkptr)p->chunk_plus_offset(psize);
- if (!p->pinuse())
- {
- mchunkptr prev;
- size_t prevsize = p->_prev_foot;
- if (p->is_mmapped())
- {
- psize += prevsize + SPP_MMAP_FOOT_PAD;
- if (SPP_CALL_MUNMAP((char*)p - prevsize, psize) == 0)
- _footprint -= psize;
- return;
- }
- prev = (mchunkptr)p->chunk_minus_offset(prevsize);
- psize += prevsize;
- p = prev;
- if (rtcheck(ok_address(prev)))
- {
- // consolidate backward
- if (p != _dv)
- unlink_chunk(p, prevsize);
- else if ((next->_head & INUSE_BITS) == INUSE_BITS)
- {
- _dvsize = psize;
- p->set_free_with_pinuse(psize, next);
- return;
- }
- }
- else
- {
- SPP_ABORT;
- return;
- }
- }
- if (rtcheck(ok_address(next)))
- {
- if (!next->cinuse())
- {
- // consolidate forward
- if (next == _top)
- {
- size_t tsize = _topsize += psize;
- _top = p;
- p->_head = tsize | PINUSE_BIT;
- if (p == _dv)
- {
- _dv = 0;
- _dvsize = 0;
- }
- return;
- }
- else if (next == _dv)
- {
- size_t dsize = _dvsize += psize;
- _dv = p;
- p->set_size_and_pinuse_of_free_chunk(dsize);
- return;
- }
- else
- {
- size_t nsize = next->chunksize();
- psize += nsize;
- unlink_chunk(next, nsize);
- p->set_size_and_pinuse_of_free_chunk(psize);
- if (p == _dv)
- {
- _dvsize = psize;
- return;
- }
- }
- }
- else
- p->set_free_with_pinuse(psize, next);
- insert_chunk(p, psize);
- }
- else
- SPP_ABORT;
-}
-
-/* ---------------------------- malloc --------------------------- */
-
-// allocate a large request from the best fitting chunk in a treebin
-void* malloc_state::tmalloc_large(size_t nb)
-{
- tchunkptr v = 0;
- size_t rsize = -nb; // Unsigned negation
- tchunkptr t;
- bindex_t idx = compute_tree_index(nb);
- if ((t = *treebin_at(idx)) != 0)
- {
- // Traverse tree for this bin looking for node with size == nb
- size_t sizebits = nb << leftshift_for_tree_index(idx);
- tchunkptr rst = 0; // The deepest untaken right subtree
- for (;;)
- {
- tchunkptr rt;
- size_t trem = t->chunksize() - nb;
- if (trem < rsize)
- {
- v = t;
- if ((rsize = trem) == 0)
- break;
- }
- rt = t->_child[1];
- t = t->_child[(sizebits >> (spp_size_t_bitsize - 1)) & 1];
- if (rt != 0 && rt != t)
- rst = rt;
- if (t == 0)
- {
- t = rst; // set t to least subtree holding sizes > nb
- break;
- }
- sizebits <<= 1;
- }
- }
- if (t == 0 && v == 0)
- {
- // set t to root of next non-empty treebin
- binmap_t leftbits = left_bits(idx2bit(idx)) & _treemap;
- if (leftbits != 0)
- {
- binmap_t leastbit = least_bit(leftbits);
- bindex_t i = compute_bit2idx(leastbit);
- t = *treebin_at(i);
- }
- }
-
- while (t != 0)
- {
- // find smallest of tree or subtree
- size_t trem = t->chunksize() - nb;
- if (trem < rsize)
- {
- rsize = trem;
- v = t;
- }
- t = t->leftmost_child();
- }
-
- // If dv is a better fit, return 0 so malloc will use it
- if (v != 0 && rsize < (size_t)(_dvsize - nb))
- {
- if (rtcheck(ok_address(v)))
- {
- // split
- mchunkptr r = (mchunkptr)v->chunk_plus_offset(nb);
- assert(v->chunksize() == rsize + nb);
- if (rtcheck(ok_next(v, r)))
- {
- unlink_large_chunk(v);
- if (rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(v, (rsize + nb));
- else
- {
- set_size_and_pinuse_of_inuse_chunk(v, nb);
- r->set_size_and_pinuse_of_free_chunk(rsize);
- insert_chunk(r, rsize);
- }
- return chunk2mem(v);
- }
- }
- SPP_ABORT;
- }
- return 0;
-}
-
-// allocate a small request from the best fitting chunk in a treebin
-void* malloc_state::tmalloc_small(size_t nb)
-{
- tchunkptr t, v;
- size_t rsize;
- binmap_t leastbit = least_bit(_treemap);
- bindex_t i = compute_bit2idx(leastbit);
- v = t = *treebin_at(i);
- rsize = t->chunksize() - nb;
-
- while ((t = t->leftmost_child()) != 0)
- {
- size_t trem = t->chunksize() - nb;
- if (trem < rsize)
- {
- rsize = trem;
- v = t;
- }
- }
-
- if (rtcheck(ok_address(v)))
- {
- mchunkptr r = (mchunkptr)v->chunk_plus_offset(nb);
- assert(v->chunksize() == rsize + nb);
- if (rtcheck(ok_next(v, r)))
- {
- unlink_large_chunk(v);
- if (rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(v, (rsize + nb));
- else
- {
- set_size_and_pinuse_of_inuse_chunk(v, nb);
- r->set_size_and_pinuse_of_free_chunk(rsize);
- replace_dv(r, rsize);
- }
- return chunk2mem(v);
- }
- }
-
- SPP_ABORT;
- return 0;
-}
-
-/* ---------------------------- malloc --------------------------- */
-
-void* malloc_state::_malloc(size_t bytes)
-{
- if (1)
- {
- void* mem;
- size_t nb;
- if (bytes <= MAX_SMALL_REQUEST)
- {
- bindex_t idx;
- binmap_t smallbits;
- nb = (bytes < MIN_REQUEST) ? MIN_CHUNK_SIZE : pad_request(bytes);
- idx = small_index(nb);
- smallbits = _smallmap >> idx;
-
- if ((smallbits & 0x3U) != 0)
- {
- // Remainderless fit to a smallbin.
- mchunkptr b, p;
- idx += ~smallbits & 1; // Uses next bin if idx empty
- b = smallbin_at(idx);
- p = b->_fd;
- assert(p->chunksize() == small_index2size(idx));
- unlink_first_small_chunk(b, p, idx);
- set_inuse_and_pinuse(p, small_index2size(idx));
- mem = chunk2mem(p);
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
-
- else if (nb > _dvsize)
- {
- if (smallbits != 0)
- {
- // Use chunk in next nonempty smallbin
- mchunkptr b, p, r;
- size_t rsize;
- binmap_t leftbits = (smallbits << idx) & left_bits(malloc_state::idx2bit(idx));
- binmap_t leastbit = least_bit(leftbits);
- bindex_t i = compute_bit2idx(leastbit);
- b = smallbin_at(i);
- p = b->_fd;
- assert(p->chunksize() == small_index2size(i));
- unlink_first_small_chunk(b, p, i);
- rsize = small_index2size(i) - nb;
- // Fit here cannot be remainderless if 4byte sizes
- if (sizeof(size_t) != 4 && rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(p, small_index2size(i));
- else
- {
- set_size_and_pinuse_of_inuse_chunk(p, nb);
- r = (mchunkptr)p->chunk_plus_offset(nb);
- r->set_size_and_pinuse_of_free_chunk(rsize);
- replace_dv(r, rsize);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
-
- else if (_treemap != 0 && (mem = tmalloc_small(nb)) != 0)
- {
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
- }
- }
- else if (bytes >= MAX_REQUEST)
- nb = spp_max_size_t; // Too big to allocate. Force failure (in sys alloc)
- else
- {
- nb = pad_request(bytes);
- if (_treemap != 0 && (mem = tmalloc_large(nb)) != 0)
- {
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
- }
-
- if (nb <= _dvsize)
- {
- size_t rsize = _dvsize - nb;
- mchunkptr p = _dv;
- if (rsize >= MIN_CHUNK_SIZE)
- {
- // split dv
- mchunkptr r = _dv = (mchunkptr)p->chunk_plus_offset(nb);
- _dvsize = rsize;
- r->set_size_and_pinuse_of_free_chunk(rsize);
- set_size_and_pinuse_of_inuse_chunk(p, nb);
- }
- else // exhaust dv
- {
- size_t dvs = _dvsize;
- _dvsize = 0;
- _dv = 0;
- set_inuse_and_pinuse(p, dvs);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
-
- else if (nb < _topsize)
- {
- // Split top
- size_t rsize = _topsize -= nb;
- mchunkptr p = _top;
- mchunkptr r = _top = (mchunkptr)p->chunk_plus_offset(nb);
- r->_head = rsize | PINUSE_BIT;
- set_size_and_pinuse_of_inuse_chunk(p, nb);
- mem = chunk2mem(p);
- check_top_chunk(_top);
- check_malloced_chunk(mem, nb);
- goto postaction;
- }
-
- mem = sys_alloc(nb);
-
-postaction:
- return mem;
- }
-
- return 0;
-}
-
-/* ---------------------------- free --------------------------- */
-
-void malloc_state::_free(mchunkptr p)
-{
- if (1)
- {
- check_inuse_chunk(p);
- if (rtcheck(ok_address(p) && ok_inuse(p)))
- {
- size_t psize = p->chunksize();
- mchunkptr next = (mchunkptr)p->chunk_plus_offset(psize);
- if (!p->pinuse())
- {
- size_t prevsize = p->_prev_foot;
- if (p->is_mmapped())
- {
- psize += prevsize + SPP_MMAP_FOOT_PAD;
- if (SPP_CALL_MUNMAP((char*)p - prevsize, psize) == 0)
- _footprint -= psize;
- goto postaction;
- }
- else
- {
- mchunkptr prev = (mchunkptr)p->chunk_minus_offset(prevsize);
- psize += prevsize;
- p = prev;
- if (rtcheck(ok_address(prev)))
- {
- // consolidate backward
- if (p != _dv)
- unlink_chunk(p, prevsize);
- else if ((next->_head & INUSE_BITS) == INUSE_BITS)
- {
- _dvsize = psize;
- p->set_free_with_pinuse(psize, next);
- goto postaction;
- }
- }
- else
- goto erroraction;
- }
- }
-
- if (rtcheck(ok_next(p, next) && ok_pinuse(next)))
- {
- if (!next->cinuse())
- {
- // consolidate forward
- if (next == _top)
- {
- size_t tsize = _topsize += psize;
- _top = p;
- p->_head = tsize | PINUSE_BIT;
- if (p == _dv)
- {
- _dv = 0;
- _dvsize = 0;
- }
- if (should_trim(tsize))
- sys_trim(0);
- goto postaction;
- }
- else if (next == _dv)
- {
- size_t dsize = _dvsize += psize;
- _dv = p;
- p->set_size_and_pinuse_of_free_chunk(dsize);
- goto postaction;
- }
- else
- {
- size_t nsize = next->chunksize();
- psize += nsize;
- unlink_chunk(next, nsize);
- p->set_size_and_pinuse_of_free_chunk(psize);
- if (p == _dv)
- {
- _dvsize = psize;
- goto postaction;
- }
- }
- }
- else
- p->set_free_with_pinuse(psize, next);
-
- if (is_small(psize))
- {
- insert_small_chunk(p, psize);
- check_free_chunk(p);
- }
- else
- {
- tchunkptr tp = (tchunkptr)p;
- insert_large_chunk(tp, psize);
- check_free_chunk(p);
- if (--_release_checks == 0)
- release_unused_segments();
- }
- goto postaction;
- }
- }
-erroraction:
- SPP_USAGE_ERROR_ACTION(this, p);
-postaction:
- ;
- }
-}
-
-/* ------------ Internal support for realloc, memalign, etc -------------- */
-
-// Try to realloc; only in-place unless can_move true
-mchunkptr malloc_state::try_realloc_chunk(mchunkptr p, size_t nb, int can_move)
-{
- mchunkptr newp = 0;
- size_t oldsize = p->chunksize();
- mchunkptr next = (mchunkptr)p->chunk_plus_offset(oldsize);
- if (rtcheck(ok_address(p) && ok_inuse(p) &&
- ok_next(p, next) && ok_pinuse(next)))
- {
- if (p->is_mmapped())
- newp = mmap_resize(p, nb, can_move);
- else if (oldsize >= nb)
- {
- // already big enough
- size_t rsize = oldsize - nb;
- if (rsize >= MIN_CHUNK_SIZE)
- {
- // split off remainder
- mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb);
- set_inuse(p, nb);
- set_inuse(r, rsize);
- dispose_chunk(r, rsize);
- }
- newp = p;
- }
- else if (next == _top)
- {
- // extend into top
- if (oldsize + _topsize > nb)
- {
- size_t newsize = oldsize + _topsize;
- size_t newtopsize = newsize - nb;
- mchunkptr newtop = (mchunkptr)p->chunk_plus_offset(nb);
- set_inuse(p, nb);
- newtop->_head = newtopsize | PINUSE_BIT;
- _top = newtop;
- _topsize = newtopsize;
- newp = p;
- }
- }
- else if (next == _dv)
- {
- // extend into dv
- size_t dvs = _dvsize;
- if (oldsize + dvs >= nb)
- {
- size_t dsize = oldsize + dvs - nb;
- if (dsize >= MIN_CHUNK_SIZE)
- {
- mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb);
- mchunkptr n = (mchunkptr)r->chunk_plus_offset(dsize);
- set_inuse(p, nb);
- r->set_size_and_pinuse_of_free_chunk(dsize);
- n->clear_pinuse();
- _dvsize = dsize;
- _dv = r;
- }
- else
- {
- // exhaust dv
- size_t newsize = oldsize + dvs;
- set_inuse(p, newsize);
- _dvsize = 0;
- _dv = 0;
- }
- newp = p;
- }
- }
- else if (!next->cinuse())
- {
- // extend into next free chunk
- size_t nextsize = next->chunksize();
- if (oldsize + nextsize >= nb)
- {
- size_t rsize = oldsize + nextsize - nb;
- unlink_chunk(next, nextsize);
- if (rsize < MIN_CHUNK_SIZE)
- {
- size_t newsize = oldsize + nextsize;
- set_inuse(p, newsize);
- }
- else
- {
- mchunkptr r = (mchunkptr)p->chunk_plus_offset(nb);
- set_inuse(p, nb);
- set_inuse(r, rsize);
- dispose_chunk(r, rsize);
- }
- newp = p;
- }
- }
- }
- else
- SPP_USAGE_ERROR_ACTION(m, chunk2mem(p));
- return newp;
-}
-
-void* malloc_state::internal_memalign(size_t alignment, size_t bytes)
-{
- void* mem = 0;
- if (alignment < MIN_CHUNK_SIZE) // must be at least a minimum chunk size
- alignment = MIN_CHUNK_SIZE;
- if ((alignment & (alignment - 1)) != 0)
- {
- // Ensure a power of 2
- size_t a = SPP_MALLOC_ALIGNMENT << 1;
- while (a < alignment)
- a <<= 1;
- alignment = a;
- }
- if (bytes >= MAX_REQUEST - alignment)
- SPP_MALLOC_FAILURE_ACTION;
- else
- {
- size_t nb = request2size(bytes);
- size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD;
- mem = internal_malloc(req);
- if (mem != 0)
- {
- mchunkptr p = mem2chunk(mem);
- if ((((size_t)(mem)) & (alignment - 1)) != 0)
- {
- // misaligned
- /*
- Find an aligned spot inside chunk. Since we need to give
- back leading space in a chunk of at least MIN_CHUNK_SIZE, if
- the first calculation places us at a spot with less than
- MIN_CHUNK_SIZE leader, we can move to the next aligned spot.
- We've allocated enough total room so that this is always
- possible.
- */
- char* br = (char*)mem2chunk((void *)(((size_t)((char*)mem + alignment - 1)) &
- -alignment));
- char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE) ?
- br : br + alignment;
- mchunkptr newp = (mchunkptr)pos;
- size_t leadsize = pos - (char*)(p);
- size_t newsize = p->chunksize() - leadsize;
-
- if (p->is_mmapped())
- {
- // For mmapped chunks, just adjust offset
- newp->_prev_foot = p->_prev_foot + leadsize;
- newp->_head = newsize;
- }
- else
- {
- // Otherwise, give back leader, use the rest
- set_inuse(newp, newsize);
- set_inuse(p, leadsize);
- dispose_chunk(p, leadsize);
- }
- p = newp;
- }
-
- // Give back spare room at the end
- if (!p->is_mmapped())
- {
- size_t size = p->chunksize();
- if (size > nb + MIN_CHUNK_SIZE)
- {
- size_t remainder_size = size - nb;
- mchunkptr remainder = (mchunkptr)p->chunk_plus_offset(nb);
- set_inuse(p, nb);
- set_inuse(remainder, remainder_size);
- dispose_chunk(remainder, remainder_size);
- }
- }
-
- mem = chunk2mem(p);
- assert(p->chunksize() >= nb);
- assert(((size_t)mem & (alignment - 1)) == 0);
- check_inuse_chunk(p);
- }
- }
- return mem;
-}
-
-/*
- Common support for independent_X routines, handling
- all of the combinations that can result.
- The opts arg has:
- bit 0 set if all elements are same size (using sizes[0])
- bit 1 set if elements should be zeroed
-*/
-void** malloc_state::ialloc(size_t n_elements, size_t* sizes, int opts,
- void* chunks[])
-{
-
- size_t element_size; // chunksize of each element, if all same
- size_t contents_size; // total size of elements
- size_t array_size; // request size of pointer array
- void* mem; // malloced aggregate space
- mchunkptr p; // corresponding chunk
- size_t remainder_size; // remaining bytes while splitting
- void** marray; // either "chunks" or malloced ptr array
- mchunkptr array_chunk; // chunk for malloced ptr array
- flag_t was_enabled; // to disable mmap
- size_t size;
- size_t i;
-
- mparams.ensure_initialization();
- // compute array length, if needed
- if (chunks != 0)
- {
- if (n_elements == 0)
- return chunks; // nothing to do
- marray = chunks;
- array_size = 0;
- }
- else
- {
- // if empty req, must still return chunk representing empty array
- if (n_elements == 0)
- return (void**)internal_malloc(0);
- marray = 0;
- array_size = request2size(n_elements * (sizeof(void*)));
- }
-
- // compute total element size
- if (opts & 0x1)
- {
- // all-same-size
- element_size = request2size(*sizes);
- contents_size = n_elements * element_size;
- }
- else
- {
- // add up all the sizes
- element_size = 0;
- contents_size = 0;
- for (i = 0; i != n_elements; ++i)
- contents_size += request2size(sizes[i]);
- }
-
- size = contents_size + array_size;
-
- /*
- Allocate the aggregate chunk. First disable direct-mmapping so
- malloc won't use it, since we would not be able to later
- free/realloc space internal to a segregated mmap region.
- */
- was_enabled = use_mmap();
- disable_mmap();
- mem = internal_malloc(size - CHUNK_OVERHEAD);
- if (was_enabled)
- enable_mmap();
- if (mem == 0)
- return 0;
-
- p = mem2chunk(mem);
- remainder_size = p->chunksize();
-
- assert(!p->is_mmapped());
-
- if (opts & 0x2)
- {
- // optionally clear the elements
- memset((size_t*)mem, 0, remainder_size - sizeof(size_t) - array_size);
- }
-
- // If not provided, allocate the pointer array as final part of chunk
- if (marray == 0)
- {
- size_t array_chunk_size;
- array_chunk = (mchunkptr)p->chunk_plus_offset(contents_size);
- array_chunk_size = remainder_size - contents_size;
- marray = (void**)(chunk2mem(array_chunk));
- set_size_and_pinuse_of_inuse_chunk(array_chunk, array_chunk_size);
- remainder_size = contents_size;
- }
-
- // split out elements
- for (i = 0; ; ++i)
- {
- marray[i] = chunk2mem(p);
- if (i != n_elements - 1)
- {
- if (element_size != 0)
- size = element_size;
- else
- size = request2size(sizes[i]);
- remainder_size -= size;
- set_size_and_pinuse_of_inuse_chunk(p, size);
- p = (mchunkptr)p->chunk_plus_offset(size);
- }
- else
- {
- // the final element absorbs any overallocation slop
- set_size_and_pinuse_of_inuse_chunk(p, remainder_size);
- break;
- }
- }
-
-#if SPP_DEBUG
- if (marray != chunks)
- {
- // final element must have exactly exhausted chunk
- if (element_size != 0)
- assert(remainder_size == element_size);
- else
- assert(remainder_size == request2size(sizes[i]));
- check_inuse_chunk(mem2chunk(marray));
- }
- for (i = 0; i != n_elements; ++i)
- check_inuse_chunk(mem2chunk(marray[i]));
-
-#endif
-
- return marray;
-}
-
-/* Try to free all pointers in the given array.
- Note: this could be made faster, by delaying consolidation,
- at the price of disabling some user integrity checks, We
- still optimize some consolidations by combining adjacent
- chunks before freeing, which will occur often if allocated
- with ialloc or the array is sorted.
-*/
-size_t malloc_state::internal_bulk_free(void* array[], size_t nelem)
-{
- size_t unfreed = 0;
- if (1)
- {
- void** a;
- void** fence = &(array[nelem]);
- for (a = array; a != fence; ++a)
- {
- void* mem = *a;
- if (mem != 0)
- {
- mchunkptr p = mem2chunk(mem);
- size_t psize = p->chunksize();
-#if SPP_FOOTERS
- if (get_mstate_for(p) != m)
- {
- ++unfreed;
- continue;
- }
-#endif
- check_inuse_chunk(p);
- *a = 0;
- if (rtcheck(ok_address(p) && ok_inuse(p)))
- {
- void ** b = a + 1; // try to merge with next chunk
- mchunkptr next = (mchunkptr)p->next_chunk();
- if (b != fence && *b == chunk2mem(next))
- {
- size_t newsize = next->chunksize() + psize;
- set_inuse(p, newsize);
- *b = chunk2mem(p);
- }
- else
- dispose_chunk(p, psize);
- }
- else
- {
- SPP_ABORT;
- break;
- }
- }
- }
- if (should_trim(_topsize))
- sys_trim(0);
- }
- return unfreed;
-}
-
-void malloc_state::init(char* tbase, size_t tsize)
-{
- _seg._base = _least_addr = tbase;
- _seg._size = _footprint = _max_footprint = tsize;
- _magic = mparams._magic;
- _release_checks = SPP_MAX_RELEASE_CHECK_RATE;
- _mflags = mparams._default_mflags;
- _extp = 0;
- _exts = 0;
- disable_contiguous();
- init_bins();
- mchunkptr mn = (mchunkptr)mem2chunk(this)->next_chunk();
- init_top(mn, (size_t)((tbase + tsize) - (char*)mn) - top_foot_size());
- check_top_chunk(_top);
-}
-
-/* Traversal */
-#if SPP_MALLOC_INSPECT_ALL
-void malloc_state::internal_inspect_all(void(*handler)(void *start, void *end,
- size_t used_bytes,
- void* callback_arg),
- void* arg)
-{
- if (is_initialized())
- {
- mchunkptr top = top;
- msegmentptr s;
- for (s = &seg; s != 0; s = s->next)
- {
- mchunkptr q = align_as_chunk(s->base);
- while (segment_holds(s, q) && q->head != FENCEPOST_HEAD)
- {
- mchunkptr next = (mchunkptr)q->next_chunk();
- size_t sz = q->chunksize();
- size_t used;
- void* start;
- if (q->is_inuse())
- {
- used = sz - CHUNK_OVERHEAD; // must not be mmapped
- start = chunk2mem(q);
- }
- else
- {
- used = 0;
- if (is_small(sz))
- {
- // offset by possible bookkeeping
- start = (void*)((char*)q + sizeof(struct malloc_chunk));
- }
- else
- start = (void*)((char*)q + sizeof(struct malloc_tree_chunk));
- }
- if (start < (void*)next) // skip if all space is bookkeeping
- handler(start, next, used, arg);
- if (q == top)
- break;
- q = next;
- }
- }
- }
-}
-#endif // SPP_MALLOC_INSPECT_ALL
-
-
-
-/* ----------------------------- user mspaces ---------------------------- */
-
-static mstate init_user_mstate(char* tbase, size_t tsize)
-{
- size_t msize = pad_request(sizeof(malloc_state));
- mchunkptr msp = align_as_chunk(tbase);
- mstate m = (mstate)(chunk2mem(msp));
- memset(m, 0, msize);
- msp->_head = (msize | INUSE_BITS);
- m->init(tbase, tsize);
- return m;
-}
-
-SPP_API mspace create_mspace(size_t capacity, int locked)
-{
- mstate m = 0;
- size_t msize;
- mparams.ensure_initialization();
- msize = pad_request(sizeof(malloc_state));
- if (capacity < (size_t) - (msize + top_foot_size() + mparams._page_size))
- {
- size_t rs = ((capacity == 0) ? mparams._granularity :
- (capacity + top_foot_size() + msize));
- size_t tsize = mparams.granularity_align(rs);
- char* tbase = (char*)(SPP_CALL_MMAP(tsize));
- if (tbase != cmfail)
- {
- m = init_user_mstate(tbase, tsize);
- m->_seg._sflags = USE_MMAP_BIT;
- m->set_lock(locked);
- }
- }
- return (mspace)m;
-}
-
-SPP_API size_t destroy_mspace(mspace msp)
-{
- size_t freed = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- {
- msegmentptr sp = &ms->_seg;
- while (sp != 0)
- {
- char* base = sp->_base;
- size_t size = sp->_size;
- flag_t flag = sp->_sflags;
- (void)base; // placate people compiling -Wunused-variable
- sp = sp->_next;
- if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) &&
- SPP_CALL_MUNMAP(base, size) == 0)
- freed += size;
- }
- }
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return freed;
-}
-
-/* ---------------------------- mspace versions of malloc/calloc/free routines -------------------- */
-SPP_API void* mspace_malloc(mspace msp, size_t bytes)
-{
- mstate ms = (mstate)msp;
- if (!ms->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return 0;
- }
- return ms->_malloc(bytes);
-}
-
-SPP_API void mspace_free(mspace msp, void* mem)
-{
- if (mem != 0)
- {
- mchunkptr p = mem2chunk(mem);
-#if SPP_FOOTERS
- mstate fm = get_mstate_for(p);
- (void)msp; // placate people compiling -Wunused
-#else
- mstate fm = (mstate)msp;
-#endif
- if (!fm->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(fm, p);
- return;
- }
- fm->_free(p);
- }
-}
-
-SPP_API inline void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size)
-{
- void* mem;
- size_t req = 0;
- mstate ms = (mstate)msp;
- if (!ms->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return 0;
- }
- if (n_elements != 0)
- {
- req = n_elements * elem_size;
- if (((n_elements | elem_size) & ~(size_t)0xffff) &&
- (req / n_elements != elem_size))
- req = spp_max_size_t; // force downstream failure on overflow
- }
- mem = ms->internal_malloc(req);
- if (mem != 0 && mem2chunk(mem)->calloc_must_clear())
- memset(mem, 0, req);
- return mem;
-}
-
-SPP_API inline void* mspace_realloc(mspace msp, void* oldmem, size_t bytes)
-{
- void* mem = 0;
- if (oldmem == 0)
- mem = mspace_malloc(msp, bytes);
- else if (bytes >= MAX_REQUEST)
- SPP_MALLOC_FAILURE_ACTION;
-#ifdef REALLOC_ZERO_BYTES_FREES
- else if (bytes == 0)
- mspace_free(msp, oldmem);
-#endif
- else
- {
- size_t nb = request2size(bytes);
- mchunkptr oldp = mem2chunk(oldmem);
-#if ! SPP_FOOTERS
- mstate m = (mstate)msp;
-#else
- mstate m = get_mstate_for(oldp);
- if (!m->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(m, oldmem);
- return 0;
- }
-#endif
- if (1)
- {
- mchunkptr newp = m->try_realloc_chunk(oldp, nb, 1);
- if (newp != 0)
- {
- m->check_inuse_chunk(newp);
- mem = chunk2mem(newp);
- }
- else
- {
- mem = mspace_malloc(m, bytes);
- if (mem != 0)
- {
- size_t oc = oldp->chunksize() - oldp->overhead_for();
- memcpy(mem, oldmem, (oc < bytes) ? oc : bytes);
- mspace_free(m, oldmem);
- }
- }
- }
- }
- return mem;
-}
-
-#if 0
-
-SPP_API mspace create_mspace_with_base(void* base, size_t capacity, int locked)
-{
- mstate m = 0;
- size_t msize;
- mparams.ensure_initialization();
- msize = pad_request(sizeof(malloc_state));
- if (capacity > msize + top_foot_size() &&
- capacity < (size_t) - (msize + top_foot_size() + mparams._page_size))
- {
- m = init_user_mstate((char*)base, capacity);
- m->_seg._sflags = EXTERN_BIT;
- m->set_lock(locked);
- }
- return (mspace)m;
-}
-
-SPP_API int mspace_track_large_chunks(mspace msp, int enable)
-{
- int ret = 0;
- mstate ms = (mstate)msp;
- if (1)
- {
- if (!ms->use_mmap())
- ret = 1;
- if (!enable)
- ms->enable_mmap();
- else
- ms->disable_mmap();
- }
- return ret;
-}
-
-SPP_API void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes)
-{
- void* mem = 0;
- if (oldmem != 0)
- {
- if (bytes >= MAX_REQUEST)
- SPP_MALLOC_FAILURE_ACTION;
- else
- {
- size_t nb = request2size(bytes);
- mchunkptr oldp = mem2chunk(oldmem);
-#if ! SPP_FOOTERS
- mstate m = (mstate)msp;
-#else
- mstate m = get_mstate_for(oldp);
- (void)msp; // placate people compiling -Wunused
- if (!m->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(m, oldmem);
- return 0;
- }
-#endif
- if (1)
- {
- mchunkptr newp = m->try_realloc_chunk(oldp, nb, 0);
- if (newp == oldp)
- {
- m->check_inuse_chunk(newp);
- mem = oldmem;
- }
- }
- }
- }
- return mem;
-}
-
-SPP_API void* mspace_memalign(mspace msp, size_t alignment, size_t bytes)
-{
- mstate ms = (mstate)msp;
- if (!ms->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return 0;
- }
- if (alignment <= SPP_MALLOC_ALIGNMENT)
- return mspace_malloc(msp, bytes);
- return ms->internal_memalign(alignment, bytes);
-}
-
-SPP_API void** mspace_independent_calloc(mspace msp, size_t n_elements,
- size_t elem_size, void* chunks[])
-{
- size_t sz = elem_size; // serves as 1-element array
- mstate ms = (mstate)msp;
- if (!ms->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return 0;
- }
- return ms->ialloc(n_elements, &sz, 3, chunks);
-}
-
-SPP_API void** mspace_independent_comalloc(mspace msp, size_t n_elements,
- size_t sizes[], void* chunks[])
-{
- mstate ms = (mstate)msp;
- if (!ms->ok_magic())
- {
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return 0;
- }
- return ms->ialloc(n_elements, sizes, 0, chunks);
-}
-
-#endif
-
-SPP_API inline size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem)
-{
- return ((mstate)msp)->internal_bulk_free(array, nelem);
-}
-
-#if SPP_MALLOC_INSPECT_ALL
-SPP_API void mspace_inspect_all(mspace msp,
- void(*handler)(void *start,
- void *end,
- size_t used_bytes,
- void* callback_arg),
- void* arg)
-{
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- internal_inspect_all(ms, handler, arg);
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
-}
-#endif
-
-SPP_API inline int mspace_trim(mspace msp, size_t pad)
-{
- int result = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- result = ms->sys_trim(pad);
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return result;
-}
-
-SPP_API inline size_t mspace_footprint(mspace msp)
-{
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- result = ms->_footprint;
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return result;
-}
-
-SPP_API inline size_t mspace_max_footprint(mspace msp)
-{
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- result = ms->_max_footprint;
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return result;
-}
-
-SPP_API inline size_t mspace_footprint_limit(mspace msp)
-{
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- {
- size_t maf = ms->_footprint_limit;
- result = (maf == 0) ? spp_max_size_t : maf;
- }
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return result;
-}
-
-SPP_API inline size_t mspace_set_footprint_limit(mspace msp, size_t bytes)
-{
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ms->ok_magic())
- {
- if (bytes == 0)
- result = mparams.granularity_align(1); // Use minimal size
- if (bytes == spp_max_size_t)
- result = 0; // disable
- else
- result = mparams.granularity_align(bytes);
- ms->_footprint_limit = result;
- }
- else
- SPP_USAGE_ERROR_ACTION(ms, ms);
- return result;
-}
-
-SPP_API inline size_t mspace_usable_size(const void* mem)
-{
- if (mem != 0)
- {
- mchunkptr p = mem2chunk(mem);
- if (p->is_inuse())
- return p->chunksize() - p->overhead_for();
- }
- return 0;
-}
-
-SPP_API inline int mspace_mallopt(int param_number, int value)
-{
- return mparams.change(param_number, value);
-}
-
-} // spp_ namespace
-
-
-#endif // SPP_EXCLUDE_IMPLEMENTATION
-
-#endif // spp_dlalloc__h_
diff --git a/benchmarks/others/sparsepp/spp_smartptr.h b/benchmarks/others/sparsepp/spp_smartptr.h
deleted file mode 100644
index fba3acfb..00000000
--- a/benchmarks/others/sparsepp/spp_smartptr.h
+++ /dev/null
@@ -1,71 +0,0 @@
-#if !defined(spp_smartptr_h_guard)
-#define spp_smartptr_h_guard
-
-
-/* -----------------------------------------------------------------------------------------------
- * quick version of intrusive_ptr
- * -----------------------------------------------------------------------------------------------
- */
-
-#include <cassert>
-#include "spp_config.h"
-
-// ------------------------------------------------------------------------
-class spp_rc
-{
-public:
- spp_rc() : _cnt(0) {}
- spp_rc(const spp_rc &) : _cnt(0) {}
- void increment() const { ++_cnt; }
- void decrement() const { assert(_cnt); if (--_cnt == 0) delete this; }
- unsigned count() const { return _cnt; }
-
-protected:
- virtual ~spp_rc() {}
-
-private:
- mutable unsigned _cnt;
-};
-
-// ------------------------------------------------------------------------
-template <class T>
-class spp_sptr
-{
-public:
- spp_sptr() : _p(0) {}
- spp_sptr(T *p) : _p(p) { if (_p) _p->increment(); }
- spp_sptr(const spp_sptr &o) : _p(o._p) { if (_p) _p->increment(); }
-#ifndef SPP_NO_CXX11_RVALUE_REFERENCES
- spp_sptr(spp_sptr &&o) : _p(o._p) { o._p = (T *)0; }
- spp_sptr& operator=(spp_sptr &&o) { this->swap(o); return *this; }
-#endif
- ~spp_sptr() { if (_p) _p->decrement(); }
- spp_sptr& operator=(const spp_sptr &o) { reset(o._p); return *this; }
- T* get() const { return _p; }
- void swap(spp_sptr &o) { T *tmp = _p; _p = o._p; o._p = tmp; }
- void reset(const T *p = 0)
- {
- if (p == _p)
- return;
- if (_p) _p->decrement();
- _p = (T *)p;
- if (_p) _p->increment();
- }
- T* operator->() const { return const_cast<T *>(_p); }
- bool operator!() const { return _p == 0; }
-
-private:
- T *_p;
-};
-
-// ------------------------------------------------------------------------
-namespace std
-{
- template <class T>
- inline void swap(spp_sptr<T> &a, spp_sptr<T> &b)
- {
- a.swap(b);
- }
-}
-
-#endif // spp_smartptr_h_guard
diff --git a/benchmarks/others/sparsepp/spp_stdint.h b/benchmarks/others/sparsepp/spp_stdint.h
deleted file mode 100644
index 3adced9c..00000000
--- a/benchmarks/others/sparsepp/spp_stdint.h
+++ /dev/null
@@ -1,16 +0,0 @@
-#if !defined(spp_stdint_h_guard)
-#define spp_stdint_h_guard
-
-#include "spp_config.h"
-
-#if defined(SPP_HAS_CSTDINT) && (__cplusplus >= 201103)
- #include <cstdint>
-#else
- #if defined(__FreeBSD__) || defined(__IBMCPP__) || defined(_AIX)
- #include <inttypes.h>
- #else
- #include <stdint.h>
- #endif
-#endif
-
-#endif // spp_stdint_h_guard
diff --git a/benchmarks/others/sparsepp/spp_timer.h b/benchmarks/others/sparsepp/spp_timer.h
deleted file mode 100644
index 48180f4d..00000000
--- a/benchmarks/others/sparsepp/spp_timer.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- Copyright (c) 2016 Mariano Gonzalez
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in all
- copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#ifndef spp_timer_h_guard
-#define spp_timer_h_guard
-
-#include <chrono>
-
-namespace spp
-{
- template<typename time_unit = std::milli>
- class Timer
- {
- public:
- Timer() { reset(); }
- void reset() { _start = _snap = clock::now(); }
- void snap() { _snap = clock::now(); }
-
- float get_total() const { return get_diff<float>(_start, clock::now()); }
- float get_delta() const { return get_diff<float>(_snap, clock::now()); }
-
- private:
- using clock = std::chrono::high_resolution_clock;
- using point = std::chrono::time_point<clock>;
-
- template<typename T>
- static T get_diff(const point& start, const point& end)
- {
- using duration_t = std::chrono::duration<T, time_unit>;
-
- return std::chrono::duration_cast<duration_t>(end - start).count();
- }
-
- point _start;
- point _snap;
- };
-}
-
-#endif // spp_timer_h_guard
diff --git a/benchmarks/others/sparsepp/spp_traits.h b/benchmarks/others/sparsepp/spp_traits.h
deleted file mode 100644
index 792f52f2..00000000
--- a/benchmarks/others/sparsepp/spp_traits.h
+++ /dev/null
@@ -1,125 +0,0 @@
-#if !defined(spp_traits_h_guard)
-#define spp_traits_h_guard
-
-#include "spp_config.h"
-
-template<int S, int H> class HashObject; // for Google's benchmark, not in spp namespace!
-
-namespace spp_
-{
-
-// ---------------------------------------------------------------------------
-// type_traits we need
-// ---------------------------------------------------------------------------
-template<class T, T v>
-struct integral_constant { static const T value = v; };
-
-template <class T, T v> const T integral_constant<T, v>::value;
-
-typedef integral_constant<bool, true> true_type;
-typedef integral_constant<bool, false> false_type;
-
-typedef integral_constant<int, 0> zero_type;
-typedef integral_constant<int, 1> one_type;
-typedef integral_constant<int, 2> two_type;
-typedef integral_constant<int, 3> three_type;
-
-template<typename T, typename U> struct is_same : public false_type { };
-template<typename T> struct is_same<T, T> : public true_type { };
-
-template<typename T> struct remove_const { typedef T type; };
-template<typename T> struct remove_const<T const> { typedef T type; };
-
-template<typename T> struct remove_volatile { typedef T type; };
-template<typename T> struct remove_volatile<T volatile> { typedef T type; };
-
-template<typename T> struct remove_cv
-{
- typedef typename remove_const<typename remove_volatile<T>::type>::type type;
-};
-
-// ---------------- is_integral ----------------------------------------
-template <class T> struct is_integral;
-template <class T> struct is_integral : false_type { };
-template<> struct is_integral<bool> : true_type { };
-template<> struct is_integral<char> : true_type { };
-template<> struct is_integral<unsigned char> : true_type { };
-template<> struct is_integral<signed char> : true_type { };
-template<> struct is_integral<short> : true_type { };
-template<> struct is_integral<unsigned short> : true_type { };
-template<> struct is_integral<int> : true_type { };
-template<> struct is_integral<unsigned int> : true_type { };
-template<> struct is_integral<long> : true_type { };
-template<> struct is_integral<unsigned long> : true_type { };
-#ifdef SPP_HAS_LONG_LONG
- template<> struct is_integral<long long> : true_type { };
- template<> struct is_integral<unsigned long long> : true_type { };
-#endif
-template <class T> struct is_integral<const T> : is_integral<T> { };
-template <class T> struct is_integral<volatile T> : is_integral<T> { };
-template <class T> struct is_integral<const volatile T> : is_integral<T> { };
-
-// ---------------- is_floating_point ----------------------------------------
-template <class T> struct is_floating_point;
-template <class T> struct is_floating_point : false_type { };
-template<> struct is_floating_point<float> : true_type { };
-template<> struct is_floating_point<double> : true_type { };
-template<> struct is_floating_point<long double> : true_type { };
-template <class T> struct is_floating_point<const T> : is_floating_point<T> { };
-template <class T> struct is_floating_point<volatile T> : is_floating_point<T> { };
-template <class T> struct is_floating_point<const volatile T> : is_floating_point<T> { };
-
-// ---------------- is_pointer ----------------------------------------
-template <class T> struct is_pointer;
-template <class T> struct is_pointer : false_type { };
-template <class T> struct is_pointer<T*> : true_type { };
-template <class T> struct is_pointer<const T> : is_pointer<T> { };
-template <class T> struct is_pointer<volatile T> : is_pointer<T> { };
-template <class T> struct is_pointer<const volatile T> : is_pointer<T> { };
-
-// ---------------- is_reference ----------------------------------------
-template <class T> struct is_reference;
-template<typename T> struct is_reference : false_type {};
-template<typename T> struct is_reference<T&> : true_type {};
-
-// ---------------- is_relocatable ----------------------------------------
-// relocatable values can be moved around in memory using memcpy and remain
-// correct. Most types are relocatable, an example of a type who is not would
-// be a struct which contains a pointer to a buffer inside itself - this is the
-// case for std::string in gcc 5.
-// ------------------------------------------------------------------------
-template <class T> struct is_relocatable;
-template <class T> struct is_relocatable :
- integral_constant<bool, (is_integral<T>::value ||
- is_floating_point<T>::value ||
- is_pointer<T>::value
- )>
-{ };
-
-template<int S, int H> struct is_relocatable<HashObject<S, H> > : true_type { };
-
-template <class T> struct is_relocatable<const T> : is_relocatable<T> { };
-template <class T> struct is_relocatable<volatile T> : is_relocatable<T> { };
-template <class T> struct is_relocatable<const volatile T> : is_relocatable<T> { };
-template <class A, int N> struct is_relocatable<A[N]> : is_relocatable<A> { };
-template <class T, class U> struct is_relocatable<std::pair<T, U> > :
- integral_constant<bool, (is_relocatable<T>::value && is_relocatable<U>::value)>
-{ };
-
-// A template helper used to select A or B based on a condition.
-// ------------------------------------------------------------
-template<bool cond, typename A, typename B>
-struct if_
-{
- typedef A type;
-};
-
-template<typename A, typename B>
-struct if_<false, A, B>
-{
- typedef B type;
-};
-
-} // spp_ namespace
-
-#endif // spp_traits_h_guard
diff --git a/benchmarks/others/sparsepp/spp_utils.h b/benchmarks/others/sparsepp/spp_utils.h
deleted file mode 100644
index 4f2e9257..00000000
--- a/benchmarks/others/sparsepp/spp_utils.h
+++ /dev/null
@@ -1,477 +0,0 @@
-// ----------------------------------------------------------------------
-// Copyright (c) 2016, Steven Gregory Popovitch - [email protected]
-// All rights reserved.
-//
-// Code derived derived from Boost libraries.
-// Boost software licence reproduced below.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * The name of Steven Gregory Popovitch may not be used to
-// endorse or promote products derived from this software without
-// specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// ----------------------------------------------------------------------
-
-// ---------------------------------------------------------------------------
-// Boost Software License - Version 1.0 - August 17th, 2003
-//
-// Permission is hereby granted, free of charge, to any person or organization
-// obtaining a copy of the software and accompanying documentation covered by
-// this license (the "Software") to use, reproduce, display, distribute,
-// execute, and transmit the Software, and to prepare derivative works of the
-// Software, and to permit third-parties to whom the Software is furnished to
-// do so, all subject to the following:
-//
-// The copyright notices in the Software and this entire statement, including
-// the above license grant, this restriction and the following disclaimer,
-// must be included in all copies of the Software, in whole or in part, and
-// all derivative works of the Software, unless such copies or derivative
-// works are solely in the form of machine-executable object code generated by
-// a source language processor.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-// DEALINGS IN THE SOFTWARE.
-// ---------------------------------------------------------------------------
-
-// ----------------------------------------------------------------------
-// H A S H F U N C T I O N S
-// ----------------------------
-//
-// Implements spp::spp_hash() and spp::hash_combine()
-// ----------------------------------------------------------------------
-
-#if !defined(spp_utils_h_guard_)
-#define spp_utils_h_guard_
-
-#if defined(_MSC_VER)
- #if (_MSC_VER >= 1600 ) // vs2010 (1900 is vs2015)
- #include <functional>
- #define SPP_HASH_CLASS std::hash
- #else
- #include <hash_map>
- #define SPP_HASH_CLASS stdext::hash_compare
- #endif
- #if (_MSC_FULL_VER < 190021730)
- #define SPP_NO_CXX11_NOEXCEPT
- #endif
-#elif defined __clang__
- #if __has_feature(cxx_noexcept) || defined(SPP_CXX11) // define SPP_CXX11 if your compiler has <functional>
- #include <functional>
- #define SPP_HASH_CLASS std::hash
- #else
- #include <tr1/functional>
- #define SPP_HASH_CLASS std::tr1::hash
- #endif
-
- #if !__has_feature(cxx_noexcept)
- #define SPP_NO_CXX11_NOEXCEPT
- #endif
-#elif defined(__GNUC__)
- #if defined(__GXX_EXPERIMENTAL_CXX0X__) || (__cplusplus >= 201103L)
- #include <functional>
- #define SPP_HASH_CLASS std::hash
-
- #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100) < 40600
- #define SPP_NO_CXX11_NOEXCEPT
- #endif
- #else
- #include <tr1/unordered_map>
- #define SPP_HASH_CLASS std::tr1::hash
- #define SPP_NO_CXX11_NOEXCEPT
- #endif
-#else
- #include <functional>
- #define SPP_HASH_CLASS std::hash
-#endif
-
-#ifdef SPP_NO_CXX11_NOEXCEPT
- #define SPP_NOEXCEPT
-#else
- #define SPP_NOEXCEPT noexcept
-#endif
-
-#ifdef SPP_NO_CXX11_CONSTEXPR
- #define SPP_CONSTEXPR
-#else
- #define SPP_CONSTEXPR constexpr
-#endif
-
-#ifdef SPP_NO_CXX14_CONSTEXPR
- #define SPP_CXX14_CONSTEXPR
-#else
- #define SPP_CXX14_CONSTEXPR constexpr
-#endif
-
-#define SPP_INLINE
-
-#ifndef spp_
- #define spp_ spp
-#endif
-
-namespace spp_
-{
-
-template <class T> T spp_min(T a, T b) { return a < b ? a : b; }
-template <class T> T spp_max(T a, T b) { return a >= b ? a : b; }
-
-template <class T>
-struct spp_hash
-{
- SPP_INLINE size_t operator()(const T &__v) const SPP_NOEXCEPT
- {
- SPP_HASH_CLASS<T> hasher;
- return hasher(__v);
- }
-};
-
-template <class T>
-struct spp_hash<T *>
-{
- static size_t spp_log2 (size_t val) SPP_NOEXCEPT
- {
- size_t res = 0;
- while (val > 1)
- {
- val >>= 1;
- res++;
- }
- return res;
- }
-
- SPP_INLINE size_t operator()(const T *__v) const SPP_NOEXCEPT
- {
- static const size_t shift = 3; // spp_log2(1 + sizeof(T)); // T might be incomplete!
- const uintptr_t i = (const uintptr_t)__v;
- return static_cast<size_t>(i >> shift);
- }
-};
-
-// from http://burtleburtle.net/bob/hash/integer.html
-// fast and efficient for power of two table sizes where we always
-// consider the last bits.
-// ---------------------------------------------------------------
-inline size_t spp_mix_32(uint32_t a)
-{
- a = a ^ (a >> 4);
- a = (a ^ 0xdeadbeef) + (a << 5);
- a = a ^ (a >> 11);
- return static_cast<size_t>(a);
-}
-
-// More thorough scrambling as described in
-// https://gist.github.com/badboy/6267743
-// ----------------------------------------
-inline size_t spp_mix_64(uint64_t a)
-{
- a = (~a) + (a << 21); // a = (a << 21) - a - 1;
- a = a ^ (a >> 24);
- a = (a + (a << 3)) + (a << 8); // a * 265
- a = a ^ (a >> 14);
- a = (a + (a << 2)) + (a << 4); // a * 21
- a = a ^ (a >> 28);
- a = a + (a << 31);
- return static_cast<size_t>(a);
-}
-
-template<class ArgumentType, class ResultType>
-struct spp_unary_function
-{
- typedef ArgumentType argument_type;
- typedef ResultType result_type;
-};
-
-template <>
-struct spp_hash<bool> : public spp_unary_function<bool, size_t>
-{
- SPP_INLINE size_t operator()(bool __v) const SPP_NOEXCEPT
- { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct spp_hash<char> : public spp_unary_function<char, size_t>
-{
- SPP_INLINE size_t operator()(char __v) const SPP_NOEXCEPT
- { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct spp_hash<signed char> : public spp_unary_function<signed char, size_t>
-{
- SPP_INLINE size_t operator()(signed char __v) const SPP_NOEXCEPT
- { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct spp_hash<unsigned char> : public spp_unary_function<unsigned char, size_t>
-{
- SPP_INLINE size_t operator()(unsigned char __v) const SPP_NOEXCEPT
- { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct spp_hash<wchar_t> : public spp_unary_function<wchar_t, size_t>
-{
- SPP_INLINE size_t operator()(wchar_t __v) const SPP_NOEXCEPT
- { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct spp_hash<int16_t> : public spp_unary_function<int16_t, size_t>
-{
- SPP_INLINE size_t operator()(int16_t __v) const SPP_NOEXCEPT
- { return spp_mix_32(static_cast<uint32_t>(__v)); }
-};
-
-template <>
-struct spp_hash<uint16_t> : public spp_unary_function<uint16_t, size_t>
-{
- SPP_INLINE size_t operator()(uint16_t __v) const SPP_NOEXCEPT
- { return spp_mix_32(static_cast<uint32_t>(__v)); }
-};
-
-template <>
-struct spp_hash<int32_t> : public spp_unary_function<int32_t, size_t>
-{
- SPP_INLINE size_t operator()(int32_t __v) const SPP_NOEXCEPT
- { return spp_mix_32(static_cast<uint32_t>(__v)); }
-};
-
-template <>
-struct spp_hash<uint32_t> : public spp_unary_function<uint32_t, size_t>
-{
- SPP_INLINE size_t operator()(uint32_t __v) const SPP_NOEXCEPT
- { return spp_mix_32(static_cast<uint32_t>(__v)); }
-};
-
-template <>
-struct spp_hash<int64_t> : public spp_unary_function<int64_t, size_t>
-{
- SPP_INLINE size_t operator()(int64_t __v) const SPP_NOEXCEPT
- { return spp_mix_64(static_cast<uint64_t>(__v)); }
-};
-
-template <>
-struct spp_hash<uint64_t> : public spp_unary_function<uint64_t, size_t>
-{
- SPP_INLINE size_t operator()(uint64_t __v) const SPP_NOEXCEPT
- { return spp_mix_64(static_cast<uint64_t>(__v)); }
-};
-
-template <>
-struct spp_hash<float> : public spp_unary_function<float, size_t>
-{
- SPP_INLINE size_t operator()(float __v) const SPP_NOEXCEPT
- {
- // -0.0 and 0.0 should return same hash
- uint32_t *as_int = reinterpret_cast<uint32_t *>(&__v);
- return (__v == 0) ? static_cast<size_t>(0) : spp_mix_32(*as_int);
- }
-};
-
-template <>
-struct spp_hash<double> : public spp_unary_function<double, size_t>
-{
- SPP_INLINE size_t operator()(double __v) const SPP_NOEXCEPT
- {
- // -0.0 and 0.0 should return same hash
- uint64_t *as_int = reinterpret_cast<uint64_t *>(&__v);
- return (__v == 0) ? static_cast<size_t>(0) : spp_mix_64(*as_int);
- }
-};
-
-template <class T, int sz> struct Combiner
-{
- inline void operator()(T& seed, T value);
-};
-
-template <class T> struct Combiner<T, 4>
-{
- inline void operator()(T& seed, T value)
- {
- seed ^= value + 0x9e3779b9 + (seed << 6) + (seed >> 2);
- }
-};
-
-template <class T> struct Combiner<T, 8>
-{
- inline void operator()(T& seed, T value)
- {
- seed ^= value + T(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2);
- }
-};
-
-template <class T>
-inline void hash_combine(std::size_t& seed, T const& v)
-{
- spp_::spp_hash<T> hasher;
- Combiner<std::size_t, sizeof(std::size_t)> combiner;
-
- combiner(seed, hasher(v));
-}
-
-static inline uint32_t s_spp_popcount_default(uint32_t i) SPP_NOEXCEPT
-{
- i = i - ((i >> 1) & 0x55555555);
- i = (i & 0x33333333) + ((i >> 2) & 0x33333333);
- return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
-}
-
-static inline uint32_t s_spp_popcount_default(uint64_t x) SPP_NOEXCEPT
-{
- const uint64_t m1 = uint64_t(0x5555555555555555); // binary: 0101...
- const uint64_t m2 = uint64_t(0x3333333333333333); // binary: 00110011..
- const uint64_t m4 = uint64_t(0x0f0f0f0f0f0f0f0f); // binary: 4 zeros, 4 ones ...
- const uint64_t h01 = uint64_t(0x0101010101010101); // the sum of 256 to the power of 0,1,2,3...
-
- x -= (x >> 1) & m1; // put count of each 2 bits into those 2 bits
- x = (x & m2) + ((x >> 2) & m2); // put count of each 4 bits into those 4 bits
- x = (x + (x >> 4)) & m4; // put count of each 8 bits into those 8 bits
- return (x * h01)>>56; // returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24)+...
-}
-
-#ifdef __APPLE__
- static inline uint32_t count_trailing_zeroes(size_t v) SPP_NOEXCEPT
- {
- size_t x = (v & -v) - 1;
- // sadly sizeof() required to build on macos
- return sizeof(size_t) == 8 ? s_spp_popcount_default((uint64_t)x) : s_spp_popcount_default((uint32_t)x);
- }
-
- static inline uint32_t s_popcount(size_t v) SPP_NOEXCEPT
- {
- // sadly sizeof() required to build on macos
- return sizeof(size_t) == 8 ? s_spp_popcount_default((uint64_t)v) : s_spp_popcount_default((uint32_t)v);
- }
-#else
- static inline uint32_t count_trailing_zeroes(size_t v) SPP_NOEXCEPT
- {
- return s_spp_popcount_default((v & -(intptr_t)v) - 1);
- }
-
- static inline uint32_t s_popcount(size_t v) SPP_NOEXCEPT
- {
- return s_spp_popcount_default(v);
- }
-#endif
-
-// -----------------------------------------------------------
-// -----------------------------------------------------------
-template<class T>
-class libc_allocator
-{
-public:
- typedef T value_type;
- typedef T* pointer;
- typedef ptrdiff_t difference_type;
- typedef const T* const_pointer;
- typedef size_t size_type;
-
- libc_allocator() {}
- libc_allocator(const libc_allocator&) {}
-
- template<class U>
- libc_allocator(const libc_allocator<U> &) {}
-
- libc_allocator& operator=(const libc_allocator &) { return *this; }
-
- template<class U>
- libc_allocator& operator=(const libc_allocator<U> &) { return *this; }
-
-#ifndef SPP_NO_CXX11_RVALUE_REFERENCES
- libc_allocator(libc_allocator &&) {}
- libc_allocator& operator=(libc_allocator &&) { return *this; }
-#endif
-
- pointer allocate(size_t n, const_pointer /* unused */= 0)
- {
- pointer res = static_cast<pointer>(malloc(n * sizeof(T)));
- if (!res)
- throw std::bad_alloc();
- return res;
- }
-
- void deallocate(pointer p, size_t /* unused */)
- {
- free(p);
- }
-
- pointer reallocate(pointer p, size_t new_size)
- {
- pointer res = static_cast<pointer>(realloc(p, new_size * sizeof(T)));
- if (!res)
- throw std::bad_alloc();
- return res;
- }
-
- // extra API to match spp_allocator interface
- pointer reallocate(pointer p, size_t /* old_size */, size_t new_size)
- {
- return static_cast<pointer>(realloc(p, new_size * sizeof(T)));
- }
-
- size_type max_size() const
- {
- return static_cast<size_type>(-1) / sizeof(value_type);
- }
-
- void construct(pointer p, const value_type& val)
- {
- new(p) value_type(val);
- }
-
- void destroy(pointer p) { p->~value_type(); }
-
- template<class U>
- struct rebind
- {
- typedef spp_::libc_allocator<U> other;
- };
-
-};
-
-// forward declaration
-// -------------------
-template<class T>
-class spp_allocator;
-
-}
-
-template<class T>
-inline bool operator==(const spp_::libc_allocator<T> &, const spp_::libc_allocator<T> &)
-{
- return true;
-}
-
-template<class T>
-inline bool operator!=(const spp_::libc_allocator<T> &, const spp_::libc_allocator<T> &)
-{
- return false;
-}
-
-#endif // spp_utils_h_guard_
-
diff --git a/benchmarks/others/update.sh b/benchmarks/others/update.sh
index 6af246d7..1ac8ad91 100644
--- a/benchmarks/others/update.sh
+++ b/benchmarks/others/update.sh
@@ -1,31 +1,30 @@
-tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl/"
-tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl/"
-greg="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp/"
-martinus="https://raw.github.com/martinus/robin-hood-hashing/master/src/include/"
-skarupke="https://raw.github.com/skarupke/flat_hash_map/master/"
-
-mkdir -p tsl sparsepp skarupke
-
-wget $martinus"robin_hood.h" -O "robin_hood.h"
-
-wget $skarupke"bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp"
-wget $skarupke"flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp"
-
-exit
-wget $greg"spp.h" -O "sparsepp/spp.h"
-wget $greg"spp_config.h" -O "sparsepp/spp_config.h"
-wget $greg"spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h"
-wget $greg"spp_memory.h" -O "sparsepp/spp_memory.h"
-wget $greg"spp_smartptr.h" -O "sparsepp/spp_smartptr.h"
-wget $greg"spp_stdint.h" -O "sparsepp/spp_stdint.h"
-wget $greg"spp_timer.h" -O "sparsepp/spp_timer.h"
-wget $greg"spp_traits.h" -O "sparsepp/spp_traits.h"
-wget $greg"spp_utils.h" -O "sparsepp/spp_utils.h"
-
-wget $tsl_h"hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h"
-wget $tsl_h"hopscotch_hash.h" -O "tsl/hopscotch_hash.h"
-wget $tsl_h"hopscotch_map.h" -O "tsl/hopscotch_map.h"
-
-wget $tsl_r"robin_growth_policy.h" -O "tsl/robin_growth_policy.h"
-wget $tsl_r"robin_hash.h" -O "tsl/robin_hash.h"
-wget $tsl_r"robin_map.h" -O "tsl/robin_map.h"
+tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl/"
+tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl/"
+greg="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp/"
+martinus="https://raw.github.com/martinus/robin-hood-hashing/master/src/include/"
+skarupke="https://raw.github.com/skarupke/flat_hash_map/master/"
+
+mkdir -p tsl sparsepp skarupke
+
+wget $martinus"robin_hood.h" -O "robin_hood.h"
+
+wget $skarupke"bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp"
+wget $skarupke"flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp"
+
+wget $greg"spp.h" -O "sparsepp/spp.h"
+wget $greg"spp_config.h" -O "sparsepp/spp_config.h"
+wget $greg"spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h"
+wget $greg"spp_memory.h" -O "sparsepp/spp_memory.h"
+wget $greg"spp_smartptr.h" -O "sparsepp/spp_smartptr.h"
+wget $greg"spp_stdint.h" -O "sparsepp/spp_stdint.h"
+wget $greg"spp_timer.h" -O "sparsepp/spp_timer.h"
+wget $greg"spp_traits.h" -O "sparsepp/spp_traits.h"
+wget $greg"spp_utils.h" -O "sparsepp/spp_utils.h"
+
+wget $tsl_h"hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h"
+wget $tsl_h"hopscotch_hash.h" -O "tsl/hopscotch_hash.h"
+wget $tsl_h"hopscotch_map.h" -O "tsl/hopscotch_map.h"
+
+wget $tsl_r"robin_growth_policy.h" -O "tsl/robin_growth_policy.h"
+wget $tsl_r"robin_hash.h" -O "tsl/robin_hash.h"
+wget $tsl_r"robin_map.h" -O "tsl/robin_map.h"
diff --git a/benchmarks/shootout1_cmap.cpp b/benchmarks/shootout1_cmap.cpp
index fab25096..87790844 100644
--- a/benchmarks/shootout1_cmap.cpp
+++ b/benchmarks/shootout1_cmap.cpp
@@ -7,7 +7,7 @@
#include "others/robin_hood.hpp"
#include "others/skarupke/bytell_hash_map.hpp"
#include "others/tsl/hopscotch_map.h"
-#include "others/sparsepp/spp.h"
+#include "others/parallel_hashmap/phmap.h"
#define PICOBENCH_IMPLEMENT_WITH_MAIN
#include "picobench.hpp"
@@ -19,15 +19,15 @@ template <class K, class V> using umap = std::unordered_map<K, V>;
template <class K, class V> using bmap = ska::bytell_hash_map<K, V>;
template <class K, class V> using fmap = ska::flat_hash_map<K, V>;
template <class K, class V> using hmap = tsl::hopscotch_map<K, V>;
-template <class K, class V> using smap = spp::sparse_hash_map<K, V>;
-template <class K, class V> using rmap = robin_hood::unordered_flat_map<K, V, robin_hood::hash<K>,
- std::equal_to<K>, MaxLoadFactor100>;
+template <class K, class V> using pmap = phmap::flat_hash_map<K, V>;
+template <class K, class V> using rmap = robin_hood::unordered_flat_map<K, V,
+ robin_hood::hash<K>, std::equal_to<K>, MaxLoadFactor100>;
#define DEFMAP(map, ...) \
using u##map = umap __VA_ARGS__; \
using b##map = bmap __VA_ARGS__; \
using f##map = fmap __VA_ARGS__; \
using h##map = hmap __VA_ARGS__; \
- using s##map = smap __VA_ARGS__; \
+ using p##map = pmap __VA_ARGS__; \
using r##map = rmap __VA_ARGS__
@@ -119,7 +119,7 @@ PICOBENCH(ins_and_erase_i<umap_x>).P;
PICOBENCH(ins_and_erase_i<bmap_x>).P;
PICOBENCH(ins_and_erase_i<fmap_x>).P;
PICOBENCH(ins_and_erase_i<hmap_x>).P;
-PICOBENCH(ins_and_erase_i<smap_x>).P;
+PICOBENCH(ins_and_erase_i<pmap_x>).P;
PICOBENCH(ins_and_erase_i<rmap_x>).P;
PICOBENCH(ins_and_erase_cmap_x).P;
#undef P
@@ -161,7 +161,7 @@ PICOBENCH(ins_and_access_i<umap_i>).P;
PICOBENCH(ins_and_access_i<bmap_i>).P;
PICOBENCH(ins_and_access_i<fmap_i>).P;
PICOBENCH(ins_and_access_i<hmap_i>).P;
-PICOBENCH(ins_and_access_i<smap_i>).P;
+PICOBENCH(ins_and_access_i<pmap_i>).P;
PICOBENCH(ins_and_access_i<rmap_i>).P;
PICOBENCH(ins_and_access_cmap_i).P;
#undef P
@@ -169,9 +169,11 @@ PICOBENCH(ins_and_access_cmap_i).P;
PICOBENCH_SUITE("Map3");
static void randomize(char* str, size_t len) {
- union {uint64_t i; char c[8];} r = {.i = stc64_random()};
- for (int i = len - 7, j = 0; i < len; ++j, ++i)
- str[i] = (r.c[j] & 63) + 48;
+ for (int k=0; k < len; ++k) {
+ union {uint64_t i; char c[8];} r = {.i = stc64_random()};
+ for (int i=0; i<8 && k<len; ++k, ++i)
+ str[k] = (r.c[i] & 63) + 48;
+ }
}
template <class MapStr>
@@ -204,8 +206,9 @@ static void ins_and_access_cmap_s(picobench::state& s)
picobench::scope scope(s);
c_forrange (s.iterations()) {
randomize(str.str, cstr_size(str));
+ //if (cstr_size(str) > 30) { printf("%s\n", str.str); exit(0); }
cmap_str_emplace(&map, str.str, str.str);
- randomize(str.str, cstr_size(str));
+ //randomize(str.str, cstr_size(str));
result += cmap_str_erase(&map, str.str);
}
s.set_result(result + cmap_str_size(map));
@@ -218,7 +221,7 @@ PICOBENCH(ins_and_access_s<umap_s>).P;
PICOBENCH(ins_and_access_s<bmap_s>).P;
PICOBENCH(ins_and_access_s<fmap_s>).P;
PICOBENCH(ins_and_access_s<hmap_s>).P;
-PICOBENCH(ins_and_access_s<smap_s>).P;
+PICOBENCH(ins_and_access_s<pmap_s>).P;
PICOBENCH(ins_and_access_s<rmap_s>).P;
PICOBENCH(ins_and_access_cmap_s).P;
#undef P
@@ -291,7 +294,7 @@ PICOBENCH(iterate_x<umap_x>).P;
PICOBENCH(iterate_x<bmap_x>).P;
PICOBENCH(iterate_x<fmap_x>).P;
PICOBENCH(iterate_x<hmap_x>).P;
-PICOBENCH(iterate_x<smap_x>).P;
+PICOBENCH(iterate_x<pmap_x>).P;
PICOBENCH(iterate_x<rmap_x>).P;
PICOBENCH(iterate_cmap_x).P;
#undef P
diff --git a/benchmarks/shootout2_cmap.cpp b/benchmarks/shootout2_cmap.cpp
index 6ad6b293..1704079d 100644
--- a/benchmarks/shootout2_cmap.cpp
+++ b/benchmarks/shootout2_cmap.cpp
@@ -10,11 +10,10 @@
#include "others/robin_hood.hpp"
#include "others/skarupke/bytell_hash_map.hpp"
#include "others/tsl/hopscotch_map.h"
-#include "others/sparsepp/spp.h"
+#include "others/parallel_hashmap/phmap.h"
template<typename C> inline void destroy_me(C& c) { C().swap(c); }
#endif
-// Visual Studio: compile with -TP to force C++: cl -TP -EHsc -O2 benchmark.c
// cmap and khash template expansion
#define i_key int64_t
@@ -117,29 +116,45 @@ stc64_t rng;
#define RMAP_CLEAR(X) UMAP_CLEAR(X)
#define RMAP_DTOR(X) UMAP_DTOR(X)
-#define SMAP_SETUP(X, Key, Value) spp::sparse_hash_map<Key, Value> map; map.max_load_factor(max_load_factor)
-#define SMAP_PUT(X, key, val) UMAP_PUT(X, key, val)
-#define SMAP_EMPLACE(X, key, val) UMAP_EMPLACE(X, key, val)
-#define SMAP_FIND(X, key) UMAP_FIND(X, key)
-#define SMAP_ERASE(X, key) UMAP_ERASE(X, key)
-#define SMAP_FOR(X, i) UMAP_FOR(X, i)
-#define SMAP_ITEM(X, i) UMAP_ITEM(X, i)
-#define SMAP_SIZE(X) UMAP_SIZE(X)
-#define SMAP_BUCKETS(X) UMAP_BUCKETS(X)
-#define SMAP_CLEAR(X) UMAP_CLEAR(X)
-#define SMAP_DTOR(X) UMAP_DTOR(X)
+#define PMAP_SETUP(X, Key, Value) phmap::flat_hash_map<Key, Value> map; map.max_load_factor(max_load_factor)
+#define PMAP_PUT(X, key, val) UMAP_PUT(X, key, val)
+#define PMAP_EMPLACE(X, key, val) UMAP_EMPLACE(X, key, val)
+#define PMAP_FIND(X, key) UMAP_FIND(X, key)
+#define PMAP_ERASE(X, key) UMAP_ERASE(X, key)
+#define PMAP_FOR(X, i) UMAP_FOR(X, i)
+#define PMAP_ITEM(X, i) UMAP_ITEM(X, i)
+#define PMAP_SIZE(X) UMAP_SIZE(X)
+#define PMAP_BUCKETS(X) UMAP_BUCKETS(X)
+#define PMAP_CLEAR(X) UMAP_CLEAR(X)
+#define PMAP_DTOR(X) UMAP_DTOR(X)
enum {
+ RR = 27,
FAC = 3,
+ N0 = 10000000 * FAC,
N1 = 10000000 * FAC,
N2 = 10000000 * FAC,
N3 = 10000000 * FAC,
N4 = 10000000 * FAC,
- RR = 24
};
int rr = RR;
+#define MAP_TEST0(M, X) \
+{ \
+ M##_SETUP(X, int64_t, int64_t); \
+ uint64_t checksum = 0; \
+ SEED(seed); \
+ clock_t difference, before = clock(); \
+ for (size_t i = 0; i < N0; ++i) { \
+ checksum += ++ M##_EMPLACE(X, RAND(rr), i); \
+ } \
+ difference = clock() - before; \
+ printf(#M ": time: %5.02f, sum: %zu, size: %zu, buckets: %8zu\n", \
+ (float) difference / CLOCKS_PER_SEC, checksum, (size_t) M##_SIZE(X), (size_t) M##_BUCKETS(X)); \
+ M##_DTOR(X); \
+}
+
#define MAP_TEST1(M, X) \
{ \
M##_SETUP(X, int64_t, int64_t); \
@@ -204,26 +219,11 @@ int rr = RR;
M##_DTOR(X); \
}
-#define MAP_TEST5(M, X) \
-{ \
- M##_SETUP(X, int64_t, int64_t); \
- uint64_t checksum = 0; \
- SEED(seed); \
- clock_t difference, before = clock(); \
- for (size_t i = 0; i < N1; ++i) { \
- checksum += ++ M##_EMPLACE(X, RAND(rr), i); \
- } \
- difference = clock() - before; \
- printf(#M ": time: %5.02f, sum: %zu, size: %zu, buckets: %8zu\n", \
- (float) difference / CLOCKS_PER_SEC, checksum, (size_t) M##_SIZE(X), (size_t) M##_BUCKETS(X)); \
- M##_DTOR(X); \
-}
-
#ifdef __cplusplus
-#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii) MAP_TEST##n(UMAP, ii) /*MAP_TEST##n(SMAP, ii)*/ \
+#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii) MAP_TEST##n(UMAP, ii) MAP_TEST##n(PMAP, ii) \
+ MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) MAP_TEST##n(RMAP, ii) /*MAP_TEST##n(HMAP, ii)*/
+#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii) /*MAP_TEST##n(KMAP, ii)*/ MAP_TEST##n(UMAP, ii) MAP_TEST##n(PMAP, ii) \
MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) MAP_TEST##n(RMAP, ii) /*MAP_TEST##n(HMAP, ii)*/
-#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii) /*MAP_TEST##n(KMAP, ii)*/ MAP_TEST##n(UMAP, ii) MAP_TEST##n(SMAP, ii) \
- MAP_TEST##n(BMAP, ii) MAP_TEST##n(FMAP, ii) /*MAP_TEST##n(RMAP, ii)*/ /*MAP_TEST##n(HMAP, ii)*/
#else
#define RUN_TEST(n) MAP_TEST##n(CMAP, ii) MAP_TEST##n(KMAP, ii)
#define RUNX_TEST(n) MAP_TEST##n(CMAP, ii)
@@ -235,14 +235,21 @@ int main(int argc, char* argv[])
rr = argc == 2 ? atoi(argv[1]) : RR;
seed = time(NULL);
- printf("\nUnordered maps: Insert %d random (2^%d) keys:\n", N1, rr);
- RUN_TEST(5)
+ printf("\nRandom keys are in range [0, 2^%d), seed = %zu:\n", rr, seed);
+ printf("CMAP = STC cmap\n"
+ "KMAP = Klib khash\n"
+ "UMAP = std::unordered_map\n"
+ "PMAP = phmap::flat_hash_map\n"
+ "BMAP = ska::bytell_hash_map\n"
+ "FMAP = ska::flat_hash_map\n"
+ "RMAP = robin_hood::unordered_map\n");
+ printf("\nUnordered maps: Insert %d random keys:\n", N0);
+ RUN_TEST(0)
- printf("\nRandom keys are in range [0, 2^%d), seed = %zu:\n", rr, seed);
- printf("\nUnordered maps: %d repeats of Insert random key + try to remove a random key:\n", N1);
+ printf("\nUnordered maps: %d insert random key + try to remove another random key:\n", N1);
RUN_TEST(1)
- printf("\nUnordered maps: Insert %d index keys, then remove them in same order:\n", N2);
+ printf("\nUnordered maps: Insert %d sequential keys, then remove them in same order:\n", N2);
RUN_TEST(2)
printf("\nUnordered maps: Insert %d random keys, then remove them in same order:\n", N3);