diff options
Diffstat (limited to 'benchmarks/external/parallel_hashmap')
| -rw-r--r-- | benchmarks/external/parallel_hashmap/btree.h | 4050 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/conanfile.py | 36 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/meminfo.h | 195 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap.h | 4788 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_base.h | 5171 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_bits.h | 663 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_config.h | 771 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_dump.h | 227 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_fwd_decl.h | 154 | ||||
| -rw-r--r-- | benchmarks/external/parallel_hashmap/phmap_utils.h | 378 |
10 files changed, 0 insertions, 16433 deletions
diff --git a/benchmarks/external/parallel_hashmap/btree.h b/benchmarks/external/parallel_hashmap/btree.h deleted file mode 100644 index b8c95433..00000000 --- a/benchmarks/external/parallel_hashmap/btree.h +++ /dev/null @@ -1,4050 +0,0 @@ -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_ -#define PHMAP_BTREE_BTREE_CONTAINER_H_ - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4355) // 'this': used in base member initializer list - #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - - -#include <cstdint> -#include <cstdlib> -#include <cstring> -#include <limits> -#include <new> - -#include "phmap_fwd_decl.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include <string_view> -#endif - -// MSVC constructibility traits do not detect destructor properties and so our -// implementations should not use them as a source-of-truth. -#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) - #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 -#endif - -namespace phmap { - - // Defined and documented later on in this file. - template <typename T> - struct is_trivially_destructible; - - // Defined and documented later on in this file. - template <typename T> - struct is_trivially_move_assignable; - - namespace type_traits_internal { - - // Silence MSVC warnings about the destructor being defined as deleted. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(push) - #pragma warning(disable : 4624) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template <class T> - union SingleMemberUnion { - T t; - }; - - // Restore the state of the destructor warning that was silenced above. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(pop) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template <class T> - struct IsTriviallyMoveConstructibleObject - : std::integral_constant< - bool, std::is_move_constructible< - type_traits_internal::SingleMemberUnion<T>>::value && - phmap::is_trivially_destructible<T>::value> {}; - - template <class T> - struct IsTriviallyCopyConstructibleObject - : std::integral_constant< - bool, std::is_copy_constructible< - type_traits_internal::SingleMemberUnion<T>>::value && - phmap::is_trivially_destructible<T>::value> {}; - - template <class T> - struct IsTriviallyMoveAssignableReference : std::false_type {}; - - template <class T> - struct IsTriviallyMoveAssignableReference<T&> - : phmap::is_trivially_move_assignable<T>::type {}; - - template <class T> - struct IsTriviallyMoveAssignableReference<T&&> - : phmap::is_trivially_move_assignable<T>::type {}; - - } // namespace type_traits_internal - - - template <typename... Ts> - using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; - - - template <typename T> - struct is_function - : std::integral_constant< - bool, !(std::is_reference<T>::value || - std::is_const<typename std::add_const<T>::type>::value)> {}; - - - namespace type_traits_internal { - - template <typename T> - class is_trivially_copyable_impl { - using ExtentsRemoved = typename std::remove_all_extents<T>::type; - static constexpr bool kIsCopyOrMoveConstructible = - std::is_copy_constructible<ExtentsRemoved>::value || - std::is_move_constructible<ExtentsRemoved>::value; - static constexpr bool kIsCopyOrMoveAssignable = - phmap::is_copy_assignable<ExtentsRemoved>::value || - phmap::is_move_assignable<ExtentsRemoved>::value; - - public: - static constexpr bool kValue = - (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && - (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && - (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && - is_trivially_destructible<ExtentsRemoved>::value && - // We need to check for this explicitly because otherwise we'll say - // references are trivial copyable when compiled by MSVC. - !std::is_reference<ExtentsRemoved>::value; - }; - - template <typename T> - struct is_trivially_copyable - : std::integral_constant< - bool, type_traits_internal::is_trivially_copyable_impl<T>::kValue> {}; - } // namespace type_traits_internal - - namespace swap_internal { - - // Necessary for the traits. - using std::swap; - - // This declaration prevents global `swap` and `phmap::swap` overloads from being - // considered unless ADL picks them up. - void swap(); - - template <class T> - using IsSwappableImpl = decltype(swap(std::declval<T&>(), std::declval<T&>())); - - // NOTE: This dance with the default template parameter is for MSVC. - template <class T, - class IsNoexcept = std::integral_constant< - bool, noexcept(swap(std::declval<T&>(), std::declval<T&>()))>> - using IsNothrowSwappableImpl = typename std::enable_if<IsNoexcept::value>::type; - - template <class T> - struct IsSwappable - : phmap::type_traits_internal::is_detected<IsSwappableImpl, T> {}; - - template <class T> - struct IsNothrowSwappable - : phmap::type_traits_internal::is_detected<IsNothrowSwappableImpl, T> {}; - - template <class T, phmap::enable_if_t<IsSwappable<T>::value, int> = 0> - void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable<T>::value) { - swap(lhs, rhs); - } - - using StdSwapIsUnconstrained = IsSwappable<void()>; - - } // namespace swap_internal - - namespace type_traits_internal { - - // Make the swap-related traits/function accessible from this namespace. - using swap_internal::IsNothrowSwappable; - using swap_internal::IsSwappable; - using swap_internal::Swap; - using swap_internal::StdSwapIsUnconstrained; - - } // namespace type_traits_internal - - namespace compare_internal { - - using value_type = int8_t; - - template <typename T> - struct Fail { - static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); - }; - - template <typename NullPtrT = std::nullptr_t> - struct OnlyLiteralZero { - constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT - - template < - typename T, - typename = typename std::enable_if< - std::is_same<T, std::nullptr_t>::value || - (std::is_integral<T>::value && !std::is_same<T, int>::value)>::type, - typename = typename Fail<T>::type> - OnlyLiteralZero(T); // NOLINT - }; - - enum class eq : value_type { - equal = 0, - equivalent = equal, - nonequal = 1, - nonequivalent = nonequal, - }; - - enum class ord : value_type { less = -1, greater = 1 }; - - enum class ncmp : value_type { unordered = -127 }; - -#if defined(__cpp_inline_variables) && !defined(_MSC_VER) - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ - static const type name; - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - inline constexpr type type::name(init) - -#else // __cpp_inline_variables - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ - static const T name; - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - template <typename T> \ - const T compare_internal::type##_base<T>::name(init) - -#endif // __cpp_inline_variables - - // These template base classes allow for defining the values of the constants - // in the header file (for performance) without using inline variables (which - // aren't available in C++11). - template <typename T> - struct weak_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template <typename T> - struct strong_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template <typename T> - struct partial_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) - }; - - template <typename T> - struct weak_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - template <typename T> - struct strong_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - } // namespace compare_internal - - class weak_equality - : public compare_internal::weak_equality_base<weak_equality> { - explicit constexpr weak_equality(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::weak_equality_base<weak_equality>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) - - // Comparisons - friend constexpr bool operator==( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class strong_equality - : public compare_internal::strong_equality_base<strong_equality> { - explicit constexpr strong_equality(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::strong_equality_base<strong_equality>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, - compare_internal::eq::nonequal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class partial_ordering - : public compare_internal::partial_ordering_base<partial_ordering> { - explicit constexpr partial_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr partial_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::partial_ordering_base<partial_ordering>; - - constexpr bool is_ordered() const noexcept { - return value_ != - compare_internal::value_type(compare_internal::ncmp::unordered); - } - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ == 0; - } - friend constexpr bool operator!=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return !v.is_ordered() || v.value_ != 0; - } - friend constexpr bool operator<( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ < 0; - } - friend constexpr bool operator<=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ <= 0; - } - friend constexpr bool operator>( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ > 0; - } - friend constexpr bool operator>=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return !v.is_ordered() || 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, - compare_internal::ord::greater); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, - compare_internal::ncmp::unordered); - - class weak_ordering - : public compare_internal::weak_ordering_base<weak_ordering> { - explicit constexpr weak_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr weak_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::weak_ordering_base<weak_ordering>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, - compare_internal::ord::greater); - - class strong_ordering - : public compare_internal::strong_ordering_base<strong_ordering> { - explicit constexpr strong_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr strong_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::strong_ordering_base<strong_ordering>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator strong_equality() const noexcept { // NOLINT - return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - constexpr operator weak_ordering() const noexcept { // NOLINT - return value_ == 0 - ? weak_ordering::equivalent - : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, - compare_internal::ord::greater); - -#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL -#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL -#undef PHMAP_COMPARE_INLINE_INIT - - namespace compare_internal { - // We also provide these comparator adapter functions for internal phmap use. - - // Helper functions to do a boolean comparison of two keys given a boolean - // or three-way comparator. - // SFINAE prevents implicit conversions to bool (such as from int). - template <typename BoolType, - phmap::enable_if_t<std::is_same<bool, BoolType>::value, int> = 0> - constexpr bool compare_result_as_less_than(const BoolType r) { return r; } - constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { - return r < 0; - } - - template <typename Compare, typename K, typename LK> - constexpr bool do_less_than_comparison(const Compare &compare, const K &x, - const LK &y) { - return compare_result_as_less_than(compare(x, y)); - } - - // Helper functions to do a three-way comparison of two keys given a boolean or - // three-way comparator. - // SFINAE prevents implicit conversions to int (such as from bool). - template <typename Int, - phmap::enable_if_t<std::is_same<int, Int>::value, int> = 0> - constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { - return c < 0 ? phmap::weak_ordering::less - : c == 0 ? phmap::weak_ordering::equivalent - : phmap::weak_ordering::greater; - } - constexpr phmap::weak_ordering compare_result_as_ordering( - const phmap::weak_ordering c) { - return c; - } - - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t<!std::is_same<bool, phmap::invoke_result< - Compare, const K &, const LK &>>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare_result_as_ordering(compare(x, y)); - } - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t<std::is_same<bool, phmap::invoke_result<Compare, - const K &, const LK &>>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare(x, y) ? phmap::weak_ordering::less - : compare(y, x) ? phmap::weak_ordering::greater - : phmap::weak_ordering::equivalent; - } - - } // namespace compare_internal -} - - -namespace phmap { - -namespace priv { - - // A helper class that indicates if the Compare parameter is a key-compare-to - // comparator. - template <typename Compare, typename T> - using btree_is_key_compare_to = - std::is_convertible<phmap::invoke_result<Compare, const T &, const T &>, - phmap::weak_ordering>; - - struct StringBtreeDefaultLess { - using is_transparent = void; - - StringBtreeDefaultLess() = default; - - // Compatibility constructor. - StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultLess(std::less<std::string_view>) {} // NOLINT - StringBtreeDefaultLess(phmap::Less<std::string_view>) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#endif - }; - - struct StringBtreeDefaultGreater { - using is_transparent = void; - - StringBtreeDefaultGreater() = default; - - StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultGreater(std::greater<std::string_view>) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#endif - }; - - // A helper class to convert a boolean comparison into a three-way "compare-to" - // comparison that returns a negative value to indicate less-than, zero to - // indicate equality and a positive value to indicate greater-than. This helper - // class is specialized for less<std::string>, greater<std::string>, - // less<std::string_view>, and greater<std::string_view>. - // - // key_compare_to_adapter is provided so that btree users - // automatically get the more efficient compare-to code when using common - // google string types with common comparison functors. - // These string-like specializations also turn on heterogeneous lookup by - // default. - template <typename Compare> - struct key_compare_to_adapter { - using type = Compare; - }; - - template <> - struct key_compare_to_adapter<std::less<std::string>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<phmap::Less<std::string>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<std::greater<std::string>> { - using type = StringBtreeDefaultGreater; - }; - -#if PHMAP_HAVE_STD_STRING_VIEW - template <> - struct key_compare_to_adapter<std::less<std::string_view>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<phmap::Less<std::string_view>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<std::greater<std::string_view>> { - using type = StringBtreeDefaultGreater; - }; -#endif - - template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi, typename SlotPolicy> - struct common_params { - // If Compare is a common comparator for a std::string-like type, then we adapt it - // to use heterogeneous lookup and to be a key-compare-to comparator. - using key_compare = typename key_compare_to_adapter<Compare>::type; - // A type which indicates if we have a key-compare-to functor or a plain old - // key-compare functor. - using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; - - using allocator_type = Alloc; - using key_type = Key; - using size_type = std::size_t ; - using difference_type = ptrdiff_t; - - // True if this is a multiset or multimap. - using is_multi_container = std::integral_constant<bool, Multi>; - - using slot_policy = SlotPolicy; - using slot_type = typename slot_policy::slot_type; - using value_type = typename slot_policy::value_type; - using init_type = typename slot_policy::mutable_value_type; - using pointer = value_type *; - using const_pointer = const value_type *; - using reference = value_type &; - using const_reference = const value_type &; - - enum { - kTargetNodeSize = TargetNodeSize, - - // Upper bound for the available space for values. This is largest for leaf - // nodes, which have overhead of at least a pointer + 4 bytes (for storing - // 3 field_types and an enum). - kNodeValueSpace = - TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), - }; - - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. - using node_count_type = - phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > - (std::numeric_limits<uint8_t>::max)()), - uint16_t, uint8_t>; // NOLINT - - // The following methods are necessary for passing this struct as PolicyTraits - // for node_handle and/or are used within btree. - static value_type &element(slot_type *slot) { - return slot_policy::element(slot); - } - static const value_type &element(const slot_type *slot) { - return slot_policy::element(slot); - } - template <class... Args> - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - slot_policy::construct(alloc, slot, std::forward<Args>(args)...); - } - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - slot_policy::construct(alloc, slot, other); - } - static void destroy(Alloc *alloc, slot_type *slot) { - slot_policy::destroy(alloc, slot); - } - static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { - construct(alloc, new_slot, old_slot); - destroy(alloc, old_slot); - } - static void swap(Alloc *alloc, slot_type *a, slot_type *b) { - slot_policy::swap(alloc, a, b); - } - static void move(Alloc *alloc, slot_type *src, slot_type *dest) { - slot_policy::move(alloc, src, dest); - } - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - slot_policy::move(alloc, first, last, result); - } - }; - - // A parameters structure for holding the type parameters for a btree_map. - // Compare and Alloc should be nothrow copy-constructible. - template <typename Key, typename Data, typename Compare, typename Alloc, - int TargetNodeSize, bool Multi> - struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - phmap::priv::map_slot_policy<Key, Data>> { - using super_type = typename map_params::common_params; - using mapped_type = Data; - // This type allows us to move keys when it is safe to do so. It is safe - // for maps in which value_type and mutable_value_type are layout compatible. - using slot_policy = typename super_type::slot_policy; - using slot_type = typename super_type::slot_type; - using value_type = typename super_type::value_type; - using init_type = typename super_type::init_type; - - using key_compare = typename super_type::key_compare; - // Inherit from key_compare for empty base class optimization. - struct value_compare : private key_compare { - value_compare() = default; - explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} - - template <typename T, typename U> - auto operator()(const T &left, const U &right) const - -> decltype(std::declval<key_compare>()(left.first, right.first)) { - return key_compare::operator()(left.first, right.first); - } - }; - using is_map_container = std::true_type; - - static const Key &key(const value_type &x) { return x.first; } - static const Key &key(const init_type &x) { return x.first; } - static const Key &key(const slot_type *x) { return slot_policy::key(x); } - static mapped_type &value(value_type *value) { return value->second; } - }; - - // This type implements the necessary functions from the - // btree::priv::slot_type interface. - template <typename Key> - struct set_slot_policy { - using slot_type = Key; - using value_type = Key; - using mutable_value_type = Key; - - static value_type &element(slot_type *slot) { return *slot; } - static const value_type &element(const slot_type *slot) { return *slot; } - - template <typename Alloc, class... Args> - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - phmap::allocator_traits<Alloc>::construct(*alloc, slot, - std::forward<Args>(args)...); - } - - template <typename Alloc> - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - phmap::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); - } - - template <typename Alloc> - static void destroy(Alloc *alloc, slot_type *slot) { - phmap::allocator_traits<Alloc>::destroy(*alloc, slot); - } - - template <typename Alloc> - static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { - using std::swap; - swap(*a, *b); - } - - template <typename Alloc> - static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { - *dest = std::move(*src); - } - - template <typename Alloc> - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } - }; - - // A parameters structure for holding the type parameters for a btree_set. - // Compare and Alloc should be nothrow copy-constructible. - template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi> - struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - set_slot_policy<Key>> { - using value_type = Key; - using slot_type = typename set_params::common_params::slot_type; - using value_compare = typename set_params::common_params::key_compare; - using is_map_container = std::false_type; - - static const Key &key(const value_type &x) { return x; } - static const Key &key(const slot_type *x) { return *x; } - }; - - // An adapter class that converts a lower-bound compare into an upper-bound - // compare. Note: there is no need to make a version of this adapter specialized - // for key-compare-to functors because the upper-bound (the first value greater - // than the input) is never an exact match. - template <typename Compare> - struct upper_bound_adapter { - explicit upper_bound_adapter(const Compare &c) : comp(c) {} - template <typename K, typename LK> - bool operator()(const K &a, const LK &b) const { - // Returns true when a is not greater than b. - return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); - } - - private: - Compare comp; - }; - - enum class MatchKind : uint8_t { kEq, kNe }; - - template <typename V, bool IsCompareTo> - struct SearchResult { - V value; - MatchKind match; - - static constexpr bool HasMatch() { return true; } - bool IsEq() const { return match == MatchKind::kEq; } - }; - - // When we don't use CompareTo, `match` is not present. - // This ensures that callers can't use it accidentally when it provides no - // useful information. - template <typename V> - struct SearchResult<V, false> { - V value; - - static constexpr bool HasMatch() { return false; } - static constexpr bool IsEq() { return false; } - }; - - // A node in the btree holding. The same node type is used for both internal - // and leaf nodes in the btree, though the nodes are allocated in such a way - // that the children array is only valid in internal nodes. - template <typename Params> - class btree_node { - using is_key_compare_to = typename Params::is_key_compare_to; - using is_multi_container = typename Params::is_multi_container; - using field_type = typename Params::node_count_type; - using allocator_type = typename Params::allocator_type; - using slot_type = typename Params::slot_type; - - public: - using params_type = Params; - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using key_compare = typename Params::key_compare; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - - // Btree decides whether to use linear node search as follows: - // - If the key is arithmetic and the comparator is std::less or - // std::greater, choose linear. - // - Otherwise, choose binary. - // TODO(ezb): Might make sense to add condition(s) based on node-size. - using use_linear_search = std::integral_constant< - bool, - std::is_arithmetic<key_type>::value && - (std::is_same<phmap::Less<key_type>, key_compare>::value || - std::is_same<std::less<key_type>, key_compare>::value || - std::is_same<std::greater<key_type>, key_compare>::value)>; - - - ~btree_node() = default; - btree_node(btree_node const &) = delete; - btree_node &operator=(btree_node const &) = delete; - - // Public for EmptyNodeType. - constexpr static size_type Alignment() { - static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), - "Alignment of all nodes must be equal."); - return (size_type)InternalLayout().Alignment(); - } - - protected: - btree_node() = default; - - private: - using layout_type = phmap::priv::Layout<btree_node *, field_type, - slot_type, btree_node *>; - constexpr static size_type SizeWithNValues(size_type n) { - return (size_type)layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)n, - /*children*/ 0) - .AllocSize(); - } - // A lower bound for the overhead of fields other than values in a leaf node. - constexpr static size_type MinimumOverhead() { - return (size_type)(SizeWithNValues(1) - sizeof(value_type)); - } - - // Compute how many values we can fit onto a leaf node taking into account - // padding. - constexpr static size_type NodeTargetValues(const int begin, const int end) { - return begin == end ? begin - : SizeWithNValues((begin + end) / 2 + 1) > - params_type::kTargetNodeSize - ? NodeTargetValues(begin, (begin + end) / 2) - : NodeTargetValues((begin + end) / 2 + 1, end); - } - - enum { - kTargetNodeSize = params_type::kTargetNodeSize, - kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), - - // We need a minimum of 3 values per internal node in order to perform - // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, - - // The node is internal (i.e. is not a leaf node) if and only if `max_count` - // has this value. - kInternalNodeMaxCount = 0, - }; - - // Leaves can have less than kNodeValues values. - constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)max_values, - /*children*/ 0); - } - constexpr static layout_type InternalLayout() { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ kNodeValues, - /*children*/ kNodeValues + 1); - } - constexpr static size_type LeafSize(const int max_values = kNodeValues) { - return (size_type)LeafLayout(max_values).AllocSize(); - } - constexpr static size_type InternalSize() { - return (size_type)InternalLayout().AllocSize(); - } - - // N is the index of the type in the Layout definition. - // ElementType<N> is the Nth type in the Layout definition. - template <size_type N> - inline typename layout_type::template ElementType<N> *GetField() { - // We assert that we don't read from values that aren't there. - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this)); - } - - template <size_type N> - inline const typename layout_type::template ElementType<N> *GetField() const { - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer<N>( - reinterpret_cast<const char *>(this)); - } - - void set_parent(btree_node *p) { *GetField<0>() = p; } - field_type &mutable_count() { return GetField<1>()[2]; } - slot_type *slot(size_type i) { return &GetField<2>()[i]; } - const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } - void set_position(field_type v) { GetField<1>()[0] = v; } - void set_start(field_type v) { GetField<1>()[1] = v; } - void set_count(field_type v) { GetField<1>()[2] = v; } - void set_max_count(field_type v) { GetField<1>()[3] = v; } - - public: - // Whether this is a leaf node or not. This value doesn't change after the - // node is created. - bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } - - // Getter for the position of this node in its parent. - field_type position() const { return GetField<1>()[0]; } - - // Getter for the offset of the first value in the `values` array. - field_type start() const { return GetField<1>()[1]; } - - // Getters for the number of values stored in this node. - field_type count() const { return GetField<1>()[2]; } - field_type max_count() const { - // Internal nodes have max_count==kInternalNodeMaxCount. - // Leaf nodes have max_count in [1, kNodeValues]. - const field_type max_cnt = GetField<1>()[3]; - return max_cnt == field_type{kInternalNodeMaxCount} - ? field_type{kNodeValues} - : max_cnt; - } - - // Getter for the parent of this node. - btree_node *parent() const { return *GetField<0>(); } - // Getter for whether the node is the root of the tree. The parent of the - // root of the tree is the leftmost node in the tree which is guaranteed to - // be a leaf. - bool is_root() const { return parent()->leaf(); } - void make_root() { - assert(parent()->is_root()); - set_parent(parent()->parent()); - } - - // Getters for the key/value at position i in the node. - const key_type &key(size_type i) const { return params_type::key(slot(i)); } - reference value(size_type i) { return params_type::element(slot(i)); } - const_reference value(size_type i) const { return params_type::element(slot(i)); } - - // Getters/setter for the child at position i in the node. - btree_node *child(size_type i) const { return GetField<3>()[i]; } - btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } - void clear_child(size_type i) { - phmap::priv::SanitizerPoisonObject(&mutable_child(i)); - } - void set_child(size_type i, btree_node *c) { - phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); - mutable_child(i) = c; - c->set_position((field_type)i); - } - void init_child(int i, btree_node *c) { - set_child(i, c); - c->set_parent(this); - } - - // Returns the position of the first value whose key is not less than k. - template <typename K> - SearchResult<int, is_key_compare_to::value> lower_bound( - const K &k, const key_compare &comp) const { - return use_linear_search::value ? linear_search(k, comp) - : binary_search(k, comp); - } - // Returns the position of the first value whose key is greater than k. - template <typename K> - int upper_bound(const K &k, const key_compare &comp) const { - auto upper_compare = upper_bound_adapter<key_compare>(comp); - return use_linear_search::value ? linear_search(k, upper_compare).value - : binary_search(k, upper_compare).value; - } - - template <typename K, typename Compare> - SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> - linear_search(const K &k, const Compare &comp) const { - return linear_search_impl(k, 0, count(), comp, - btree_is_key_compare_to<Compare, key_type>()); - } - - template <typename K, typename Compare> - SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> - binary_search(const K &k, const Compare &comp) const { - return binary_search_impl(k, 0, count(), comp, - btree_is_key_compare_to<Compare, key_type>()); - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using plain compare. - template <typename K, typename Compare> - SearchResult<int, false> linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s < e) { - if (!comp(key(s), k)) { - break; - } - ++s; - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using compare-to. - template <typename K, typename Compare> - SearchResult<int, true> linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::true_type /* IsCompareTo */) const { - while (s < e) { - const phmap::weak_ordering c = comp(key(s), k); - if (c == 0) { - return {s, MatchKind::kEq}; - } else if (c > 0) { - break; - } - ++s; - } - return {s, MatchKind::kNe}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using plain compare. - template <typename K, typename Compare> - SearchResult<int, false> binary_search_impl( - const K &k, int s, int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s != e) { - const int mid = (s + e) >> 1; - if (comp(key(mid), k)) { - s = mid + 1; - } else { - e = mid; - } - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using compare-to. - template <typename K, typename CompareTo> - SearchResult<int, true> binary_search_impl( - const K &k, int s, int e, const CompareTo &comp, - std::true_type /* IsCompareTo */) const { - if (is_multi_container::value) { - MatchKind exact_match = MatchKind::kNe; - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else { - e = mid; - if (c == 0) { - // Need to return the first value whose key is not less than k, - // which requires continuing the binary search if this is a - // multi-container. - exact_match = MatchKind::kEq; - } - } - } - return {s, exact_match}; - } else { // Not a multi-container. - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else if (c > 0) { - e = mid; - } else { - return {mid, MatchKind::kEq}; - } - } - return {s, MatchKind::kNe}; - } - } - - // Emplaces a value at position i, shifting all existing values and - // children at positions >= i to the right by 1. - template <typename... Args> - void emplace_value(size_type i, allocator_type *alloc, Args &&... args); - - // Removes the value at position i, shifting all existing values and children - // at positions > i to the left by 1. - void remove_value(int i, allocator_type *alloc); - - // Removes the values at positions [i, i + to_erase), shifting all values - // after that range to the left by to_erase. Does not change children at all. - void remove_values_ignore_children(int i, size_type to_erase, - allocator_type *alloc); - - // Rebalances a node with its right sibling. - void rebalance_right_to_left(int to_move, btree_node *right, - allocator_type *alloc); - void rebalance_left_to_right(int to_move, btree_node *right, - allocator_type *alloc); - - // Splits a node, moving a portion of the node's values to its right sibling. - void split(int insert_position, btree_node *dest, allocator_type *alloc); - - // Merges a node with its right sibling, moving all of the values and the - // delimiting key in the parent node onto itself. - void merge(btree_node *sibling, allocator_type *alloc); - - // Swap the contents of "this" and "src". - void swap(btree_node *src, allocator_type *alloc); - - // Node allocation/deletion routines. - static btree_node *init_leaf(btree_node *n, btree_node *parent, - int max_cnt) { - n->set_parent(parent); - n->set_position(0); - n->set_start(0); - n->set_count(0); - n->set_max_count((field_type)max_cnt); - phmap::priv::SanitizerPoisonMemoryRegion( - n->slot(0), max_cnt * sizeof(slot_type)); - return n; - } - static btree_node *init_internal(btree_node *n, btree_node *parent) { - init_leaf(n, parent, kNodeValues); - // Set `max_count` to a sentinel value to indicate that this node is - // internal. - n->set_max_count(kInternalNodeMaxCount); - phmap::priv::SanitizerPoisonMemoryRegion( - &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); - return n; - } - void destroy(allocator_type *alloc) { - for (int i = 0; i < count(); ++i) { - value_destroy(i, alloc); - } - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return use_linear_search::value; - } - - private: - template <typename... Args> - void value_init(const size_type i, allocator_type *alloc, Args &&... args) { - phmap::priv::SanitizerUnpoisonObject(slot(i)); - params_type::construct(alloc, slot(i), std::forward<Args>(args)...); - } - void value_destroy(const size_type i, allocator_type *alloc) { - params_type::destroy(alloc, slot(i)); - phmap::priv::SanitizerPoisonObject(slot(i)); - } - - // Move n values starting at value i in this node into the values starting at - // value j in node x. - void uninitialized_move_n(const size_type n, const size_type i, - const size_type j, btree_node *x, - allocator_type *alloc) { - phmap::priv::SanitizerUnpoisonMemoryRegion( - x->slot(j), n * sizeof(slot_type)); - for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); - src != end; ++src, ++dest) { - params_type::construct(alloc, dest, src); - } - } - - // Destroys a range of n values, starting at index i. - void value_destroy_n(const size_type i, const size_type n, - allocator_type *alloc) { - for (int j = 0; j < n; ++j) { - value_destroy(i + j, alloc); - } - } - - template <typename P> - friend class btree; - template <typename N, typename R, typename P> - friend struct btree_iterator; - friend class BtreeNodePeer; - }; - - template <typename Node, typename Reference, typename Pointer> - struct btree_iterator { - private: - using key_type = typename Node::key_type; - using size_type = typename Node::size_type; - using params_type = typename Node::params_type; - - using node_type = Node; - using normal_node = typename std::remove_const<Node>::type; - using const_node = const Node; - using normal_pointer = typename params_type::pointer; - using normal_reference = typename params_type::reference; - using const_pointer = typename params_type::const_pointer; - using const_reference = typename params_type::const_reference; - using slot_type = typename params_type::slot_type; - - using iterator = - btree_iterator<normal_node, normal_reference, normal_pointer>; - using const_iterator = - btree_iterator<const_node, const_reference, const_pointer>; - - public: - // These aliases are public for std::iterator_traits. - using difference_type = typename Node::difference_type; - using value_type = typename params_type::value_type; - using pointer = Pointer; - using reference = Reference; - using iterator_category = std::bidirectional_iterator_tag; - - btree_iterator() : node(nullptr), position(-1) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} - - // NOTE: this SFINAE allows for implicit conversions from iterator to - // const_iterator, but it specifically avoids defining copy constructors so - // that btree_iterator can be trivially copyable. This is for performance and - // binary size reasons. - template <typename N, typename R, typename P, - phmap::enable_if_t< - std::is_same<btree_iterator<N, R, P>, iterator>::value && - std::is_same<btree_iterator, const_iterator>::value, - int> = 0> - btree_iterator(const btree_iterator<N, R, P> &x) // NOLINT - : node(x.node), position(x.position) {} - - private: - // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids defining a copy constructor. - // NOTE: the const_cast is safe because this constructor is only called by - // non-const methods and the container owns the nodes. - template <typename N, typename R, typename P, - phmap::enable_if_t< - std::is_same<btree_iterator<N, R, P>, const_iterator>::value && - std::is_same<btree_iterator, iterator>::value, - int> = 0> - explicit btree_iterator(const btree_iterator<N, R, P> &x) - : node(const_cast<node_type *>(x.node)), position(x.position) {} - - // Increment/decrement the iterator. - void increment() { - if (node->leaf() && ++position < node->count()) { - return; - } - increment_slow(); - } - void increment_slow(); - - void decrement() { - if (node->leaf() && --position >= 0) { - return; - } - decrement_slow(); - } - void decrement_slow(); - - public: - bool operator==(const const_iterator &x) const { - return node == x.node && position == x.position; - } - bool operator!=(const const_iterator &x) const { - return node != x.node || position != x.position; - } - - // Accessors for the key/value the iterator is pointing at. - reference operator*() const { - return node->value(position); - } - pointer operator->() const { - return &node->value(position); - } - - btree_iterator& operator++() { - increment(); - return *this; - } - btree_iterator& operator--() { - decrement(); - return *this; - } - btree_iterator operator++(int) { - btree_iterator tmp = *this; - ++*this; - return tmp; - } - btree_iterator operator--(int) { - btree_iterator tmp = *this; - --*this; - return tmp; - } - - private: - template <typename Params> - friend class btree; - template <typename Tree> - friend class btree_container; - template <typename Tree> - friend class btree_set_container; - template <typename Tree> - friend class btree_map_container; - template <typename Tree> - friend class btree_multiset_container; - template <typename N, typename R, typename P> - friend struct btree_iterator; - template <typename TreeType, typename CheckerType> - friend class base_checker; - - const key_type &key() const { return node->key(position); } - slot_type *slot() { return node->slot(position); } - - // The node in the tree the iterator is pointing at. - Node *node; - // The position within the node of the tree the iterator is pointing at. - // TODO(ezb): make this a field_type - int position; - }; - - template <typename Params> - class btree { - using node_type = btree_node<Params>; - using is_key_compare_to = typename Params::is_key_compare_to; - - // We use a static empty node for the root/leftmost/rightmost of empty btrees - // in order to avoid branching in begin()/end(). - struct alignas(node_type::Alignment()) EmptyNodeType : node_type { - using field_type = typename node_type::field_type; - node_type *parent; - field_type position = 0; - field_type start = 0; - field_type count = 0; - // max_count must be != kInternalNodeMaxCount (so that this node is regarded - // as a leaf node). max_count() is never called when the tree is empty. - field_type max_count = node_type::kInternalNodeMaxCount + 1; - -#ifdef _MSC_VER - // MSVC has constexpr code generations bugs here. - EmptyNodeType() : parent(this) {} -#else - constexpr EmptyNodeType(node_type *p) : parent(p) {} -#endif - }; - - static node_type *EmptyNode() { -#ifdef _MSC_VER - static EmptyNodeType empty_node; - // This assert fails on some other construction methods. - assert(empty_node.parent == &empty_node); - return &empty_node; -#else - static constexpr EmptyNodeType empty_node( - const_cast<EmptyNodeType *>(&empty_node)); - return const_cast<EmptyNodeType *>(&empty_node); -#endif - } - - enum { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, - }; - - struct node_stats { - using size_type = typename Params::size_type; - - node_stats(size_type l, size_type i) - : leaf_nodes(l), - internal_nodes(i) { - } - - node_stats& operator+=(const node_stats &x) { - leaf_nodes += x.leaf_nodes; - internal_nodes += x.internal_nodes; - return *this; - } - - size_type leaf_nodes; - size_type internal_nodes; - }; - - public: - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - using key_compare = typename Params::key_compare; - using value_compare = typename Params::value_compare; - using allocator_type = typename Params::allocator_type; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using iterator = btree_iterator<node_type, reference, pointer>; - using const_iterator = typename iterator::const_iterator; - using reverse_iterator = std::reverse_iterator<iterator>; - using const_reverse_iterator = std::reverse_iterator<const_iterator>; - using node_handle_type = node_handle<Params, Params, allocator_type>; - - // Internal types made public for use by btree_container types. - using params_type = Params; - using slot_type = typename Params::slot_type; - - private: - // For use in copy_or_move_values_in_order. - const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } - value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } - - // Copies or moves (depending on the template parameter) the values in - // x into this btree in their order in x. This btree must be empty before this - // method is called. This method is used in copy construction, copy - // assignment, and move assignment. - template <typename Btree> - void copy_or_move_values_in_order(Btree *x); - - // Validates that various assumptions/requirements are true at compile time. - constexpr static bool static_assert_validation(); - - public: - btree(const key_compare &comp, const allocator_type &alloc); - - btree(const btree &x); - btree(btree &&x) noexcept - : root_(std::move(x.root_)), - rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), - size_(phmap::exchange(x.size_, 0)) { - x.mutable_root() = EmptyNode(); - } - - ~btree() { - // Put static_asserts in destructor to avoid triggering them before the type - // is complete. - static_assert(static_assert_validation(), "This call must be elided."); - clear(); - } - - // Assign the contents of x to *this. - btree &operator=(const btree &x); - btree &operator=(btree &&x) noexcept; - - iterator begin() { - return iterator(leftmost(), 0); - } - const_iterator begin() const { - return const_iterator(leftmost(), 0); - } - iterator end() { return iterator(rightmost_, rightmost_->count()); } - const_iterator end() const { - return const_iterator(rightmost_, rightmost_->count()); - } - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - reverse_iterator rend() { - return reverse_iterator(begin()); - } - const_reverse_iterator rend() const { - return const_reverse_iterator(begin()); - } - - // Finds the first element whose key is not less than key. - template <typename K> - iterator lower_bound(const K &key) { - return internal_end(internal_lower_bound(key)); - } - template <typename K> - const_iterator lower_bound(const K &key) const { - return internal_end(internal_lower_bound(key)); - } - - // Finds the first element whose key is greater than key. - template <typename K> - iterator upper_bound(const K &key) { - return internal_end(internal_upper_bound(key)); - } - template <typename K> - const_iterator upper_bound(const K &key) const { - return internal_end(internal_upper_bound(key)); - } - - // Finds the range of values which compare equal to key. The first member of - // the returned pair is equal to lower_bound(key). The second member pair of - // the pair is equal to upper_bound(key). - template <typename K> - std::pair<iterator, iterator> equal_range(const K &key) { - return {lower_bound(key), upper_bound(key)}; - } - template <typename K> - std::pair<const_iterator, const_iterator> equal_range(const K &key) const { - return {lower_bound(key), upper_bound(key)}; - } - - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template <typename... Args> - std::pair<iterator, bool> insert_unique(const key_type &key, Args &&... args); - - // Inserts with hint. Checks to see if the value should be placed immediately - // before `position` in the tree. If so, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_unique() were made. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template <typename... Args> - std::pair<iterator, bool> insert_hint_unique(iterator position, - const key_type &key, - Args &&... args); - - // Insert a range of values into the btree. - template <typename InputIterator> - void insert_iterator_unique(InputIterator b, InputIterator e); - - // Inserts a value into the btree. - template <typename ValueType> - iterator insert_multi(const key_type &key, ValueType &&v); - - // Inserts a value into the btree. - template <typename ValueType> - iterator insert_multi(ValueType &&v) { - return insert_multi(params_type::key(v), std::forward<ValueType>(v)); - } - - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_multi(v) were made. - template <typename ValueType> - iterator insert_hint_multi(iterator position, ValueType &&v); - - // Insert a range of values into the btree. - template <typename InputIterator> - void insert_iterator_multi(InputIterator b, InputIterator e); - - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - // Requirement: does not read the value at `*iter`. - iterator erase(iterator iter); - - // Erases range. Returns the number of keys erased and an iterator pointing - // to the element after the last erased element. - std::pair<size_type, iterator> erase(iterator begin, iterator end); - - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - template <typename K> - size_type erase_unique(const K &key); - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - template <typename K> - size_type erase_multi(const K &key); - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. - template <typename K> - iterator find(const K &key) { - return internal_end(internal_find(key)); - } - template <typename K> - const_iterator find(const K &key) const { - return internal_end(internal_find(key)); - } - - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_unique(const K &key) const { - const iterator beg = internal_find(key); - if (beg.node == nullptr) { - // The key doesn't exist in the tree. - return 0; - } - return 1; - } - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_multi(const K &key) const { - const auto range = equal_range(key); - return std::distance(range.first, range.second); - } - - // Clear the btree, deleting all of the values it contains. - void clear(); - - // Swap the contents of *this and x. - void swap(btree &x); - - const key_compare &key_comp() const noexcept { - return root_.template get<0>(); - } - template <typename K, typename LK> - bool compare_keys(const K &x, const LK &y) const { - return compare_internal::compare_result_as_less_than(key_comp()(x, y)); - } - - value_compare value_comp() const { return value_compare(key_comp()); } - - // Verifies the structure of the btree. - void verify() const; - - // Size routines. - size_type size() const { return size_; } - size_type max_size() const { return (std::numeric_limits<size_type>::max)(); } - bool empty() const { return size_ == 0; } - - // The height of the btree. An empty tree will have height 0. - size_type height() const { - size_type h = 0; - if (!empty()) { - // Count the length of the chain from the leftmost node up to the - // root. We actually count from the root back around to the level below - // the root, but the calculation is the same because of the circularity - // of that traversal. - const node_type *n = root(); - do { - ++h; - n = n->parent(); - } while (n != root()); - } - return h; - } - - // The number of internal, leaf and total nodes used by the btree. - size_type leaf_nodes() const { - return internal_stats(root()).leaf_nodes; - } - size_type internal_nodes() const { - return internal_stats(root()).internal_nodes; - } - size_type nodes() const { - node_stats stats = internal_stats(root()); - return stats.leaf_nodes + stats.internal_nodes; - } - - // The total number of bytes used by the btree. - size_type bytes_used() const { - node_stats stats = internal_stats(root()); - if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { - return sizeof(*this) + - node_type::LeafSize(root()->max_count()); - } else { - return sizeof(*this) + - stats.leaf_nodes * node_type::LeafSize() + - stats.internal_nodes * node_type::InternalSize(); - } - } - - // The average number of bytes used per value stored in the btree. - static double average_bytes_per_value() { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return node_type::LeafSize() / (kNodeValues * 0.75); - } - - // The fullness of the btree. Computed as the number of elements in the btree - // divided by the maximum number of elements a tree with the current number - // of nodes could hold. A value of 1 indicates perfect space - // utilization. Smaller values indicate space wastage. - // Returns 0 for empty trees. - double fullness() const { - if (empty()) return 0.0; - return static_cast<double>(size()) / (nodes() * kNodeValues); - } - // The overhead of the btree structure in bytes per node. Computed as the - // total number of bytes used by the btree minus the number of bytes used for - // storing elements divided by the number of elements. - // Returns 0 for empty trees. - double overhead() const { - if (empty()) return 0.0; - return (bytes_used() - size() * sizeof(value_type)) / - static_cast<double>(size()); - } - - // The allocator used by the btree. - allocator_type get_allocator() const { - return allocator(); - } - - private: - // Internal accessor routines. - node_type *root() { return root_.template get<2>(); } - const node_type *root() const { return root_.template get<2>(); } - node_type *&mutable_root() noexcept { return root_.template get<2>(); } - key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } - - // The leftmost node is stored as the parent of the root node. - node_type *leftmost() { return root()->parent(); } - const node_type *leftmost() const { return root()->parent(); } - - // Allocator routines. - allocator_type *mutable_allocator() noexcept { - return &root_.template get<1>(); - } - const allocator_type &allocator() const noexcept { - return root_.template get<1>(); - } - - // Allocates a correctly aligned node of at least size bytes using the - // allocator. - node_type *allocate(const size_type sz) { - return reinterpret_cast<node_type *>( - phmap::priv::Allocate<node_type::Alignment()>( - mutable_allocator(), (size_t)sz)); - } - - // Node creation/deletion routines. - node_type* new_internal_node(node_type *parent) { - node_type *p = allocate(node_type::InternalSize()); - return node_type::init_internal(p, parent); - } - node_type* new_leaf_node(node_type *parent) { - node_type *p = allocate(node_type::LeafSize()); - return node_type::init_leaf(p, parent, kNodeValues); - } - node_type *new_leaf_root_node(const int max_count) { - node_type *p = allocate(node_type::LeafSize(max_count)); - return node_type::init_leaf(p, p, max_count); - } - - // Deletion helper routines. - void erase_same_node(iterator begin, iterator end); - iterator erase_from_leaf_node(iterator begin, size_type to_erase); - iterator rebalance_after_delete(iterator iter); - - // Deallocates a node of a certain size in bytes using the allocator. - void deallocate(const size_type sz, node_type *node) { - phmap::priv::Deallocate<node_type::Alignment()>( - mutable_allocator(), node, (size_t)sz); - } - - void delete_internal_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::InternalSize(), node); - } - void delete_leaf_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::LeafSize(node->max_count()), node); - } - - // Rebalances or splits the node iter points to. - void rebalance_or_split(iterator *iter); - - // Merges the values of left, right and the delimiting key on their parent - // onto left, removing the delimiting key and deleting right. - void merge_nodes(node_type *left, node_type *right); - - // Tries to merge node with its left or right sibling, and failing that, - // rebalance with its left or right sibling. Returns true if a merge - // occurred, at which point it is no longer valid to access node. Returns - // false if no merging took place. - bool try_merge_or_rebalance(iterator *iter); - - // Tries to shrink the height of the tree by 1. - void try_shrink(); - - iterator internal_end(iterator iter) { - return iter.node != nullptr ? iter : end(); - } - const_iterator internal_end(const_iterator iter) const { - return iter.node != nullptr ? iter : end(); - } - - // Emplaces a value into the btree immediately before iter. Requires that - // key(v) <= iter.key() and (--iter).key() <= key(v). - template <typename... Args> - iterator internal_emplace(iterator iter, Args &&... args); - - // Returns an iterator pointing to the first value >= the value "iter" is - // pointing at. Note that "iter" might be pointing to an invalid location as - // iter.position == iter.node->count(). This routine simply moves iter up in - // the tree to a valid location. - // Requires: iter.node is non-null. - template <typename IterType> - static IterType internal_last(IterType iter); - - // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version uses a less-than comparator and is incapable of distinguishing when - // there is an exact match. The second version is for the key-compare-to - // specialization and distinguishes exact matches. The key-compare-to - // specialization allows the caller to avoid a subsequent comparison to - // determine if an exact match was made, which is important for keys with - // expensive comparison, such as strings. - template <typename K> - SearchResult<iterator, is_key_compare_to::value> internal_locate( - const K &key) const; - - template <typename K> - SearchResult<iterator, false> internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const; - - template <typename K> - SearchResult<iterator, true> internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const; - - // Internal routine which implements lower_bound(). - template <typename K> - iterator internal_lower_bound(const K &key) const; - - // Internal routine which implements upper_bound(). - template <typename K> - iterator internal_upper_bound(const K &key) const; - - // Internal routine which implements find(). - template <typename K> - iterator internal_find(const K &key) const; - - // Deletes a node and all of its children. - void internal_clear(node_type *node); - - // Verifies the tree structure of node. - int internal_verify(const node_type *node, - const key_type *lo, const key_type *hi) const; - - node_stats internal_stats(const node_type *node) const { - // The root can be a static empty node. - if (node == nullptr || (node == root() && empty())) { - return node_stats(0, 0); - } - if (node->leaf()) { - return node_stats(1, 0); - } - node_stats res(0, 1); - for (int i = 0; i <= node->count(); ++i) { - res += internal_stats(node->child(i)); - } - return res; - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return node_type::testonly_uses_linear_node_search(); - } - - private: - // We use compressed tuple in order to save space because key_compare and - // allocator_type are usually empty. - phmap::priv::CompressedTuple<key_compare, allocator_type, - node_type *> - root_; - - // A pointer to the rightmost node. Note that the leftmost node is stored as - // the root's parent. - node_type *rightmost_; - - // Number of values. - size_type size_; - }; - - //// - // btree_node methods - template <typename P> - template <typename... Args> - inline void btree_node<P>::emplace_value(const size_type i, - allocator_type *alloc, - Args &&... args) { - assert(i <= count()); - // Shift old values to create space for new value and then construct it in - // place. - if (i < count()) { - value_init(count(), alloc, slot(count() - 1)); - for (size_type j = count() - 1; j > i; --j) - params_type::move(alloc, slot(j - 1), slot(j)); - value_destroy(i, alloc); - } - value_init(i, alloc, std::forward<Args>(args)...); - set_count((field_type)(count() + 1)); - - if (!leaf() && count() > i + 1) { - for (int j = count(); j > i + 1; --j) { - set_child(j, child(j - 1)); - } - clear_child(i + 1); - } - } - - template <typename P> - inline void btree_node<P>::remove_value(const int i, allocator_type *alloc) { - if (!leaf() && count() > i + 1) { - assert(child(i + 1)->count() == 0); - for (size_type j = i + 1; j < count(); ++j) { - set_child(j, child(j + 1)); - } - clear_child(count()); - } - - remove_values_ignore_children(i, /*to_erase=*/1, alloc); - } - - template <typename P> - inline void btree_node<P>::remove_values_ignore_children( - int i, size_type to_erase, allocator_type *alloc) { - params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); - value_destroy_n(count() - to_erase, to_erase, alloc); - set_count((field_type)(count() - to_erase)); - } - - template <typename P> - void btree_node<P>::rebalance_right_to_left(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(right->count() >= count()); - assert(to_move >= 1); - assert(to_move <= right->count()); - - // 1) Move the delimiting value in the parent to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // 2) Move the (to_move - 1) values from the right node to the left node. - right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); - - // 3) Move the new delimiting value to the parent from the right node. - params_type::move(alloc, right->slot(to_move - 1), - parent()->slot(position())); - - // 4) Shift the values in the right node to their correct position. - params_type::move(alloc, right->slot(to_move), right->slot(right->count()), - right->slot(0)); - - // 5) Destroy the now-empty to_move entries in the right node. - right->value_destroy_n(right->count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i < to_move; ++i) { - init_child(count() + i + 1, right->child(i)); - } - for (int i = 0; i <= right->count() - to_move; ++i) { - assert(i + to_move <= right->max_count()); - right->init_child(i, right->child(i + to_move)); - right->clear_child(i + to_move); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() + to_move)); - right->set_count((field_type)(right->count() - to_move)); - } - - template <typename P> - void btree_node<P>::rebalance_left_to_right(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(count() >= right->count()); - assert(to_move >= 1); - assert(to_move <= count()); - - // Values in the right node are shifted to the right to make room for the - // new to_move values. Then, the delimiting value in the parent and the - // other (to_move - 1) values in the left node are moved into the right node. - // Lastly, a new delimiting value is moved from the left node into the - // parent, and the remaining empty left node entries are destroyed. - - if (right->count() >= to_move) { - // The original location of the right->count() values are sufficient to hold - // the new to_move entries from the parent and left node. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(to_move, right->count() - to_move, - right->count(), right, alloc); - for (slot_type *src = right->slot(right->count() - to_move - 1), - *dest = right->slot(right->count() - 1), - *end = right->slot(0); - src >= end; --src, --dest) { - params_type::move(alloc, src, dest); - } - - // 2) Move the delimiting value in the parent to the right node. - params_type::move(alloc, parent()->slot(position()), - right->slot(to_move - 1)); - - // 3) Move the (to_move - 1) values from the left node to the right node. - params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), - right->slot(0)); - } else { - // The right node does not have enough initialized space to hold the new - // to_move entries, so part of them will move to uninitialized space. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); - - // 2) Move the delimiting value in the parent to the right node. - right->value_init(to_move - 1, alloc, parent()->slot(position())); - - // 3) Move the (to_move - 1) values from the left node to the right node. - const size_type uninitialized_remaining = to_move - right->count() - 1; - uninitialized_move_n(uninitialized_remaining, - count() - uninitialized_remaining, right->count(), - right, alloc); - params_type::move(alloc, slot(count() - (to_move - 1)), - slot(count() - uninitialized_remaining), right->slot(0)); - } - - // 4) Move the new delimiting value to the parent from the left node. - params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); - - // 5) Destroy the now-empty to_move entries in the left node. - value_destroy_n(count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the left to the right node. - for (int i = right->count(); i >= 0; --i) { - right->init_child(i + to_move, right->child(i)); - right->clear_child(i); - } - for (int i = 1; i <= to_move; ++i) { - right->init_child(i - 1, child(count() - to_move + i)); - clear_child(count() - to_move + i); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() - to_move)); - right->set_count((field_type)(right->count() + to_move)); - } - - template <typename P> - void btree_node<P>::split(const int insert_position, btree_node *dest, - allocator_type *alloc) { - assert(dest->count() == 0); - assert(max_count() == kNodeValues); - - // We bias the split based on the position being inserted. If we're - // inserting at the beginning of the left node then bias the split to put - // more values on the right node. If we're inserting at the end of the - // right node then bias the split to put more values on the left node. - if (insert_position == 0) { - dest->set_count((field_type)(count() - 1)); - } else if (insert_position == kNodeValues) { - dest->set_count(0); - } else { - dest->set_count((field_type)(count() / 2)); - } - set_count((field_type)(count() - dest->count())); - assert(count() >= 1); - - // Move values from the left sibling to the right sibling. - uninitialized_move_n(dest->count(), count(), 0, dest, alloc); - - // Destroy the now-empty entries in the left node. - value_destroy_n(count(), dest->count(), alloc); - - // The split key is the largest value in the left sibling. - set_count((field_type)(count() - 1)); - parent()->emplace_value(position(), alloc, slot(count())); - value_destroy(count(), alloc); - parent()->init_child(position() + 1, dest); - - if (!leaf()) { - for (int i = 0; i <= dest->count(); ++i) { - assert(child(count() + i + 1) != nullptr); - dest->init_child(i, child(count() + i + 1)); - clear_child(count() + i + 1); - } - } - } - - template <typename P> - void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); - - // Move the delimiting value to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // Move the values from the right to the left node. - src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); - - // Destroy the now-empty entries in the right node. - src->value_destroy_n(0, src->count(), alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i <= src->count(); ++i) { - init_child(count() + i + 1, src->child(i)); - src->clear_child(i); - } - } - - // Fixup the counts on the src and dest nodes. - set_count((field_type)(1 + count() + src->count())); - src->set_count(0); - - // Remove the value on the parent node. - parent()->remove_value(position(), alloc); - } - - template <typename P> - void btree_node<P>::swap(btree_node *x, allocator_type *alloc) { - using std::swap; - assert(leaf() == x->leaf()); - - // Determine which is the smaller/larger node. - btree_node *smaller = this, *larger = x; - if (smaller->count() > larger->count()) { - swap(smaller, larger); - } - - // Swap the values. - for (slot_type *a = smaller->slot(0), *b = larger->slot(0), - *end = a + smaller->count(); - a != end; ++a, ++b) { - params_type::swap(alloc, a, b); - } - - // Move values that can't be swapped. - const size_type to_move = larger->count() - smaller->count(); - larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), - smaller, alloc); - larger->value_destroy_n(smaller->count(), to_move, alloc); - - if (!leaf()) { - // Swap the child pointers. - std::swap_ranges(&smaller->mutable_child(0), - &smaller->mutable_child(smaller->count() + 1), - &larger->mutable_child(0)); - // Update swapped children's parent pointers. - int i = 0; - for (; i <= smaller->count(); ++i) { - smaller->child(i)->set_parent(smaller); - larger->child(i)->set_parent(larger); - } - // Move the child pointers that couldn't be swapped. - for (; i <= larger->count(); ++i) { - smaller->init_child(i, larger->child(i)); - larger->clear_child(i); - } - } - - // Swap the counts. - swap(mutable_count(), x->mutable_count()); - } - - //// - // btree_iterator methods - template <typename N, typename R, typename P> - void btree_iterator<N, R, P>::increment_slow() { - if (node->leaf()) { - assert(position >= node->count()); - btree_iterator save(*this); - while (position == node->count() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); - } - if (position == node->count()) { - *this = save; - } - } else { - assert(position < node->count()); - node = node->child(position + 1); - while (!node->leaf()) { - node = node->child(0); - } - position = 0; - } - } - - template <typename N, typename R, typename P> - void btree_iterator<N, R, P>::decrement_slow() { - if (node->leaf()) { - assert(position <= -1); - btree_iterator save(*this); - while (position < 0 && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); - } - if (position < 0) { - *this = save; - } - } else { - assert(position >= 0); - node = node->child(position); - while (!node->leaf()) { - node = node->child(node->count()); - } - position = node->count() - 1; - } - } - - //// - // btree methods - template <typename P> - template <typename Btree> - void btree<P>::copy_or_move_values_in_order(Btree *x) { - static_assert(std::is_same<btree, Btree>::value || - std::is_same<const btree, Btree>::value, - "Btree type must be same or const."); - assert(empty()); - - // We can avoid key comparisons because we know the order of the - // values is the same order we'll store them in. - auto iter = x->begin(); - if (iter == x->end()) return; - insert_multi(maybe_move_from_iterator(iter)); - ++iter; - for (; iter != x->end(); ++iter) { - // If the btree is not empty, we can just insert the new value at the end - // of the tree. - internal_emplace(end(), maybe_move_from_iterator(iter)); - } - } - - template <typename P> - constexpr bool btree<P>::static_assert_validation() { - static_assert(std::is_nothrow_copy_constructible<key_compare>::value, - "Key comparison must be nothrow copy constructible"); - static_assert(std::is_nothrow_copy_constructible<allocator_type>::value, - "Allocator must be nothrow copy constructible"); - static_assert(type_traits_internal::is_trivially_copyable<iterator>::value, - "iterator not trivially copyable."); - - // Note: We assert that kTargetValues, which is computed from - // Params::kTargetNodeSize, must fit the node_type::field_type. - static_assert( - kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), - "target node size too large"); - - // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. - using compare_result_type = - phmap::invoke_result<key_compare, key_type, key_type>; - static_assert( - std::is_same<compare_result_type, bool>::value || - std::is_convertible<compare_result_type, phmap::weak_ordering>::value, - "key comparison function must return phmap::{weak,strong}_ordering or " - "bool."); - - // Test the assumption made in setting kNodeValueSpace. - static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, - "node space assumption incorrect"); - - return true; - } - - template <typename P> - btree<P>::btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - - template <typename P> - btree<P>::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { - copy_or_move_values_in_order(&x); - } - - template <typename P> - template <typename... Args> - auto btree<P>::insert_unique(const key_type &key, Args &&... args) - -> std::pair<iterator, bool> { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - auto res = internal_locate(key); - iterator &iter = res.value; - - if (res.HasMatch()) { - if (res.IsEq()) { - // The key already exists in the tree, do nothing. - return {iter, false}; - } - } else { - iterator last = internal_last(iter); - if (last.node && !compare_keys(key, last.key())) { - // The key already exists in the tree, do nothing. - return {last, false}; - } - } - return {internal_emplace(iter, std::forward<Args>(args)...), true}; - } - - template <typename P> - template <typename... Args> - inline auto btree<P>::insert_hint_unique(iterator position, const key_type &key, - Args &&... args) - -> std::pair<iterator, bool> { - if (!empty()) { - if (position == end() || compare_keys(key, position.key())) { - iterator prev = position; - if (position == begin() || compare_keys((--prev).key(), key)) { - // prev.key() < key < position.key() - return {internal_emplace(position, std::forward<Args>(args)...), true}; - } - } else if (compare_keys(position.key(), key)) { - ++position; - if (position == end() || compare_keys(key, position.key())) { - // {original `position`}.key() < key < {current `position`}.key() - return {internal_emplace(position, std::forward<Args>(args)...), true}; - } - } else { - // position.key() == key - return {position, false}; - } - } - return insert_unique(key, std::forward<Args>(args)...); - } - - template <typename P> - template <typename InputIterator> - void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_unique(end(), params_type::key(*b), *b); - } - } - - template <typename P> - template <typename ValueType> - auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - iterator iter = internal_upper_bound(key); - if (iter.node == nullptr) { - iter = end(); - } - return internal_emplace(iter, std::forward<ValueType>(v)); - } - - template <typename P> - template <typename ValueType> - auto btree<P>::insert_hint_multi(iterator position, ValueType &&v) -> iterator { - if (!empty()) { - const key_type &key = params_type::key(v); - if (position == end() || !compare_keys(position.key(), key)) { - iterator prev = position; - if (position == begin() || !compare_keys(key, (--prev).key())) { - // prev.key() <= key <= position.key() - return internal_emplace(position, std::forward<ValueType>(v)); - } - } else { - iterator next = position; - ++next; - if (next == end() || !compare_keys(next.key(), key)) { - // position.key() < key <= next.key() - return internal_emplace(next, std::forward<ValueType>(v)); - } - } - } - return insert_multi(std::forward<ValueType>(v)); - } - - template <typename P> - template <typename InputIterator> - void btree<P>::insert_iterator_multi(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_multi(end(), *b); - } - } - - template <typename P> - auto btree<P>::operator=(const btree &x) -> btree & { - if (this != &x) { - clear(); - - *mutable_key_comp() = x.key_comp(); - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - *mutable_allocator() = x.allocator(); - } - - copy_or_move_values_in_order(&x); - } - return *this; - } - - template <typename P> - auto btree<P>::operator=(btree &&x) noexcept -> btree & { - if (this != &x) { - clear(); - - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - if (allocator() == x.allocator()) { - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - // We aren't allowed to propagate the allocator and the allocator is - // different so we can't take over its memory. We must move each element - // individually. We need both `x` and `this` to have `x`s key comparator - // while moving the values so we can't swap the key comparators. - *mutable_key_comp() = x.key_comp(); - copy_or_move_values_in_order(&x); - } - } - } - return *this; - } - - template <typename P> - auto btree<P>::erase(iterator iter) -> iterator { - bool internal_delete = false; - if (!iter.node->leaf()) { - // Deletion of a value on an internal node. First, move the largest value - // from our left child here, then delete that position (in remove_value() - // below). We can get to the largest value from our left child by - // decrementing iter. - iterator internal_iter(iter); - --iter; - assert(iter.node->leaf()); - params_type::move(mutable_allocator(), iter.node->slot(iter.position), - internal_iter.node->slot(internal_iter.position)); - internal_delete = true; - } - - // Delete the key from the leaf. - iter.node->remove_value(iter.position, mutable_allocator()); - --size_; - - // We want to return the next value after the one we just erased. If we - // erased from an internal node (internal_delete == true), then the next - // value is ++(++iter). If we erased from a leaf node (internal_delete == - // false) then the next value is ++iter. Note that ++iter may point to an - // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. - - iterator res = rebalance_after_delete(iter); - - // If we erased from an internal node, advance the iterator. - if (internal_delete) { - ++res; - } - return res; - } - - template <typename P> - auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { - // Merge/rebalance as we walk back up the tree. - iterator res(iter); - bool first_iteration = true; - for (;;) { - if (iter.node == root()) { - try_shrink(); - if (empty()) { - return end(); - } - break; - } - if (iter.node->count() >= kMinNodeValues) { - break; - } - bool merged = try_merge_or_rebalance(&iter); - // On the first iteration, we should update `res` with `iter` because `res` - // may have been invalidated. - if (first_iteration) { - res = iter; - first_iteration = false; - } - if (!merged) { - break; - } - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - } - - // Adjust our return value. If we're pointing at the end of a node, advance - // the iterator. - if (res.position == res.node->count()) { - res.position = res.node->count() - 1; - ++res; - } - - return res; - } - - template <typename P> - auto btree<P>::erase(iterator _begin, iterator _end) - -> std::pair<size_type, iterator> { - difference_type count = std::distance(_begin, _end); - assert(count >= 0); - - if (count == 0) { - return {0, _begin}; - } - - if (count == (difference_type)size_) { - clear(); - return {count, this->end()}; - } - - if (_begin.node == _end.node) { - erase_same_node(_begin, _end); - size_ -= count; - return {count, rebalance_after_delete(_begin)}; - } - - const size_type target_size = size_ - count; - while (size_ > target_size) { - if (_begin.node->leaf()) { - const size_type remaining_to_erase = size_ - target_size; - const size_type remaining_in_node = _begin.node->count() - _begin.position; - _begin = erase_from_leaf_node( - _begin, (std::min)(remaining_to_erase, remaining_in_node)); - } else { - _begin = erase(_begin); - } - } - return {count, _begin}; - } - - template <typename P> - void btree<P>::erase_same_node(iterator _begin, iterator _end) { - assert(_begin.node == _end.node); - assert(_end.position > _begin.position); - - node_type *node = _begin.node; - size_type to_erase = _end.position - _begin.position; - if (!node->leaf()) { - // Delete all children between _begin and _end. - for (size_type i = 0; i < to_erase; ++i) { - internal_clear(node->child(_begin.position + i + 1)); - } - // Rotate children after _end into new positions. - for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { - node->set_child(i - to_erase, node->child(i)); - node->clear_child(i); - } - } - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - // Do not need to update rightmost_, because - // * either _end == this->end(), and therefore node == rightmost_, and still - // exists - // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since - // it wasn't covered in [_begin, _end) - } - - template <typename P> - auto btree<P>::erase_from_leaf_node(iterator _begin, size_type to_erase) - -> iterator { - node_type *node = _begin.node; - assert(node->leaf()); - assert(node->count() > _begin.position); - assert(_begin.position + to_erase <= node->count()); - - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - size_ -= to_erase; - - return rebalance_after_delete(_begin); - } - - template <typename P> - template <typename K> - auto btree<P>::erase_unique(const K &key) -> size_type { - const iterator iter = internal_find(key); - if (iter.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - erase(iter); - return 1; - } - - template <typename P> - template <typename K> - auto btree<P>::erase_multi(const K &key) -> size_type { - const iterator _begin = internal_lower_bound(key); - if (_begin.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - // Delete all of the keys between _begin and upper_bound(key). - const iterator _end = internal_end(internal_upper_bound(key)); - return erase(_begin, _end).first; - } - - template <typename P> - void btree<P>::clear() { - if (!empty()) { - internal_clear(root()); - } - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - size_ = 0; - } - - template <typename P> - void btree<P>::swap(btree &x) { - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_swap::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - } else { - // It's undefined behavior if the allocators are unequal here. - assert(allocator() == x.allocator()); - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - } - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } - - template <typename P> - void btree<P>::verify() const { - assert(root() != nullptr); - assert(leftmost() != nullptr); - assert(rightmost_ != nullptr); - assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost_ == (--const_iterator(root(), root()->count())).node); - assert(leftmost()->leaf()); - assert(rightmost_->leaf()); - } - - template <typename P> - void btree<P>::rebalance_or_split(iterator *iter) { - node_type *&node = iter->node; - int &insert_position = iter->position; - assert(node->count() == node->max_count()); - assert(kNodeValues == node->max_count()); - - // First try to make room on the node by rebalancing. - node_type *parent = node->parent(); - if (node != root()) { - if (node->position() > 0) { - // Try rebalancing with our left sibling. - node_type *left = parent->child(node->position() - 1); - assert(left->max_count() == kNodeValues); - if (left->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the end of the right node then we bias rebalancing to - // fill up the left node. - int to_move = (kNodeValues - left->count()) / - (1 + (insert_position < kNodeValues)); - to_move = (std::max)(1, to_move); - - if (((insert_position - to_move) >= 0) || - ((left->count() + to_move) < kNodeValues)) { - left->rebalance_right_to_left(to_move, node, mutable_allocator()); - - assert(node->max_count() - node->count() == to_move); - insert_position = insert_position - to_move; - if (insert_position < 0) { - insert_position = insert_position + left->count() + 1; - node = left; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - if (node->position() < parent->count()) { - // Try rebalancing with our right sibling. - node_type *right = parent->child(node->position() + 1); - assert(right->max_count() == kNodeValues); - if (right->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the _beginning of the left node then we bias rebalancing - // to fill up the right node. - int to_move = - (kNodeValues - right->count()) / (1 + (insert_position > 0)); - to_move = (std::max)(1, to_move); - - if ((insert_position <= (node->count() - to_move)) || - ((right->count() + to_move) < kNodeValues)) { - node->rebalance_left_to_right(to_move, right, mutable_allocator()); - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = right; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - // Rebalancing failed, make sure there is room on the parent node for a new - // value. - assert(parent->max_count() == kNodeValues); - if (parent->count() == kNodeValues) { - iterator parent_iter(node->parent(), node->position()); - rebalance_or_split(&parent_iter); - } - } else { - // Rebalancing not possible because this is the root node. - // Create a new root node and set the current root node as the child of the - // new root. - parent = new_internal_node(parent); - parent->init_child(0, root()); - mutable_root() = parent; - // If the former root was a leaf node, then it's now the rightmost node. - assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); - } - - // Split the node. - node_type *split_node; - if (node->leaf()) { - split_node = new_leaf_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - if (rightmost_ == node) rightmost_ = split_node; - } else { - split_node = new_internal_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - } - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = split_node; - } - } - - template <typename P> - void btree<P>::merge_nodes(node_type *left, node_type *right) { - left->merge(right, mutable_allocator()); - if (right->leaf()) { - if (rightmost_ == right) rightmost_ = left; - delete_leaf_node(right); - } else { - delete_internal_node(right); - } - } - - template <typename P> - bool btree<P>::try_merge_or_rebalance(iterator *iter) { - node_type *parent = iter->node->parent(); - if (iter->node->position() > 0) { - // Try merging with our left sibling. - node_type *left = parent->child(iter->node->position() - 1); - assert(left->max_count() == kNodeValues); - if ((1 + left->count() + iter->node->count()) <= kNodeValues) { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; - return true; - } - } - if (iter->node->position() < parent->count()) { - // Try merging with our right sibling. - node_type *right = parent->child(iter->node->position() + 1); - assert(right->max_count() == kNodeValues); - if ((1 + iter->node->count() + right->count()) <= kNodeValues) { - merge_nodes(iter->node, right); - return true; - } - // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the front of the tree. - if ((right->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position > 0))) { - int to_move = (right->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); - return false; - } - } - if (iter->node->position() > 0) { - // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the back of the tree. - node_type *left = parent->child(iter->node->position() - 1); - if ((left->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position < iter->node->count()))) { - int to_move = (left->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, left->count() - 1); - left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); - iter->position += to_move; - return false; - } - } - return false; - } - - template <typename P> - void btree<P>::try_shrink() { - if (root()->count() > 0) { - return; - } - // Deleted the last item on the root node, shrink the height of the tree. - if (root()->leaf()) { - assert(size() == 0); - delete_leaf_node(root()); - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - } else { - node_type *child = root()->child(0); - child->make_root(); - delete_internal_node(root()); - mutable_root() = child; - } - } - - template <typename P> - template <typename IterType> - inline IterType btree<P>::internal_last(IterType iter) { - assert(iter.node != nullptr); - while (iter.position == iter.node->count()) { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - if (iter.node->leaf()) { - iter.node = nullptr; - break; - } - } - return iter; - } - - template <typename P> - template <typename... Args> - inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) - -> iterator { - if (!iter.node->leaf()) { - // We can't insert on an internal node. Instead, we'll insert after the - // previous value which is guaranteed to be on a leaf node. - --iter; - ++iter.position; - } - const int max_count = iter.node->max_count(); - if (iter.node->count() == max_count) { - // Make room in the leaf for the new item. - if (max_count < kNodeValues) { - // Insertion into the root where the root is smaller than the full node - // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = - new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count)); - iter.node->swap(root(), mutable_allocator()); - delete_leaf_node(root()); - mutable_root() = iter.node; - rightmost_ = iter.node; - } else { - rebalance_or_split(&iter); - } - } - iter.node->emplace_value(iter.position, mutable_allocator(), - std::forward<Args>(args)...); - ++size_; - return iter; - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate(const K &key) const - -> SearchResult<iterator, is_key_compare_to::value> { - return internal_locate_impl(key, is_key_compare_to()); - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const - -> SearchResult<iterator, false> { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - // NOTE: we don't need to walk all the way down the tree if the keys are - // equal, but determining equality would require doing an extra comparison - // on each node on the way down, and we will need to go all the way to the - // leaf node in the expected case. - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter}; - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const - -> SearchResult<iterator, true> { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - SearchResult<int, true> res = iter.node->lower_bound(key, key_comp()); - iter.position = res.value; - if (res.match == MatchKind::kEq) { - return {iter, MatchKind::kEq}; - } - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter, MatchKind::kNe}; - } - - template <typename P> - template <typename K> - auto btree<P>::internal_lower_bound(const K &key) const -> iterator { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template <typename P> - template <typename K> - auto btree<P>::internal_upper_bound(const K &key) const -> iterator { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->upper_bound(key, key_comp()); - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template <typename P> - template <typename K> - auto btree<P>::internal_find(const K &key) const -> iterator { - auto res = internal_locate(key); - if (res.HasMatch()) { - if (res.IsEq()) { - return res.value; - } - } else { - const iterator iter = internal_last(res.value); - if (iter.node != nullptr && !compare_keys(key, iter.key())) { - return iter; - } - } - return {nullptr, 0}; - } - - template <typename P> - void btree<P>::internal_clear(node_type *node) { - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - internal_clear(node->child(i)); - } - delete_internal_node(node); - } else { - delete_leaf_node(node); - } - } - - template <typename P> - int btree<P>::internal_verify( - const node_type *node, const key_type *lo, const key_type *hi) const { - assert(node->count() > 0); - assert(node->count() <= node->max_count()); - if (lo) { - assert(!compare_keys(node->key(0), *lo)); - } - if (hi) { - assert(!compare_keys(*hi, node->key(node->count() - 1))); - } - for (int i = 1; i < node->count(); ++i) { - assert(!compare_keys(node->key(i), node->key(i - 1))); - } - int count = node->count(); - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - assert(node->child(i) != nullptr); - assert(node->child(i)->parent() == node); - assert(node->child(i)->position() == i); - count += internal_verify( - node->child(i), - (i == 0) ? lo : &node->key(i - 1), - (i == node->count()) ? hi : &node->key(i)); - } - } - return count; - } - - // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. - // --------------------------------------------------------------------------------- - template <typename Tree> - class btree_container { - using params_type = typename Tree::params_type; - - protected: - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template <class K> - using key_arg = - typename KeyArg<IsTransparent<typename Tree::key_compare>::value>:: - template type<K, typename Tree::key_type>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using difference_type = typename Tree::difference_type; - using key_compare = typename Tree::key_compare; - using value_compare = typename Tree::value_compare; - using allocator_type = typename Tree::allocator_type; - using reference = typename Tree::reference; - using const_reference = typename Tree::const_reference; - using pointer = typename Tree::pointer; - using const_pointer = typename Tree::const_pointer; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using reverse_iterator = typename Tree::reverse_iterator; - using const_reverse_iterator = typename Tree::const_reverse_iterator; - using node_type = typename Tree::node_handle_type; - - // Constructors/assignments. - btree_container() : tree_(key_compare(), allocator_type()) {} - explicit btree_container(const key_compare &comp, - const allocator_type &alloc = allocator_type()) - : tree_(comp, alloc) {} - btree_container(const btree_container &x) = default; - btree_container(btree_container &&x) noexcept = default; - btree_container &operator=(const btree_container &x) = default; - btree_container &operator=(btree_container &&x) noexcept( - std::is_nothrow_move_assignable<Tree>::value) = default; - - // Iterator routines. - iterator begin() { return tree_.begin(); } - const_iterator begin() const { return tree_.begin(); } - const_iterator cbegin() const { return tree_.begin(); } - iterator end() { return tree_.end(); } - const_iterator end() const { return tree_.end(); } - const_iterator cend() const { return tree_.end(); } - reverse_iterator rbegin() { return tree_.rbegin(); } - const_reverse_iterator rbegin() const { return tree_.rbegin(); } - const_reverse_iterator crbegin() const { return tree_.rbegin(); } - reverse_iterator rend() { return tree_.rend(); } - const_reverse_iterator rend() const { return tree_.rend(); } - const_reverse_iterator crend() const { return tree_.rend(); } - - // Lookup routines. - template <typename K = key_type> - iterator find(const key_arg<K> &key) { - return tree_.find(key); - } - template <typename K = key_type> - const_iterator find(const key_arg<K> &key) const { return tree_.find(key); } - - template <typename K = key_type> - bool contains(const key_arg<K> &key) const { return find(key) != end(); } - - template <typename K = key_type> - iterator lower_bound(const key_arg<K> &key) { return tree_.lower_bound(key); } - - template <typename K = key_type> - const_iterator lower_bound(const key_arg<K> &key) const { return tree_.lower_bound(key); } - - template <typename K = key_type> - iterator upper_bound(const key_arg<K> &key) { return tree_.upper_bound(key); } - - template <typename K = key_type> - const_iterator upper_bound(const key_arg<K> &key) const { return tree_.upper_bound(key); } - - template <typename K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K> &key) { return tree_.equal_range(key); } - - template <typename K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K> &key) const { - return tree_.equal_range(key); - } - - iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } - iterator erase(iterator iter) { return tree_.erase(iter); } - iterator erase(const_iterator first, const_iterator last) { - return tree_.erase(iterator(first), iterator(last)).second; - } - - node_type extract(iterator position) { - // Use Move instead of Transfer, because the rebalancing code expects to - // have a valid object to scribble metadata bits on top of. - auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot()); - erase(position); - return node; - } - - node_type extract(const_iterator position) { - return extract(iterator(position)); - } - - public: - void clear() { tree_.clear(); } - void swap(btree_container &x) { tree_.swap(x.tree_); } - void verify() const { tree_.verify(); } - - size_type size() const { return tree_.size(); } - size_type max_size() const { return tree_.max_size(); } - bool empty() const { return tree_.empty(); } - - friend bool operator==(const btree_container &x, const btree_container &y) { - if (x.size() != y.size()) return false; - return std::equal(x.begin(), x.end(), y.begin()); - } - - friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } - - friend bool operator<(const btree_container &x, const btree_container &y) { - return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); - } - - friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } - - friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } - - friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } - - // The allocator used by the btree. - allocator_type get_allocator() const { return tree_.get_allocator(); } - - // The key comparator used by the btree. - key_compare key_comp() const { return tree_.key_comp(); } - value_compare value_comp() const { return tree_.value_comp(); } - - // Support absl::Hash. - template <typename State> - friend State AbslHashValue(State h, const btree_container &b) { - for (const auto &v : b) { - h = State::combine(std::move(h), v); - } - return State::combine(std::move(h), b.size()); - } - - protected: - Tree tree_; - }; - - // A common base class for btree_set and btree_map. - // ----------------------------------------------- - template <typename Tree> - class btree_set_container : public btree_container<Tree> { - using super_type = btree_container<Tree>; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - friend class BtreeNodePeer; - - protected: - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - using insert_return_type = InsertReturnType<iterator, node_type>; - using super_type::super_type; - btree_set_container() {} - - template <class InputIterator> - btree_set_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - btree_set_container(std::initializer_list<init_type> init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_set_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_unique(key); - } - - // Insertion routines. - std::pair<iterator, bool> insert(const value_type &x) { - return this->tree_.insert_unique(params_type::key(x), x); - } - std::pair<iterator, bool> insert(value_type &&x) { - return this->tree_.insert_unique(params_type::key(x), std::move(x)); - } - template <typename... Args> - std::pair<iterator, bool> emplace(Args &&... args) { - init_type v(std::forward<Args>(args)...); - return this->tree_.insert_unique(params_type::key(v), std::move(v)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), x) - .first; - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), - std::move(x)) - .first; - } - - template <typename... Args> - iterator emplace_hint(const_iterator position, Args &&... args) { - init_type v(std::forward<Args>(args)...); - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(v), - std::move(v)) - .first; - } - - template <typename InputIterator> - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_unique(b, e); - } - - void insert(std::initializer_list<init_type> init) { - this->tree_.insert_iterator_unique(init.begin(), init.end()); - } - - insert_return_type insert(node_type &&node) { - if (!node) return {this->end(), false, node_type()}; - std::pair<iterator, bool> res = - this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) { - CommonAccess::Destroy(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - std::pair<iterator, bool> res = this->tree_.insert_hint_unique( - iterator(hint), params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) CommonAccess::Destroy(&node); - return res.first; - } - - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { return this->tree_.erase_unique(key); } - using super_type::erase; - - template <typename K = key_type> - node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - - using super_type::extract; - - // Merge routines. - // Moves elements from `src` into `this`. If the element already exists in - // `this`, it is left unmodified in `src`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &src) { // NOLINT - for (auto src_it = src.begin(); src_it != src.end();) { - if (insert(std::move(*src_it)).second) { - src_it = src.erase(src_it); - } else { - ++src_it; - } - } - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &&src) { - merge(src); - } - }; - - // Base class for btree_map. - // ------------------------- - template <typename Tree> - class btree_map_container : public btree_set_container<Tree> { - using super_type = btree_set_container<Tree>; - using params_type = typename Tree::params_type; - - protected: - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using mapped_type = typename params_type::mapped_type; - using value_type = typename Tree::value_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - - // Inherit constructors. - using super_type::super_type; - btree_map_container() {} - - // Insertion routines. - template <typename... Args> - std::pair<iterator, bool> try_emplace(const key_type &k, Args &&... args) { - return this->tree_.insert_unique( - k, std::piecewise_construct, std::forward_as_tuple(k), - std::forward_as_tuple(std::forward<Args>(args)...)); - } - template <typename... Args> - std::pair<iterator, bool> try_emplace(key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_unique guarantees that `key` is never - // referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_.insert_unique( - key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - } - template <typename... Args> - iterator try_emplace(const_iterator hint, const key_type &k, - Args &&... args) { - return this->tree_ - .insert_hint_unique(iterator(hint), k, std::piecewise_construct, - std::forward_as_tuple(k), - std::forward_as_tuple(std::forward<Args>(args)...)) - .first; - } - template <typename... Args> - iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_hint_unique guarantees that `key` is - // never referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_ - .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, - std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward<Args>(args)...)) - .first; - } - mapped_type &operator[](const key_type &k) { - return try_emplace(k).first->second; - } - mapped_type &operator[](key_type &&k) { - return try_emplace(std::move(k)).first->second; - } - - template <typename K = key_type> - mapped_type &at(const key_arg<K> &key) { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - template <typename K = key_type> - const mapped_type &at(const key_arg<K> &key) const { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - }; - - // A common base class for btree_multiset and btree_multimap. - template <typename Tree> - class btree_multiset_container : public btree_container<Tree> { - using super_type = btree_container<Tree>; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - - // Inherit constructors. - using super_type::super_type; - btree_multiset_container() {} - - // Range constructor. - template <class InputIterator> - btree_multiset_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - // Initializer list constructor. - btree_multiset_container(std::initializer_list<init_type> init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_multi(key); - } - - // Insertion routines. - iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } - iterator insert(value_type &&x) { - return this->tree_.insert_multi(std::move(x)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_.insert_hint_multi(iterator(position), x); - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_.insert_hint_multi(iterator(position), std::move(x)); - } - template <typename InputIterator> - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_multi(b, e); - } - void insert(std::initializer_list<init_type> init) { - this->tree_.insert_iterator_multi(init.begin(), init.end()); - } - template <typename... Args> - iterator emplace(Args &&... args) { - return this->tree_.insert_multi(init_type(std::forward<Args>(args)...)); - } - template <typename... Args> - iterator emplace_hint(const_iterator position, Args &&... args) { - return this->tree_.insert_hint_multi( - iterator(position), init_type(std::forward<Args>(args)...)); - } - iterator insert(node_type &&node) { - if (!node) return this->end(); - iterator res = - this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - CommonAccess::Destroy(&node); - return res; - } - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - iterator res = this->tree_.insert_hint_multi( - iterator(hint), - std::move(params_type::element(CommonAccess::GetSlot(node)))); - CommonAccess::Destroy(&node); - return res; - } - - // Deletion routines. - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { - return this->tree_.erase_multi(key); - } - using super_type::erase; - - // Node extraction routines. - template <typename K = key_type> - node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - using super_type::extract; - - // Merge routines. - // Moves all elements from `src` into `this`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &src) { // NOLINT - insert(std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); - src.clear(); - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &&src) { - merge(src); - } - }; - - // A base class for btree_multimap. - template <typename Tree> - class btree_multimap_container : public btree_multiset_container<Tree> { - using super_type = btree_multiset_container<Tree>; - using params_type = typename Tree::params_type; - - public: - using mapped_type = typename params_type::mapped_type; - - // Inherit constructors. - using super_type::super_type; - btree_multimap_container() {} - }; - -} // namespace priv - - - - // ---------------------------------------------------------------------- - // btree_set - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Compare, typename Alloc> - class btree_set : public priv::btree_set_container< - priv::btree<priv::set_params< - Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> - { - using Base = typename btree_set::btree_set_container; - - public: - btree_set() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_set` containers. - // ------------------------------------------------------- - template <typename K, typename C, typename A> - void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename C, typename A, typename Pred> - void erase_if(btree_set<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multiset - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Compare, typename Alloc> - class btree_multiset : public priv::btree_multiset_container< - priv::btree<priv::set_params< - Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> - { - using Base = typename btree_multiset::btree_multiset_container; - - public: - btree_multiset() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multiset` containers. - // ------------------------------------------------------------ - template <typename K, typename C, typename A> - void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename C, typename A, typename Pred> - void erase_if(btree_multiset<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - - // ---------------------------------------------------------------------- - // btree_map - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Value, typename Compare, typename Alloc> - class btree_map : public priv::btree_map_container< - priv::btree<priv::map_params< - Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> - { - using Base = typename btree_map::btree_map_container; - - public: - btree_map() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_map` containers. - // ------------------------------------------------------- - template <typename K, typename V, typename C, typename A> - void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) { - return x.swap(y); - } - - // ---------------------------------------------------------------------- - template <typename K, typename V, typename C, typename A, typename Pred> - void erase_if(btree_map<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multimap - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Value, typename Compare, typename Alloc> - class btree_multimap : public priv::btree_multimap_container< - priv::btree<priv::map_params< - Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> - { - using Base = typename btree_multimap::btree_multimap_container; - - public: - btree_multimap() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multimap` containers. - // ------------------------------------------------------------ - template <typename K, typename V, typename C, typename A> - void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename V, typename C, typename A, typename Pred> - void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - -} // namespace btree - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/benchmarks/external/parallel_hashmap/conanfile.py b/benchmarks/external/parallel_hashmap/conanfile.py deleted file mode 100644 index c046377d..00000000 --- a/benchmarks/external/parallel_hashmap/conanfile.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from conans import ConanFile, tools -import os - -class SparseppConan(ConanFile): - name = "parallel_hashmap" - version = "1.27" - description = "A header-only, very fast and memory-friendly hash map" - - # Indicates License type of the packaged library - license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" - - # Packages the license for the conanfile.py - exports = ["LICENSE"] - - # Custom attributes for Bincrafters recipe conventions - source_subfolder = "source_subfolder" - - def source(self): - source_url = "https://github.com/greg7mdp/parallel-hashmap" - tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) - extracted_dir = self.name + "-" + self.version - - #Rename to "source_folder" is a convention to simplify later steps - os.rename(extracted_dir, self.source_subfolder) - - - def package(self): - include_folder = os.path.join(self.source_subfolder, "parallel_hashmap") - self.copy(pattern="LICENSE") - self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder) - - def package_id(self): - self.info.header_only() diff --git a/benchmarks/external/parallel_hashmap/meminfo.h b/benchmarks/external/parallel_hashmap/meminfo.h deleted file mode 100644 index 872f3c69..00000000 --- a/benchmarks/external/parallel_hashmap/meminfo.h +++ /dev/null @@ -1,195 +0,0 @@ -#if !defined(spp_memory_h_guard) -#define spp_memory_h_guard - -#include <cstdint> -#include <cstring> -#include <cstdlib> - -#if defined(_WIN32) || defined( __CYGWIN__) - #define SPP_WIN -#endif - -#ifdef SPP_WIN - #include <windows.h> - #include <Psapi.h> - #undef min - #undef max -#elif defined(__linux__) - #include <sys/types.h> - #include <sys/sysinfo.h> -#elif defined(__FreeBSD__) - #include <paths.h> - #include <fcntl.h> - #include <kvm.h> - #include <unistd.h> - #include <sys/sysctl.h> - #include <sys/user.h> -#endif - -namespace spp -{ - uint64_t GetSystemMemory(); - uint64_t GetTotalMemoryUsed(); - uint64_t GetProcessMemoryUsed(); - uint64_t GetPhysicalMemory(); - - uint64_t GetSystemMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo (&memInfo); - auto totalVirtualMem = memInfo.totalram; - - totalVirtualMem += memInfo.totalswap; - totalVirtualMem *= memInfo.mem_unit; - return static_cast<uint64_t>(totalVirtualMem); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageCnt; - size_t pageCntLen = sizeof(pageCnt); - u_int pageSize; - struct kvm_swap kswap; - uint64_t totalVirtualMem; - - pageSize = static_cast<u_int>(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - totalVirtualMem = pageCnt * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - totalVirtualMem += kswap.ksw_total * pageSize; - - return totalVirtualMem; -#else - return 0; -#endif - } - - uint64_t GetTotalMemoryUsed() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - auto virtualMemUsed = memInfo.totalram - memInfo.freeram; - - virtualMemUsed += memInfo.totalswap - memInfo.freeswap; - virtualMemUsed *= memInfo.mem_unit; - - return static_cast<uint64_t>(virtualMemUsed); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageSize; - u_int pageCnt, freeCnt; - size_t pageCntLen = sizeof(pageCnt); - size_t freeCntLen = sizeof(freeCnt); - struct kvm_swap kswap; - uint64_t virtualMemUsed; - - pageSize = static_cast<u_int>(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); - virtualMemUsed = (pageCnt - freeCnt) * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - virtualMemUsed += kswap.ksw_used * pageSize; - - return virtualMemUsed; -#else - return 0; -#endif - } - - uint64_t GetProcessMemoryUsed() - { -#ifdef SPP_WIN - PROCESS_MEMORY_COUNTERS_EX pmc; - GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PPROCESS_MEMORY_COUNTERS>(&pmc), sizeof(pmc)); - return static_cast<uint64_t>(pmc.PrivateUsage); -#elif defined(__linux__) - auto parseLine = - [](char* line)->int - { - auto i = strlen(line); - - while(*line < '0' || *line > '9') - { - line++; - } - - line[i-3] = '\0'; - i = atoi(line); - return i; - }; - - auto file = fopen("/proc/self/status", "r"); - auto result = -1; - char line[128]; - - while(fgets(line, 128, file) != nullptr) - { - if(strncmp(line, "VmSize:", 7) == 0) - { - result = parseLine(line); - break; - } - } - - fclose(file); - return static_cast<uint64_t>(result) * 1024; -#elif defined(__FreeBSD__) - struct kinfo_proc info; - size_t infoLen = sizeof(info); - int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); - return static_cast<uint64_t>(info.ki_rssize * getpagesize()); -#else - return 0; -#endif - } - - uint64_t GetPhysicalMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPhys); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - - auto totalPhysMem = memInfo.totalram; - - totalPhysMem *= memInfo.mem_unit; - return static_cast<uint64_t>(totalPhysMem); -#elif defined(__FreeBSD__) - u_long physMem; - size_t physMemLen = sizeof(physMem); - int mib[] = { CTL_HW, HW_PHYSMEM }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); - return physMem; -#else - return 0; -#endif - } - -} - -#endif // spp_memory_h_guard diff --git a/benchmarks/external/parallel_hashmap/phmap.h b/benchmarks/external/parallel_hashmap/phmap.h deleted file mode 100644 index 653ae5ea..00000000 --- a/benchmarks/external/parallel_hashmap/phmap.h +++ /dev/null @@ -1,4788 +0,0 @@ -#if !defined(phmap_h_guard_) -#define phmap_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - -#include <algorithm> -#include <cmath> -#include <cstring> -#include <iterator> -#include <limits> -#include <memory> -#include <tuple> -#include <type_traits> -#include <utility> -#include <array> -#include <cassert> -#include <atomic> - -#include "phmap_fwd_decl.h" -#include "phmap_utils.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include <string_view> -#endif - -namespace phmap { - -namespace priv { - -// -------------------------------------------------------------------------- -template <size_t Width> -class probe_seq -{ -public: - probe_seq(size_t hashval, size_t mask) { - assert(((mask + 1) & mask) == 0 && "not a mask"); - mask_ = mask; - offset_ = hashval & mask_; - } - size_t offset() const { return offset_; } - size_t offset(size_t i) const { return (offset_ + i) & mask_; } - - void next() { - index_ += Width; - offset_ += index_; - offset_ &= mask_; - } - // 0-based probe index. The i-th probe in the probe sequence. - size_t getindex() const { return index_; } - -private: - size_t mask_; - size_t offset_; - size_t index_ = 0; -}; - -// -------------------------------------------------------------------------- -template <class ContainerKey, class Hash, class Eq> -struct RequireUsableKey -{ - template <class PassedKey, class... Args> - std::pair< - decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), - decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), - std::declval<const PassedKey&>()))>* - operator()(const PassedKey&, const Args&...) const; -}; - -// -------------------------------------------------------------------------- -template <class E, class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable : std::false_type {}; - -template <class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable< - phmap::void_t<decltype( - Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), - std::declval<Ts>()...))>, - Policy, Hash, Eq, Ts...> : std::true_type {}; - -// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. -// -------------------------------------------------------------------------- -template <class T> -constexpr bool IsNoThrowSwappable() { - using std::swap; - return noexcept(swap(std::declval<T&>(), std::declval<T&>())); -} - -// -------------------------------------------------------------------------- -template <typename T> -int TrailingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountTrailingZerosNonZero64(static_cast<uint64_t>(x)); - else - return base_internal::CountTrailingZerosNonZero32(static_cast<uint32_t>(x)); -} - -// -------------------------------------------------------------------------- -template <typename T> -int LeadingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountLeadingZeros64(static_cast<uint64_t>(x)); - else - return base_internal::CountLeadingZeros32(static_cast<uint32_t>(x)); -} - -// -------------------------------------------------------------------------- -// An abstraction over a bitmask. It provides an easy way to iterate through the -// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), -// this is a true bitmask. On non-SSE, platforms the arithematic used to -// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as -// either 0x00 or 0x80. -// -// For example: -// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 -// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 -// -------------------------------------------------------------------------- -template <class T, int SignificantBits, int Shift = 0> -class BitMask -{ - static_assert(std::is_unsigned<T>::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - -public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } - int LowestBitSet() const { - return priv::TrailingZeros(mask_) >> Shift; - } - int HighestBitSet() const { - return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - - 1) >> - Shift; - } - - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - - int TrailingZeros() const { - return priv::TrailingZeros(mask_) >> Shift; - } - - int LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return priv::LeadingZeros(mask_ << extra_bits) >> Shift; - } - -private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } - - T mask_; -}; - -// -------------------------------------------------------------------------- -using ctrl_t = signed char; -using h2_t = uint8_t; - -// -------------------------------------------------------------------------- -// The values here are selected for maximum performance. See the static asserts -// below for details. -// -------------------------------------------------------------------------- -enum Ctrl : ctrl_t -{ - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 -}; - -static_assert( - kEmpty & kDeleted & kSentinel & 0x80, - "Special markers need to have the MSB to make checking for them efficient"); -static_assert(kEmpty < kSentinel && kDeleted < kSentinel, - "kEmpty and kDeleted must be smaller than kSentinel to make the " - "SIMD test of IsEmptyOrDeleted() efficient"); -static_assert(kSentinel == -1, - "kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(kEmpty == -128, - "kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); -static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, - "kEmpty and kDeleted must share an unset bit that is not shared " - "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " - "efficient"); -static_assert(kDeleted == -2, - "kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - -// -------------------------------------------------------------------------- -// A single block of empty control bytes for tables without any slots allocated. -// This enables removing a branch in the hot path of find(). -// -------------------------------------------------------------------------- -inline ctrl_t* EmptyGroup() { - alignas(16) static constexpr ctrl_t empty_group[] = { - kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, - kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; - return const_cast<ctrl_t*>(empty_group); -} - -// -------------------------------------------------------------------------- -inline size_t HashSeed(const ctrl_t* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return reinterpret_cast<uintptr_t>(ctrl) >> 12; -} - -#ifdef PHMAP_NON_DETERMINISTIC - -inline size_t H1(size_t hashval, const ctrl_t* ctrl) { - // use ctrl_ pointer to add entropy to ensure - // non-deterministic iteration order. - return (hashval >> 7) ^ HashSeed(ctrl); -} - -#else - -inline size_t H1(size_t hashval, const ctrl_t* ) { - return (hashval >> 7); -} - -#endif - - -inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } - -inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= 0; } -inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } -inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } - -#if PHMAP_HAVE_SSE2 - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch -#endif - -// -------------------------------------------------------------------------- -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -// -------------------------------------------------------------------------- -inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { -#if defined(__GNUC__) && !defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Woverflow" - - if (std::is_unsigned<char>::value) { - const __m128i mask = _mm_set1_epi8(static_cast<char>(0x80)); - const __m128i diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } - - #pragma GCC diagnostic pop -#endif - return _mm_cmpgt_epi8(a, b); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupSse2Impl -{ - enum { kWidth = 16 }; // the number of slots per group - - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); - } - - // Returns a bitmask representing the positions of slots that match hash. - // ---------------------------------------------------------------------- - BitMask<uint32_t, kWidth> Match(h2_t hash) const { - auto match = _mm_set1_epi8((char)hash); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); - } - - // Returns a bitmask representing the positions of empty slots. - // ------------------------------------------------------------ - BitMask<uint32_t, kWidth> MatchEmpty() const { -#if PHMAP_HAVE_SSSE3 - // This only works because kEmpty is -128. - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); -#else - return Match(static_cast<h2_t>(kEmpty)); -#endif - } - - // Returns a bitmask representing the positions of empty or deleted slots. - // ----------------------------------------------------------------------- - BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); - } - - // Returns the number of trailing empty or deleted elements in the group. - // ---------------------------------------------------------------------- - uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return TrailingZeros( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); - } - - // ---------------------------------------------------------------------- - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast<char>(-128)); - auto x126 = _mm_set1_epi8(126); -#if PHMAP_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); -#else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); -#endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } - - __m128i ctrl; -}; - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // PHMAP_HAVE_SSE2 - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupPortableImpl -{ - enum { kWidth = 8 }; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on kEmpty, kDeleted, kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } - - uint64_t ctrl; -}; - -#if PHMAP_HAVE_SSE2 - using Group = GroupSse2Impl; -#else - using Group = GroupPortableImpl; -#endif - -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set; - -inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } - -// -------------------------------------------------------------------------- -// PRECONDITION: -// IsValidCapacity(capacity) -// ctrl[capacity] == kSentinel -// ctrl[i] != kSentinel for all i < capacity -// Applies mapping for every byte in ctrl: -// DELETED -> EMPTY -// EMPTY -> EMPTY -// FULL -> DELETED -// -------------------------------------------------------------------------- -inline void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) -{ - assert(ctrl[capacity] == kSentinel); - assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { - Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); - } - // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; -} - -// -------------------------------------------------------------------------- -// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. -// -------------------------------------------------------------------------- -inline size_t NormalizeCapacity(size_t n) -{ - return n ? ~size_t{} >> LeadingZeros(n) : 1; -} - -// -------------------------------------------------------------------------- -// We use 7/8th as maximum load factor. -// For 16-wide groups, that gives an average of two empty slots per group. -// -------------------------------------------------------------------------- -inline size_t CapacityToGrowth(size_t capacity) -{ - assert(IsValidCapacity(capacity)); - // `capacity*7/8` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (capacity == 7) - { - // x-x/8 does not work when x==7. - return 6; - } - } - return capacity - capacity / 8; -} - -// -------------------------------------------------------------------------- -// From desired "growth" to a lowerbound of the necessary capacity. -// Might not be a valid one and required NormalizeCapacity(). -// -------------------------------------------------------------------------- -inline size_t GrowthToLowerboundCapacity(size_t growth) -{ - // `growth*8/7` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (growth == 7) - { - // x+(x-1)/7 does not work when x==7. - return 8; - } - } - return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); -} - -namespace hashtable_debug_internal { - -// If it is a map, call get<0>(). -using std::get; -template <typename T, typename = typename T::mapped_type> -auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { - return get<0>(pair); -} - -// If it is not a map, return the value directly. -template <typename T> -const typename T::key_type& GetKey(const typename T::key_type& key, char) { - return key; -} - -// -------------------------------------------------------------------------- -// Containers should specialize this to provide debug information for that -// container. -// -------------------------------------------------------------------------- -template <class Container, typename Enabler = void> -struct HashtableDebugAccess -{ - // Returns the number of probes required to find `key` in `c`. The "number of - // probes" is a concept that can vary by container. Implementations should - // return 0 when `key` was found in the minimum number of operations and - // should increment the result for each non-trivial operation required to find - // `key`. - // - // The default implementation uses the bucket api from the standard and thus - // works for `std::unordered_*` containers. - // -------------------------------------------------------------------------- - static size_t GetNumProbes(const Container& c, - const typename Container::key_type& key) { - if (!c.bucket_count()) return {}; - size_t num_probes = 0; - size_t bucket = c.bucket(key); - for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { - if (it == e) return num_probes; - if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes; - } - } -}; - -} // namespace hashtable_debug_internal - -// ---------------------------------------------------------------------------- -// I N F O Z S T U B S -// ---------------------------------------------------------------------------- -struct HashtablezInfo -{ - void PrepareForSampling() {} -}; - -inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} - -static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} - -static inline void RecordEraseSlow(HashtablezInfo*) {} - -static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } -static inline void UnsampleSlow(HashtablezInfo* ) {} - -class HashtablezInfoHandle -{ -public: - inline void RecordStorageChanged(size_t , size_t ) {} - inline void RecordRehash(size_t ) {} - inline void RecordInsert(size_t , size_t ) {} - inline void RecordErase() {} - friend inline void swap(HashtablezInfoHandle& , - HashtablezInfoHandle& ) noexcept {} -}; - -static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } - -class HashtablezSampler -{ -public: - // Returns a global Sampler. - static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } - HashtablezInfo* Register() { static HashtablezInfo info; return &info; } - void Unregister(HashtablezInfo* ) {} - - using DisposeCallback = void (*)(const HashtablezInfo&); - DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } - int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& ) { return 0; } -}; - -static inline void SetHashtablezEnabled(bool ) {} -static inline void SetHashtablezSampleParameter(int32_t ) {} -static inline void SetHashtablezMaxSamples(int32_t ) {} - - -namespace memory_internal { - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template <class Alloc, class T, class Tuple, size_t... I> -void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, - phmap::index_sequence<I...>) { - phmap::allocator_traits<Alloc>::construct( - *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...); -} - -template <class T, class F> -struct WithConstructedImplF { - template <class... Args> - decltype(std::declval<F>()(std::declval<T>())) operator()( - Args&&... args) const { - return std::forward<F>(f)(T(std::forward<Args>(args)...)); - } - F&& f; -}; - -template <class T, class Tuple, size_t... Is, class F> -decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl( - Tuple&& t, phmap::index_sequence<Is...>, F&& f) { - return WithConstructedImplF<T, F>{std::forward<F>(f)}( - std::get<Is>(std::forward<Tuple>(t))...); -} - -template <class T, size_t... Is> -auto TupleRefImpl(T&& t, phmap::index_sequence<Is...>) - -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) { - return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...); -} - -// Returns a tuple of references to the elements of the input tuple. T must be a -// tuple. -// ---------------------------------------------------------------------------- -template <class T> -auto TupleRef(T&& t) -> decltype( - TupleRefImpl(std::forward<T>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<T>::type>::value>())) { - return TupleRefImpl( - std::forward<T>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<T>::type>::value>()); -} - -template <class F, class K, class V> -decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct, - std::declval<std::tuple<K>>(), std::declval<V>())) -DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) { - const auto& key = std::get<0>(p.first); - return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), - std::move(p.second)); -} - -} // namespace memory_internal - - -// ---------------------------------------------------------------------------- -// R A W _ H A S H _ S E T -// ---------------------------------------------------------------------------- -// An open-addressing -// hashtable with quadratic probing. -// -// This is a low level hashtable on top of which different interfaces can be -// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. -// -// The table interface is similar to that of std::unordered_set. Notable -// differences are that most member functions support heterogeneous keys when -// BOTH the hash and eq functions are marked as transparent. They do so by -// providing a typedef called `is_transparent`. -// -// When heterogeneous lookup is enabled, functions that take key_type act as if -// they have an overload set like: -// -// iterator find(const key_type& key); -// template <class K> -// iterator find(const K& key); -// -// size_type erase(const key_type& key); -// template <class K> -// size_type erase(const K& key); -// -// std::pair<iterator, iterator> equal_range(const key_type& key); -// template <class K> -// std::pair<iterator, iterator> equal_range(const K& key); -// -// When heterogeneous lookup is disabled, only the explicit `key_type` overloads -// exist. -// -// find() also supports passing the hash explicitly: -// -// iterator find(const key_type& key, size_t hash); -// template <class U> -// iterator find(const U& key, size_t hash); -// -// In addition the pointer to element and iterator stability guarantees are -// weaker: all iterators and pointers are invalidated after a new element is -// inserted. -// -// IMPLEMENTATION DETAILS -// -// The table stores elements inline in a slot array. In addition to the slot -// array the table maintains some control state per slot. The extra state is one -// byte per slot and stores empty or deleted marks, or alternatively 7 bits from -// the hash of an occupied slot. The table is split into logical groups of -// slots, like so: -// -// Group 1 Group 2 Group 3 -// +---------------+---------------+---------------+ -// | | | | | | | | | | | | | | | | | | | | | | | | | -// +---------------+---------------+---------------+ -// -// On lookup the hash is split into two parts: -// - H2: 7 bits (those stored in the control bytes) -// - H1: the rest of the bits -// The groups are probed using H1. For each group the slots are matched to H2 in -// parallel. Because H2 is 7 bits (128 states) and the number of slots per group -// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. -// -// On insert, once the right group is found (as in lookup), its slots are -// filled in order. -// -// On erase a slot is cleared. In case the group did not have any empty slots -// before the erase, the erased slot is marked as deleted. -// -// Groups without empty slots (but maybe with deleted slots) extend the probe -// sequence. The probing algorithm is quadratic. Given N the number of groups, -// the probing function for the i'th probe is: -// -// P(0) = H1 % N -// -// P(i) = (P(i - 1) + i) % N -// -// This probing function guarantees that after N probes, all the groups of the -// table will be probed exactly once. -// ---------------------------------------------------------------------------- -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set -{ - using PolicyTraits = hash_policy_traits<Policy>; - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - -public: - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - -private: - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using Layout = phmap::priv::Layout<ctrl_t, slot_type>; - - static Layout MakeLayout(size_t capacity) { - assert(IsValidCapacity(capacity)); - return Layout(capacity + Group::kWidth + 1, capacity); - } - - using AllocTraits = phmap::allocator_traits<allocator_type>; - using SlotAlloc = typename phmap::allocator_traits< - allocator_type>::template rebind_alloc<slot_type>; - using SlotAllocTraits = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<slot_type>; - - static_assert(std::is_lvalue_reference<reference>::value, - "Policy::element() must return a reference"); - - template <typename T> - struct SameAsElementReference - : std::is_same<typename std::remove_cv< - typename std::remove_reference<reference>::type>::type, - typename std::remove_cv< - typename std::remove_reference<T>::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - template <class T> - using RequiresInsertable = typename std::enable_if< - phmap::disjunction<std::is_convertible<T, init_type>, - SameAsElementReference<T>>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template <class T> - using RequiresNotInit = - typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; - - template <class... Ts> - using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; - -public: - static_assert(std::is_same<pointer, value_type*>::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same<const_pointer, const value_type*>::value, - "Allocators with custom pointer types are not supported"); - - class iterator - { - friend class raw_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename raw_hash_set::value_type; - using reference = - phmap::conditional_t<PolicyTraits::constant_iterators::value, - const value_type&, value_type&>; - using pointer = phmap::remove_reference_t<reference>*; - using difference_type = typename raw_hash_set::difference_type; - - iterator() {} - - // PRECONDITION: not an end() iterator. - reference operator*() const { return PolicyTraits::element(slot_); } - - // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } - - // PRECONDITION: not an end() iterator. - iterator& operator++() { - ++ctrl_; - ++slot_; - skip_empty_or_deleted(); - return *this; - } - // PRECONDITION: not an end() iterator. - iterator operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } - -#if PHMAP_BIDIRECTIONAL - // PRECONDITION: not a begin() iterator. - iterator& operator--() { - assert(ctrl_); - do { - --ctrl_; - --slot_; - } while (IsEmptyOrDeleted(*ctrl_)); - return *this; - } - - // PRECONDITION: not a begin() iterator. - iterator operator--(int) { - auto tmp = *this; - --*this; - return tmp; - } -#endif - - friend bool operator==(const iterator& a, const iterator& b) { - return a.ctrl_ == b.ctrl_; - } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() - iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} - - void skip_empty_or_deleted() { - while (IsEmptyOrDeleted(*ctrl_)) { - // ctrl is not necessarily aligned to Group::kWidth. It is also likely - // to read past the space for ctrl bytes and into slots. This is ok - // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there - // is no way to read outside the combined slot array. - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); - ctrl_ += shift; - slot_ += shift; - } - } - - ctrl_t* ctrl_ = nullptr; - // To avoid uninitialized member warnings, put slot_ in an anonymous union. - // The member is not initialized on singleton and end iterators. - union { - slot_type* slot_; - }; - }; - - class const_iterator - { - friend class raw_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename raw_hash_set::value_type; - using reference = typename raw_hash_set::const_reference; - using pointer = typename raw_hash_set::const_pointer; - using difference_type = typename raw_hash_set::difference_type; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} - - reference operator*() const { return *inner_; } - pointer operator->() const { return inner_.operator->(); } - - const_iterator& operator++() { - ++inner_; - return *this; - } - const_iterator operator++(int) { return inner_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.inner_ == b.inner_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} - - iterator inner_; - }; - - using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; - using insert_return_type = InsertReturnType<iterator, node_type>; - - raw_hash_set() noexcept( - std::is_nothrow_default_constructible<hasher>::value&& - std::is_nothrow_default_constructible<key_equal>::value&& - std::is_nothrow_default_constructible<allocator_type>::value) {} - - explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { - if (bucket_cnt) { - capacity_ = NormalizeCapacity(bucket_cnt); - reset_growth_left(); - initialize_slots(); - } - } - - raw_hash_set(size_t bucket_cnt, const hasher& hashfn, - const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit raw_hash_set(const allocator_type& alloc) - : raw_hash_set(0, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { - insert(first, last); - } - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list<value_type> as the first - // argument like std::unordered_set<value_type> does, we have two overloads - // that accept std::initializer_list<T> and std::initializer_list<init_type>. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list<std::string>, then - // // copies the strings into the set. - // std::unordered_set<std::string> s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list<const char*>, then - // // copies the strings into the set. - // phmap::flat_hash_set<std::string> s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set<int> a, b{a}; - // - // RequiresNotInit<T> is a workaround for gcc prior to 7.1. - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, - const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(const raw_hash_set& that) - : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - reserve(that.size()); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(hashval); - set_ctrl(target.offset, H2(hashval)); - emplace_at(target.offset, v); - infoz_.RecordInsert(hashval, target.probe_length); - } - size_ = that.size(); - growth_left() -= that.size(); - } - - raw_hash_set(raw_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible<hasher>::value&& - std::is_nothrow_copy_constructible<key_equal>::value&& - std::is_nothrow_copy_constructible<allocator_type>::value) - : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), - slots_(phmap::exchange(that.slots_, nullptr)), - size_(phmap::exchange(that.size_, 0)), - capacity_(phmap::exchange(that.capacity_, 0)), - infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function<Key>, moving it - // would create a nullptr functor that cannot be called. - settings_(that.settings_) { - // growth_left was copied above, reset the one from `that`. - that.growth_left() = 0; - } - - raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), - settings_(0, that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); - std::swap(infoz_, that.infoz_); - } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); - } - } - - raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; - } - - raw_hash_set& operator=(raw_hash_set&& that) noexcept( - phmap::allocator_traits<allocator_type>::is_always_equal::value&& - std::is_nothrow_move_assignable<hasher>::value&& - std::is_nothrow_move_assignable<key_equal>::value) { - // TODO(sbenza): We should only use the operations from the noexcept clause - // to make sure we actually adhere to that contract. - return move_assign( - std::move(that), - typename AllocTraits::propagate_on_container_move_assignment()); - } - - ~raw_hash_set() { destroy_slots(); } - - iterator begin() { - auto it = iterator_at(0); - it.skip_empty_or_deleted(); - return it; - } - iterator end() - { -#if PHMAP_BIDIRECTIONAL - return iterator_at(capacity_); -#else - return {ctrl_ + capacity_}; -#endif - } - - const_iterator begin() const { - return const_cast<raw_hash_set*>(this)->begin(); - } - const_iterator end() const { return const_cast<raw_hash_set*>(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } - size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (empty()) - return; - if (capacity_ > 127) { - destroy_slots(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - size_ = 0; - reset_ctrl(); - reset_growth_left(); - } - assert(empty()); - infoz_.RecordStorageChanged(0, capacity_); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map<std::string, int> m; - // m.insert(std::make_pair("abc", 42)); - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - std::pair<iterator, bool> insert(T&& value) { - return emplace(std::forward<T>(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set<int> s; - // s.insert(n); - // - // flat_hash_set<std::string> s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - std::pair<iterator, bool> insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set<std::string, int> s; - // s.insert({"abc", 42}); - std::pair<iterator, bool> insert(init_type&& value) { - return emplace(std::move(value)); - } - - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward<T>(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template <typename It> - using IsRandomAccess = std::is_same<typename std::iterator_traits<It>::iterator_category, - std::random_access_iterator_tag>; - - - template<typename T> - struct has_difference_operator - { - private: - using yes = std::true_type; - using no = std::false_type; - - template<typename U> static auto test(int) -> decltype(std::declval<U>() - std::declval<U>() == 1, yes()); - template<typename> static no test(...); - - public: - static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; - }; - - template <class InputIt, typename phmap::enable_if_t<has_difference_operator<InputIt>::value, int> = 0> - void insert(InputIt first, InputIt last) { - this->reserve(this->size() + (last - first)); - for (; first != last; ++first) - emplace(*first); - } - - template <class InputIt, typename phmap::enable_if_t<!has_difference_operator<InputIt>::value, int> = 0> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) - emplace(*first); - } - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> - void insert(std::initializer_list<T> ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list<init_type> ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - insert_return_type insert(node_type&& node, size_t hashval) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlotWithHash<false>{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward<Args>(args)...); - } - - template <class... Args, typename std::enable_if<IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type - raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); - } - - template <class... Args, typename std::enable_if<!IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlotWithHash<true>{*this, std::move(*slot), hashval}, elem); - } - - template <class... Args> - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward<Args>(args)...).first; - } - - template <class... Args> - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward<Args>(args)...).first; - } - - // Extension API: support for lazy emplace. - // - // Looks up key in the table. If found, returns the iterator to the element. - // Otherwise calls f with one argument of type raw_hash_set::constructor. f - // MUST call raw_hash_set::constructor with arguments as if a - // raw_hash_set::value_type is constructed, otherwise the behavior is - // undefined. - // - // For example: - // - // std::unordered_set<ArenaString> s; - // // Makes ArenaStr even if "abc" is in the map. - // s.insert(ArenaString(&arena, "abc")); - // - // flat_hash_set<ArenaStr> s; - // // Makes ArenaStr only if "abc" is not in the map. - // s.lazy_emplace("abc", [&](const constructor& ctor) { - // ctor(&arena, "abc"); - // }); - // - // WARNING: This API is currently experimental. If there is a way to implement - // the same thing with the rest of the API, prefer that. - class constructor - { - friend class raw_hash_set; - - public: - template <class... Args> - void operator()(Args&&... args) const { - assert(*slot_); - PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); - *slot_ = nullptr; - } - - private: - constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - - allocator_type* alloc_; - slot_type** slot_; - }; - - template <class K = key_type, class F> - iterator lazy_emplace(const key_arg<K>& key, F&& f) { - auto res = find_or_prepare_insert(key); - if (res.second) { - lazy_emplace_at(res.first, std::forward<F>(f)); - } - return iterator_at(res.first); - } - - template <class K = key_type, class F> - iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t &hashval, F&& f) { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - lazy_emplace_at(res.first, std::forward<F>(f)); - } - return iterator_at(res.first); - } - - template <class K = key_type, class F> - void lazy_emplace_at(size_t& idx, F&& f) { - slot_type* slot = slots_ + idx; - std::forward<F>(f)(constructor(&alloc_ref(), &slot)); - assert(!slot); - } - - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - template <class K = key_type> - size_type erase(const key_arg<K>& key) { - auto it = find(key); - if (it == end()) return 0; - _erase(it); - return 1; - } - - - iterator erase(const_iterator cit) { return erase(cit.inner_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if (<pred>) { - // m._erase(it++); - // } else { - // ++it; - // } - // } - void _erase(iterator it) { - assert(it != end()); - PolicyTraits::destroy(&alloc_ref(), it.slot_); - erase_meta_only(it); - } - void _erase(const_iterator cit) { _erase(cit.inner_); } - - // This overload is necessary because otherwise erase<K>(const K&) would be - // a better match if non-const iterator is passed as an argument. - iterator erase(iterator it) { - auto res = it; - ++res; - _erase(it); - return res; - } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.inner_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT - assert(this != &src); - for (auto it = src.begin(), e = src.end(); it != e; ++it) { - if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) - .second) { - src.erase_meta_only(it); - } - } - } - - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - auto node = - CommonAccess::Make<node_type>(alloc_ref(), position.inner_.slot_); - erase_meta_only(position); - return node; - } - - template < - class K = key_type, - typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> - node_type extract(const key_arg<K>& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable<allocator_type>())) { - using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); - swap(hash_ref(), that.hash_ref()); - swap(eq_ref(), that.eq_ref()); - swap(infoz_, that.infoz_); - if (AllocTraits::propagate_on_container_swap::value) { - swap(alloc_ref(), that.alloc_ref()); - } else { - // If the allocators do not compare equal it is officially undefined - // behavior. We choose to do nothing. - } - } - -#ifndef PHMAP_NON_DETERMINISTIC - template<typename OutputArchive> - bool dump(OutputArchive&) const; - - template<typename InputArchive> - bool load(InputArchive&); -#endif - - void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); - infoz_.RecordStorageChanged(0, 0); - return; - } - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity((std::max)(n, size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { - resize(m); - } - } - - void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - template <class K = key_type> - size_t count(const key_arg<K>& key) const { - return find(key) == end() ? size_t(0) : size_t(1); - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - void prefetch_hash(size_t hashval) const { - (void)hashval; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - auto seq = probe(hashval); - _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); - _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); -#elif defined(__GNUC__) - auto seq = probe(hashval); - __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); -#endif // __GNUC__ - } - - template <class K = key_type> - void prefetch(const key_arg<K>& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - template <class K = key_type> - iterator find(const key_arg<K>& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return iterator_at(offset); - else - return end(); - } - - template <class K = key_type> - pointer find_ptr(const key_arg<K>& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return &PolicyTraits::element(slots_ + offset); - else - return nullptr; - } - - template <class K = key_type> - iterator find(const key_arg<K>& key) { - return find(key, this->hash(key)); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key, size_t hashval) const { - return const_cast<raw_hash_set*>(this)->find(key, hashval); - } - template <class K = key_type> - const_iterator find(const key_arg<K>& key) const { - return find(key, this->hash(key)); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key) const { - return find(key) != end(); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template <class K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - template <class K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K>& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { return capacity_; } - float load_factor() const { - return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; - } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { - if (a.size() != b.size()) return false; - const raw_hash_set* outer = &a; - const raw_hash_set* inner = &b; - if (outer->capacity() > inner->capacity()) - std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; - return true; - } - - friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { - return !(a == b); - } - - friend void swap(raw_hash_set& a, - raw_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template <class K> - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -private: - template <class Container, typename Enabler> - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - template <class K = key_type> - bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) { - auto seq = probe(hashval); - while (true) { - Group g{ ctrl_ + seq.offset() }; - for (int i : g.Match((h2_t)H2(hashval))) { - offset = seq.offset((size_t)i); - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + offset)))) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) - return false; - seq.next(); - } - } - - struct FindElement - { - template <class K, class... Args> - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const raw_hash_set& s; - }; - - struct HashElement - { - template <class K, class... Args> - size_t operator()(const K& key, Args&&...) const { - return phmap_mix<sizeof(size_t)>()(h(key)); - } - const hasher& h; - }; - - template <class K1> - struct EqualElement - { - template <class K2, class... Args> - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable(const K& key, size_t hashval, - Args&&... args) - { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - emplace_at(res.first, std::forward<Args>(args)...); - } - return {iterator_at(res.first), res.second}; - } - - struct EmplaceDecomposable - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, s.hash(key), std::forward<Args>(args)...); - } - raw_hash_set& s; - }; - - struct EmplaceDecomposableHashval { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, hashval, std::forward<Args>(args)...); - } - raw_hash_set& s; - size_t hashval; - }; - - template <bool do_destroy> - struct InsertSlot - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - }; - - template <bool do_destroy> - struct InsertSlotWithHash - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key, hashval); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - size_t &hashval; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast<size_t>(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - - set_ctrl(index, was_never_full ? kEmpty : kDeleted); - growth_left() += was_never_full; - infoz_.RecordErase(); - } - - void initialize_slots() { - assert(capacity_); - if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && - slots_ == nullptr) { - infoz_ = Sample(); - } - - auto layout = MakeLayout(capacity_); - char* mem = static_cast<char*>( - Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize())); - ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem)); - slots_ = layout.template Pointer<1>(mem); - reset_ctrl(); - reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); - } - - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - auto layout = MakeLayout(capacity_); - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize()); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; - } - - void resize(size_t new_capacity) { - assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - capacity_ = new_capacity; - initialize_slots(); - - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); - } - } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); - auto layout = MakeLayout(old_capacity); - Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, - layout.AllocSize()); - } - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); - assert(!is_small()); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type - raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - - // Verify if the old and new i fall within the same group wrt the hashval. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const auto probe_index = [&](size_t pos) { - return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - set_ctrl(i, H2(hashval)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. - // set_ctrl poisons/unpoisons the slots so we have to call it at the - // right time. - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - set_ctrl(i, kEmpty); - } else { - assert(IsDeleted(ctrl_[new_i])); - set_ctrl(new_i, H2(hashval)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(); - } - - void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); - } else if (size() <= CapacityToGrowth(capacity()) / 2) { - // Squash DELETED without growing if there is enough capacity. - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(capacity_ * 2 + 1); - } - } - - bool has_element(const value_type& elem, size_t hashval) const { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == - elem)) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; - seq.next(); - assert(seq.getindex() < capacity_ && "full table!"); - } - return false; - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - return has_element(elem, hashval); - } - - // Probes the raw_hash_set with the probe sequence for hash and returns the - // pointer to the first empty or deleted slot. - // NOTE: this function must work with tables having both kEmpty and kDelete - // in one group. Such tables appears during drop_deletes_without_resize. - // - // This function is very useful when insertions happen and: - // - the input is already a set - // - there are enough slots - // - the element with the hash is not in the table - struct FindInfo - { - size_t offset; - size_t probe_length; - }; - FindInfo find_first_non_full(size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); - if (mask) { - return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; - } - assert(seq.getindex() < capacity_ && "full table!"); - seq.next(); - } - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset((size_t)i))))) - return {seq.offset((size_t)i), false}; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; - seq.next(); - } - return {prepare_insert(hashval), true}; - } - - template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key) { - return find_or_prepare_insert(key, this->hash(key)); - } - - size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(hashval); - if (PHMAP_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { - rehash_and_grow_if_necessary(); - target = find_first_non_full(hashval); - } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); - set_ctrl(target.offset, H2(hashval)); - infoz_.RecordInsert(hashval, target.probe_length); - return target.offset; - } - - // Constructs the value in the space pointed by the iterator. This only works - // after an unsuccessful find_or_prepare_insert() and before any other - // modifications happen in the raw_hash_set. - // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward<Args>(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. - // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). - template <class... Args> - void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, - std::forward<Args>(args)...); - - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && - "constructed value does not match the lookup key"); - } - - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } - -private: - friend struct RawHashSetTestOnlyAccess; - - probe_seq<Group::kWidth> probe(size_t hashval) const { - return probe_seq<Group::kWidth>(H1(hashval, ctrl_), capacity_); - } - - // Reset all ctrl bytes back to kEmpty, except the sentinel. - void reset_ctrl() { - std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); - ctrl_[capacity_] = kSentinel; - SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - } - - void reset_growth_left() { - growth_left() = CapacityToGrowth(capacity()) - size_; - } - - // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at - // the end too. - void set_ctrl(size_t i, ctrl_t h) { - assert(i < capacity_); - - if (IsFull(h)) { - SanitizerUnpoisonObject(slots_ + i); - } else { - SanitizerPoisonObject(slots_ + i); - } - - ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + 1 + - ((Group::kWidth - 1) & capacity_)] = h; - } - - size_t& growth_left() { return settings_.template get<0>(); } - - template <size_t N, - template <class, class, class, class> class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_set; - - template <size_t N, - template <class, class, class, class> class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_map; - - // The representation of the object has two modes: - // - small: For capacities < kWidth-1 - // - large: For the rest. - // - // Differences: - // - In small mode we are able to use the whole capacity. The extra control - // bytes give us at least one "empty" control byte to stop the iteration. - // This is important to make 1 a valid capacity. - // - // - In small mode only the first `capacity()` control bytes after the - // sentinel are valid. The rest contain dummy kEmpty values that do not - // represent a real slot. This is important to take into account on - // find_first_non_full(), where we never try ShouldInsertBackwards() for - // small tables. - bool is_small() const { return capacity_ < Group::kWidth - 1; } - - hasher& hash_ref() { return settings_.template get<1>(); } - const hasher& hash_ref() const { return settings_.template get<1>(); } - key_equal& eq_ref() { return settings_.template get<2>(); } - const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } - const allocator_type& alloc_ref() const { - return settings_.template get<3>(); - } - - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array - ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] - slot_type* slots_ = nullptr; // [capacity * slot_type] - size_t size_ = 0; // number of full slots - size_t capacity_ = 0; // total number of slots - HashtablezInfoHandle infoz_; - phmap::priv::CompressedTuple<size_t /* growth_left */, hasher, - key_equal, allocator_type> - settings_{0, hasher{}, key_equal{}, allocator_type{}}; -}; - - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map<K, IncompleteType>. - // MappedReference<> may be a non-reference type. - template <class P> - using MappedReference = decltype(P::value( - std::addressof(std::declval<typename raw_hash_map::reference>()))); - - // MappedConstReference<> may be a non-reference type. - template <class P> - using MappedConstReference = decltype(P::value( - std::addressof(std::declval<typename raw_hash_map::const_reference>()))); - - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - using Base = raw_hash_set<Policy, Hash, Eq, Alloc>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - - static_assert(!std::is_reference<key_type>::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference<mapped_type>::value, ""); - - using iterator = typename raw_hash_map::raw_hash_set::iterator; - using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; - - raw_hash_map() {} - using Base::raw_hash_set; // use raw_hash_set constructor - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map<int, int> m; - // m.insert_or_assign(n, n); - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { - return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { - return insert_or_assign_impl(std::forward<K>(k), v); - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { - return insert_or_assign_impl(k, std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { - return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { - return insert_or_assign(std::forward<K>(k), v).first; - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { - return insert_or_assign(k, std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { - return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { - return try_emplace_impl(k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace(k, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class P = Policy> - MappedReference<P> at(const key_arg<K>& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy> - MappedConstReference<P> at(const key_arg<K>& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy, K* = nullptr> - MappedReference<P> operator[](key_arg<K>&& key) { - return Policy::value(&*try_emplace(std::forward<K>(key)).first); - } - - template <class K = key_type, class P = Policy> - MappedReference<P> operator[](const key_arg<K>& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template <class K, class V> - std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); - else - Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v); - return {this->iterator_at(res.first), res.second}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(res.first), res.second}; - } -}; - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -// Returns "random" seed. -inline size_t RandomSeed() -{ -#if PHMAP_HAVE_THREAD_LOCAL - static thread_local size_t counter = 0; - size_t value = ++counter; -#else // PHMAP_HAVE_THREAD_LOCAL - static std::atomic<size_t> counter(0); - size_t value = counter.fetch_add(1, std::memory_order_relaxed); -#endif // PHMAP_HAVE_THREAD_LOCAL - return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); -} - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_set -{ - using PolicyTraits = hash_policy_traits<Policy>; - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - static_assert(N <= 12, "N = 12 means 4096 hash tables!"); - constexpr static size_t num_tables = 1 << N; - constexpr static size_t mask = num_tables - 1; - -public: - using EmbeddedSet = RefSet<Policy, Hash, Eq, Alloc>; - using EmbeddedIterator= typename EmbeddedSet::iterator; - using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; - using constructor = typename EmbeddedSet::constructor; - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - // -------------------------------------------------------------------- - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - -protected: - using Lockable = phmap::LockableImpl<Mtx_>; - - // -------------------------------------------------------------------- - struct Inner : public Lockable - { - bool operator==(const Inner& o) const - { - typename Lockable::SharedLocks l(const_cast<Inner &>(*this), const_cast<Inner &>(o)); - return set_ == o.set_; - } - - EmbeddedSet set_; - }; - -private: - // Give an early error when key_type is not hashable/eq. - // -------------------------------------------------------------------- - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using AllocTraits = phmap::allocator_traits<allocator_type>; - - static_assert(std::is_lvalue_reference<reference>::value, - "Policy::element() must return a reference"); - - template <typename T> - struct SameAsElementReference : std::is_same< - typename std::remove_cv<typename std::remove_reference<reference>::type>::type, - typename std::remove_cv<typename std::remove_reference<T>::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - // -------------------------------------------------------------------- - template <class T> - using RequiresInsertable = typename std::enable_if< - phmap::disjunction<std::is_convertible<T, init_type>, - SameAsElementReference<T>>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template <class T> - using RequiresNotInit = - typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; - - template <class... Ts> - using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; - -public: - static_assert(std::is_same<pointer, value_type*>::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same<const_pointer, const value_type*>::value, - "Allocators with custom pointer types are not supported"); - - // --------------------- i t e r a t o r ------------------------------ - class iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename parallel_hash_set::value_type; - using reference = - phmap::conditional_t<PolicyTraits::constant_iterators::value, - const value_type&, value_type&>; - using pointer = phmap::remove_reference_t<reference>*; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; - using EmbeddedIterator = typename EmbeddedSet::iterator; - - iterator() {} - - reference operator*() const { return *it_; } - pointer operator->() const { return &operator*(); } - - iterator& operator++() { - assert(inner_); // null inner means we are already at the end - ++it_; - skip_empty(); - return *this; - } - - iterator operator++(int) { - assert(inner_); // null inner means we are already at the end - auto tmp = *this; - ++*this; - return tmp; - } - - friend bool operator==(const iterator& a, const iterator& b) { - return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); - } - - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : - inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() - if (inner) - it_end_ = inner->set_.end(); - } - - void skip_empty() { - while (it_ == it_end_) { - ++inner_; - if (inner_ == inner_end_) { - inner_ = nullptr; // marks end() - break; - } - else { - it_ = inner_->set_.begin(); - it_end_ = inner_->set_.end(); - } - } - } - - Inner *inner_ = nullptr; - Inner *inner_end_ = nullptr; - EmbeddedIterator it_, it_end_; - }; - - // --------------------- c o n s t i t e r a t o r ----------------- - class const_iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename parallel_hash_set::value_type; - using reference = typename parallel_hash_set::const_reference; - using pointer = typename parallel_hash_set::const_pointer; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : iter_(std::move(i)) {} - - reference operator*() const { return *(iter_); } - pointer operator->() const { return iter_.operator->(); } - - const_iterator& operator++() { - ++iter_; - return *this; - } - const_iterator operator++(int) { return iter_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.iter_ == b.iter_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) - : iter_(const_cast<Inner**>(inner), - const_cast<Inner**>(inner_end), - const_cast<EmbeddedIterator*>(it)) {} - - iterator iter_; - }; - - using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; - using insert_return_type = InsertReturnType<iterator, node_type>; - - // ------------------------- c o n s t r u c t o r s ------------------ - - parallel_hash_set() noexcept( - std::is_nothrow_default_constructible<hasher>::value&& - std::is_nothrow_default_constructible<key_equal>::value&& - std::is_nothrow_default_constructible<allocator_type>::value) {} - - explicit parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) { - for (auto& inner : sets_) - inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); - } - - parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param, - const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit parallel_hash_set(const allocator_type& alloc) - : parallel_hash_set(0, hasher(), key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { - insert(first, last); - } - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list<value_type> as the first - // argument like std::unordered_set<value_type> does, we have two overloads - // that accept std::initializer_list<T> and std::initializer_list<init_type>. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list<std::string>, then copies - // // the strings into the set. - // std::unordered_set<std::string> s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list<const char*>, then - // // copies the strings into the set. - // phmap::flat_hash_set<std::string> s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set<int> a, b{a}; - // - // RequiresNotInit<T> is a workaround for gcc prior to 7.1. - // -------------------------------------------------------------------- - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, - const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(const parallel_hash_set& that) - : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) - : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = { that.sets_[i].set_, a }; - } - - parallel_hash_set(parallel_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible<hasher>::value&& - std::is_nothrow_copy_constructible<key_equal>::value&& - std::is_nothrow_copy_constructible<allocator_type>::value) - : parallel_hash_set(std::move(that), that.alloc_ref()) { - } - - parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) - { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = { std::move(that.sets_[i]).set_, a }; - } - - parallel_hash_set& operator=(const parallel_hash_set& that) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = that.sets_[i].set_; - return *this; - } - - parallel_hash_set& operator=(parallel_hash_set&& that) noexcept( - phmap::allocator_traits<allocator_type>::is_always_equal::value && - std::is_nothrow_move_assignable<hasher>::value && - std::is_nothrow_move_assignable<key_equal>::value) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = std::move(that.sets_[i].set_); - return *this; - } - - ~parallel_hash_set() {} - - iterator begin() { - auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin()); - it.skip_empty(); - return it; - } - - iterator end() { return iterator(); } - const_iterator begin() const { return const_cast<parallel_hash_set *>(this)->begin(); } - const_iterator end() const { return const_cast<parallel_hash_set *>(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - - size_t size() const { - size_t sz = 0; - for (const auto& inner : sets_) - sz += inner.set_.size(); - return sz; - } - - size_t capacity() const { - size_t c = 0; - for (const auto& inner : sets_) - c += inner.set_.capacity(); - return c; - } - - size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - } - - // extension - clears only soecified submap - // ---------------------------------------- - void clear(std::size_t submap_index) { - Inner& inner = sets_[submap_index]; - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map<std::string, int> m; - // m.insert(std::make_pair("abc", 42)); - // -------------------------------------------------------------------- - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - std::pair<iterator, bool> insert(T&& value) { - return emplace(std::forward<T>(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set<int> s; - // s.insert(n); - // - // flat_hash_set<std::string> s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - std::pair<iterator, bool> insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set<std::pair<std::string, int>> s; - // s.insert({"abc", 42}); - // -------------------------------------------------------------------- - std::pair<iterator, bool> insert(init_type&& value) { - return emplace(std::move(value)); - } - - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward<T>(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template <class InputIt> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) insert(*first); - } - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> - void insert(std::initializer_list<T> ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list<init_type> ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) - return {end(), false, node_type()}; - auto& key = node.key(); - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - - typename Lockable::UniqueLock m(inner); - auto res = set.insert(std::move(node), hashval); - return { make_iterator(&inner, res.position), - res.inserted, - res.inserted ? node_type() : std::move(res.node) }; - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - struct ReturnKey_ - { - template <class Key, class... Args> - Key operator()(Key&& k, const Args&...) const { - return std::forward<Key>(k); - } - }; - - // -------------------------------------------------------------------- - // phmap expension: emplace_with_hash - // ---------------------------------- - // same as emplace, but hashval is provided - // -------------------------------------------------------------------- - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); - } - - struct EmplaceDecomposableHashval - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable_with_hash(key, hashval, std::forward<Args>(args)...); - } - parallel_hash_set& s; - size_t hashval; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, - std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash<true> f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template <class... Args> - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class F> - iterator lazy_emplace_with_hash(size_t hashval, const key_arg<K>& key, F&& f) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); - } - - // -------------------------------------------------------------------- - // end of phmap expension - // -------------------------------------------------------------------- - - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable(const K& key, Args&&... args) - { - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); - } - - struct EmplaceDecomposable - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, std::forward<Args>(args)...); - } - parallel_hash_set& s; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - size_t hashval = this->hash(PolicyTraits::key(slot)); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash<true> f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template <class... Args> - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward<Args>(args)...).first; - } - - iterator make_iterator(Inner* inner, const EmbeddedIterator it) - { - if (it == inner->set_.end()) - return iterator(); - return iterator(inner, &sets_[0] + num_tables, it); - } - - std::pair<iterator, bool> make_rv(Inner* inner, - const std::pair<EmbeddedIterator, bool>& res) - { - return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; - } - - template <class K = key_type, class F> - iterator lazy_emplace(const key_arg<K>& key, F&& f) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); - } - - template <class K = key_type, class FExists, class FEmplace> - bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(key, m); - Inner* inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.lazy_emplace_at(std::get<1>(res), std::forward<FEmplace>(fEmplace)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward<FExists>(fExists)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - // -------------------------------------------------------------------- - template <class K = key_type> - size_type erase(const key_arg<K>& key) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UpgradeLock m(inner); - auto it = set.find(key, hashval); - if (it == set.end()) - return 0; - - typename Lockable::UpgradeToUnique unique(m); - set._erase(it); - return 1; - } - - // -------------------------------------------------------------------- - iterator erase(const_iterator cit) { return erase(cit.iter_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if (<pred>) { - // m._erase(it++); - // } else { - // ++it; - // } - // } - // -------------------------------------------------------------------- - void _erase(iterator it) { - assert(it.inner_ != nullptr); - it.inner_->set_._erase(it.it_); - } - void _erase(const_iterator cit) { _erase(cit.iter_); } - - // This overload is necessary because otherwise erase<K>(const K&) would be - // a better match if non-const iterator is passed as an argument. - // -------------------------------------------------------------------- - iterator erase(iterator it) { _erase(it++); return it; } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.iter_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - // -------------------------------------------------------------------- - template <typename E = Eq> - void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>& src) { // NOLINT - assert(this != &src); - if (this != &src) - { - for (size_t i=0; i<num_tables; ++i) - { - typename Lockable::UniqueLocks l(sets_[i], src.sets_[i]); - sets_[i].set_.merge(src.sets_[i].set_); - } - } - } - - template <typename E = Eq> - void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); - } - - template < - class K = key_type, - typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> - node_type extract(const key_arg<K>& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(parallel_hash_set& that) noexcept( - IsNoThrowSwappable<EmbeddedSet>() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable<allocator_type>())) { - using std::swap; - for (size_t i=0; i<num_tables; ++i) - { - typename Lockable::UniqueLocks l(sets_[i], that.sets_[i]); - swap(sets_[i].set_, that.sets_[i].set_); - } - } - - void rehash(size_t n) { - size_t nn = n / num_tables; - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.rehash(nn); - } - } - - void reserve(size_t n) - { - size_t target = GrowthToLowerboundCapacity(n); - size_t normalized = 16 * NormalizeCapacity(n / num_tables); - rehash(normalized > target ? normalized : target); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - // -------------------------------------------------------------------- - template <class K = key_type> - size_t count(const key_arg<K>& key) const { - return find(key) == end() ? 0 : 1; - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - // -------------------------------------------------------------------- - void prefetch_hash(size_t hashval) const { - const Inner& inner = sets_[subidx(hashval)]; - const auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - set.prefetch_hash(hashval); - } - - template <class K = key_type> - void prefetch(const key_arg<K>& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - // -------------------------------------------------------------------- - template <class K = key_type> - iterator find(const key_arg<K>& key, size_t hashval) { - typename Lockable::SharedLock m; - return find(key, hashval, m); - } - - template <class K = key_type> - iterator find(const key_arg<K>& key) { - return find(key, this->hash(key)); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key, size_t hashval) const { - return const_cast<parallel_hash_set*>(this)->find(key, hashval); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key) const { - return find(key, this->hash(key)); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key) const { - return find(key) != end(); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template <class K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - template <class K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K>& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { - size_t sz = 0; - for (const auto& inner : sets_) - { - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - sz += inner.set_.bucket_count(); - } - return sz; - } - - float load_factor() const { - size_t _capacity = bucket_count(); - return _capacity ? static_cast<float>(static_cast<double>(size()) / _capacity) : 0; - } - - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { - return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); - } - - friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { - return !(a == b); - } - - friend void swap(parallel_hash_set& a, - parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template <class K> - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -#ifndef PHMAP_NON_DETERMINISTIC - template<typename OutputArchive> - bool dump(OutputArchive& ar) const; - - template<typename InputArchive> - bool load(InputArchive& ar); -#endif - -private: - template <class Container, typename Enabler> - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - struct FindElement - { - template <class K, class... Args> - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const parallel_hash_set& s; - }; - - struct HashElement - { - template <class K, class... Args> - size_t operator()(const K& key, Args&&...) const { - return phmap_mix<sizeof(size_t)>()(h(key)); - } - const hasher& h; - }; - - template <class K1> - struct EqualElement - { - template <class K2, class... Args> - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - // -------------------------------------------------------------------- - void erase_meta_only(const_iterator cit) { - auto &it = cit.iter_; - assert(it.set_ != nullptr); - it.set_.erase_meta_only(const_iterator(it.it_)); - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.drop_deletes_without_resize(); - } - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - return set.has_element(elem, hashval); - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - // -------------------------------------------------------------------- - parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { - parallel_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - - parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { - parallel_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template <class K = key_type, class L = typename Lockable::SharedLock> - pointer find_ptr(const key_arg<K>& key, size_t hashval, L& mutexlock) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return set.find_ptr(key, hashval); - } - - template <class K = key_type, class L = typename Lockable::SharedLock> - iterator find(const key_arg<K>& key, size_t hashval, L& mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return make_iterator(&inner, set.find(key, hashval)); - } - - template <class K> - std::tuple<Inner*, size_t, bool> - find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(typename Lockable::UniqueLock(inner)); - auto p = set.find_or_prepare_insert(key, hashval); // std::pair<size_t, bool> - return std::make_tuple(&inner, p.first, p.second); - } - - template <class K> - std::tuple<Inner*, size_t, bool> - find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { - return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock); - } - - iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) { - return {inner, &sets_[0] + num_tables, it}; - } - const_iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) const { - return {inner, &sets_[0] + num_tables, it}; - } - - static size_t subidx(size_t hashval) { - return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; - } - - static size_t subcnt() { - return num_tables; - } - -private: - friend struct RawHashSetTestOnlyAccess; - - size_t growth_left() { - size_t sz = 0; - for (const auto& set : sets_) - sz += set.growth_left(); - return sz; - } - - hasher& hash_ref() { return sets_[0].set_.hash_ref(); } - const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } - key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } - const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } - allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } - const allocator_type& alloc_ref() const { - return sets_[0].set_.alloc_ref(); - } - -protected: // protected in case users want to derive fromm this - std::array<Inner, num_tables> sets_; -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc> -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map<K, IncompleteType>. - // MappedReference<> may be a non-reference type. - template <class P> - using MappedReference = decltype(P::value( - std::addressof(std::declval<typename parallel_hash_map::reference>()))); - - // MappedConstReference<> may be a non-reference type. - template <class P> - using MappedConstReference = decltype(P::value( - std::addressof(std::declval<typename parallel_hash_map::const_reference>()))); - - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - using Base = typename parallel_hash_map::parallel_hash_set; - using Lockable = phmap::LockableImpl<Mtx_>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - - static_assert(!std::is_reference<key_type>::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference<mapped_type>::value, ""); - - using iterator = typename parallel_hash_map::parallel_hash_set::iterator; - using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; - - parallel_hash_map() {} - -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using parallel_hash_map::parallel_hash_set::parallel_hash_set; -#endif - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map<int, int> m; - // m.insert_or_assign(n, n); - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { - return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { - return insert_or_assign_impl(std::forward<K>(k), v); - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { - return insert_or_assign_impl(k, std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { - return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { - return insert_or_assign(std::forward<K>(k), v).first; - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { - return insert_or_assign(k, std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { - return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { - return try_emplace_impl(k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace(k, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class P = Policy> - MappedReference<P> at(const key_arg<K>& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy> - MappedConstReference<P> at(const key_arg<K>& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - // ----------- phmap extensions -------------------------- - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, key_arg<K>&& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, const key_arg<K>& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace_with_hash(hashval, k, std::forward<Args>(args)...).first; - } - - // if map contains key, lambda is called with the mapped value (under read lock protection), - // and if_contains returns true. This is a const API and lambda should not modify the value - // ----------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool if_contains(const key_arg<K>& key, F&& f) const { - return const_cast<parallel_hash_map*>(this)->template - modify_if_impl<K, F, typename Lockable::SharedLock>(key, std::forward<F>(f)); - } - - // if map contains key, lambda is called with the mapped value without read lock protection, - // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value - // This should be used only if we know that no other thread may be mutating the map at the time. - // ----------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool if_contains_unsafe(const key_arg<K>& key, F&& f) const { - return const_cast<parallel_hash_map*>(this)->template - modify_if_impl<K, F, LockableBaseImpl<phmap::NullMutex>::DoNothing>(key, std::forward<F>(f)); - } - - // if map contains key, lambda is called with the mapped value (under write lock protection), - // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value - // ---------------------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool modify_if(const key_arg<K>& key, F&& f) { - return modify_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); - } - - - // if map contains key, lambda is called with the mapped value (under write lock protection). - // If the lambda returns true, the key is subsequently erased from the map (the write lock - // is only released after erase). - // returns true if key was erased, false otherwise. - // ---------------------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool erase_if(const key_arg<K>& key, F&& f) { - return erase_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); - } - - // if map does not contains key, it is inserted and the mapped value is value-constructed - // with the provided arguments (if any), as with try_emplace. - // if map already contains key, then the lambda is called with the mapped value (under - // write lock protection) and can update the mapped value. - // returns true if key was not already present, false otherwise. - // --------------------------------------------------------------------------------------- - template <class K = key_type, class F, class... Args> - bool try_emplace_l(K&& k, F&& f, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward<F>(f)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // ----------- end of phmap extensions -------------------------- - - template <class K = key_type, class P = Policy, K* = nullptr> - MappedReference<P> operator[](key_arg<K>&& key) { - return Policy::value(&*try_emplace(std::forward<K>(key)).first); - } - - template <class K = key_type, class P = Policy> - MappedReference<P> operator[](const key_arg<K>& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template <class K = key_type, class F, class L> - bool modify_if_impl(const key_arg<K>& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable<F, mapped_type&>::value); -#endif - L m; - auto ptr = this->template find_ptr<K, L>(key, this->hash(key), m); - if (ptr == nullptr) - return false; - std::forward<F>(f)(Policy::value(ptr)); - return true; - } - - template <class K = key_type, class F, class L> - bool erase_if_impl(const key_arg<K>& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable<F, mapped_type&>::value); -#endif - L m; - auto it = this->template find<K, L>(key, this->hash(key), m); - if (it == this->end()) return false; - if (std::forward<F>(f)(Policy::value(&*it))) - { - this->erase(it); - return true; - } - return false; - } - - - template <class K, class V> - std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::forward<K>(k), std::forward<V>(v)); - else - Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward<V>(v); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - -}; - - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template <class Alloc, class T, class Tuple> -void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { - memory_internal::ConstructFromTupleImpl( - alloc, ptr, std::forward<Tuple>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<Tuple>::type>::value>()); -} - -// Constructs T using the args specified in the tuple and calls F with the -// constructed value. -// ---------------------------------------------------------------------------- -template <class T, class Tuple, class F> -decltype(std::declval<F>()(std::declval<T>())) WithConstructed( - Tuple&& t, F&& f) { - return memory_internal::WithConstructedImpl<T>( - std::forward<Tuple>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<Tuple>::type>::value>(), - std::forward<F>(f)); -} - -// ---------------------------------------------------------------------------- -// Given arguments of an std::pair's consructor, PairArgs() returns a pair of -// tuples with references to the passed arguments. The tuples contain -// constructor arguments for the first and the second elements of the pair. -// -// The following two snippets are equivalent. -// -// 1. std::pair<F, S> p(args...); -// -// 2. auto a = PairArgs(args...); -// std::pair<F, S> p(std::piecewise_construct, -// std::move(p.first), std::move(p.second)); -// ---------------------------------------------------------------------------- -inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; } - -template <class F, class S> -std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) { - return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)), - std::forward_as_tuple(std::forward<S>(s))}; -} - -template <class F, class S> -std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs( - const std::pair<F, S>& p) { - return PairArgs(p.first, p.second); -} - -template <class F, class S> -std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) { - return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second)); -} - -template <class F, class S> -auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) - -> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), - memory_internal::TupleRef(std::forward<S>(s)))) { - return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), - memory_internal::TupleRef(std::forward<S>(s))); -} - -// A helper function for implementing apply() in map policies. -// ---------------------------------------------------------------------------- -template <class F, class... Args> -auto DecomposePair(F&& f, Args&&... args) - -> decltype(memory_internal::DecomposePairImpl( - std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) { - return memory_internal::DecomposePairImpl( - std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); -} - -// A helper function for implementing apply() in set policies. -// ---------------------------------------------------------------------------- -template <class F, class Arg> -decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>())) -DecomposeValue(F&& f, Arg&& arg) { - const auto& key = arg; - return std::forward<F>(f)(key, std::forward<Arg>(arg)); -} - - -// -------------------------------------------------------------------------- -// Policy: a policy defines how to perform different operations on -// the slots of the hashtable (see hash_policy_traits.h for the full interface -// of policy). -// -// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The -// functor should accept a key and return size_t as hash. For best performance -// it is important that the hash function provides high entropy across all bits -// of the hash. -// -// Eq: a (possibly polymorphic) functor that compares two keys for equality. It -// should accept two (of possibly different type) keys and return a bool: true -// if they are equal, false if they are not. If two keys compare equal, then -// their hash values as defined by Hash MUST be equal. -// -// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which -// the storage of the hashtable will be allocated and the elements will be -// constructed and destroyed. -// -------------------------------------------------------------------------- -template <class T> -struct FlatHashSetPolicy -{ - using slot_type = T; - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - phmap::allocator_traits<Allocator>::construct(*alloc, slot, - std::forward<Args>(args)...); - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - phmap::allocator_traits<Allocator>::destroy(*alloc, slot); - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - construct(alloc, new_slot, std::move(*old_slot)); - destroy(alloc, old_slot); - } - - static T& element(slot_type* slot) { return *slot; } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposeValue( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward<F>(f), std::forward<Args>(args)...); - } - - static size_t space_used(const T*) { return 0; } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class K, class V> -struct FlatHashMapPolicy -{ - using slot_policy = priv::map_slot_policy<K, V>; - using slot_type = typename slot_policy::slot_type; - using key_type = K; - using mapped_type = V; - using init_type = std::pair</*non const*/ key_type, mapped_type>; - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - slot_policy::construct(alloc, slot, std::forward<Args>(args)...); - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - slot_policy::transfer(alloc, new_slot, old_slot); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposePair( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward<F>(f), - std::forward<Args>(args)...); - } - - static size_t space_used(const slot_type*) { return 0; } - - static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } - - static V& value(std::pair<const K, V>* kv) { return kv->second; } - static const V& value(const std::pair<const K, V>* kv) { return kv->second; } -}; - -template <class Reference, class Policy> -struct node_hash_policy { - static_assert(std::is_lvalue_reference<Reference>::value, ""); - - using slot_type = typename std::remove_cv< - typename std::remove_reference<Reference>::type>::type*; - - template <class Alloc, class... Args> - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - *slot = Policy::new_element(alloc, std::forward<Args>(args)...); - } - - template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::delete_element(alloc, *slot); - } - - template <class Alloc> - static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { - *new_slot = *old_slot; - } - - static size_t space_used(const slot_type* slot) { - if (slot == nullptr) return Policy::element_space_used(nullptr); - return Policy::element_space_used(*slot); - } - - static Reference element(slot_type* slot) { return **slot; } - - template <class T, class P = Policy> - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - - template <class... Ts, class P = Policy> - static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward<Ts>(ts)...)) { - return P::apply(std::forward<Ts>(ts)...); - } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class T> -struct NodeHashSetPolicy - : phmap::priv::node_hash_policy<T&, NodeHashSetPolicy<T>> -{ - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template <class Allocator, class... Args> - static T* new_element(Allocator* alloc, Args&&... args) { - using ValueAlloc = - typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; - ValueAlloc value_alloc(*alloc); - T* res = phmap::allocator_traits<ValueAlloc>::allocate(value_alloc, 1); - phmap::allocator_traits<ValueAlloc>::construct(value_alloc, res, - std::forward<Args>(args)...); - return res; - } - - template <class Allocator> - static void delete_element(Allocator* alloc, T* elem) { - using ValueAlloc = - typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; - ValueAlloc value_alloc(*alloc); - phmap::allocator_traits<ValueAlloc>::destroy(value_alloc, elem); - phmap::allocator_traits<ValueAlloc>::deallocate(value_alloc, elem, 1); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposeValue( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward<F>(f), std::forward<Args>(args)...); - } - - static size_t element_space_used(const T*) { return sizeof(T); } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class Key, class Value> -class NodeHashMapPolicy - : public phmap::priv::node_hash_policy< - std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>> -{ - using value_type = std::pair<const Key, Value>; - -public: - using key_type = Key; - using mapped_type = Value; - using init_type = std::pair</*non const*/ key_type, mapped_type>; - - template <class Allocator, class... Args> - static value_type* new_element(Allocator* alloc, Args&&... args) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc<value_type>; - PairAlloc pair_alloc(*alloc); - value_type* res = - phmap::allocator_traits<PairAlloc>::allocate(pair_alloc, 1); - phmap::allocator_traits<PairAlloc>::construct(pair_alloc, res, - std::forward<Args>(args)...); - return res; - } - - template <class Allocator> - static void delete_element(Allocator* alloc, value_type* pair) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc<value_type>; - PairAlloc pair_alloc(*alloc); - phmap::allocator_traits<PairAlloc>::destroy(pair_alloc, pair); - phmap::allocator_traits<PairAlloc>::deallocate(pair_alloc, pair, 1); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposePair( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward<F>(f), - std::forward<Args>(args)...); - } - - static size_t element_space_used(const value_type*) { - return sizeof(value_type); - } - - static Value& value(value_type* elem) { return elem->second; } - static const Value& value(const value_type* elem) { return elem->second; } -}; - - -// -------------------------------------------------------------------------- -// hash_default -// -------------------------------------------------------------------------- - -#if PHMAP_HAVE_STD_STRING_VIEW - -// support char16_t wchar_t .... -template<class CharT> -struct StringHashT -{ - using is_transparent = void; - - size_t operator()(std::basic_string_view<CharT> v) const { - std::string_view bv{reinterpret_cast<const char*>(v.data()), v.size() * sizeof(CharT)}; - return std::hash<std::string_view>()(bv); - } -}; - -// Supports heterogeneous lookup for basic_string<T>-like elements. -template<class CharT> -struct StringHashEqT -{ - using Hash = StringHashT<CharT>; - - struct Eq { - using is_transparent = void; - - bool operator()(std::basic_string_view<CharT> lhs, std::basic_string_view<CharT> rhs) const { - return lhs == rhs; - } - }; -}; - -template <> -struct HashEq<std::string> : StringHashEqT<char> {}; - -template <> -struct HashEq<std::string_view> : StringHashEqT<char> {}; - -// char16_t -template <> -struct HashEq<std::u16string> : StringHashEqT<char16_t> {}; - -template <> -struct HashEq<std::u16string_view> : StringHashEqT<char16_t> {}; - -// wchar_t -template <> -struct HashEq<std::wstring> : StringHashEqT<wchar_t> {}; - -template <> -struct HashEq<std::wstring_view> : StringHashEqT<wchar_t> {}; - -#endif - -// Supports heterogeneous lookup for pointers and smart pointers. -// ------------------------------------------------------------- -template <class T> -struct HashEq<T*> -{ - struct Hash { - using is_transparent = void; - template <class U> - size_t operator()(const U& ptr) const { - return phmap::Hash<const T*>{}(HashEq::ToPtr(ptr)); - } - }; - - struct Eq { - using is_transparent = void; - template <class A, class B> - bool operator()(const A& a, const B& b) const { - return HashEq::ToPtr(a) == HashEq::ToPtr(b); - } - }; - -private: - static const T* ToPtr(const T* ptr) { return ptr; } - - template <class U, class D> - static const T* ToPtr(const std::unique_ptr<U, D>& ptr) { - return ptr.get(); - } - - template <class U> - static const T* ToPtr(const std::shared_ptr<U>& ptr) { - return ptr.get(); - } -}; - -template <class T, class D> -struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {}; - -template <class T> -struct HashEq<std::shared_ptr<T>> : HashEq<T*> {}; - -namespace hashtable_debug_internal { - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <typename Set> -struct HashtableDebugAccess<Set, phmap::void_t<typename Set::raw_hash_set>> -{ - using Traits = typename Set::PolicyTraits; - using Slot = typename Traits::slot_type; - - static size_t GetNumProbes(const Set& set, - const typename Set::key_type& key) { - size_t num_probes = 0; - size_t hashval = set.hash(key); - auto seq = set.probe(hashval); - while (true) { - priv::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(priv::H2(hashval))) { - if (Traits::apply( - typename Set::template EqualElement<typename Set::key_type>{ - key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset((size_t)i)))) - return num_probes; - ++num_probes; - } - if (g.MatchEmpty()) return num_probes; - seq.next(); - ++num_probes; - } - } - - static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); - size_t m = layout.AllocSize(); - - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * c.size(); - } else { - for (size_t i = 0; i != capacity; ++i) { - if (priv::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); - } - } - } - return m; - } - - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); - size_t m = layout.AllocSize(); - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; - } -}; - -} // namespace hashtable_debug_internal -} // namespace priv - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::flat_hash_set<T>` is an unordered associative container which has -// been optimized for both speed and memory footprint in most common use cases. -// Its interface is similar to that of `std::unordered_set<T>` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the set is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class flat_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::FlatHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename flat_hash_set::raw_hash_set; - -public: - flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::flat_hash_map<K, V>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map<K, V>` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class K, class V, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class flat_hash_map : public phmap::priv::raw_hash_map< - phmap::priv::FlatHashMapPolicy<K, V>, - Hash, Eq, Alloc> { - using Base = typename flat_hash_map::raw_hash_map; - -public: - flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::node_hash_set<T>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_set<T>` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class node_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename node_hash_set::raw_hash_set; - -public: - node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::node_hash_map<K, V>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map<K, V>` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class Key, class Value, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class node_hash_map - : public phmap::priv::raw_hash_map< - phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, - Alloc> -{ - using Base = typename node_hash_map::raw_hash_map; - -public: - node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_set -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> // default values in phmap_fwd_decl.h -class parallel_flat_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashSetPolicy<T>, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_set::parallel_hash_set; - -public: - parallel_flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h -// ----------------------------------------------------------------------------- -template <class K, class V, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashMapPolicy<K, V>, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_map::parallel_hash_map; - -public: - parallel_flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_set -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_node_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename parallel_node_hash_set::parallel_hash_set; - -public: - parallel_node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_map -// ----------------------------------------------------------------------------- -template <class Key, class Value, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_node_hash_map - : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, - Alloc> -{ - using Base = typename parallel_node_hash_map::parallel_hash_map; - -public: - parallel_node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // phmap_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_base.h b/benchmarks/external/parallel_hashmap/phmap_base.h deleted file mode 100644 index d0c6f3ce..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_base.h +++ /dev/null @@ -1,5171 +0,0 @@ -#if !defined(phmap_base_h_guard_) -#define phmap_base_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#include <algorithm> -#include <cassert> -#include <cstddef> -#include <initializer_list> -#include <iterator> -#include <string> -#include <type_traits> -#include <utility> -#include <functional> -#include <tuple> -#include <utility> -#include <memory> -#include <mutex> // for std::lock - -#include "phmap_config.h" - -#ifdef PHMAP_HAVE_SHARED_MUTEX - #include <shared_mutex> // after "phmap_config.h" -#endif - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4582) // constructor is not implicitly called - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member -#endif // _MSC_VER - -namespace phmap { - -template <class T> using Allocator = typename std::allocator<T>; - -template<class T1, class T2> using Pair = typename std::pair<T1, T2>; - -template <class T> -struct EqualTo -{ - inline bool operator()(const T& a, const T& b) const - { - return std::equal_to<T>()(a, b); - } -}; - -template <class T> -struct Less -{ - inline bool operator()(const T& a, const T& b) const - { - return std::less<T>()(a, b); - } -}; - -namespace type_traits_internal { - -template <typename... Ts> -struct VoidTImpl { - using type = void; -}; - -// This trick to retrieve a default alignment is necessary for our -// implementation of aligned_storage_t to be consistent with any implementation -// of std::aligned_storage. -// --------------------------------------------------------------------------- -template <size_t Len, typename T = std::aligned_storage<Len>> -struct default_alignment_of_aligned_storage; - -template <size_t Len, size_t Align> -struct default_alignment_of_aligned_storage<Len, - std::aligned_storage<Len, Align>> { - static constexpr size_t value = Align; -}; - -// NOTE: The `is_detected` family of templates here differ from the library -// fundamentals specification in that for library fundamentals, `Op<Args...>` is -// evaluated as soon as the type `is_detected<Op, Args...>` undergoes -// substitution, regardless of whether or not the `::value` is accessed. That -// is inconsistent with all other standard traits and prevents lazy evaluation -// in larger contexts (such as if the `is_detected` check is a trailing argument -// of a `conjunction`. This implementation opts to instead be lazy in the same -// way that the standard traits are (this "defect" of the detection idiom -// specifications has been reported). -// --------------------------------------------------------------------------- - -template <class Enabler, template <class...> class Op, class... Args> -struct is_detected_impl { - using type = std::false_type; -}; - -template <template <class...> class Op, class... Args> -struct is_detected_impl<typename VoidTImpl<Op<Args...>>::type, Op, Args...> { - using type = std::true_type; -}; - -template <template <class...> class Op, class... Args> -struct is_detected : is_detected_impl<void, Op, Args...>::type {}; - -template <class Enabler, class To, template <class...> class Op, class... Args> -struct is_detected_convertible_impl { - using type = std::false_type; -}; - -template <class To, template <class...> class Op, class... Args> -struct is_detected_convertible_impl< - typename std::enable_if<std::is_convertible<Op<Args...>, To>::value>::type, - To, Op, Args...> { - using type = std::true_type; -}; - -template <class To, template <class...> class Op, class... Args> -struct is_detected_convertible - : is_detected_convertible_impl<void, To, Op, Args...>::type {}; - -template <typename T> -using IsCopyAssignableImpl = - decltype(std::declval<T&>() = std::declval<const T&>()); - -template <typename T> -using IsMoveAssignableImpl = decltype(std::declval<T&>() = std::declval<T&&>()); - -} // namespace type_traits_internal - -template <typename T> -struct is_copy_assignable : type_traits_internal::is_detected< - type_traits_internal::IsCopyAssignableImpl, T> { -}; - -template <typename T> -struct is_move_assignable : type_traits_internal::is_detected< - type_traits_internal::IsMoveAssignableImpl, T> { -}; - -// --------------------------------------------------------------------------- -// void_t() -// -// Ignores the type of any its arguments and returns `void`. In general, this -// metafunction allows you to create a general case that maps to `void` while -// allowing specializations that map to specific types. -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::void_t` metafunction. -// -// NOTE: `phmap::void_t` does not use the standard-specified implementation so -// that it can remain compatible with gcc < 5.1. This can introduce slightly -// different behavior, such as when ordering partial specializations. -// --------------------------------------------------------------------------- -template <typename... Ts> -using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; - -// --------------------------------------------------------------------------- -// conjunction -// -// Performs a compile-time logical AND operation on the passed types (which -// must have `::value` members convertible to `bool`. Short-circuits if it -// encounters any `false` members (and does not compare the `::value` members -// of any remaining arguments). -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::conjunction` metafunction. -// --------------------------------------------------------------------------- -template <typename... Ts> -struct conjunction; - -template <typename T, typename... Ts> -struct conjunction<T, Ts...> - : std::conditional<T::value, conjunction<Ts...>, T>::type {}; - -template <typename T> -struct conjunction<T> : T {}; - -template <> -struct conjunction<> : std::true_type {}; - -// --------------------------------------------------------------------------- -// disjunction -// -// Performs a compile-time logical OR operation on the passed types (which -// must have `::value` members convertible to `bool`. Short-circuits if it -// encounters any `true` members (and does not compare the `::value` members -// of any remaining arguments). -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::disjunction` metafunction. -// --------------------------------------------------------------------------- -template <typename... Ts> -struct disjunction; - -template <typename T, typename... Ts> -struct disjunction<T, Ts...> : - std::conditional<T::value, T, disjunction<Ts...>>::type {}; - -template <typename T> -struct disjunction<T> : T {}; - -template <> -struct disjunction<> : std::false_type {}; - -template <typename T> -struct negation : std::integral_constant<bool, !T::value> {}; - -template <typename T> -struct is_trivially_destructible - : std::integral_constant<bool, __has_trivial_destructor(T) && - std::is_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_default_constructible - : std::integral_constant<bool, __has_trivial_constructor(T) && - std::is_default_constructible<T>::value && - is_trivially_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_copy_constructible - : std::integral_constant<bool, __has_trivial_copy(T) && - std::is_copy_constructible<T>::value && - is_trivially_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_copy_assignable - : std::integral_constant< - bool, __has_trivial_assign(typename std::remove_reference<T>::type) && - phmap::is_copy_assignable<T>::value> {}; - -// ----------------------------------------------------------------------------- -// C++14 "_t" trait aliases -// ----------------------------------------------------------------------------- - -template <typename T> -using remove_cv_t = typename std::remove_cv<T>::type; - -template <typename T> -using remove_const_t = typename std::remove_const<T>::type; - -template <typename T> -using remove_volatile_t = typename std::remove_volatile<T>::type; - -template <typename T> -using add_cv_t = typename std::add_cv<T>::type; - -template <typename T> -using add_const_t = typename std::add_const<T>::type; - -template <typename T> -using add_volatile_t = typename std::add_volatile<T>::type; - -template <typename T> -using remove_reference_t = typename std::remove_reference<T>::type; - -template <typename T> -using add_lvalue_reference_t = typename std::add_lvalue_reference<T>::type; - -template <typename T> -using add_rvalue_reference_t = typename std::add_rvalue_reference<T>::type; - -template <typename T> -using remove_pointer_t = typename std::remove_pointer<T>::type; - -template <typename T> -using add_pointer_t = typename std::add_pointer<T>::type; - -template <typename T> -using make_signed_t = typename std::make_signed<T>::type; - -template <typename T> -using make_unsigned_t = typename std::make_unsigned<T>::type; - -template <typename T> -using remove_extent_t = typename std::remove_extent<T>::type; - -template <typename T> -using remove_all_extents_t = typename std::remove_all_extents<T>::type; - -template <size_t Len, size_t Align = type_traits_internal:: - default_alignment_of_aligned_storage<Len>::value> -using aligned_storage_t = typename std::aligned_storage<Len, Align>::type; - -template <typename T> -using decay_t = typename std::decay<T>::type; - -template <bool B, typename T = void> -using enable_if_t = typename std::enable_if<B, T>::type; - -template <bool B, typename T, typename F> -using conditional_t = typename std::conditional<B, T, F>::type; - - -template <typename... T> -using common_type_t = typename std::common_type<T...>::type; - -template <typename T> -using underlying_type_t = typename std::underlying_type<T>::type; - -template< class F, class... ArgTypes> -#if PHMAP_HAVE_CC17 - using invoke_result = typename std::invoke_result<F, ArgTypes...>; -#else - using invoke_result = typename std::result_of<F(ArgTypes...)>::type; -#endif - -namespace type_traits_internal { - -// ---------------------------------------------------------------------- -// In MSVC we can't probe std::hash or stdext::hash because it triggers a -// static_assert instead of failing substitution. Libc++ prior to 4.0 -// also used a static_assert. -// ---------------------------------------------------------------------- -#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \ - _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 -#else - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 -#endif - -#if !PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template <typename Key, typename = size_t> - struct IsHashable : std::true_type {}; -#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template <typename Key, typename = void> - struct IsHashable : std::false_type {}; - - template <typename Key> - struct IsHashable<Key, - phmap::enable_if_t<std::is_convertible< - decltype(std::declval<std::hash<Key>&>()(std::declval<Key const&>())), - std::size_t>::value>> : std::true_type {}; -#endif - -struct AssertHashEnabledHelper -{ -private: - static void Sink(...) {} - struct NAT {}; - - template <class Key> - static auto GetReturnType(int) - -> decltype(std::declval<std::hash<Key>>()(std::declval<Key const&>())); - template <class Key> - static NAT GetReturnType(...); - - template <class Key> - static std::nullptr_t DoIt() { - static_assert(IsHashable<Key>::value, - "std::hash<Key> does not provide a call operator"); - static_assert( - std::is_default_constructible<std::hash<Key>>::value, - "std::hash<Key> must be default constructible when it is enabled"); - static_assert( - std::is_copy_constructible<std::hash<Key>>::value, - "std::hash<Key> must be copy constructible when it is enabled"); - static_assert(phmap::is_copy_assignable<std::hash<Key>>::value, - "std::hash<Key> must be copy assignable when it is enabled"); - // is_destructible is unchecked as it's implied by each of the - // is_constructible checks. - using ReturnType = decltype(GetReturnType<Key>(0)); - static_assert(std::is_same<ReturnType, NAT>::value || - std::is_same<ReturnType, size_t>::value, - "std::hash<Key> must return size_t"); - return nullptr; - } - - template <class... Ts> - friend void AssertHashEnabled(); -}; - -template <class... Ts> -inline void AssertHashEnabled -() -{ - using Helper = AssertHashEnabledHelper; - Helper::Sink(Helper::DoIt<Ts>()...); -} - -} // namespace type_traits_internal - -} // namespace phmap - - -// ----------------------------------------------------------------------------- -// hash_policy_traits -// ----------------------------------------------------------------------------- -namespace phmap { -namespace priv { - -// Defines how slots are initialized/destroyed/moved. -template <class Policy, class = void> -struct hash_policy_traits -{ -private: - struct ReturnKey - { - // We return `Key` here. - // When Key=T&, we forward the lvalue reference. - // When Key=T, we return by value to avoid a dangling reference. - // eg, for string_hash_map. - template <class Key, class... Args> - Key operator()(Key&& k, const Args&...) const { - return std::forward<Key>(k); - } - }; - - template <class P = Policy, class = void> - struct ConstantIteratorsImpl : std::false_type {}; - - template <class P> - struct ConstantIteratorsImpl<P, phmap::void_t<typename P::constant_iterators>> - : P::constant_iterators {}; - -public: - // The actual object stored in the hash table. - using slot_type = typename Policy::slot_type; - - // The type of the keys stored in the hashtable. - using key_type = typename Policy::key_type; - - // The argument type for insertions into the hashtable. This is different - // from value_type for increased performance. See initializer_list constructor - // and insert() member functions for more details. - using init_type = typename Policy::init_type; - - using reference = decltype(Policy::element(std::declval<slot_type*>())); - using pointer = typename std::remove_reference<reference>::type*; - using value_type = typename std::remove_reference<reference>::type; - - // Policies can set this variable to tell raw_hash_set that all iterators - // should be constant, even `iterator`. This is useful for set-like - // containers. - // Defaults to false if not provided by the policy. - using constant_iterators = ConstantIteratorsImpl<>; - - // PRECONDITION: `slot` is UNINITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class Alloc, class... Args> - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - Policy::construct(alloc, slot, std::forward<Args>(args)...); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is UNINITIALIZED - template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); - } - - // Transfers the `old_slot` to `new_slot`. Any memory allocated by the - // allocator inside `old_slot` to `new_slot` can be transferred. - // - // OPTIONAL: defaults to: - // - // clone(new_slot, std::move(*old_slot)); - // destroy(old_slot); - // - // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED - // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is - // UNINITIALIZED - template <class Alloc> - static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, 0); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class P = Policy> - static auto element(slot_type* slot) -> decltype(P::element(slot)) { - return P::element(slot); - } - - // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. - // - // If `slot` is nullptr, returns the constant amount of memory owned by any - // full slot or -1 if slots own variable amounts of memory. - // - // PRECONDITION: `slot` is INITIALIZED or nullptr - template <class P = Policy> - static size_t space_used(const slot_type* slot) { - return P::space_used(slot); - } - - // Provides generalized access to the key for elements, both for elements in - // the table and for elements that have not yet been inserted (or even - // constructed). We would like an API that allows us to say: `key(args...)` - // but we cannot do that for all cases, so we use this more general API that - // can be used for many things, including the following: - // - // - Given an element in a table, get its key. - // - Given an element initializer, get its key. - // - Given `emplace()` arguments, get the element key. - // - // Implementations of this must adhere to a very strict technical - // specification around aliasing and consuming arguments: - // - // Let `value_type` be the result type of `element()` without ref- and - // cv-qualifiers. The first argument is a functor, the rest are constructor - // arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where - // `k` is the element key, and `xs...` are the new constructor arguments for - // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias - // `ts...`. The key won't be touched once `xs...` are used to construct an - // element; `ts...` won't be touched at all, which allows `apply()` to consume - // any rvalues among them. - // - // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not - // trigger a hard compile error unless it originates from `f`. In other words, - // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not - // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. - // - // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, - // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. - template <class F, class... Ts, class P = Policy> - static auto apply(F&& f, Ts&&... ts) - -> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) { - return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...); - } - - // Returns the "key" portion of the slot. - // Used for node handle manipulation. - template <class P = Policy> - static auto key(slot_type* slot) - -> decltype(P::apply(ReturnKey(), element(slot))) { - return P::apply(ReturnKey(), element(slot)); - } - - // Returns the "value" (as opposed to the "key") portion of the element. Used - // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. - template <class T, class P = Policy> - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - -private: - - // Use auto -> decltype as an enabler. - template <class Alloc, class P = Policy> - static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, int) - -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { - P::transfer(alloc, new_slot, old_slot); - } - - template <class Alloc> - static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, char) { - construct(alloc, new_slot, std::move(element(old_slot))); - destroy(alloc, old_slot); - } -}; - -} // namespace priv -} // namespace phmap - -// ----------------------------------------------------------------------------- -// file utility.h -// ----------------------------------------------------------------------------- - -// --------- identity.h -namespace phmap { -namespace internal { - -template <typename T> -struct identity { - typedef T type; -}; - -template <typename T> -using identity_t = typename identity<T>::type; - -} // namespace internal -} // namespace phmap - - -// --------- inline_variable.h - -#ifdef __cpp_inline_variables - -#if defined(__clang__) - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) \ - extern const ::phmap::internal::identity_t<type> name; -#else // Otherwise, just define the macro to do nothing. - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) -#endif // defined(__clang__) - -// See above comment at top of file for details. -#define PHMAP_INTERNAL_INLINE_CONSTEXPR(type, name, init) \ - PHMAP_INTERNAL_EXTERN_DECL(type, name) \ - inline constexpr ::phmap::internal::identity_t<type> name = init - -#else - -// See above comment at top of file for details. -// -// Note: -// identity_t is used here so that the const and name are in the -// appropriate place for pointer types, reference types, function pointer -// types, etc.. -#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ - template <class /*PhmapInternalDummy*/ = void> \ - struct PhmapInternalInlineVariableHolder##name { \ - static constexpr ::phmap::internal::identity_t<var_type> kInstance = init; \ - }; \ - \ - template <class PhmapInternalDummy> \ - constexpr ::phmap::internal::identity_t<var_type> \ - PhmapInternalInlineVariableHolder##name<PhmapInternalDummy>::kInstance; \ - \ - static constexpr const ::phmap::internal::identity_t<var_type>& \ - name = /* NOLINT */ \ - PhmapInternalInlineVariableHolder##name<>::kInstance; \ - static_assert(sizeof(void (*)(decltype(name))) != 0, \ - "Silence unused variable warnings.") - -#endif // __cpp_inline_variables - -// ----------- throw_delegate - -namespace phmap { -namespace base_internal { - -namespace { -template <typename T> -#ifdef PHMAP_HAVE_EXCEPTIONS -[[noreturn]] void Throw(const T& error) { - throw error; -} -#else -[[noreturn]] void Throw(const T&) { - std::abort(); -} -#endif -} // namespace - -static inline void ThrowStdLogicError(const std::string& what_arg) { - Throw(std::logic_error(what_arg)); -} -static inline void ThrowStdLogicError(const char* what_arg) { - Throw(std::logic_error(what_arg)); -} -static inline void ThrowStdInvalidArgument(const std::string& what_arg) { - Throw(std::invalid_argument(what_arg)); -} -static inline void ThrowStdInvalidArgument(const char* what_arg) { - Throw(std::invalid_argument(what_arg)); -} - -static inline void ThrowStdDomainError(const std::string& what_arg) { - Throw(std::domain_error(what_arg)); -} -static inline void ThrowStdDomainError(const char* what_arg) { - Throw(std::domain_error(what_arg)); -} - -static inline void ThrowStdLengthError(const std::string& what_arg) { - Throw(std::length_error(what_arg)); -} -static inline void ThrowStdLengthError(const char* what_arg) { - Throw(std::length_error(what_arg)); -} - -static inline void ThrowStdOutOfRange(const std::string& what_arg) { - Throw(std::out_of_range(what_arg)); -} -static inline void ThrowStdOutOfRange(const char* what_arg) { - Throw(std::out_of_range(what_arg)); -} - -static inline void ThrowStdRuntimeError(const std::string& what_arg) { - Throw(std::runtime_error(what_arg)); -} -static inline void ThrowStdRuntimeError(const char* what_arg) { - Throw(std::runtime_error(what_arg)); -} - -static inline void ThrowStdRangeError(const std::string& what_arg) { - Throw(std::range_error(what_arg)); -} -static inline void ThrowStdRangeError(const char* what_arg) { - Throw(std::range_error(what_arg)); -} - -static inline void ThrowStdOverflowError(const std::string& what_arg) { - Throw(std::overflow_error(what_arg)); -} -static inline void ThrowStdOverflowError(const char* what_arg) { - Throw(std::overflow_error(what_arg)); -} - -static inline void ThrowStdUnderflowError(const std::string& what_arg) { - Throw(std::underflow_error(what_arg)); -} -static inline void ThrowStdUnderflowError(const char* what_arg) { - Throw(std::underflow_error(what_arg)); -} - -static inline void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); } - -static inline void ThrowStdBadAlloc() { Throw(std::bad_alloc()); } - -} // namespace base_internal -} // namespace phmap - -// ----------- invoke.h - -namespace phmap { -namespace base_internal { - -template <typename Derived> -struct StrippedAccept -{ - template <typename... Args> - struct Accept : Derived::template AcceptImpl<typename std::remove_cv< - typename std::remove_reference<Args>::type>::type...> {}; -}; - -// (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T -// and t1 is an object of type T or a reference to an object of type T or a -// reference to an object of a type derived from T. -struct MemFunAndRef : StrippedAccept<MemFunAndRef> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename... Params, typename Obj, - typename... Args> - struct AcceptImpl<R (C::*)(Params...), Obj, Args...> - : std::is_base_of<C, Obj> {}; - - template <typename R, typename C, typename... Params, typename Obj, - typename... Args> - struct AcceptImpl<R (C::*)(Params...) const, Obj, Args...> - : std::is_base_of<C, Obj> {}; - - template <typename MemFun, typename Obj, typename... Args> - static decltype((std::declval<Obj>().* - std::declval<MemFun>())(std::declval<Args>()...)) - Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { - return (std::forward<Obj>(obj).* - std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); - } -}; - -// ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a -// class T and t1 is not one of the types described in the previous item. -struct MemFunAndPtr : StrippedAccept<MemFunAndPtr> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename... Params, typename Ptr, - typename... Args> - struct AcceptImpl<R (C::*)(Params...), Ptr, Args...> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename R, typename C, typename... Params, typename Ptr, - typename... Args> - struct AcceptImpl<R (C::*)(Params...) const, Ptr, Args...> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename MemFun, typename Ptr, typename... Args> - static decltype(((*std::declval<Ptr>()).* - std::declval<MemFun>())(std::declval<Args>()...)) - Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { - return ((*std::forward<Ptr>(ptr)).* - std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); - } -}; - -// t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is -// an object of type T or a reference to an object of type T or a reference -// to an object of a type derived from T. -struct DataMemAndRef : StrippedAccept<DataMemAndRef> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename Obj> - struct AcceptImpl<R C::*, Obj> : std::is_base_of<C, Obj> {}; - - template <typename DataMem, typename Ref> - static decltype(std::declval<Ref>().*std::declval<DataMem>()) Invoke( - DataMem&& data_mem, Ref&& ref) { - return std::forward<Ref>(ref).*std::forward<DataMem>(data_mem); - } -}; - -// (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 -// is not one of the types described in the previous item. -struct DataMemAndPtr : StrippedAccept<DataMemAndPtr> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename Ptr> - struct AcceptImpl<R C::*, Ptr> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename DataMem, typename Ptr> - static decltype((*std::declval<Ptr>()).*std::declval<DataMem>()) Invoke( - DataMem&& data_mem, Ptr&& ptr) { - return (*std::forward<Ptr>(ptr)).*std::forward<DataMem>(data_mem); - } -}; - -// f(t1, t2, ..., tN) in all other cases. -struct Callable -{ - // Callable doesn't have Accept because it's the last clause that gets picked - // when none of the previous clauses are applicable. - template <typename F, typename... Args> - static decltype(std::declval<F>()(std::declval<Args>()...)) Invoke( - F&& f, Args&&... args) { - return std::forward<F>(f)(std::forward<Args>(args)...); - } -}; - -// Resolves to the first matching clause. -template <typename... Args> -struct Invoker -{ - typedef typename std::conditional< - MemFunAndRef::Accept<Args...>::value, MemFunAndRef, - typename std::conditional< - MemFunAndPtr::Accept<Args...>::value, MemFunAndPtr, - typename std::conditional< - DataMemAndRef::Accept<Args...>::value, DataMemAndRef, - typename std::conditional<DataMemAndPtr::Accept<Args...>::value, - DataMemAndPtr, Callable>::type>::type>:: - type>::type type; -}; - -// The result type of Invoke<F, Args...>. -template <typename F, typename... Args> -using InvokeT = decltype(Invoker<F, Args...>::type::Invoke( - std::declval<F>(), std::declval<Args>()...)); - -// Invoke(f, args...) is an implementation of INVOKE(f, args...) from section -// [func.require] of the C++ standard. -template <typename F, typename... Args> -InvokeT<F, Args...> Invoke(F&& f, Args&&... args) { - return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), - std::forward<Args>(args)...); -} -} // namespace base_internal -} // namespace phmap - - -// ----------- utility.h - -namespace phmap { - -// integer_sequence -// -// Class template representing a compile-time integer sequence. An instantiation -// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its -// type through its template arguments (which is a common need when -// working with C++11 variadic templates). `phmap::integer_sequence` is designed -// to be a drop-in replacement for C++14's `std::integer_sequence`. -// -// Example: -// -// template< class T, T... Ints > -// void user_function(integer_sequence<T, Ints...>); -// -// int main() -// { -// // user_function's `T` will be deduced to `int` and `Ints...` -// // will be deduced to `0, 1, 2, 3, 4`. -// user_function(make_integer_sequence<int, 5>()); -// } -template <typename T, T... Ints> -struct integer_sequence -{ - using value_type = T; - static constexpr size_t size() noexcept { return sizeof...(Ints); } -}; - -// index_sequence -// -// A helper template for an `integer_sequence` of `size_t`, -// `phmap::index_sequence` is designed to be a drop-in replacement for C++14's -// `std::index_sequence`. -template <size_t... Ints> -using index_sequence = integer_sequence<size_t, Ints...>; - -namespace utility_internal { - -template <typename Seq, size_t SeqSize, size_t Rem> -struct Extend; - -// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>; -}; - -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>; -}; - -// Recursion helper for 'make_integer_sequence<T, N>'. -// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'. -template <typename T, size_t N> -struct Gen { - using type = - typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type; -}; - -template <typename T> -struct Gen<T, 0> { - using type = integer_sequence<T>; -}; - -} // namespace utility_internal - -// Compile-time sequences of integers - -// make_integer_sequence -// -// This template alias is equivalent to -// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in -// replacement for C++14's `std::make_integer_sequence`. -template <typename T, T N> -using make_integer_sequence = typename utility_internal::Gen<T, N>::type; - -// make_index_sequence -// -// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, -// and is designed to be a drop-in replacement for C++14's -// `std::make_index_sequence`. -template <size_t N> -using make_index_sequence = make_integer_sequence<size_t, N>; - -// index_sequence_for -// -// Converts a typename pack into an index sequence of the same length, and -// is designed to be a drop-in replacement for C++14's -// `std::index_sequence_for()` -template <typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; - -// Tag types - -#ifdef PHMAP_HAVE_STD_OPTIONAL - -using std::in_place_t; -using std::in_place; - -#else // PHMAP_HAVE_STD_OPTIONAL - -// in_place_t -// -// Tag type used to specify in-place construction, such as with -// `phmap::optional`, designed to be a drop-in replacement for C++17's -// `std::in_place_t`. -struct in_place_t {}; - -PHMAP_INTERNAL_INLINE_CONSTEXPR(in_place_t, in_place, {}); - -#endif // PHMAP_HAVE_STD_OPTIONAL - -#if defined(PHMAP_HAVE_STD_ANY) || defined(PHMAP_HAVE_STD_VARIANT) -using std::in_place_type_t; -#else - -// in_place_type_t -// -// Tag type used for in-place construction when the type to construct needs to -// be specified, such as with `phmap::any`, designed to be a drop-in replacement -// for C++17's `std::in_place_type_t`. -template <typename T> -struct in_place_type_t {}; -#endif // PHMAP_HAVE_STD_ANY || PHMAP_HAVE_STD_VARIANT - -#ifdef PHMAP_HAVE_STD_VARIANT -using std::in_place_index_t; -#else - -// in_place_index_t -// -// Tag type used for in-place construction when the type to construct needs to -// be specified, such as with `phmap::any`, designed to be a drop-in replacement -// for C++17's `std::in_place_index_t`. -template <size_t I> -struct in_place_index_t {}; -#endif // PHMAP_HAVE_STD_VARIANT - -// Constexpr move and forward - -// move() -// -// A constexpr version of `std::move()`, designed to be a drop-in replacement -// for C++14's `std::move()`. -template <typename T> -constexpr phmap::remove_reference_t<T>&& move(T&& t) noexcept { - return static_cast<phmap::remove_reference_t<T>&&>(t); -} - -// forward() -// -// A constexpr version of `std::forward()`, designed to be a drop-in replacement -// for C++14's `std::forward()`. -template <typename T> -constexpr T&& forward( - phmap::remove_reference_t<T>& t) noexcept { // NOLINT(runtime/references) - return static_cast<T&&>(t); -} - -namespace utility_internal { -// Helper method for expanding tuple into a called method. -template <typename Functor, typename Tuple, std::size_t... Indexes> -auto apply_helper(Functor&& functor, Tuple&& t, index_sequence<Indexes...>) - -> decltype(phmap::base_internal::Invoke( - phmap::forward<Functor>(functor), - std::get<Indexes>(phmap::forward<Tuple>(t))...)) { - return phmap::base_internal::Invoke( - phmap::forward<Functor>(functor), - std::get<Indexes>(phmap::forward<Tuple>(t))...); -} - -} // namespace utility_internal - -// apply -// -// Invokes a Callable using elements of a tuple as its arguments. -// Each element of the tuple corresponds to an argument of the call (in order). -// Both the Callable argument and the tuple argument are perfect-forwarded. -// For member-function Callables, the first tuple element acts as the `this` -// pointer. `phmap::apply` is designed to be a drop-in replacement for C++17's -// `std::apply`. Unlike C++17's `std::apply`, this is not currently `constexpr`. -// -// Example: -// -// class Foo { -// public: -// void Bar(int); -// }; -// void user_function1(int, std::string); -// void user_function2(std::unique_ptr<Foo>); -// auto user_lambda = [](int, int) {}; -// -// int main() -// { -// std::tuple<int, std::string> tuple1(42, "bar"); -// // Invokes the first user function on int, std::string. -// phmap::apply(&user_function1, tuple1); -// -// std::tuple<std::unique_ptr<Foo>> tuple2(phmap::make_unique<Foo>()); -// // Invokes the user function that takes ownership of the unique -// // pointer. -// phmap::apply(&user_function2, std::move(tuple2)); -// -// auto foo = phmap::make_unique<Foo>(); -// std::tuple<Foo*, int> tuple3(foo.get(), 42); -// // Invokes the method Bar on foo with one argument, 42. -// phmap::apply(&Foo::Bar, tuple3); -// -// std::tuple<int, int> tuple4(8, 9); -// // Invokes a lambda. -// phmap::apply(user_lambda, tuple4); -// } -template <typename Functor, typename Tuple> -auto apply(Functor&& functor, Tuple&& t) - -> decltype(utility_internal::apply_helper( - phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), - phmap::make_index_sequence<std::tuple_size< - typename std::remove_reference<Tuple>::type>::value>{})) { - return utility_internal::apply_helper( - phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), - phmap::make_index_sequence<std::tuple_size< - typename std::remove_reference<Tuple>::type>::value>{}); -} - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch -#endif // _MSC_VER - -// exchange -// -// Replaces the value of `obj` with `new_value` and returns the old value of -// `obj`. `phmap::exchange` is designed to be a drop-in replacement for C++14's -// `std::exchange`. -// -// Example: -// -// Foo& operator=(Foo&& other) { -// ptr1_ = phmap::exchange(other.ptr1_, nullptr); -// int1_ = phmap::exchange(other.int1_, -1); -// return *this; -// } -template <typename T, typename U = T> -T exchange(T& obj, U&& new_value) -{ - T old_value = phmap::move(obj); - obj = phmap::forward<U>(new_value); - return old_value; -} - -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - - -} // namespace phmap - -// ----------------------------------------------------------------------------- -// memory.h -// ----------------------------------------------------------------------------- - -namespace phmap { - -template <typename T> -std::unique_ptr<T> WrapUnique(T* ptr) -{ - static_assert(!std::is_array<T>::value, "array types are unsupported"); - static_assert(std::is_object<T>::value, "non-object types are unsupported"); - return std::unique_ptr<T>(ptr); -} - -namespace memory_internal { - -// Traits to select proper overload and return type for `phmap::make_unique<>`. -template <typename T> -struct MakeUniqueResult { - using scalar = std::unique_ptr<T>; -}; -template <typename T> -struct MakeUniqueResult<T[]> { - using array = std::unique_ptr<T[]>; -}; -template <typename T, size_t N> -struct MakeUniqueResult<T[N]> { - using invalid = void; -}; - -} // namespace memory_internal - -#if (__cplusplus > 201103L || defined(_MSC_VER)) && \ - !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) - using std::make_unique; -#else - - template <typename T, typename... Args> - typename memory_internal::MakeUniqueResult<T>::scalar make_unique( - Args&&... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); - } - - template <typename T> - typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) { - return std::unique_ptr<T>(new typename phmap::remove_extent_t<T>[n]()); - } - - template <typename T, typename... Args> - typename memory_internal::MakeUniqueResult<T>::invalid make_unique( - Args&&... /* args */) = delete; -#endif - -template <typename T> -auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) -{ - // ptr is a forwarding reference to support Ts with non-const operators. - return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; -} - -inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; } - -template <typename T, typename D> -std::shared_ptr<T> ShareUniquePtr(std::unique_ptr<T, D>&& ptr) { - return ptr ? std::shared_ptr<T>(std::move(ptr)) : std::shared_ptr<T>(); -} - -template <typename T> -std::weak_ptr<T> WeakenPtr(const std::shared_ptr<T>& ptr) { - return std::weak_ptr<T>(ptr); -} - -namespace memory_internal { - -// ExtractOr<E, O, D>::type evaluates to E<O> if possible. Otherwise, D. -template <template <typename> class Extract, typename Obj, typename Default, - typename> -struct ExtractOr { - using type = Default; -}; - -template <template <typename> class Extract, typename Obj, typename Default> -struct ExtractOr<Extract, Obj, Default, void_t<Extract<Obj>>> { - using type = Extract<Obj>; -}; - -template <template <typename> class Extract, typename Obj, typename Default> -using ExtractOrT = typename ExtractOr<Extract, Obj, Default, void>::type; - -// Extractors for the features of allocators. -template <typename T> -using GetPointer = typename T::pointer; - -template <typename T> -using GetConstPointer = typename T::const_pointer; - -template <typename T> -using GetVoidPointer = typename T::void_pointer; - -template <typename T> -using GetConstVoidPointer = typename T::const_void_pointer; - -template <typename T> -using GetDifferenceType = typename T::difference_type; - -template <typename T> -using GetSizeType = typename T::size_type; - -template <typename T> -using GetPropagateOnContainerCopyAssignment = - typename T::propagate_on_container_copy_assignment; - -template <typename T> -using GetPropagateOnContainerMoveAssignment = - typename T::propagate_on_container_move_assignment; - -template <typename T> -using GetPropagateOnContainerSwap = typename T::propagate_on_container_swap; - -template <typename T> -using GetIsAlwaysEqual = typename T::is_always_equal; - -template <typename T> -struct GetFirstArg; - -template <template <typename...> class Class, typename T, typename... Args> -struct GetFirstArg<Class<T, Args...>> { - using type = T; -}; - -template <typename Ptr, typename = void> -struct ElementType { - using type = typename GetFirstArg<Ptr>::type; -}; - -template <typename T> -struct ElementType<T, void_t<typename T::element_type>> { - using type = typename T::element_type; -}; - -template <typename T, typename U> -struct RebindFirstArg; - -template <template <typename...> class Class, typename T, typename... Args, - typename U> -struct RebindFirstArg<Class<T, Args...>, U> { - using type = Class<U, Args...>; -}; - -template <typename T, typename U, typename = void> -struct RebindPtr { - using type = typename RebindFirstArg<T, U>::type; -}; - -template <typename T, typename U> -struct RebindPtr<T, U, void_t<typename T::template rebind<U>>> { - using type = typename T::template rebind<U>; -}; - -template <typename T, typename U> -constexpr bool HasRebindAlloc(...) { - return false; -} - -template <typename T, typename U> -constexpr bool HasRebindAlloc(typename std::allocator_traits<T>::template rebind_alloc<U>*) { - return true; -} - -template <typename T, typename U, bool = HasRebindAlloc<T, U>(nullptr)> -struct RebindAlloc { - using type = typename RebindFirstArg<T, U>::type; -}; - -template <typename A, typename U> -struct RebindAlloc<A, U, true> { - using type = typename std::allocator_traits<A>::template rebind_alloc<U>; -}; - - -} // namespace memory_internal - -template <typename Ptr> -struct pointer_traits -{ - using pointer = Ptr; - - // element_type: - // Ptr::element_type if present. Otherwise T if Ptr is a template - // instantiation Template<T, Args...> - using element_type = typename memory_internal::ElementType<Ptr>::type; - - // difference_type: - // Ptr::difference_type if present, otherwise std::ptrdiff_t - using difference_type = - memory_internal::ExtractOrT<memory_internal::GetDifferenceType, Ptr, - std::ptrdiff_t>; - - // rebind: - // Ptr::rebind<U> if exists, otherwise Template<U, Args...> if Ptr is a - // template instantiation Template<T, Args...> - template <typename U> - using rebind = typename memory_internal::RebindPtr<Ptr, U>::type; - - // pointer_to: - // Calls Ptr::pointer_to(r) - static pointer pointer_to(element_type& r) { // NOLINT(runtime/references) - return Ptr::pointer_to(r); - } -}; - -// Specialization for T*. -template <typename T> -struct pointer_traits<T*> -{ - using pointer = T*; - using element_type = T; - using difference_type = std::ptrdiff_t; - - template <typename U> - using rebind = U*; - - // pointer_to: - // Calls std::addressof(r) - static pointer pointer_to( - element_type& r) noexcept { // NOLINT(runtime/references) - return std::addressof(r); - } -}; - -// ----------------------------------------------------------------------------- -// Class Template: allocator_traits -// ----------------------------------------------------------------------------- -// -// A C++11 compatible implementation of C++17's std::allocator_traits. -// -template <typename Alloc> -struct allocator_traits -{ - using allocator_type = Alloc; - - // value_type: - // Alloc::value_type - using value_type = typename Alloc::value_type; - - // pointer: - // Alloc::pointer if present, otherwise value_type* - using pointer = memory_internal::ExtractOrT<memory_internal::GetPointer, - Alloc, value_type*>; - - // const_pointer: - // Alloc::const_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<const value_type> - using const_pointer = - memory_internal::ExtractOrT<memory_internal::GetConstPointer, Alloc, - typename phmap::pointer_traits<pointer>:: - template rebind<const value_type>>; - - // void_pointer: - // Alloc::void_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<void> - using void_pointer = memory_internal::ExtractOrT< - memory_internal::GetVoidPointer, Alloc, - typename phmap::pointer_traits<pointer>::template rebind<void>>; - - // const_void_pointer: - // Alloc::const_void_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<const void> - using const_void_pointer = memory_internal::ExtractOrT< - memory_internal::GetConstVoidPointer, Alloc, - typename phmap::pointer_traits<pointer>::template rebind<const void>>; - - // difference_type: - // Alloc::difference_type if present, otherwise - // phmap::pointer_traits<pointer>::difference_type - using difference_type = memory_internal::ExtractOrT< - memory_internal::GetDifferenceType, Alloc, - typename phmap::pointer_traits<pointer>::difference_type>; - - // size_type: - // Alloc::size_type if present, otherwise - // std::make_unsigned<difference_type>::type - using size_type = memory_internal::ExtractOrT< - memory_internal::GetSizeType, Alloc, - typename std::make_unsigned<difference_type>::type>; - - // propagate_on_container_copy_assignment: - // Alloc::propagate_on_container_copy_assignment if present, otherwise - // std::false_type - using propagate_on_container_copy_assignment = memory_internal::ExtractOrT< - memory_internal::GetPropagateOnContainerCopyAssignment, Alloc, - std::false_type>; - - // propagate_on_container_move_assignment: - // Alloc::propagate_on_container_move_assignment if present, otherwise - // std::false_type - using propagate_on_container_move_assignment = memory_internal::ExtractOrT< - memory_internal::GetPropagateOnContainerMoveAssignment, Alloc, - std::false_type>; - - // propagate_on_container_swap: - // Alloc::propagate_on_container_swap if present, otherwise std::false_type - using propagate_on_container_swap = - memory_internal::ExtractOrT<memory_internal::GetPropagateOnContainerSwap, - Alloc, std::false_type>; - - // is_always_equal: - // Alloc::is_always_equal if present, otherwise std::is_empty<Alloc>::type - using is_always_equal = - memory_internal::ExtractOrT<memory_internal::GetIsAlwaysEqual, Alloc, - typename std::is_empty<Alloc>::type>; - - // rebind_alloc: - // Alloc::rebind<T>::other if present, otherwise Alloc<T, Args> if this Alloc - // is Alloc<U, Args> - template <typename T> - using rebind_alloc = typename memory_internal::RebindAlloc<Alloc, T>::type; - - // rebind_traits: - // phmap::allocator_traits<rebind_alloc<T>> - template <typename T> - using rebind_traits = phmap::allocator_traits<rebind_alloc<T>>; - - // allocate(Alloc& a, size_type n): - // Calls a.allocate(n) - static pointer allocate(Alloc& a, // NOLINT(runtime/references) - size_type n) { - return a.allocate(n); - } - - // allocate(Alloc& a, size_type n, const_void_pointer hint): - // Calls a.allocate(n, hint) if possible. - // If not possible, calls a.allocate(n) - static pointer allocate(Alloc& a, size_type n, // NOLINT(runtime/references) - const_void_pointer hint) { - return allocate_impl(0, a, n, hint); - } - - // deallocate(Alloc& a, pointer p, size_type n): - // Calls a.deallocate(p, n) - static void deallocate(Alloc& a, pointer p, // NOLINT(runtime/references) - size_type n) { - a.deallocate(p, n); - } - - // construct(Alloc& a, T* p, Args&&... args): - // Calls a.construct(p, std::forward<Args>(args)...) if possible. - // If not possible, calls - // ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...) - template <typename T, typename... Args> - static void construct(Alloc& a, T* p, // NOLINT(runtime/references) - Args&&... args) { - construct_impl(0, a, p, std::forward<Args>(args)...); - } - - // destroy(Alloc& a, T* p): - // Calls a.destroy(p) if possible. If not possible, calls p->~T(). - template <typename T> - static void destroy(Alloc& a, T* p) { // NOLINT(runtime/references) - destroy_impl(0, a, p); - } - - // max_size(const Alloc& a): - // Returns a.max_size() if possible. If not possible, returns - // std::numeric_limits<size_type>::max() / sizeof(value_type) - static size_type max_size(const Alloc& a) { return max_size_impl(0, a); } - - // select_on_container_copy_construction(const Alloc& a): - // Returns a.select_on_container_copy_construction() if possible. - // If not possible, returns a. - static Alloc select_on_container_copy_construction(const Alloc& a) { - return select_on_container_copy_construction_impl(0, a); - } - -private: - template <typename A> - static auto allocate_impl(int, A& a, // NOLINT(runtime/references) - size_type n, const_void_pointer hint) - -> decltype(a.allocate(n, hint)) { - return a.allocate(n, hint); - } - static pointer allocate_impl(char, Alloc& a, // NOLINT(runtime/references) - size_type n, const_void_pointer) { - return a.allocate(n); - } - - template <typename A, typename... Args> - static auto construct_impl(int, A& a, // NOLINT(runtime/references) - Args&&... args) - -> decltype(std::allocator_traits<A>::construct(a, std::forward<Args>(args)...)) { - std::allocator_traits<A>::construct(a, std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static void construct_impl(char, Alloc&, T* p, Args&&... args) { - ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...); - } - - template <typename A, typename T> - static auto destroy_impl(int, A& a, // NOLINT(runtime/references) - T* p) -> decltype(std::allocator_traits<A>::destroy(a, p)) { - std::allocator_traits<A>::destroy(a, p); - } - template <typename T> - static void destroy_impl(char, Alloc&, T* p) { - p->~T(); - } - - template <typename A> - static auto max_size_impl(int, const A& a) -> decltype(a.max_size()) { - return a.max_size(); - } - static size_type max_size_impl(char, const Alloc&) { - return (std::numeric_limits<size_type>::max)() / sizeof(value_type); - } - - template <typename A> - static auto select_on_container_copy_construction_impl(int, const A& a) - -> decltype(a.select_on_container_copy_construction()) { - return a.select_on_container_copy_construction(); - } - static Alloc select_on_container_copy_construction_impl(char, - const Alloc& a) { - return a; - } -}; - -namespace memory_internal { - -// This template alias transforms Alloc::is_nothrow into a metafunction with -// Alloc as a parameter so it can be used with ExtractOrT<>. -template <typename Alloc> -using GetIsNothrow = typename Alloc::is_nothrow; - -} // namespace memory_internal - -// PHMAP_ALLOCATOR_NOTHROW is a build time configuration macro for user to -// specify whether the default allocation function can throw or never throws. -// If the allocation function never throws, user should define it to a non-zero -// value (e.g. via `-DPHMAP_ALLOCATOR_NOTHROW`). -// If the allocation function can throw, user should leave it undefined or -// define it to zero. -// -// allocator_is_nothrow<Alloc> is a traits class that derives from -// Alloc::is_nothrow if present, otherwise std::false_type. It's specialized -// for Alloc = std::allocator<T> for any type T according to the state of -// PHMAP_ALLOCATOR_NOTHROW. -// -// default_allocator_is_nothrow is a class that derives from std::true_type -// when the default allocator (global operator new) never throws, and -// std::false_type when it can throw. It is a convenience shorthand for writing -// allocator_is_nothrow<std::allocator<T>> (T can be any type). -// NOTE: allocator_is_nothrow<std::allocator<T>> is guaranteed to derive from -// the same type for all T, because users should specialize neither -// allocator_is_nothrow nor std::allocator. -template <typename Alloc> -struct allocator_is_nothrow - : memory_internal::ExtractOrT<memory_internal::GetIsNothrow, Alloc, - std::false_type> {}; - -#if defined(PHMAP_ALLOCATOR_NOTHROW) && PHMAP_ALLOCATOR_NOTHROW - template <typename T> - struct allocator_is_nothrow<std::allocator<T>> : std::true_type {}; - struct default_allocator_is_nothrow : std::true_type {}; -#else - struct default_allocator_is_nothrow : std::false_type {}; -#endif - -namespace memory_internal { -template <typename Allocator, typename Iterator, typename... Args> -void ConstructRange(Allocator& alloc, Iterator first, Iterator last, - const Args&... args) -{ - for (Iterator cur = first; cur != last; ++cur) { - PHMAP_INTERNAL_TRY { - std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), - args...); - } - PHMAP_INTERNAL_CATCH_ANY { - while (cur != first) { - --cur; - std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); - } - PHMAP_INTERNAL_RETHROW; - } - } -} - -template <typename Allocator, typename Iterator, typename InputIterator> -void CopyRange(Allocator& alloc, Iterator destination, InputIterator first, - InputIterator last) -{ - for (Iterator cur = destination; first != last; - static_cast<void>(++cur), static_cast<void>(++first)) { - PHMAP_INTERNAL_TRY { - std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), - *first); - } - PHMAP_INTERNAL_CATCH_ANY { - while (cur != destination) { - --cur; - std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); - } - PHMAP_INTERNAL_RETHROW; - } - } -} -} // namespace memory_internal -} // namespace phmap - - -// ----------------------------------------------------------------------------- -// optional.h -// ----------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_OPTIONAL - -#include <optional> // IWYU pragma: export - -namespace phmap { -using std::bad_optional_access; -using std::optional; -using std::make_optional; -using std::nullopt_t; -using std::nullopt; -} // namespace phmap - -#else - -#if defined(__clang__) - #if __has_feature(cxx_inheriting_constructors) - #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 - #endif -#elif (defined(__GNUC__) && \ - (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8)) || \ - (__cpp_inheriting_constructors >= 200802) || \ - (defined(_MSC_VER) && _MSC_VER >= 1910) - - #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 -#endif - -namespace phmap { - -class bad_optional_access : public std::exception -{ -public: - bad_optional_access() = default; - ~bad_optional_access() override; - const char* what() const noexcept override; -}; - -template <typename T> -class optional; - -// -------------------------------- -struct nullopt_t -{ - struct init_t {}; - static init_t init; - - explicit constexpr nullopt_t(init_t& /*unused*/) {} -}; - -constexpr nullopt_t nullopt(nullopt_t::init); - -namespace optional_internal { - -// throw delegator -[[noreturn]] void throw_bad_optional_access(); - - -struct empty_struct {}; - -// This class stores the data in optional<T>. -// It is specialized based on whether T is trivially destructible. -// This is the specialization for non trivially destructible type. -template <typename T, bool unused = std::is_trivially_destructible<T>::value> -class optional_data_dtor_base -{ - struct dummy_type { - static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); - // Use an array to avoid GCC 6 placement-new warning. - empty_struct data[sizeof(T) / sizeof(empty_struct)]; - }; - -protected: - // Whether there is data or not. - bool engaged_; - // Data storage - union { - dummy_type dummy_; - T data_; - }; - - void destruct() noexcept { - if (engaged_) { - data_.~T(); - engaged_ = false; - } - } - - // dummy_ must be initialized for constexpr constructor. - constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} - - template <typename... Args> - constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(phmap::forward<Args>(args)...) {} - - ~optional_data_dtor_base() { destruct(); } -}; - -// Specialization for trivially destructible type. -template <typename T> -class optional_data_dtor_base<T, true> -{ - struct dummy_type { - static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); - // Use array to avoid GCC 6 placement-new warning. - empty_struct data[sizeof(T) / sizeof(empty_struct)]; - }; - -protected: - // Whether there is data or not. - bool engaged_; - // Data storage - union { - dummy_type dummy_; - T data_; - }; - void destruct() noexcept { engaged_ = false; } - - // dummy_ must be initialized for constexpr constructor. - constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} - - template <typename... Args> - constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(phmap::forward<Args>(args)...) {} -}; - -template <typename T> -class optional_data_base : public optional_data_dtor_base<T> -{ -protected: - using base = optional_data_dtor_base<T>; -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using base::base; -#else - optional_data_base() = default; - - template <typename... Args> - constexpr explicit optional_data_base(in_place_t t, Args&&... args) - : base(t, phmap::forward<Args>(args)...) {} -#endif - - template <typename... Args> - void construct(Args&&... args) { - // Use dummy_'s address to work around casting cv-qualified T* to void*. - ::new (static_cast<void*>(&this->dummy_)) T(std::forward<Args>(args)...); - this->engaged_ = true; - } - - template <typename U> - void assign(U&& u) { - if (this->engaged_) { - this->data_ = std::forward<U>(u); - } else { - construct(std::forward<U>(u)); - } - } -}; - -// TODO: Add another class using -// std::is_trivially_move_constructible trait when available to match -// http://cplusplus.github.io/LWG/lwg-defects.html#2900, for types that -// have trivial move but nontrivial copy. -// Also, we should be checking is_trivially_copyable here, which is not -// supported now, so we use is_trivially_* traits instead. -template <typename T, - bool unused = phmap::is_trivially_copy_constructible<T>::value&& - phmap::is_trivially_copy_assignable<typename std::remove_cv< - T>::type>::value&& std::is_trivially_destructible<T>::value> -class optional_data; - -// Trivially copyable types -template <typename T> -class optional_data<T, true> : public optional_data_base<T> -{ -protected: -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using optional_data_base<T>::optional_data_base; -#else - optional_data() = default; - - template <typename... Args> - constexpr explicit optional_data(in_place_t t, Args&&... args) - : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} -#endif -}; - -template <typename T> -class optional_data<T, false> : public optional_data_base<T> -{ -protected: -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using optional_data_base<T>::optional_data_base; -#else - template <typename... Args> - constexpr explicit optional_data(in_place_t t, Args&&... args) - : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} -#endif - - optional_data() = default; - - optional_data(const optional_data& rhs) : optional_data_base<T>() { - if (rhs.engaged_) { - this->construct(rhs.data_); - } - } - - optional_data(optional_data&& rhs) noexcept( - phmap::default_allocator_is_nothrow::value || - std::is_nothrow_move_constructible<T>::value) - : optional_data_base<T>() { - if (rhs.engaged_) { - this->construct(std::move(rhs.data_)); - } - } - - optional_data& operator=(const optional_data& rhs) { - if (rhs.engaged_) { - this->assign(rhs.data_); - } else { - this->destruct(); - } - return *this; - } - - optional_data& operator=(optional_data&& rhs) noexcept( - std::is_nothrow_move_assignable<T>::value&& - std::is_nothrow_move_constructible<T>::value) { - if (rhs.engaged_) { - this->assign(std::move(rhs.data_)); - } else { - this->destruct(); - } - return *this; - } -}; - -// Ordered by level of restriction, from low to high. -// Copyable implies movable. -enum class copy_traits { copyable = 0, movable = 1, non_movable = 2 }; - -// Base class for enabling/disabling copy/move constructor. -template <copy_traits> -class optional_ctor_base; - -template <> -class optional_ctor_base<copy_traits::copyable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = default; - optional_ctor_base(optional_ctor_base&&) = default; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -template <> -class optional_ctor_base<copy_traits::movable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = delete; - optional_ctor_base(optional_ctor_base&&) = default; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -template <> -class optional_ctor_base<copy_traits::non_movable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = delete; - optional_ctor_base(optional_ctor_base&&) = delete; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -// Base class for enabling/disabling copy/move assignment. -template <copy_traits> -class optional_assign_base; - -template <> -class optional_assign_base<copy_traits::copyable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = default; - optional_assign_base& operator=(optional_assign_base&&) = default; -}; - -template <> -class optional_assign_base<copy_traits::movable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = delete; - optional_assign_base& operator=(optional_assign_base&&) = default; -}; - -template <> -class optional_assign_base<copy_traits::non_movable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = delete; - optional_assign_base& operator=(optional_assign_base&&) = delete; -}; - -template <typename T> -constexpr copy_traits get_ctor_copy_traits() -{ - return std::is_copy_constructible<T>::value - ? copy_traits::copyable - : std::is_move_constructible<T>::value ? copy_traits::movable - : copy_traits::non_movable; -} - -template <typename T> -constexpr copy_traits get_assign_copy_traits() -{ - return phmap::is_copy_assignable<T>::value && - std::is_copy_constructible<T>::value - ? copy_traits::copyable - : phmap::is_move_assignable<T>::value && - std::is_move_constructible<T>::value - ? copy_traits::movable - : copy_traits::non_movable; -} - -// Whether T is constructible or convertible from optional<U>. -template <typename T, typename U> -struct is_constructible_convertible_from_optional - : std::integral_constant< - bool, std::is_constructible<T, optional<U>&>::value || - std::is_constructible<T, optional<U>&&>::value || - std::is_constructible<T, const optional<U>&>::value || - std::is_constructible<T, const optional<U>&&>::value || - std::is_convertible<optional<U>&, T>::value || - std::is_convertible<optional<U>&&, T>::value || - std::is_convertible<const optional<U>&, T>::value || - std::is_convertible<const optional<U>&&, T>::value> {}; - -// Whether T is constructible or convertible or assignable from optional<U>. -template <typename T, typename U> -struct is_constructible_convertible_assignable_from_optional - : std::integral_constant< - bool, is_constructible_convertible_from_optional<T, U>::value || - std::is_assignable<T&, optional<U>&>::value || - std::is_assignable<T&, optional<U>&&>::value || - std::is_assignable<T&, const optional<U>&>::value || - std::is_assignable<T&, const optional<U>&&>::value> {}; - -// Helper function used by [optional.relops], [optional.comp_with_t], -// for checking whether an expression is convertible to bool. -bool convertible_to_bool(bool); - -// Base class for std::hash<phmap::optional<T>>: -// If std::hash<std::remove_const_t<T>> is enabled, it provides operator() to -// compute the hash; Otherwise, it is disabled. -// Reference N4659 23.14.15 [unord.hash]. -template <typename T, typename = size_t> -struct optional_hash_base -{ - optional_hash_base() = delete; - optional_hash_base(const optional_hash_base&) = delete; - optional_hash_base(optional_hash_base&&) = delete; - optional_hash_base& operator=(const optional_hash_base&) = delete; - optional_hash_base& operator=(optional_hash_base&&) = delete; -}; - -template <typename T> -struct optional_hash_base<T, decltype(std::hash<phmap::remove_const_t<T> >()( - std::declval<phmap::remove_const_t<T> >()))> -{ - using argument_type = phmap::optional<T>; - using result_type = size_t; - size_t operator()(const phmap::optional<T>& opt) const { - phmap::type_traits_internal::AssertHashEnabled<phmap::remove_const_t<T>>(); - if (opt) { - return std::hash<phmap::remove_const_t<T> >()(*opt); - } else { - return static_cast<size_t>(0x297814aaad196e6dULL); - } - } -}; - -} // namespace optional_internal - - -// ----------------------------------------------------------------------------- -// phmap::optional class definition -// ----------------------------------------------------------------------------- - -template <typename T> -class optional : private optional_internal::optional_data<T>, - private optional_internal::optional_ctor_base< - optional_internal::get_ctor_copy_traits<T>()>, - private optional_internal::optional_assign_base< - optional_internal::get_assign_copy_traits<T>()> -{ - using data_base = optional_internal::optional_data<T>; - -public: - typedef T value_type; - - // Constructors - - // Constructs an `optional` holding an empty value, NOT a default constructed - // `T`. - constexpr optional() noexcept {} - - // Constructs an `optional` initialized with `nullopt` to hold an empty value. - constexpr optional(nullopt_t) noexcept {} // NOLINT(runtime/explicit) - - // Copy constructor, standard semantics - optional(const optional& src) = default; - - // Move constructor, standard semantics - optional(optional&& src) = default; - - // Constructs a non-empty `optional` direct-initialized value of type `T` from - // the arguments `std::forward<Args>(args)...` within the `optional`. - // (The `in_place_t` is a tag used to indicate that the contained object - // should be constructed in-place.) - template <typename InPlaceT, typename... Args, - phmap::enable_if_t<phmap::conjunction< - std::is_same<InPlaceT, in_place_t>, - std::is_constructible<T, Args&&...> >::value>* = nullptr> - constexpr explicit optional(InPlaceT, Args&&... args) - : data_base(in_place_t(), phmap::forward<Args>(args)...) {} - - // Constructs a non-empty `optional` direct-initialized value of type `T` from - // the arguments of an initializer_list and `std::forward<Args>(args)...`. - // (The `in_place_t` is a tag used to indicate that the contained object - // should be constructed in-place.) - template <typename U, typename... Args, - typename = typename std::enable_if<std::is_constructible< - T, std::initializer_list<U>&, Args&&...>::value>::type> - constexpr explicit optional(in_place_t, std::initializer_list<U> il, - Args&&... args) - : data_base(in_place_t(), il, phmap::forward<Args>(args)...) { - } - - // Value constructor (implicit) - template < - typename U = T, - typename std::enable_if< - phmap::conjunction<phmap::negation<std::is_same< - in_place_t, typename std::decay<U>::type> >, - phmap::negation<std::is_same< - optional<T>, typename std::decay<U>::type> >, - std::is_convertible<U&&, T>, - std::is_constructible<T, U&&> >::value, - bool>::type = false> - constexpr optional(U&& v) : data_base(in_place_t(), phmap::forward<U>(v)) {} - - // Value constructor (explicit) - template < - typename U = T, - typename std::enable_if< - phmap::conjunction<phmap::negation<std::is_same< - in_place_t, typename std::decay<U>::type>>, - phmap::negation<std::is_same< - optional<T>, typename std::decay<U>::type>>, - phmap::negation<std::is_convertible<U&&, T>>, - std::is_constructible<T, U&&>>::value, - bool>::type = false> - explicit constexpr optional(U&& v) - : data_base(in_place_t(), phmap::forward<U>(v)) {} - - // Converting copy constructor (implicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U> >, - std::is_constructible<T, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U> >, - std::is_convertible<const U&, T> >::value, - bool>::type = false> - optional(const optional<U>& rhs) { - if (rhs) { - this->construct(*rhs); - } - } - - // Converting copy constructor (explicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U>>, - std::is_constructible<T, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U>>, - phmap::negation<std::is_convertible<const U&, T>>>::value, - bool>::type = false> - explicit optional(const optional<U>& rhs) { - if (rhs) { - this->construct(*rhs); - } - } - - // Converting move constructor (implicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U> >, - std::is_constructible<T, U&&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U> >, - std::is_convertible<U&&, T> >::value, - bool>::type = false> - optional(optional<U>&& rhs) { - if (rhs) { - this->construct(std::move(*rhs)); - } - } - - // Converting move constructor (explicit) - template < - typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, - phmap::negation< - optional_internal::is_constructible_convertible_from_optional< - T, U>>, - phmap::negation<std::is_convertible<U&&, T>>>::value, - bool>::type = false> - explicit optional(optional<U>&& rhs) { - if (rhs) { - this->construct(std::move(*rhs)); - } - } - - // Destructor. Trivial if `T` is trivially destructible. - ~optional() = default; - - // Assignment Operators - - // Assignment from `nullopt` - // - // Example: - // - // struct S { int value; }; - // optional<S> opt = phmap::nullopt; // Could also use opt = { }; - optional& operator=(nullopt_t) noexcept { - this->destruct(); - return *this; - } - - // Copy assignment operator, standard semantics - optional& operator=(const optional& src) = default; - - // Move assignment operator, standard semantics - optional& operator=(optional&& src) = default; - - // Value assignment operators - template < - typename U = T, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation< - std::is_same<optional<T>, typename std::decay<U>::type>>, - phmap::negation< - phmap::conjunction<std::is_scalar<T>, - std::is_same<T, typename std::decay<U>::type>>>, - std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type> - optional& operator=(U&& v) { - this->assign(std::forward<U>(v)); - return *this; - } - - template < - typename U, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation<std::is_same<T, U>>, - std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> - optional& operator=(const optional<U>& rhs) { - if (rhs) { - this->assign(*rhs); - } else { - this->destruct(); - } - return *this; - } - - template <typename U, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U>, - std::is_assignable<T&, U>, - phmap::negation< - optional_internal:: - is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> - optional& operator=(optional<U>&& rhs) { - if (rhs) { - this->assign(std::move(*rhs)); - } else { - this->destruct(); - } - return *this; - } - - // Modifiers - - // optional::reset() - // - // Destroys the inner `T` value of an `phmap::optional` if one is present. - PHMAP_ATTRIBUTE_REINITIALIZES void reset() noexcept { this->destruct(); } - - // optional::emplace() - // - // (Re)constructs the underlying `T` in-place with the given forwarded - // arguments. - // - // Example: - // - // optional<Foo> opt; - // opt.emplace(arg1,arg2,arg3); // Constructs Foo(arg1,arg2,arg3) - // - // If the optional is non-empty, and the `args` refer to subobjects of the - // current object, then behaviour is undefined, because the current object - // will be destructed before the new object is constructed with `args`. - template <typename... Args, - typename = typename std::enable_if< - std::is_constructible<T, Args&&...>::value>::type> - T& emplace(Args&&... args) { - this->destruct(); - this->construct(std::forward<Args>(args)...); - return reference(); - } - - // Emplace reconstruction overload for an initializer list and the given - // forwarded arguments. - // - // Example: - // - // struct Foo { - // Foo(std::initializer_list<int>); - // }; - // - // optional<Foo> opt; - // opt.emplace({1,2,3}); // Constructs Foo({1,2,3}) - template <typename U, typename... Args, - typename = typename std::enable_if<std::is_constructible< - T, std::initializer_list<U>&, Args&&...>::value>::type> - T& emplace(std::initializer_list<U> il, Args&&... args) { - this->destruct(); - this->construct(il, std::forward<Args>(args)...); - return reference(); - } - - // Swaps - - // Swap, standard semantics - void swap(optional& rhs) noexcept( - std::is_nothrow_move_constructible<T>::value&& - std::is_trivial<T>::value) { - if (*this) { - if (rhs) { - using std::swap; - swap(**this, *rhs); - } else { - rhs.construct(std::move(**this)); - this->destruct(); - } - } else { - if (rhs) { - this->construct(std::move(*rhs)); - rhs.destruct(); - } else { - // No effect (swap(disengaged, disengaged)). - } - } - } - - // Observers - - // optional::operator->() - // - // Accesses the underlying `T` value's member `m` of an `optional`. If the - // `optional` is empty, behavior is undefined. - // - // If you need myOpt->foo in constexpr, use (*myOpt).foo instead. - const T* operator->() const { - assert(this->engaged_); - return std::addressof(this->data_); - } - T* operator->() { - assert(this->engaged_); - return std::addressof(this->data_); - } - - // optional::operator*() - // - // Accesses the underlying `T` value of an `optional`. If the `optional` is - // empty, behavior is undefined. - constexpr const T& operator*() const & { return reference(); } - T& operator*() & { - assert(this->engaged_); - return reference(); - } - constexpr const T&& operator*() const && { - return phmap::move(reference()); - } - T&& operator*() && { - assert(this->engaged_); - return std::move(reference()); - } - - // optional::operator bool() - // - // Returns false if and only if the `optional` is empty. - // - // if (opt) { - // // do something with opt.value(); - // } else { - // // opt is empty. - // } - // - constexpr explicit operator bool() const noexcept { return this->engaged_; } - - // optional::has_value() - // - // Determines whether the `optional` contains a value. Returns `false` if and - // only if `*this` is empty. - constexpr bool has_value() const noexcept { return this->engaged_; } - -// Suppress bogus warning on MSVC: MSVC complains call to reference() after -// throw_bad_optional_access() is unreachable. -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4702) -#endif // _MSC_VER - // optional::value() - // - // Returns a reference to an `optional`s underlying value. The constness - // and lvalue/rvalue-ness of the `optional` is preserved to the view of - // the `T` sub-object. Throws `phmap::bad_optional_access` when the `optional` - // is empty. - constexpr const T& value() const & { - return static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference()); - } - T& value() & { - return static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference()); - } - T&& value() && { // NOLINT(build/c++11) - return std::move( - static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference())); - } - constexpr const T&& value() const && { // NOLINT(build/c++11) - return phmap::move( - static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference())); - } -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - - // optional::value_or() - // - // Returns either the value of `T` or a passed default `v` if the `optional` - // is empty. - template <typename U> - constexpr T value_or(U&& v) const& { - static_assert(std::is_copy_constructible<value_type>::value, - "optional<T>::value_or: T must by copy constructible"); - static_assert(std::is_convertible<U&&, value_type>::value, - "optional<T>::value_or: U must be convertible to T"); - return static_cast<bool>(*this) - ? **this - : static_cast<T>(phmap::forward<U>(v)); - } - template <typename U> - T value_or(U&& v) && { // NOLINT(build/c++11) - static_assert(std::is_move_constructible<value_type>::value, - "optional<T>::value_or: T must by move constructible"); - static_assert(std::is_convertible<U&&, value_type>::value, - "optional<T>::value_or: U must be convertible to T"); - return static_cast<bool>(*this) ? std::move(**this) - : static_cast<T>(std::forward<U>(v)); - } - -private: - // Private accessors for internal storage viewed as reference to T. - constexpr const T& reference() const { return this->data_; } - T& reference() { return this->data_; } - - // T constraint checks. You can't have an optional of nullopt_t, in_place_t - // or a reference. - static_assert( - !std::is_same<nullopt_t, typename std::remove_cv<T>::type>::value, - "optional<nullopt_t> is not allowed."); - static_assert( - !std::is_same<in_place_t, typename std::remove_cv<T>::type>::value, - "optional<in_place_t> is not allowed."); - static_assert(!std::is_reference<T>::value, - "optional<reference> is not allowed."); -}; - -// Non-member functions - -// swap() -// -// Performs a swap between two `phmap::optional` objects, using standard -// semantics. -// -// NOTE: we assume `is_swappable()` is always `true`. A compile error will -// result if this is not the case. -template <typename T, - typename std::enable_if<std::is_move_constructible<T>::value, - bool>::type = false> -void swap(optional<T>& a, optional<T>& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); -} - -// make_optional() -// -// Creates a non-empty `optional<T>` where the type of `T` is deduced. An -// `phmap::optional` can also be explicitly instantiated with -// `make_optional<T>(v)`. -// -// Note: `make_optional()` constructions may be declared `constexpr` for -// trivially copyable types `T`. Non-trivial types require copy elision -// support in C++17 for `make_optional` to support `constexpr` on such -// non-trivial types. -// -// Example: -// -// constexpr phmap::optional<int> opt = phmap::make_optional(1); -// static_assert(opt.value() == 1, ""); -template <typename T> -constexpr optional<typename std::decay<T>::type> make_optional(T&& v) { - return optional<typename std::decay<T>::type>(phmap::forward<T>(v)); -} - -template <typename T, typename... Args> -constexpr optional<T> make_optional(Args&&... args) { - return optional<T>(in_place_t(), phmap::forward<Args>(args)...); -} - -template <typename T, typename U, typename... Args> -constexpr optional<T> make_optional(std::initializer_list<U> il, - Args&&... args) { - return optional<T>(in_place_t(), il, - phmap::forward<Args>(args)...); -} - -// Relational operators [optional.relops] - -// Empty optionals are considered equal to each other and less than non-empty -// optionals. Supports relations between optional<T> and optional<U>, between -// optional<T> and U, and between optional<T> and nullopt. -// -// Note: We're careful to support T having non-bool relationals. - -// Requires: The expression, e.g. "*x == *y" shall be well-formed and its result -// shall be convertible to bool. -// The C++17 (N4606) "Returns:" statements are translated into -// code in an obvious way here, and the original text retained as function docs. -// Returns: If bool(x) != bool(y), false; otherwise if bool(x) == false, true; -// otherwise *x == *y. -template <typename T, typename U> -constexpr auto operator==(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x == *y)) { - return static_cast<bool>(x) != static_cast<bool>(y) - ? false - : static_cast<bool>(x) == false ? true - : static_cast<bool>(*x == *y); -} - -// Returns: If bool(x) != bool(y), true; otherwise, if bool(x) == false, false; -// otherwise *x != *y. -template <typename T, typename U> -constexpr auto operator!=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x != *y)) { - return static_cast<bool>(x) != static_cast<bool>(y) - ? true - : static_cast<bool>(x) == false ? false - : static_cast<bool>(*x != *y); -} -// Returns: If !y, false; otherwise, if !x, true; otherwise *x < *y. -template <typename T, typename U> -constexpr auto operator<(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x < *y)) { - return !y ? false : !x ? true : static_cast<bool>(*x < *y); -} -// Returns: If !x, false; otherwise, if !y, true; otherwise *x > *y. -template <typename T, typename U> -constexpr auto operator>(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x > *y)) { - return !x ? false : !y ? true : static_cast<bool>(*x > *y); -} -// Returns: If !x, true; otherwise, if !y, false; otherwise *x <= *y. -template <typename T, typename U> -constexpr auto operator<=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x <= *y)) { - return !x ? true : !y ? false : static_cast<bool>(*x <= *y); -} -// Returns: If !y, true; otherwise, if !x, false; otherwise *x >= *y. -template <typename T, typename U> -constexpr auto operator>=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x >= *y)) { - return !y ? true : !x ? false : static_cast<bool>(*x >= *y); -} - -// Comparison with nullopt [optional.nullops] -// The C++17 (N4606) "Returns:" statements are used directly here. -template <typename T> -constexpr bool operator==(const optional<T>& x, nullopt_t) noexcept { - return !x; -} -template <typename T> -constexpr bool operator==(nullopt_t, const optional<T>& x) noexcept { - return !x; -} -template <typename T> -constexpr bool operator!=(const optional<T>& x, nullopt_t) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator!=(nullopt_t, const optional<T>& x) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator<(const optional<T>&, nullopt_t) noexcept { - return false; -} -template <typename T> -constexpr bool operator<(nullopt_t, const optional<T>& x) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator<=(const optional<T>& x, nullopt_t) noexcept { - return !x; -} -template <typename T> -constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept { - return true; -} -template <typename T> -constexpr bool operator>(const optional<T>& x, nullopt_t) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator>(nullopt_t, const optional<T>&) noexcept { - return false; -} -template <typename T> -constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept { - return true; -} -template <typename T> -constexpr bool operator>=(nullopt_t, const optional<T>& x) noexcept { - return !x; -} - -// Comparison with T [optional.comp_with_t] - -// Requires: The expression, e.g. "*x == v" shall be well-formed and its result -// shall be convertible to bool. -// The C++17 (N4606) "Equivalent to:" statements are used directly here. -template <typename T, typename U> -constexpr auto operator==(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x == v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x == v) : false; -} -template <typename T, typename U> -constexpr auto operator==(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v == *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v == *x) : false; -} -template <typename T, typename U> -constexpr auto operator!=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x != v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x != v) : true; -} -template <typename T, typename U> -constexpr auto operator!=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v != *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v != *x) : true; -} -template <typename T, typename U> -constexpr auto operator<(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x < v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x < v) : true; -} -template <typename T, typename U> -constexpr auto operator<(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v < *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v < *x) : false; -} -template <typename T, typename U> -constexpr auto operator<=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x <= v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x <= v) : true; -} -template <typename T, typename U> -constexpr auto operator<=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v <= *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v <= *x) : false; -} -template <typename T, typename U> -constexpr auto operator>(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x > v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x > v) : false; -} -template <typename T, typename U> -constexpr auto operator>(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v > *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v > *x) : true; -} -template <typename T, typename U> -constexpr auto operator>=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x >= v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x >= v) : false; -} -template <typename T, typename U> -constexpr auto operator>=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v >= *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v >= *x) : true; -} - -} // namespace phmap - -namespace std { - -// std::hash specialization for phmap::optional. -template <typename T> -struct hash<phmap::optional<T> > - : phmap::optional_internal::optional_hash_base<T> {}; - -} // namespace std - -#endif - -// ----------------------------------------------------------------------------- -// common.h -// ----------------------------------------------------------------------------- -namespace phmap { -namespace priv { - -template <class, class = void> -struct IsTransparent : std::false_type {}; -template <class T> -struct IsTransparent<T, phmap::void_t<typename T::is_transparent>> - : std::true_type {}; - -template <bool is_transparent> -struct KeyArg -{ - // Transparent. Forward `K`. - template <typename K, typename key_type> - using type = K; -}; - -template <> -struct KeyArg<false> -{ - // Not transparent. Always use `key_type`. - template <typename K, typename key_type> - using type = key_type; -}; - -#ifdef _MSC_VER - #pragma warning(push) - // warning C4820: '6' bytes padding added after data member - #pragma warning(disable : 4820) -#endif - -// The node_handle concept from C++17. -// We specialize node_handle for sets and maps. node_handle_base holds the -// common API of both. -// ----------------------------------------------------------------------- -template <typename PolicyTraits, typename Alloc> -class node_handle_base -{ -protected: - using slot_type = typename PolicyTraits::slot_type; - -public: - using allocator_type = Alloc; - - constexpr node_handle_base() {} - - node_handle_base(node_handle_base&& other) noexcept { - *this = std::move(other); - } - - ~node_handle_base() { destroy(); } - - node_handle_base& operator=(node_handle_base&& other) noexcept { - destroy(); - if (!other.empty()) { - alloc_ = other.alloc_; - PolicyTraits::transfer(alloc(), slot(), other.slot()); - other.reset(); - } - return *this; - } - - bool empty() const noexcept { return !alloc_; } - explicit operator bool() const noexcept { return !empty(); } - allocator_type get_allocator() const { return *alloc_; } - -protected: - friend struct CommonAccess; - - struct transfer_tag_t {}; - node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s) - : alloc_(a) { - PolicyTraits::transfer(alloc(), slot(), s); - } - - struct move_tag_t {}; - node_handle_base(move_tag_t, const allocator_type& a, slot_type* s) - : alloc_(a) { - PolicyTraits::construct(alloc(), slot(), s); - } - - node_handle_base(const allocator_type& a, slot_type* s) : alloc_(a) { - PolicyTraits::transfer(alloc(), slot(), s); - } - - //node_handle_base(const node_handle_base&) = delete; - //node_handle_base& operator=(const node_handle_base&) = delete; - - void destroy() { - if (!empty()) { - PolicyTraits::destroy(alloc(), slot()); - reset(); - } - } - - void reset() { - assert(alloc_.has_value()); - alloc_ = phmap::nullopt; - } - - slot_type* slot() const { - assert(!empty()); - return reinterpret_cast<slot_type*>(std::addressof(slot_space_)); - } - - allocator_type* alloc() { return std::addressof(*alloc_); } - -private: - phmap::optional<allocator_type> alloc_; - mutable phmap::aligned_storage_t<sizeof(slot_type), alignof(slot_type)> slot_space_; -}; - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -// For sets. -// --------- -template <typename Policy, typename PolicyTraits, typename Alloc, - typename = void> -class node_handle : public node_handle_base<PolicyTraits, Alloc> -{ - using Base = node_handle_base<PolicyTraits, Alloc>; - -public: - using value_type = typename PolicyTraits::value_type; - - constexpr node_handle() {} - - value_type& value() const { return PolicyTraits::element(this->slot()); } - - value_type& key() const { return PolicyTraits::element(this->slot()); } - -private: - friend struct CommonAccess; - - using Base::Base; -}; - -// For maps. -// --------- -template <typename Policy, typename PolicyTraits, typename Alloc> -class node_handle<Policy, PolicyTraits, Alloc, - phmap::void_t<typename Policy::mapped_type>> - : public node_handle_base<PolicyTraits, Alloc> -{ - using Base = node_handle_base<PolicyTraits, Alloc>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - - constexpr node_handle() {} - - auto key() const -> decltype(PolicyTraits::key(this->slot())) { - return PolicyTraits::key(this->slot()); - } - - mapped_type& mapped() const { - return PolicyTraits::value(&PolicyTraits::element(this->slot())); - } - -private: - friend struct CommonAccess; - - using Base::Base; -}; - -// Provide access to non-public node-handle functions. -struct CommonAccess -{ - template <typename Node> - static auto GetSlot(const Node& node) -> decltype(node.slot()) { - return node.slot(); - } - - template <typename Node> - static void Destroy(Node* node) { - node->destroy(); - } - - template <typename Node> - static void Reset(Node* node) { - node->reset(); - } - - template <typename T, typename... Args> - static T Make(Args&&... args) { - return T(std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static T Transfer(Args&&... args) { - return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static T Move(Args&&... args) { - return T(typename T::move_tag_t{}, std::forward<Args>(args)...); - } -}; - -// Implement the insert_return_type<> concept of C++17. -template <class Iterator, class NodeType> -struct InsertReturnType -{ - Iterator position; - bool inserted; - NodeType node; -}; - -} // namespace priv -} // namespace phmap - - -#ifdef ADDRESS_SANITIZER - #include <sanitizer/asan_interface.h> -#endif - -// --------------------------------------------------------------------------- -// span.h -// --------------------------------------------------------------------------- - -namespace phmap { - -template <typename T> -class Span; - -namespace span_internal { -// A constexpr min function -constexpr size_t Min(size_t a, size_t b) noexcept { return a < b ? a : b; } - -// Wrappers for access to container data pointers. -template <typename C> -constexpr auto GetDataImpl(C& c, char) noexcept // NOLINT(runtime/references) - -> decltype(c.data()) { - return c.data(); -} - -// Before C++17, std::string::data returns a const char* in all cases. -inline char* GetDataImpl(std::string& s, // NOLINT(runtime/references) - int) noexcept { - return &s[0]; -} - -template <typename C> -constexpr auto GetData(C& c) noexcept // NOLINT(runtime/references) - -> decltype(GetDataImpl(c, 0)) { - return GetDataImpl(c, 0); -} - -// Detection idioms for size() and data(). -template <typename C> -using HasSize = - std::is_integral<phmap::decay_t<decltype(std::declval<C&>().size())>>; - -// We want to enable conversion from vector<T*> to Span<const T* const> but -// disable conversion from vector<Derived> to Span<Base>. Here we use -// the fact that U** is convertible to Q* const* if and only if Q is the same -// type or a more cv-qualified version of U. We also decay the result type of -// data() to avoid problems with classes which have a member function data() -// which returns a reference. -template <typename T, typename C> -using HasData = - std::is_convertible<phmap::decay_t<decltype(GetData(std::declval<C&>()))>*, - T* const*>; - -// Extracts value type from a Container -template <typename C> -struct ElementType { - using type = typename phmap::remove_reference_t<C>::value_type; -}; - -template <typename T, size_t N> -struct ElementType<T (&)[N]> { - using type = T; -}; - -template <typename C> -using ElementT = typename ElementType<C>::type; - -template <typename T> -using EnableIfMutable = - typename std::enable_if<!std::is_const<T>::value, int>::type; - -template <typename T> -bool EqualImpl(Span<T> a, Span<T> b) { - static_assert(std::is_const<T>::value, ""); - return std::equal(a.begin(), a.end(), b.begin(), b.end()); -} - -template <typename T> -bool LessThanImpl(Span<T> a, Span<T> b) { - static_assert(std::is_const<T>::value, ""); - return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); -} - -// The `IsConvertible` classes here are needed because of the -// `std::is_convertible` bug in libcxx when compiled with GCC. This build -// configuration is used by Android NDK toolchain. Reference link: -// https://bugs.llvm.org/show_bug.cgi?id=27538. -template <typename From, typename To> -struct IsConvertibleHelper { - static std::true_type testval(To); - static std::false_type testval(...); - - using type = decltype(testval(std::declval<From>())); -}; - -template <typename From, typename To> -struct IsConvertible : IsConvertibleHelper<From, To>::type {}; - -// TODO(zhangxy): replace `IsConvertible` with `std::is_convertible` once the -// older version of libcxx is not supported. -template <typename From, typename To> -using EnableIfConvertibleToSpanConst = - typename std::enable_if<IsConvertible<From, Span<const To>>::value>::type; -} // namespace span_internal - -//------------------------------------------------------------------------------ -// Span -//------------------------------------------------------------------------------ -// -// A `Span` is an "array view" type for holding a view of a contiguous data -// array; the `Span` object does not and cannot own such data itself. A span -// provides an easy way to provide overloads for anything operating on -// contiguous sequences without needing to manage pointers and array lengths -// manually. - -// A span is conceptually a pointer (ptr) and a length (size) into an already -// existing array of contiguous memory; the array it represents references the -// elements "ptr[0] .. ptr[size-1]". Passing a properly-constructed `Span` -// instead of raw pointers avoids many issues related to index out of bounds -// errors. -// -// Spans may also be constructed from containers holding contiguous sequences. -// Such containers must supply `data()` and `size() const` methods (e.g -// `std::vector<T>`, `phmap::InlinedVector<T, N>`). All implicit conversions to -// `phmap::Span` from such containers will create spans of type `const T`; -// spans which can mutate their values (of type `T`) must use explicit -// constructors. -// -// A `Span<T>` is somewhat analogous to an `phmap::string_view`, but for an array -// of elements of type `T`. A user of `Span` must ensure that the data being -// pointed to outlives the `Span` itself. -// -// You can construct a `Span<T>` in several ways: -// -// * Explicitly from a reference to a container type -// * Explicitly from a pointer and size -// * Implicitly from a container type (but only for spans of type `const T`) -// * Using the `MakeSpan()` or `MakeConstSpan()` factory functions. -// -// Examples: -// -// // Construct a Span explicitly from a container: -// std::vector<int> v = {1, 2, 3, 4, 5}; -// auto span = phmap::Span<const int>(v); -// -// // Construct a Span explicitly from a C-style array: -// int a[5] = {1, 2, 3, 4, 5}; -// auto span = phmap::Span<const int>(a); -// -// // Construct a Span implicitly from a container -// void MyRoutine(phmap::Span<const int> a) { -// ... -// } -// std::vector v = {1,2,3,4,5}; -// MyRoutine(v) // convert to Span<const T> -// -// Note that `Span` objects, in addition to requiring that the memory they -// point to remains alive, must also ensure that such memory does not get -// reallocated. Therefore, to avoid undefined behavior, containers with -// associated span views should not invoke operations that may reallocate memory -// (such as resizing) or invalidate iterators into the container. -// -// One common use for a `Span` is when passing arguments to a routine that can -// accept a variety of array types (e.g. a `std::vector`, `phmap::InlinedVector`, -// a C-style array, etc.). Instead of creating overloads for each case, you -// can simply specify a `Span` as the argument to such a routine. -// -// Example: -// -// void MyRoutine(phmap::Span<const int> a) { -// ... -// } -// -// std::vector v = {1,2,3,4,5}; -// MyRoutine(v); -// -// phmap::InlinedVector<int, 4> my_inline_vector; -// MyRoutine(my_inline_vector); -// -// // Explicit constructor from pointer,size -// int* my_array = new int[10]; -// MyRoutine(phmap::Span<const int>(my_array, 10)); -template <typename T> -class Span -{ -private: - // Used to determine whether a Span can be constructed from a container of - // type C. - template <typename C> - using EnableIfConvertibleFrom = - typename std::enable_if<span_internal::HasData<T, C>::value && - span_internal::HasSize<C>::value>::type; - - // Used to SFINAE-enable a function when the slice elements are const. - template <typename U> - using EnableIfConstView = - typename std::enable_if<std::is_const<T>::value, U>::type; - - // Used to SFINAE-enable a function when the slice elements are mutable. - template <typename U> - using EnableIfMutableView = - typename std::enable_if<!std::is_const<T>::value, U>::type; - -public: - using value_type = phmap::remove_cv_t<T>; - using pointer = T*; - using const_pointer = const T*; - using reference = T&; - using const_reference = const T&; - using iterator = pointer; - using const_iterator = const_pointer; - using reverse_iterator = std::reverse_iterator<iterator>; - using const_reverse_iterator = std::reverse_iterator<const_iterator>; - using size_type = size_t; - using difference_type = ptrdiff_t; - - static const size_type npos = ~(size_type(0)); - - constexpr Span() noexcept : Span(nullptr, 0) {} - constexpr Span(pointer array, size_type lgth) noexcept - : ptr_(array), len_(lgth) {} - - // Implicit conversion constructors - template <size_t N> - constexpr Span(T (&a)[N]) noexcept // NOLINT(runtime/explicit) - : Span(a, N) {} - - // Explicit reference constructor for a mutable `Span<T>` type. Can be - // replaced with MakeSpan() to infer the type parameter. - template <typename V, typename = EnableIfConvertibleFrom<V>, - typename = EnableIfMutableView<V>> - explicit Span(V& v) noexcept // NOLINT(runtime/references) - : Span(span_internal::GetData(v), v.size()) {} - - // Implicit reference constructor for a read-only `Span<const T>` type - template <typename V, typename = EnableIfConvertibleFrom<V>, - typename = EnableIfConstView<V>> - constexpr Span(const V& v) noexcept // NOLINT(runtime/explicit) - : Span(span_internal::GetData(v), v.size()) {} - - // Implicit constructor from an initializer list, making it possible to pass a - // brace-enclosed initializer list to a function expecting a `Span`. Such - // spans constructed from an initializer list must be of type `Span<const T>`. - // - // void Process(phmap::Span<const int> x); - // Process({1, 2, 3}); - // - // Note that as always the array referenced by the span must outlive the span. - // Since an initializer list constructor acts as if it is fed a temporary - // array (cf. C++ standard [dcl.init.list]/5), it's safe to use this - // constructor only when the `std::initializer_list` itself outlives the span. - // In order to meet this requirement it's sufficient to ensure that neither - // the span nor a copy of it is used outside of the expression in which it's - // created: - // - // // Assume that this function uses the array directly, not retaining any - // // copy of the span or pointer to any of its elements. - // void Process(phmap::Span<const int> ints); - // - // // Okay: the std::initializer_list<int> will reference a temporary array - // // that isn't destroyed until after the call to Process returns. - // Process({ 17, 19 }); - // - // // Not okay: the storage used by the std::initializer_list<int> is not - // // allowed to be referenced after the first line. - // phmap::Span<const int> ints = { 17, 19 }; - // Process(ints); - // - // // Not okay for the same reason as above: even when the elements of the - // // initializer list expression are not temporaries the underlying array - // // is, so the initializer list must still outlive the span. - // const int foo = 17; - // phmap::Span<const int> ints = { foo }; - // Process(ints); - // - template <typename LazyT = T, - typename = EnableIfConstView<LazyT>> - Span( - std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) - : Span(v.begin(), v.size()) {} - - // Accessors - - // Span::data() - // - // Returns a pointer to the span's underlying array of data (which is held - // outside the span). - constexpr pointer data() const noexcept { return ptr_; } - - // Span::size() - // - // Returns the size of this span. - constexpr size_type size() const noexcept { return len_; } - - // Span::length() - // - // Returns the length (size) of this span. - constexpr size_type length() const noexcept { return size(); } - - // Span::empty() - // - // Returns a boolean indicating whether or not this span is considered empty. - constexpr bool empty() const noexcept { return size() == 0; } - - // Span::operator[] - // - // Returns a reference to the i'th element of this span. - constexpr reference operator[](size_type i) const noexcept { - // MSVC 2015 accepts this as constexpr, but not ptr_[i] - return *(data() + i); - } - - // Span::at() - // - // Returns a reference to the i'th element of this span. - constexpr reference at(size_type i) const { - return PHMAP_PREDICT_TRUE(i < size()) // - ? *(data() + i) - : (base_internal::ThrowStdOutOfRange( - "Span::at failed bounds check"), - *(data() + i)); - } - - // Span::front() - // - // Returns a reference to the first element of this span. - constexpr reference front() const noexcept { - return PHMAP_ASSERT(size() > 0), *data(); - } - - // Span::back() - // - // Returns a reference to the last element of this span. - constexpr reference back() const noexcept { - return PHMAP_ASSERT(size() > 0), *(data() + size() - 1); - } - - // Span::begin() - // - // Returns an iterator to the first element of this span. - constexpr iterator begin() const noexcept { return data(); } - - // Span::cbegin() - // - // Returns a const iterator to the first element of this span. - constexpr const_iterator cbegin() const noexcept { return begin(); } - - // Span::end() - // - // Returns an iterator to the last element of this span. - constexpr iterator end() const noexcept { return data() + size(); } - - // Span::cend() - // - // Returns a const iterator to the last element of this span. - constexpr const_iterator cend() const noexcept { return end(); } - - // Span::rbegin() - // - // Returns a reverse iterator starting at the last element of this span. - constexpr reverse_iterator rbegin() const noexcept { - return reverse_iterator(end()); - } - - // Span::crbegin() - // - // Returns a reverse const iterator starting at the last element of this span. - constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); } - - // Span::rend() - // - // Returns a reverse iterator starting at the first element of this span. - constexpr reverse_iterator rend() const noexcept { - return reverse_iterator(begin()); - } - - // Span::crend() - // - // Returns a reverse iterator starting at the first element of this span. - constexpr const_reverse_iterator crend() const noexcept { return rend(); } - - // Span mutations - - // Span::remove_prefix() - // - // Removes the first `n` elements from the span. - void remove_prefix(size_type n) noexcept { - assert(size() >= n); - ptr_ += n; - len_ -= n; - } - - // Span::remove_suffix() - // - // Removes the last `n` elements from the span. - void remove_suffix(size_type n) noexcept { - assert(size() >= n); - len_ -= n; - } - - // Span::subspan() - // - // Returns a `Span` starting at element `pos` and of length `len`. Both `pos` - // and `len` are of type `size_type` and thus non-negative. Parameter `pos` - // must be <= size(). Any `len` value that points past the end of the span - // will be trimmed to at most size() - `pos`. A default `len` value of `npos` - // ensures the returned subspan continues until the end of the span. - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).subspan(1, 2); // {11, 12} - // phmap::MakeSpan(vec).subspan(2, 8); // {12, 13} - // phmap::MakeSpan(vec).subspan(1); // {11, 12, 13} - // phmap::MakeSpan(vec).subspan(4); // {} - // phmap::MakeSpan(vec).subspan(5); // throws std::out_of_range - constexpr Span subspan(size_type pos = 0, size_type len = npos) const { - return (pos <= size()) - ? Span(data() + pos, span_internal::Min(size() - pos, len)) - : (base_internal::ThrowStdOutOfRange("pos > size()"), Span()); - } - - // Span::first() - // - // Returns a `Span` containing first `len` elements. Parameter `len` is of - // type `size_type` and thus non-negative. `len` value must be <= size(). - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).first(1); // {10} - // phmap::MakeSpan(vec).first(3); // {10, 11, 12} - // phmap::MakeSpan(vec).first(5); // throws std::out_of_range - constexpr Span first(size_type len) const { - return (len <= size()) - ? Span(data(), len) - : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); - } - - // Span::last() - // - // Returns a `Span` containing last `len` elements. Parameter `len` is of - // type `size_type` and thus non-negative. `len` value must be <= size(). - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).last(1); // {13} - // phmap::MakeSpan(vec).last(3); // {11, 12, 13} - // phmap::MakeSpan(vec).last(5); // throws std::out_of_range - constexpr Span last(size_type len) const { - return (len <= size()) - ? Span(size() - len + data(), len) - : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); - } - - // Support for phmap::Hash. - template <typename H> - friend H AbslHashValue(H h, Span v) { - return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), - v.size()); - } - -private: - pointer ptr_; - size_type len_; -}; - -template <typename T> -const typename Span<T>::size_type Span<T>::npos; - -// Span relationals - -// Equality is compared element-by-element, while ordering is lexicographical. -// We provide three overloads for each operator to cover any combination on the -// left or right hand side of mutable Span<T>, read-only Span<const T>, and -// convertible-to-read-only Span<T>. -// TODO(zhangxy): Due to MSVC overload resolution bug with partial ordering -// template functions, 5 overloads per operator is needed as a workaround. We -// should update them to 3 overloads per operator using non-deduced context like -// string_view, i.e. -// - (Span<T>, Span<T>) -// - (Span<T>, non_deduced<Span<const T>>) -// - (non_deduced<Span<const T>>, Span<T>) - -// operator== -template <typename T> -bool operator==(Span<T> a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T> -bool operator==(Span<const T> a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T> -bool operator==(Span<T> a, Span<const T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator==(const U& a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator==(Span<T> a, const U& b) { - return span_internal::EqualImpl<const T>(a, b); -} - -// operator!= -template <typename T> -bool operator!=(Span<T> a, Span<T> b) { - return !(a == b); -} - -template <typename T> -bool operator!=(Span<const T> a, Span<T> b) { - return !(a == b); -} - -template <typename T> -bool operator!=(Span<T> a, Span<const T> b) { - return !(a == b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator!=(const U& a, Span<T> b) { - return !(a == b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator!=(Span<T> a, const U& b) { - return !(a == b); -} - -// operator< -template <typename T> -bool operator<(Span<T> a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T> -bool operator<(Span<const T> a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T> -bool operator<(Span<T> a, Span<const T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<(const U& a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<(Span<T> a, const U& b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -// operator> -template <typename T> -bool operator>(Span<T> a, Span<T> b) { - return b < a; -} - -template <typename T> -bool operator>(Span<const T> a, Span<T> b) { - return b < a; -} - -template <typename T> -bool operator>(Span<T> a, Span<const T> b) { - return b < a; -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>(const U& a, Span<T> b) { - return b < a; -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>(Span<T> a, const U& b) { - return b < a; -} - -// operator<= -template <typename T> -bool operator<=(Span<T> a, Span<T> b) { - return !(b < a); -} - -template <typename T> -bool operator<=(Span<const T> a, Span<T> b) { - return !(b < a); -} - -template <typename T> -bool operator<=(Span<T> a, Span<const T> b) { - return !(b < a); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<=(const U& a, Span<T> b) { - return !(b < a); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<=(Span<T> a, const U& b) { - return !(b < a); -} - -// operator>= -template <typename T> -bool operator>=(Span<T> a, Span<T> b) { - return !(a < b); -} - -template <typename T> -bool operator>=(Span<const T> a, Span<T> b) { - return !(a < b); -} - -template <typename T> -bool operator>=(Span<T> a, Span<const T> b) { - return !(a < b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>=(const U& a, Span<T> b) { - return !(a < b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>=(Span<T> a, const U& b) { - return !(a < b); -} - -// MakeSpan() -// -// Constructs a mutable `Span<T>`, deducing `T` automatically from either a -// container or pointer+size. -// -// Because a read-only `Span<const T>` is implicitly constructed from container -// types regardless of whether the container itself is a const container, -// constructing mutable spans of type `Span<T>` from containers requires -// explicit constructors. The container-accepting version of `MakeSpan()` -// deduces the type of `T` by the constness of the pointer received from the -// container's `data()` member. Similarly, the pointer-accepting version returns -// a `Span<const T>` if `T` is `const`, and a `Span<T>` otherwise. -// -// Examples: -// -// void MyRoutine(phmap::Span<MyComplicatedType> a) { -// ... -// }; -// // my_vector is a container of non-const types -// std::vector<MyComplicatedType> my_vector; -// -// // Constructing a Span implicitly attempts to create a Span of type -// // `Span<const T>` -// MyRoutine(my_vector); // error, type mismatch -// -// // Explicitly constructing the Span is verbose -// MyRoutine(phmap::Span<MyComplicatedType>(my_vector)); -// -// // Use MakeSpan() to make an phmap::Span<T> -// MyRoutine(phmap::MakeSpan(my_vector)); -// -// // Construct a span from an array ptr+size -// phmap::Span<T> my_span() { -// return phmap::MakeSpan(&array[0], num_elements_); -// } -// -template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { - return Span<T>(ptr, size); -} - -template <int&... ExplicitArgumentBarrier, typename T> -Span<T> MakeSpan(T* begin, T* end) noexcept { - return PHMAP_ASSERT(begin <= end), Span<T>(begin, end - begin); -} - -template <int&... ExplicitArgumentBarrier, typename C> -constexpr auto MakeSpan(C& c) noexcept // NOLINT(runtime/references) - -> decltype(phmap::MakeSpan(span_internal::GetData(c), c.size())) { - return MakeSpan(span_internal::GetData(c), c.size()); -} - -template <int&... ExplicitArgumentBarrier, typename T, size_t N> -constexpr Span<T> MakeSpan(T (&array)[N]) noexcept { - return Span<T>(array, N); -} - -// MakeConstSpan() -// -// Constructs a `Span<const T>` as with `MakeSpan`, deducing `T` automatically, -// but always returning a `Span<const T>`. -// -// Examples: -// -// void ProcessInts(phmap::Span<const int> some_ints); -// -// // Call with a pointer and size. -// int array[3] = { 0, 0, 0 }; -// ProcessInts(phmap::MakeConstSpan(&array[0], 3)); -// -// // Call with a [begin, end) pair. -// ProcessInts(phmap::MakeConstSpan(&array[0], &array[3])); -// -// // Call directly with an array. -// ProcessInts(phmap::MakeConstSpan(array)); -// -// // Call with a contiguous container. -// std::vector<int> some_ints = ...; -// ProcessInts(phmap::MakeConstSpan(some_ints)); -// ProcessInts(phmap::MakeConstSpan(std::vector<int>{ 0, 0, 0 })); -// -template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept { - return Span<const T>(ptr, size); -} - -template <int&... ExplicitArgumentBarrier, typename T> -Span<const T> MakeConstSpan(T* begin, T* end) noexcept { - return PHMAP_ASSERT(begin <= end), Span<const T>(begin, end - begin); -} - -template <int&... ExplicitArgumentBarrier, typename C> -constexpr auto MakeConstSpan(const C& c) noexcept -> decltype(MakeSpan(c)) { - return MakeSpan(c); -} - -template <int&... ExplicitArgumentBarrier, typename T, size_t N> -constexpr Span<const T> MakeConstSpan(const T (&array)[N]) noexcept { - return Span<const T>(array, N); -} -} // namespace phmap - -// --------------------------------------------------------------------------- -// layout.h -// --------------------------------------------------------------------------- -#if defined(__GXX_RTTI) - #define PHMAP_INTERNAL_HAS_CXA_DEMANGLE -#endif - -#ifdef PHMAP_INTERNAL_HAS_CXA_DEMANGLE - #include <cxxabi.h> -#endif - -namespace phmap { -namespace priv { - -// A type wrapper that instructs `Layout` to use the specific alignment for the -// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API -// and behavior as `Layout<..., T, ...>` except that the first element of the -// array of `T` is aligned to `N` (the rest of the elements follow without -// padding). -// -// Requires: `N >= alignof(T)` and `N` is a power of 2. -template <class T, size_t N> -struct Aligned; - -namespace internal_layout { - -template <class T> -struct NotAligned {}; - -template <class T, size_t N> -struct NotAligned<const Aligned<T, N>> { - static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified"); -}; - -template <size_t> -using IntToSize = size_t; - -template <class> -using TypeToSize = size_t; - -template <class T> -struct Type : NotAligned<T> { - using type = T; -}; - -template <class T, size_t N> -struct Type<Aligned<T, N>> { - using type = T; -}; - -template <class T> -struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {}; - -template <class T, size_t N> -struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {}; - -// Note: workaround for https://gcc.gnu.org/PR88115 -template <class T> -struct AlignOf : NotAligned<T> { - static constexpr size_t value = alignof(T); -}; - -template <class T, size_t N> -struct AlignOf<Aligned<T, N>> { - static_assert(N % alignof(T) == 0, - "Custom alignment can't be lower than the type's alignment"); - static constexpr size_t value = N; -}; - -// Does `Ts...` contain `T`? -template <class T, class... Ts> -using Contains = phmap::disjunction<std::is_same<T, Ts>...>; - -template <class From, class To> -using CopyConst = - typename std::conditional<std::is_const<From>::value, const To, To>::type; - -// Note: We're not qualifying this with phmap:: because it doesn't compile under -// MSVC. -template <class T> -using SliceType = Span<T>; - -// This namespace contains no types. It prevents functions defined in it from -// being found by ADL. -namespace adl_barrier { - -template <class Needle, class... Ts> -constexpr size_t Find(Needle, Needle, Ts...) { - static_assert(!Contains<Needle, Ts...>(), "Duplicate element type"); - return 0; -} - -template <class Needle, class T, class... Ts> -constexpr size_t Find(Needle, T, Ts...) { - return adl_barrier::Find(Needle(), Ts()...) + 1; -} - -constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); } - -// Returns `q * m` for the smallest `q` such that `q * m >= n`. -// Requires: `m` is a power of two. It's enforced by IsLegalElementType below. -constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } - -constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; } - -constexpr size_t Max(size_t a) { return a; } - -template <class... Ts> -constexpr size_t Max(size_t a, size_t b, Ts... rest) { - return adl_barrier::Max(b < a ? a : b, rest...); -} - -} // namespace adl_barrier - -template <bool C> -using EnableIf = typename std::enable_if<C, int>::type; - -// Can `T` be a template argument of `Layout`? -// --------------------------------------------------------------------------- -template <class T> -using IsLegalElementType = std::integral_constant< - bool, !std::is_reference<T>::value && !std::is_volatile<T>::value && - !std::is_reference<typename Type<T>::type>::value && - !std::is_volatile<typename Type<T>::type>::value && - adl_barrier::IsPow2(AlignOf<T>::value)>; - -template <class Elements, class SizeSeq, class OffsetSeq> -class LayoutImpl; - -// --------------------------------------------------------------------------- -// Public base class of `Layout` and the result type of `Layout::Partial()`. -// -// `Elements...` contains all template arguments of `Layout` that created this -// instance. -// -// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments -// passed to `Layout::Partial()` or `Layout::Layout()`. -// -// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is -// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we -// can compute offsets). -// --------------------------------------------------------------------------- -template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> -class LayoutImpl<std::tuple<Elements...>, phmap::index_sequence<SizeSeq...>, - phmap::index_sequence<OffsetSeq...>> -{ -private: - static_assert(sizeof...(Elements) > 0, "At least one field is required"); - static_assert(phmap::conjunction<IsLegalElementType<Elements>...>::value, - "Invalid element type (see IsLegalElementType)"); - - enum { - NumTypes = sizeof...(Elements), - NumSizes = sizeof...(SizeSeq), - NumOffsets = sizeof...(OffsetSeq), - }; - - // These are guaranteed by `Layout`. - static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), - "Internal error"); - static_assert(NumTypes > 0, "Internal error"); - - // Returns the index of `T` in `Elements...`. Results in a compilation error - // if `Elements...` doesn't contain exactly one instance of `T`. - template <class T> - static constexpr size_t ElementIndex() { - static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(), - "Type not found"); - return adl_barrier::Find(Type<T>(), - Type<typename Type<Elements>::type>()...); - } - - template <size_t N> - using ElementAlignment = - AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>; - -public: - // Element types of all arrays packed in a tuple. - using ElementTypes = std::tuple<typename Type<Elements>::type...>; - - // Element type of the Nth array. - template <size_t N> - using ElementType = typename std::tuple_element<N, ElementTypes>::type; - - constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) - : size_{sizes...} {} - - // Alignment of the layout, equal to the strictest alignment of all elements. - // All pointers passed to the methods of layout must be aligned to this value. - static constexpr size_t Alignment() { - return adl_barrier::Max(AlignOf<Elements>::value...); - } - - // Offset in bytes of the Nth array. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<0>() == 0); // The ints starts from 0. - // assert(x.Offset<1>() == 16); // The doubles starts from 16. - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - template <size_t N, EnableIf<N == 0> = 0> - constexpr size_t Offset() const { - return 0; - } - - template <size_t N, EnableIf<N != 0> = 0> - constexpr size_t Offset() const { - static_assert(N < NumOffsets, "Index out of bounds"); - return adl_barrier::Align( - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1], - ElementAlignment<N>::value); - } - - // Offset in bytes of the array with the specified element type. There must - // be exactly one such array and its zero-based index must be at most - // `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<int>() == 0); // The ints starts from 0. - // assert(x.Offset<double>() == 16); // The doubles starts from 16. - template <class T> - constexpr size_t Offset() const { - return Offset<ElementIndex<T>()>(); - } - - // Offsets in bytes of all arrays for which the offsets are known. - constexpr std::array<size_t, NumOffsets> Offsets() const { - return {{Offset<OffsetSeq>()...}}; - } - - // The number of elements in the Nth array. This is the Nth argument of - // `Layout::Partial()` or `Layout::Layout()` (zero-based). - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<0>() == 3); - // assert(x.Size<1>() == 4); - // - // Requires: `N < NumSizes`. - template <size_t N> - constexpr size_t Size() const { - static_assert(N < NumSizes, "Index out of bounds"); - return size_[N]; - } - - // The number of elements in the array with the specified element type. - // There must be exactly one such array and its zero-based index must be - // at most `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<int>() == 3); - // assert(x.Size<double>() == 4); - template <class T> - constexpr size_t Size() const { - return Size<ElementIndex<T>()>(); - } - - // The number of elements of all arrays for which they are known. - constexpr std::array<size_t, NumSizes> Sizes() const { - return {{Size<SizeSeq>()...}}; - } - - // Pointer to the beginning of the Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<0>(p); - // double* doubles = x.Pointer<1>(p); - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - CopyConst<Char, ElementType<N>>* Pointer(Char* p) const { - using C = typename std::remove_const<Char>::type; - static_assert( - std::is_same<C, char>() || std::is_same<C, unsigned char>() || - std::is_same<C, signed char>(), - "The argument must be a pointer to [const] [signed|unsigned] char"); - constexpr size_t alignment = Alignment(); - (void)alignment; - assert(reinterpret_cast<uintptr_t>(p) % alignment == 0); - return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>()); - } - - // Pointer to the beginning of the array with the specified element type. - // There must be exactly one such array and its zero-based index must be at - // most `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<int>(p); - // double* doubles = x.Pointer<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - CopyConst<Char, T>* Pointer(Char* p) const { - return Pointer<ElementIndex<T>()>(p); - } - - // Pointers to all arrays for which pointers are known. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // int* ints; - // double* doubles; - // std::tie(ints, doubles) = x.Pointers(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<CopyConst< - Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> - Pointers(Char* p) const { - return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( - Pointer<OffsetSeq>(p)...); - } - - // The Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<0>(p); - // Span<double> doubles = x.Slice<1>(p); - // - // Requires: `N < NumSizes`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const { - return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>()); - } - - // The array with the specified element type. There must be exactly one - // such array and its zero-based index must be less than `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<int>(p); - // Span<double> doubles = x.Slice<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - SliceType<CopyConst<Char, T>> Slice(Char* p) const { - return Slice<ElementIndex<T>()>(p); - } - - // All arrays with known sizes. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // Span<int> ints; - // Span<double> doubles; - // std::tie(ints, doubles) = x.Slices(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<SliceType<CopyConst< - Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> - Slices(Char* p) const { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed - // in 6.1). - (void)p; - return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( - Slice<SizeSeq>(p)...); - } - - // The size of the allocation that fits all arrays. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes - // - // Requires: `NumSizes == sizeof...(Ts)`. - constexpr size_t AllocSize() const { - static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); - return Offset<NumTypes - 1>() + - SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1]; - } - - // If built with --config=asan, poisons padding bytes (if any) in the - // allocation. The pointer must point to a memory block at least - // `AllocSize()` bytes in length. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // Requires: `p` is aligned to `Alignment()`. - template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0> - void PoisonPadding(const Char* p) const { - Pointer<0>(p); // verify the requirements on `Char` and `p` - } - - template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0> - void PoisonPadding(const Char* p) const { - static_assert(N < NumOffsets, "Index out of bounds"); - (void)p; -#ifdef ADDRESS_SANITIZER - PoisonPadding<Char, N - 1>(p); - // The `if` is an optimization. It doesn't affect the observable behaviour. - if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { - size_t start = - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1]; - ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); - } -#endif - } - -private: - // Arguments of `Layout::Partial()` or `Layout::Layout()`. - size_t size_[NumSizes > 0 ? NumSizes : 1]; -}; - -template <size_t NumSizes, class... Ts> -using LayoutType = LayoutImpl< - std::tuple<Ts...>, phmap::make_index_sequence<NumSizes>, - phmap::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; - -} // namespace internal_layout - -// --------------------------------------------------------------------------- -// Descriptor of arrays of various types and sizes laid out in memory one after -// another. See the top of the file for documentation. -// -// Check out the public API of internal_layout::LayoutImpl above. The type is -// internal to the library but its methods are public, and they are inherited -// by `Layout`. -// --------------------------------------------------------------------------- -template <class... Ts> -class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> -{ -public: - static_assert(sizeof...(Ts) > 0, "At least one field is required"); - static_assert( - phmap::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, - "Invalid element type (see IsLegalElementType)"); - - template <size_t NumSizes> - using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; - - template <class... Sizes> - static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { - static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType<sizeof...(Sizes)>(phmap::forward<Sizes>(sizes)...); - } - - // Creates a layout with the sizes of all arrays specified. If you know - // only the sizes of the first N arrays (where N can be zero), you can use - // `Partial()` defined above. The constructor is essentially equivalent to - // calling `Partial()` and passing in all array sizes; the constructor is - // provided as a convenient abbreviation. - // - // Note: The sizes of the arrays must be specified in number of elements, - // not in bytes. - constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) - : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} -}; - -} // namespace priv -} // namespace phmap - -// --------------------------------------------------------------------------- -// compressed_tuple.h -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - // We need to mark these classes with this declspec to ensure that - // CompressedTuple happens. - #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases) -#else // _MSC_VER - #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC -#endif // _MSC_VER - -namespace phmap { -namespace priv { - -template <typename... Ts> -class CompressedTuple; - -namespace internal_compressed_tuple { - -template <typename D, size_t I> -struct Elem; -template <typename... B, size_t I> -struct Elem<CompressedTuple<B...>, I> - : std::tuple_element<I, std::tuple<B...>> {}; -template <typename D, size_t I> -using ElemT = typename Elem<D, I>::type; - -// --------------------------------------------------------------------------- -// Use the __is_final intrinsic if available. Where it's not available, classes -// declared with the 'final' specifier cannot be used as CompressedTuple -// elements. -// TODO(sbenza): Replace this with std::is_final in C++14. -// --------------------------------------------------------------------------- -template <typename T> -constexpr bool IsFinal() { -#if defined(__clang__) || defined(__GNUC__) - return __is_final(T); -#else - return false; -#endif -} - -template <typename T> -constexpr bool ShouldUseBase() { -#ifdef __INTEL_COMPILER - // avoid crash in Intel compiler - // assertion failed at: "shared/cfe/edgcpfe/lower_init.c", line 7013 - return false; -#else - return std::is_class<T>::value && std::is_empty<T>::value && !IsFinal<T>(); -#endif -} - -// The storage class provides two specializations: -// - For empty classes, it stores T as a base class. -// - For everything else, it stores T as a member. -// ------------------------------------------------ -template <typename D, size_t I, bool = ShouldUseBase<ElemT<D, I>>()> -struct Storage -{ - using T = ElemT<D, I>; - T value; - constexpr Storage() = default; - explicit constexpr Storage(T&& v) : value(phmap::forward<T>(v)) {} - constexpr const T& get() const& { return value; } - T& get() & { return value; } - constexpr const T&& get() const&& { return phmap::move(*this).value; } - T&& get() && { return std::move(*this).value; } -}; - -template <typename D, size_t I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<D, I, true> - : ElemT<D, I> -{ - using T = internal_compressed_tuple::ElemT<D, I>; - constexpr Storage() = default; - explicit constexpr Storage(T&& v) : T(phmap::forward<T>(v)) {} - constexpr const T& get() const& { return *this; } - T& get() & { return *this; } - constexpr const T&& get() const&& { return phmap::move(*this); } - T&& get() && { return std::move(*this); } -}; - -template <typename D, typename I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl; - -template <typename... Ts, size_t... I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC - CompressedTupleImpl<CompressedTuple<Ts...>, phmap::index_sequence<I...>> - // We use the dummy identity function through std::integral_constant to - // convince MSVC of accepting and expanding I in that context. Without it - // you would get: - // error C3548: 'I': parameter pack cannot be used in this context - : Storage<CompressedTuple<Ts...>, - std::integral_constant<size_t, I>::value>... -{ - constexpr CompressedTupleImpl() = default; - explicit constexpr CompressedTupleImpl(Ts&&... args) - : Storage<CompressedTuple<Ts...>, I>(phmap::forward<Ts>(args))... {} -}; - -} // namespace internal_compressed_tuple - -// --------------------------------------------------------------------------- -// Helper class to perform the Empty Base Class Optimization. -// Ts can contain classes and non-classes, empty or not. For the ones that -// are empty classes, we perform the CompressedTuple. If all types in Ts are -// empty classes, then CompressedTuple<Ts...> is itself an empty class. -// -// To access the members, use member .get<N>() function. -// -// Eg: -// phmap::priv::CompressedTuple<int, T1, T2, T3> value(7, t1, t2, -// t3); -// assert(value.get<0>() == 7); -// T1& t1 = value.get<1>(); -// const T2& t2 = value.get<2>(); -// ... -// -// https://en.cppreference.com/w/cpp/language/ebo -// --------------------------------------------------------------------------- -template <typename... Ts> -class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple - : private internal_compressed_tuple::CompressedTupleImpl< - CompressedTuple<Ts...>, phmap::index_sequence_for<Ts...>> -{ -private: - template <int I> - using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>; - -public: - constexpr CompressedTuple() = default; - explicit constexpr CompressedTuple(Ts... base) - : CompressedTuple::CompressedTupleImpl(phmap::forward<Ts>(base)...) {} - - template <int I> - ElemT<I>& get() & { - return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); - } - - template <int I> - constexpr const ElemT<I>& get() const& { - return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); - } - - template <int I> - ElemT<I>&& get() && { - return std::move(*this) - .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); - } - - template <int I> - constexpr const ElemT<I>&& get() const&& { - return phmap::move(*this) - .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); - } -}; - -// Explicit specialization for a zero-element tuple -// (needed to avoid ambiguous overloads for the default constructor). -// --------------------------------------------------------------------------- -template <> -class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {}; - -} // namespace priv -} // namespace phmap - - -namespace phmap { -namespace priv { - -#ifdef _MSC_VER - #pragma warning(push) - // warning warning C4324: structure was padded due to alignment specifier - #pragma warning(disable : 4324) -#endif - - -// ---------------------------------------------------------------------------- -// Allocates at least n bytes aligned to the specified alignment. -// Alignment must be a power of 2. It must be positive. -// -// Note that many allocators don't honor alignment requirements above certain -// threshold (usually either alignof(std::max_align_t) or alignof(void*)). -// Allocate() doesn't apply alignment corrections. If the underlying allocator -// returns insufficiently alignment pointer, that's what you are going to get. -// ---------------------------------------------------------------------------- -template <size_t Alignment, class Alloc> -void* Allocate(Alloc* alloc, size_t n) { - static_assert(Alignment > 0, ""); - assert(n && "n must be positive"); - struct alignas(Alignment) M {}; - using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; - using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; - A mem_alloc(*alloc); - void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); - assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 && - "allocator does not respect alignment"); - return p; -} - -// ---------------------------------------------------------------------------- -// The pointer must have been previously obtained by calling -// Allocate<Alignment>(alloc, n). -// ---------------------------------------------------------------------------- -template <size_t Alignment, class Alloc> -void Deallocate(Alloc* alloc, void* p, size_t n) { - static_assert(Alignment > 0, ""); - assert(n && "n must be positive"); - struct alignas(Alignment) M {}; - using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; - using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; - A mem_alloc(*alloc); - AT::deallocate(mem_alloc, static_cast<M*>(p), - (n + sizeof(M) - 1) / sizeof(M)); -} - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -// Helper functions for asan and msan. -// ---------------------------------------------------------------------------- -inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) { -#ifdef ADDRESS_SANITIZER - ASAN_POISON_MEMORY_REGION(m, s); -#endif -#ifdef MEMORY_SANITIZER - __msan_poison(m, s); -#endif - (void)m; - (void)s; -} - -inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) { -#ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(m, s); -#endif -#ifdef MEMORY_SANITIZER - __msan_unpoison(m, s); -#endif - (void)m; - (void)s; -} - -template <typename T> -inline void SanitizerPoisonObject(const T* object) { - SanitizerPoisonMemoryRegion(object, sizeof(T)); -} - -template <typename T> -inline void SanitizerUnpoisonObject(const T* object) { - SanitizerUnpoisonMemoryRegion(object, sizeof(T)); -} - -} // namespace priv -} // namespace phmap - - -// --------------------------------------------------------------------------- -// thread_annotations.h -// --------------------------------------------------------------------------- - -#if defined(__clang__) - #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#else - #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op -#endif - -#define PHMAP_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) -#define PHMAP_PT_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) - -#define PHMAP_ACQUIRED_AFTER(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) - -#define PHMAP_ACQUIRED_BEFORE(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) - -#define PHMAP_EXCLUSIVE_LOCKS_REQUIRED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) - -#define PHMAP_SHARED_LOCKS_REQUIRED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) - -#define PHMAP_LOCKS_EXCLUDED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) - -#define PHMAP_LOCK_RETURNED(x) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -#define PHMAP_LOCKABLE \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lockable) - -#define PHMAP_SCOPED_LOCKABLE \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -#define PHMAP_EXCLUSIVE_LOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) - -#define PHMAP_SHARED_LOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) - -#define PHMAP_UNLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) - -#define PHMAP_EXCLUSIVE_TRYLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) - -#define PHMAP_SHARED_TRYLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) - -#define PHMAP_ASSERT_EXCLUSIVE_LOCK(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) - -#define PHMAP_ASSERT_SHARED_LOCK(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) - -#define PHMAP_NO_THREAD_SAFETY_ANALYSIS \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -//------------------------------------------------------------------------------ -// Tool-Supplied Annotations -//------------------------------------------------------------------------------ - -// TS_UNCHECKED should be placed around lock expressions that are not valid -// C++ syntax, but which are present for documentation purposes. These -// annotations will be ignored by the analysis. -#define PHMAP_TS_UNCHECKED(x) "" - -// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. -// It is used by automated tools to mark and disable invalid expressions. -// The annotation should either be fixed, or changed to TS_UNCHECKED. -#define PHMAP_TS_FIXME(x) "" - -// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of -// a particular function. However, this attribute is used to mark functions -// that are incorrect and need to be fixed. It is used by automated tools to -// avoid breaking the build when the analysis is updated. -// Code owners are expected to eventually fix the routine. -#define PHMAP_NO_THREAD_SAFETY_ANALYSIS_FIXME PHMAP_NO_THREAD_SAFETY_ANALYSIS - -// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY -// annotation that needs to be fixed, because it is producing thread safety -// warning. It disables the GUARDED_BY. -#define PHMAP_GUARDED_BY_FIXME(x) - -// Disables warnings for a single read operation. This can be used to avoid -// warnings when it is known that the read is not actually involved in a race, -// but the compiler cannot confirm that. -#define PHMAP_TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) - - -namespace phmap { -namespace thread_safety_analysis { - -// Takes a reference to a guarded data member, and returns an unguarded -// reference. -template <typename T> -inline const T& ts_unchecked_read(const T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -template <typename T> -inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -} // namespace thread_safety_analysis - -namespace priv { - -namespace memory_internal { - -// ---------------------------------------------------------------------------- -// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and -// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and -// offsetof(Pair, second) respectively. Otherwise they are -1. -// -// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout -// type, which is non-portable. -// ---------------------------------------------------------------------------- -template <class Pair, class = std::true_type> -struct OffsetOf { - static constexpr size_t kFirst = (size_t)-1; - static constexpr size_t kSecond = (size_t)-1; -}; - -template <class Pair> -struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type> -{ - static constexpr size_t kFirst = offsetof(Pair, first); - static constexpr size_t kSecond = offsetof(Pair, second); -}; - -// ---------------------------------------------------------------------------- -template <class K, class V> -struct IsLayoutCompatible -{ -private: - struct Pair { - K first; - V second; - }; - - // Is P layout-compatible with Pair? - template <class P> - static constexpr bool LayoutCompatible() { - return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) && - alignof(P) == alignof(Pair) && - memory_internal::OffsetOf<P>::kFirst == - memory_internal::OffsetOf<Pair>::kFirst && - memory_internal::OffsetOf<P>::kSecond == - memory_internal::OffsetOf<Pair>::kSecond; - } - -public: - // Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are, - // then it is safe to store them in a union and read from either. - static constexpr bool value = std::is_standard_layout<K>() && - std::is_standard_layout<Pair>() && - memory_internal::OffsetOf<Pair>::kFirst == 0 && - LayoutCompatible<std::pair<K, V>>() && - LayoutCompatible<std::pair<const K, V>>(); -}; - -} // namespace memory_internal - -// ---------------------------------------------------------------------------- -// The internal storage type for key-value containers like flat_hash_map. -// -// It is convenient for the value_type of a flat_hash_map<K, V> to be -// pair<const K, V>; the "const K" prevents accidental modification of the key -// when dealing with the reference returned from find() and similar methods. -// However, this creates other problems; we want to be able to emplace(K, V) -// efficiently with move operations, and similarly be able to move a -// pair<K, V> in insert(). -// -// The solution is this union, which aliases the const and non-const versions -// of the pair. This also allows flat_hash_map<const K, V> to work, even though -// that has the same efficiency issues with move in emplace() and insert() - -// but people do it anyway. -// -// If kMutableKeys is false, only the value member can be accessed. -// -// If kMutableKeys is true, key can be accessed through all slots while value -// and mutable_value must be accessed only via INITIALIZED slots. Slots are -// created and destroyed via mutable_value so that the key can be moved later. -// -// Accessing one of the union fields while the other is active is safe as -// long as they are layout-compatible, which is guaranteed by the definition of -// kMutableKeys. For C++11, the relevant section of the standard is -// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19) -// ---------------------------------------------------------------------------- -template <class K, class V> -union map_slot_type -{ - map_slot_type() {} - ~map_slot_type() = delete; - map_slot_type(const map_slot_type&) = delete; - map_slot_type& operator=(const map_slot_type&) = delete; - - using value_type = std::pair<const K, V>; - using mutable_value_type = std::pair<K, V>; - - value_type value; - mutable_value_type mutable_value; - K key; -}; - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -template <class K, class V> -struct map_slot_policy -{ - using slot_type = map_slot_type<K, V>; - using value_type = std::pair<const K, V>; - using mutable_value_type = std::pair<K, V>; - -private: - static void emplace(slot_type* slot) { - // The construction of union doesn't do anything at runtime but it allows us - // to access its members without violating aliasing rules. - new (slot) slot_type; - } - // If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one - // or the other via slot_type. We are also free to access the key via - // slot_type::key in this case. - using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>; - -public: - static value_type& element(slot_type* slot) { return slot->value; } - static const value_type& element(const slot_type* slot) { - return slot->value; - } - - static const K& key(const slot_type* slot) { - return kMutableKeys::value ? slot->key : slot->value.first; - } - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - emplace(slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value, - std::forward<Args>(args)...); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, - std::forward<Args>(args)...); - } - } - - // Construct this slot by moving from another slot. - template <class Allocator> - static void construct(Allocator* alloc, slot_type* slot, slot_type* other) { - emplace(slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct( - *alloc, &slot->mutable_value, std::move(other->mutable_value)); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, - std::move(other->value)); - } - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value); - } else { - phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->value); - } - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - emplace(new_slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct( - *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &new_slot->value, - std::move(old_slot->value)); - } - destroy(alloc, old_slot); - } - - template <class Allocator> - static void swap(Allocator* alloc, slot_type* a, slot_type* b) { - if (kMutableKeys::value) { - using std::swap; - swap(a->mutable_value, b->mutable_value); - } else { - value_type tmp = std::move(a->value); - phmap::allocator_traits<Allocator>::destroy(*alloc, &a->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &a->value, - std::move(b->value)); - phmap::allocator_traits<Allocator>::destroy(*alloc, &b->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &b->value, - std::move(tmp)); - } - } - - template <class Allocator> - static void move(Allocator* alloc, slot_type* src, slot_type* dest) { - if (kMutableKeys::value) { - dest->mutable_value = std::move(src->mutable_value); - } else { - phmap::allocator_traits<Allocator>::destroy(*alloc, &dest->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &dest->value, - std::move(src->value)); - } - } - - template <class Allocator> - static void move(Allocator* alloc, slot_type* first, slot_type* last, - slot_type* result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } -}; - -} // namespace priv -} // phmap - - -namespace phmap { - -#ifdef BOOST_THREAD_LOCK_OPTIONS_HPP - using defer_lock_t = boost::defer_lock_t; - using try_to_lock_t = boost::try_to_lock_t; - using adopt_lock_t = boost::adopt_lock_t; -#else - struct adopt_lock_t { explicit adopt_lock_t() = default; }; - struct defer_lock_t { explicit defer_lock_t() = default; }; - struct try_to_lock_t { explicit try_to_lock_t() = default; }; -#endif - -// ----------------------------------------------------------------------------- -// NullMutex -// ----------------------------------------------------------------------------- -// A class that implements the Mutex interface, but does nothing. This is to be -// used as a default template parameters for classes who provide optional -// internal locking (like phmap::parallel_flat_hash_map). -// ----------------------------------------------------------------------------- -class NullMutex { -public: - NullMutex() {} - ~NullMutex() {} - void lock() {} - void unlock() {} - bool try_lock() { return true; } - void lock_shared() {} - void unlock_shared() {} - bool try_lock_shared() { return true; } -}; - -// ------------------------ lockable object used internally ------------------------- -template <class MutexType> -class LockableBaseImpl -{ -public: - // ---------------------------------------------------- - struct DoNothing - { - using mutex_type = MutexType; - DoNothing() noexcept {} - explicit DoNothing(mutex_type& ) noexcept {} - explicit DoNothing(mutex_type& , mutex_type&) noexcept {} - DoNothing(mutex_type&, phmap::adopt_lock_t) noexcept {} - DoNothing(mutex_type&, phmap::defer_lock_t) noexcept {} - DoNothing(mutex_type&, phmap::try_to_lock_t) {} - template<class T> explicit DoNothing(T&&) {} - DoNothing& operator=(const DoNothing&) { return *this; } - DoNothing& operator=(DoNothing&&) { return *this; } - void swap(DoNothing &) {} - bool owns_lock() const noexcept { return true; } - }; - - // ---------------------------------------------------- - class WriteLock - { - public: - using mutex_type = MutexType; - - WriteLock() : m_(nullptr), locked_(false) {} - - explicit WriteLock(mutex_type &m) : m_(&m) { - m_->lock(); - locked_ = true; - } - - WriteLock(mutex_type& m, adopt_lock_t) noexcept : - m_(&m), locked_(true) - {} - - WriteLock(mutex_type& m, defer_lock_t) noexcept : - m_(&m), locked_(false) - {} - - WriteLock(mutex_type& m, try_to_lock_t) : - m_(&m), locked_(false) { - m_->try_lock(); - } - - WriteLock(WriteLock &&o) : - m_(std::move(o.m_)), locked_(std::move(o.locked_)) { - o.locked_ = false; - o.m_ = nullptr; - } - - WriteLock& operator=(WriteLock&& other) { - WriteLock temp(std::move(other)); - swap(temp); - return *this; - } - - ~WriteLock() { - if (locked_) - m_->unlock(); - } - - void lock() { - if (!locked_) { - m_->lock(); - locked_ = true; - } - } - - void unlock() { - if (locked_) { - m_->unlock(); - locked_ = false; - } - } - - bool try_lock() { - if (locked_) - return true; - locked_ = m_->try_lock(); - return locked_; - } - - bool owns_lock() const noexcept { return locked_; } - - void swap(WriteLock &o) noexcept { - std::swap(m_, o.m_); - std::swap(locked_, o.locked_); - } - - mutex_type *mutex() const noexcept { return m_; } - - private: - mutex_type *m_; - bool locked_; - }; - - // ---------------------------------------------------- - class ReadLock - { - public: - using mutex_type = MutexType; - - ReadLock() : m_(nullptr), locked_(false) {} - - explicit ReadLock(mutex_type &m) : m_(&m) { - m_->lock_shared(); - locked_ = true; - } - - ReadLock(mutex_type& m, adopt_lock_t) noexcept : - m_(&m), locked_(true) - {} - - ReadLock(mutex_type& m, defer_lock_t) noexcept : - m_(&m), locked_(false) - {} - - ReadLock(mutex_type& m, try_to_lock_t) : - m_(&m), locked_(false) { - m_->try_lock_shared(); - } - - ReadLock(ReadLock &&o) : - m_(std::move(o.m_)), locked_(std::move(o.locked_)) { - o.locked_ = false; - o.m_ = nullptr; - } - - ReadLock& operator=(ReadLock&& other) { - ReadLock temp(std::move(other)); - swap(temp); - return *this; - } - - ~ReadLock() { - if (locked_) - m_->unlock_shared(); - } - - void lock() { - if (!locked_) { - m_->lock_shared(); - locked_ = true; - } - } - - void unlock() { - if (locked_) { - m_->unlock_shared(); - locked_ = false; - } - } - - bool try_lock() { - if (locked_) - return true; - locked_ = m_->try_lock_shared(); - return locked_; - } - - bool owns_lock() const noexcept { return locked_; } - - void swap(ReadLock &o) noexcept { - std::swap(m_, o.m_); - std::swap(locked_, o.locked_); - } - - mutex_type *mutex() const noexcept { return m_; } - - private: - mutex_type *m_; - bool locked_; - }; - - // ---------------------------------------------------- - class WriteLocks - { - public: - using mutex_type = MutexType; - - explicit WriteLocks(mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { - std::lock(m1, m2); - } - - WriteLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { // adopt means we already own the mutexes - } - - ~WriteLocks() - { - _m1.unlock(); - _m2.unlock(); - } - - WriteLocks(WriteLocks const&) = delete; - WriteLocks& operator=(WriteLocks const&) = delete; - private: - mutex_type& _m1; - mutex_type& _m2; - }; - - // ---------------------------------------------------- - class ReadLocks - { - public: - using mutex_type = MutexType; - - explicit ReadLocks(mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { - _m1.lock_shared(); - _m2.lock_shared(); - } - - ReadLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { // adopt means we already own the mutexes - } - - ~ReadLocks() - { - _m1.unlock_shared(); - _m2.unlock_shared(); - } - - ReadLocks(ReadLocks const&) = delete; - ReadLocks& operator=(ReadLocks const&) = delete; - private: - mutex_type& _m1; - mutex_type& _m2; - }; -}; - -// ------------------------ holds a mutex ------------------------------------ -// Default implementation for Lockable, should work fine for std::mutex -// ----------------------------------- -// use as: -// using Lockable = phmap::LockableImpl<mutex_type>; -// Lockable m; -// -// Lockable::UpgradeLock read_lock(m); // take a upgradable lock -// -// { -// Lockable::UpgradeToUnique unique_lock(read_lock); -// // now locked for write -// } -// -// --------------------------------------------------------------------------- -// Generic mutex support (always write locks) -// -------------------------------------------------------------------------- -template <class Mtx_> -class LockableImpl : public Mtx_ -{ -public: - using mutex_type = Mtx_; - using Base = LockableBaseImpl<Mtx_>; - using SharedLock = typename Base::WriteLock; - using UpgradeLock = typename Base::WriteLock; - using UniqueLock = typename Base::WriteLock; - using SharedLocks = typename Base::WriteLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership -}; - -// --------------------------------------------------------------------------- -// Null mutex (no-op) - when we don't want internal synchronization -// --------------------------------------------------------------------------- -template <> -class LockableImpl<phmap::NullMutex>: public phmap::NullMutex -{ -public: - using mutex_type = phmap::NullMutex; - using Base = LockableBaseImpl<phmap::NullMutex>; - using SharedLock = typename Base::DoNothing; - using UpgradeLock = typename Base::DoNothing; - using UniqueLock = typename Base::DoNothing; - using UpgradeToUnique = typename Base::DoNothing; - using SharedLocks = typename Base::DoNothing; - using UniqueLocks = typename Base::DoNothing; -}; - -// -------------------------------------------------------------------------- -// Abseil Mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef ABSL_SYNCHRONIZATION_MUTEX_H_ - - struct AbslMutex : protected absl::Mutex - { - void lock() { this->Lock(); } - void unlock() { this->Unlock(); } - void try_lock() { this->TryLock(); } - void lock_shared() { this->ReaderLock(); } - void unlock_shared() { this->ReaderUnlock(); } - void try_lock_shared() { this->ReaderTryLock(); } - }; - - template <> - class LockableImpl<absl::Mutex> : public AbslMutex - { - public: - using mutex_type = phmap::AbslMutex; - using Base = LockableBaseImpl<phmap::AbslMutex>; - using SharedLock = typename Base::ReadLock; - using UpgradeLock = typename Base::WriteLock; - using UniqueLock = typename Base::WriteLock; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; - -#endif - -// -------------------------------------------------------------------------- -// Boost shared_mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef BOOST_THREAD_SHARED_MUTEX_HPP - -#if 1 - // --------------------------------------------------------------------------- - template <> - class LockableImpl<boost::shared_mutex> : public boost::shared_mutex - { - public: - using mutex_type = boost::shared_mutex; - using Base = LockableBaseImpl<boost::shared_mutex>; - using SharedLock = boost::shared_lock<mutex_type>; - using UpgradeLock = boost::unique_lock<mutex_type>; // assume can't upgrade - using UniqueLock = boost::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; -#else - // --------------------------------------------------------------------------- - template <> - class LockableImpl<boost::upgrade_mutex> : public boost::upgrade_mutex - { - public: - using mutex_type = boost::upgrade_mutex; - using SharedLock = boost::shared_lock<mutex_type>; - using UpgradeLock = boost::upgrade_lock<mutex_type>; - using UniqueLock = boost::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = boost::upgrade_to_unique_lock<mutex_type>; - }; -#endif - -#endif // BOOST_THREAD_SHARED_MUTEX_HPP - -// -------------------------------------------------------------------------- -// std::shared_mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_SHARED_MUTEX - - // --------------------------------------------------------------------------- - template <> - class LockableImpl<std::shared_mutex> : public std::shared_mutex - { - public: - using mutex_type = std::shared_mutex; - using Base = LockableBaseImpl<std::shared_mutex>; - using SharedLock = std::shared_lock<mutex_type>; - using UpgradeLock = std::unique_lock<mutex_type>; // assume can't upgrade - using UniqueLock = std::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; -#endif // PHMAP_HAVE_SHARED_MUTEX - - -} // phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // phmap_base_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_bits.h b/benchmarks/external/parallel_hashmap/phmap_bits.h deleted file mode 100644 index 6b765fff..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_bits.h +++ /dev/null @@ -1,663 +0,0 @@ -#if !defined(phmap_bits_h_guard_) -#define phmap_bits_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -// The following guarantees declaration of the byte swap functions -#ifdef _MSC_VER - #include <stdlib.h> // NOLINT(build/include) -#elif defined(__APPLE__) - // Mac OS X / Darwin features - #include <libkern/OSByteOrder.h> -#elif defined(__FreeBSD__) - #include <sys/endian.h> -#elif defined(__GLIBC__) - #include <byteswap.h> // IWYU pragma: export -#endif - -#include <string.h> -#include <cstdint> -#include "phmap_config.h" - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed -#endif - -// ----------------------------------------------------------------------------- -// unaligned APIs -// ----------------------------------------------------------------------------- -// Portable handling of unaligned loads, stores, and copies. -// On some platforms, like ARM, the copy functions can be more efficient -// then a load and a store. -// ----------------------------------------------------------------------------- - -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) -#include <stdint.h> - -extern "C" { - uint16_t __sanitizer_unaligned_load16(const void *p); - uint32_t __sanitizer_unaligned_load32(const void *p); - uint64_t __sanitizer_unaligned_load64(const void *p); - void __sanitizer_unaligned_store16(void *p, uint16_t v); - void __sanitizer_unaligned_store32(void *p, uint32_t v); - void __sanitizer_unaligned_store64(void *p, uint64_t v); -} // extern "C" - -namespace phmap { -namespace bits { - -inline uint16_t UnalignedLoad16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32_t UnalignedLoad32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64_t UnalignedLoad64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void UnalignedStore16(void *p, uint16_t v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void UnalignedStore32(void *p, uint32_t v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void UnalignedStore64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); -} - -} // namespace bits -} // namespace phmap - -#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) - -#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) - -#else - -namespace phmap { -namespace bits { - -inline uint16_t UnalignedLoad16(const void *p) { - uint16_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint32_t UnalignedLoad32(const void *p) { - uint32_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint64_t UnalignedLoad64(const void *p) { - uint64_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); } - -inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } - -inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } - -} // namespace bits -} // namespace phmap - -#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) - -#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) - -#endif - -// ----------------------------------------------------------------------------- -// File: optimization.h -// ----------------------------------------------------------------------------- - -#if defined(__pnacl__) - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } -#elif defined(__clang__) - // Clang will not tail call given inline volatile assembly. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") -#elif defined(__GNUC__) - // GCC will not tail call given inline volatile assembly. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") -#elif defined(_MSC_VER) - #include <intrin.h> - // The __nop() intrinsic blocks the optimisation. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __nop() -#else - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } -#endif - -#if defined(__GNUC__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wpedantic" -#endif - -#ifdef PHMAP_HAVE_INTRINSIC_INT128 - __extension__ typedef unsigned __int128 phmap_uint128; - inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) - { - auto result = static_cast<phmap_uint128>(a) * static_cast<phmap_uint128>(b); - *high = static_cast<uint64_t>(result >> 64); - return static_cast<uint64_t>(result); - } - #define PHMAP_HAS_UMUL128 1 -#elif (defined(_MSC_VER)) - #if defined(_M_X64) - #pragma intrinsic(_umul128) - inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) - { - return _umul128(a, b, high); - } - #define PHMAP_HAS_UMUL128 1 - #endif -#endif - -#if defined(__GNUC__) - #pragma GCC diagnostic pop -#endif - -#if defined(__GNUC__) - // Cache line alignment - #if defined(__i386__) || defined(__x86_64__) - #define PHMAP_CACHELINE_SIZE 64 - #elif defined(__powerpc64__) - #define PHMAP_CACHELINE_SIZE 128 - #elif defined(__aarch64__) - // We would need to read special register ctr_el0 to find out L1 dcache size. - // This value is a good estimate based on a real aarch64 machine. - #define PHMAP_CACHELINE_SIZE 64 - #elif defined(__arm__) - // Cache line sizes for ARM: These values are not strictly correct since - // cache line sizes depend on implementations, not architectures. There - // are even implementations with cache line sizes configurable at boot - // time. - #if defined(__ARM_ARCH_5T__) - #define PHMAP_CACHELINE_SIZE 32 - #elif defined(__ARM_ARCH_7A__) - #define PHMAP_CACHELINE_SIZE 64 - #endif - #endif - - #ifndef PHMAP_CACHELINE_SIZE - // A reasonable default guess. Note that overestimates tend to waste more - // space, while underestimates tend to waste more time. - #define PHMAP_CACHELINE_SIZE 64 - #endif - - #define PHMAP_CACHELINE_ALIGNED __attribute__((aligned(PHMAP_CACHELINE_SIZE))) -#elif defined(_MSC_VER) - #define PHMAP_CACHELINE_SIZE 64 - #define PHMAP_CACHELINE_ALIGNED __declspec(align(PHMAP_CACHELINE_SIZE)) -#else - #define PHMAP_CACHELINE_SIZE 64 - #define PHMAP_CACHELINE_ALIGNED -#endif - - -#if PHMAP_HAVE_BUILTIN(__builtin_expect) || \ - (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_PREDICT_FALSE(x) (__builtin_expect(x, 0)) - #define PHMAP_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) -#else - #define PHMAP_PREDICT_FALSE(x) (x) - #define PHMAP_PREDICT_TRUE(x) (x) -#endif - -// ----------------------------------------------------------------------------- -// File: bits.h -// ----------------------------------------------------------------------------- - -#if defined(_MSC_VER) - // We can achieve something similar to attribute((always_inline)) with MSVC by - // using the __forceinline keyword, however this is not perfect. MSVC is - // much less aggressive about inlining, and even with the __forceinline keyword. - #define PHMAP_BASE_INTERNAL_FORCEINLINE __forceinline -#else - // Use default attribute inline. - #define PHMAP_BASE_INTERNAL_FORCEINLINE inline PHMAP_ATTRIBUTE_ALWAYS_INLINE -#endif - - -namespace phmap { -namespace base_internal { - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { - int zeroes = 60; - if (n >> 32) zeroes -= 32, n >>= 32; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { -#if defined(_MSC_VER) && defined(_M_X64) - // MSVC does not have __buitin_clzll. Use _BitScanReverse64. - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse64(&result, n)) { - return (int)(63 - result); - } - return 64; -#elif defined(_MSC_VER) && !defined(__clang__) - // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse - unsigned long result = 0; // NOLINT(runtime/int) - if ((n >> 32) && _BitScanReverse(&result, (unsigned long)(n >> 32))) { - return 31 - result; - } - if (_BitScanReverse(&result, (unsigned long)n)) { - return 63 - result; - } - return 64; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clzll, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_clzll does not take 64-bit arg"); - - // Handle 0 as a special case because __builtin_clzll(0) is undefined. - if (n == 0) { - return 64; - } - return __builtin_clzll(n); -#else - return CountLeadingZeros64Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { - int zeroes = 28; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse(&result, n)) { - return (int)(31 - result); - } - return 32; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clz, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(int) == sizeof(n), - "__builtin_clz does not take 32-bit arg"); - - // Handle 0 as a special case because __builtin_clz(0) is undefined. - if (n == 0) { - return 32; - } - return __builtin_clz(n); -#else - return CountLeadingZeros32Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) { - int c = 63; - n &= ~n + 1; - if (n & 0x00000000FFFFFFFF) c -= 32; - if (n & 0x0000FFFF0000FFFF) c -= 16; - if (n & 0x00FF00FF00FF00FF) c -= 8; - if (n & 0x0F0F0F0F0F0F0F0F) c -= 4; - if (n & 0x3333333333333333) c -= 2; - if (n & 0x5555555555555555) c -= 1; - return c; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) { -#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward64(&result, n); - return (int)result; -#elif defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (static_cast<uint32_t>(n) == 0) { - _BitScanForward(&result, (unsigned long)(n >> 32)); - return result + 32; - } - _BitScanForward(&result, (unsigned long)n); - return result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_ctzll does not take 64-bit arg"); - return __builtin_ctzll(n); -#else - return CountTrailingZerosNonZero64Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) { - int c = 31; - n &= ~n + 1; - if (n & 0x0000FFFF) c -= 16; - if (n & 0x00FF00FF) c -= 8; - if (n & 0x0F0F0F0F) c -= 4; - if (n & 0x33333333) c -= 2; - if (n & 0x55555555) c -= 1; - return c; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward(&result, n); - return (int)result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(int) == sizeof(n), - "__builtin_ctz does not take 32-bit arg"); - return __builtin_ctz(n); -#else - return CountTrailingZerosNonZero32Slow(n); -#endif -} - -#undef PHMAP_BASE_INTERNAL_FORCEINLINE - -} // namespace base_internal -} // namespace phmap - -// ----------------------------------------------------------------------------- -// File: endian.h -// ----------------------------------------------------------------------------- - -namespace phmap { - -// Use compiler byte-swapping intrinsics if they are available. 32-bit -// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. -// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. -// For simplicity, we enable them all only for GCC 4.8.0 or later. -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) - - inline uint64_t gbswap_64(uint64_t host_int) { - return __builtin_bswap64(host_int); - } - inline uint32_t gbswap_32(uint32_t host_int) { - return __builtin_bswap32(host_int); - } - inline uint16_t gbswap_16(uint16_t host_int) { - return __builtin_bswap16(host_int); - } - -#elif defined(_MSC_VER) - - inline uint64_t gbswap_64(uint64_t host_int) { - return _byteswap_uint64(host_int); - } - inline uint32_t gbswap_32(uint32_t host_int) { - return _byteswap_ulong(host_int); - } - inline uint16_t gbswap_16(uint16_t host_int) { - return _byteswap_ushort(host_int); - } - -#elif defined(__APPLE__) - - inline uint64_t gbswap_64(uint64_t host_int) { return OSSwapInt16(host_int); } - inline uint32_t gbswap_32(uint32_t host_int) { return OSSwapInt32(host_int); } - inline uint16_t gbswap_16(uint16_t host_int) { return OSSwapInt64(host_int); } - -#else - - inline uint64_t gbswap_64(uint64_t host_int) { -#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) - // Adapted from /usr/include/byteswap.h. Not available on Mac. - if (__builtin_constant_p(host_int)) { - return __bswap_constant_64(host_int); - } else { - uint64_t result; - __asm__("bswap %0" : "=r"(result) : "0"(host_int)); - return result; - } -#elif defined(__GLIBC__) - return bswap_64(host_int); -#else - return (((host_int & uint64_t{0xFF}) << 56) | - ((host_int & uint64_t{0xFF00}) << 40) | - ((host_int & uint64_t{0xFF0000}) << 24) | - ((host_int & uint64_t{0xFF000000}) << 8) | - ((host_int & uint64_t{0xFF00000000}) >> 8) | - ((host_int & uint64_t{0xFF0000000000}) >> 24) | - ((host_int & uint64_t{0xFF000000000000}) >> 40) | - ((host_int & uint64_t{0xFF00000000000000}) >> 56)); -#endif // bswap_64 - } - - inline uint32_t gbswap_32(uint32_t host_int) { -#if defined(__GLIBC__) - return bswap_32(host_int); -#else - return (((host_int & uint32_t{0xFF}) << 24) | - ((host_int & uint32_t{0xFF00}) << 8) | - ((host_int & uint32_t{0xFF0000}) >> 8) | - ((host_int & uint32_t{0xFF000000}) >> 24)); -#endif - } - - inline uint16_t gbswap_16(uint16_t host_int) { -#if defined(__GLIBC__) - return bswap_16(host_int); -#else - return (((host_int & uint16_t{0xFF}) << 8) | - ((host_int & uint16_t{0xFF00}) >> 8)); -#endif - } - -#endif // intrinics available - -#ifdef PHMAP_IS_LITTLE_ENDIAN - - // Definitions for ntohl etc. that don't require us to include - // netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather - // than just #defining them because in debug mode, gcc doesn't - // correctly handle the (rather involved) definitions of bswap_32. - // gcc guarantees that inline functions are as fast as macros, so - // this isn't a performance hit. - inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } - inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } - inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } - -#elif defined PHMAP_IS_BIG_ENDIAN - - // These definitions are simpler on big-endian machines - // These are functions instead of macros to avoid self-assignment warnings - // on calls such as "i = ghtnol(i);". This also provides type checking. - inline uint16_t ghtons(uint16_t x) { return x; } - inline uint32_t ghtonl(uint32_t x) { return x; } - inline uint64_t ghtonll(uint64_t x) { return x; } - -#else - #error \ - "Unsupported byte order: Either PHMAP_IS_BIG_ENDIAN or " \ - "PHMAP_IS_LITTLE_ENDIAN must be defined" -#endif // byte order - -inline uint16_t gntohs(uint16_t x) { return ghtons(x); } -inline uint32_t gntohl(uint32_t x) { return ghtonl(x); } -inline uint64_t gntohll(uint64_t x) { return ghtonll(x); } - -// Utilities to convert numbers between the current hosts's native byte -// order and little-endian byte order -// -// Load/Store methods are alignment safe -namespace little_endian { -// Conversion functions. -#ifdef PHMAP_IS_LITTLE_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return x; } - inline uint16_t ToHost16(uint16_t x) { return x; } - - inline uint32_t FromHost32(uint32_t x) { return x; } - inline uint32_t ToHost32(uint32_t x) { return x; } - - inline uint64_t FromHost64(uint64_t x) { return x; } - inline uint64_t ToHost64(uint64_t x) { return x; } - - inline constexpr bool IsLittleEndian() { return true; } - -#elif defined PHMAP_IS_BIG_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } - inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } - - inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } - inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } - - inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } - inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } - - inline constexpr bool IsLittleEndian() { return false; } - -#endif /* ENDIAN */ - -// Functions to do unaligned loads and stores in little-endian order. -inline uint16_t Load16(const void *p) { - return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); -} - -inline void Store16(void *p, uint16_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); -} - -inline uint32_t Load32(const void *p) { - return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); -} - -inline void Store32(void *p, uint32_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); -} - -inline uint64_t Load64(const void *p) { - return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); -} - -inline void Store64(void *p, uint64_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); -} - -} // namespace little_endian - -// Utilities to convert numbers between the current hosts's native byte -// order and big-endian byte order (same as network byte order) -// -// Load/Store methods are alignment safe -namespace big_endian { -#ifdef PHMAP_IS_LITTLE_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } - inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } - - inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } - inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } - - inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } - inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } - - inline constexpr bool IsLittleEndian() { return true; } - -#elif defined PHMAP_IS_BIG_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return x; } - inline uint16_t ToHost16(uint16_t x) { return x; } - - inline uint32_t FromHost32(uint32_t x) { return x; } - inline uint32_t ToHost32(uint32_t x) { return x; } - - inline uint64_t FromHost64(uint64_t x) { return x; } - inline uint64_t ToHost64(uint64_t x) { return x; } - - inline constexpr bool IsLittleEndian() { return false; } - -#endif /* ENDIAN */ - -// Functions to do unaligned loads and stores in big-endian order. -inline uint16_t Load16(const void *p) { - return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); -} - -inline void Store16(void *p, uint16_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); -} - -inline uint32_t Load32(const void *p) { - return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); -} - -inline void Store32(void *p, uint32_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); -} - -inline uint64_t Load64(const void *p) { - return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); -} - -inline void Store64(void *p, uint64_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); -} - -} // namespace big_endian - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_bits_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_config.h b/benchmarks/external/parallel_hashmap/phmap_config.h deleted file mode 100644 index fa515025..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_config.h +++ /dev/null @@ -1,771 +0,0 @@ -#if !defined(phmap_config_h_guard_) -#define phmap_config_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#define PHMAP_VERSION_MAJOR 1 -#define PHMAP_VERSION_MINOR 0 -#define PHMAP_VERSION_PATCH 0 - -// Included for the __GLIBC__ macro (or similar macros on other systems). -#include <limits.h> - -#ifdef __cplusplus - // Included for __GLIBCXX__, _LIBCPP_VERSION - #include <cstddef> -#endif // __cplusplus - -#if defined(__APPLE__) - // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, - // __IPHONE_8_0. - #include <Availability.h> - #include <TargetConditionals.h> -#endif - -#define PHMAP_XSTR(x) PHMAP_STR(x) -#define PHMAP_STR(x) #x -#define PHMAP_VAR_NAME_VALUE(var) #var "=" PHMAP_STR(var) - -// ----------------------------------------------------------------------------- -// Some sanity checks -// ----------------------------------------------------------------------------- -//#if defined(__CYGWIN__) -// #error "Cygwin is not supported." -//#endif - -#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023918 && !defined(__clang__) - #error "phmap requires Visual Studio 2015 Update 2 or higher." -#endif - -// We support gcc 4.7 and later. -#if defined(__GNUC__) && !defined(__clang__) - #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) - #error "phmap requires gcc 4.7 or higher." - #endif -#endif - -// We support Apple Xcode clang 4.2.1 (version 421.11.65) and later. -// This corresponds to Apple Xcode version 4.5. -#if defined(__apple_build_version__) && __apple_build_version__ < 4211165 - #error "phmap requires __apple_build_version__ of 4211165 or higher." -#endif - -// Enforce C++11 as the minimum. -#if defined(__cplusplus) && !defined(_MSC_VER) - #if __cplusplus < 201103L - #error "C++ versions less than C++11 are not supported." - #endif -#endif - -// We have chosen glibc 2.12 as the minimum -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) - #if !__GLIBC_PREREQ(2, 12) - #error "Minimum required version of glibc is 2.12." - #endif -#endif - -#if defined(_STLPORT_VERSION) - #error "STLPort is not supported." -#endif - -#if CHAR_BIT != 8 - #error "phmap assumes CHAR_BIT == 8." -#endif - -// phmap currently assumes that an int is 4 bytes. -#if INT_MAX < 2147483647 - #error "phmap assumes that int is at least 4 bytes. " -#endif - - - -// ----------------------------------------------------------------------------- -// Compiler Feature Checks -// ----------------------------------------------------------------------------- - -#ifdef __has_builtin - #define PHMAP_HAVE_BUILTIN(x) __has_builtin(x) -#else - #define PHMAP_HAVE_BUILTIN(x) 0 -#endif - -#if (defined(_MSVC_LANG) && _MSVC_LANG >= 201703) || __cplusplus >= 201703 - #define PHMAP_HAVE_CC17 1 -#else - #define PHMAP_HAVE_CC17 0 -#endif - -#define PHMAP_BRANCHLESS 1 - -// ---------------------------------------------------------------- -// Checks whether `std::is_trivially_destructible<T>` is supported. -// ---------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE - #error PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set -#elif defined(_LIBCPP_VERSION) || \ - (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ - defined(_MSC_VER) - #define PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 -#endif - -// -------------------------------------------------------------- -// Checks whether `std::is_trivially_default_constructible<T>` is -// supported. -// -------------------------------------------------------------- -#if defined(PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) - #error PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set -#elif defined(PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) - #error PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set -#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ - (!defined(__clang__) && defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ - (defined(_MSC_VER) && !defined(__NVCC__)) - #define PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 - #define PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 -#endif - -// ------------------------------------------------------------------- -// Checks whether C++11's `thread_local` storage duration specifier is -// supported. -// ------------------------------------------------------------------- -#ifdef PHMAP_HAVE_THREAD_LOCAL - #error PHMAP_HAVE_THREAD_LOCAL cannot be directly set -#elif defined(__APPLE__) - #if __has_feature(cxx_thread_local) && \ - !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0) - #define PHMAP_HAVE_THREAD_LOCAL 1 - #endif -#else // !defined(__APPLE__) - #define PHMAP_HAVE_THREAD_LOCAL 1 -#endif - -#if defined(__ANDROID__) && defined(__clang__) - - #if __has_include(<android/ndk-version.h>) - #include <android/ndk-version.h> - #endif // __has_include(<android/ndk-version.h>) - - #if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ - defined(__NDK_MINOR__) && \ - ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) - #undef PHMAP_HAVE_TLS - #undef PHMAP_HAVE_THREAD_LOCAL - #endif -#endif - -// ------------------------------------------------------------ -// Checks whether the __int128 compiler extension for a 128-bit -// integral type is supported. -// ------------------------------------------------------------ -#ifdef PHMAP_HAVE_INTRINSIC_INT128 - #error PHMAP_HAVE_INTRINSIC_INT128 cannot be directly set -#elif defined(__SIZEOF_INT128__) - #if (defined(__clang__) && !defined(_WIN32) && !defined(__aarch64__)) || \ - (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ - (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)) - #define PHMAP_HAVE_INTRINSIC_INT128 1 - #elif defined(__CUDACC__) - #if __CUDACC_VER__ >= 70000 - #define PHMAP_HAVE_INTRINSIC_INT128 1 - #endif // __CUDACC_VER__ >= 70000 - #endif // defined(__CUDACC__) -#endif - -// ------------------------------------------------------------------ -// Checks whether the compiler both supports and enables exceptions. -// ------------------------------------------------------------------ -#ifdef PHMAP_HAVE_EXCEPTIONS - #error PHMAP_HAVE_EXCEPTIONS cannot be directly set. -#elif defined(__clang__) - #if defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) - #define PHMAP_HAVE_EXCEPTIONS 1 - #endif // defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) -#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ - !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ - !(defined(_MSC_VER) && !defined(_CPPUNWIND)) - #define PHMAP_HAVE_EXCEPTIONS 1 -#endif - - -// ----------------------------------------------------------------------- -// Checks whether the platform has an mmap(2) implementation as defined in -// POSIX.1-2001. -// ----------------------------------------------------------------------- -#ifdef PHMAP_HAVE_MMAP - #error PHMAP_HAVE_MMAP cannot be directly set -#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \ - defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \ - defined(__ASYLO__) - #define PHMAP_HAVE_MMAP 1 -#endif - -// ----------------------------------------------------------------------- -// Checks the endianness of the platform. -// ----------------------------------------------------------------------- -#if defined(PHMAP_IS_BIG_ENDIAN) - #error "PHMAP_IS_BIG_ENDIAN cannot be directly set." -#endif - -#if defined(PHMAP_IS_LITTLE_ENDIAN) - #error "PHMAP_IS_LITTLE_ENDIAN cannot be directly set." -#endif - -#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - #define PHMAP_IS_LITTLE_ENDIAN 1 -#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - #define PHMAP_IS_BIG_ENDIAN 1 -#elif defined(_WIN32) - #define PHMAP_IS_LITTLE_ENDIAN 1 -#else - #error "phmap endian detection needs to be set up for your compiler" -#endif - -#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ - defined(__MAC_OS_X_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400 - #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 1 -#else - #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 0 -#endif - -// --------------------------------------------------------------------------- -// Checks whether C++17 std::any is available by checking whether <any> exists. -// --------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_ANY - #error "PHMAP_HAVE_STD_ANY cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<any>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_ANY 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_OPTIONAL - #error "PHMAP_HAVE_STD_OPTIONAL cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<optional>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_OPTIONAL 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_VARIANT - #error "PHMAP_HAVE_STD_VARIANT cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<variant>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_VARIANT 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_STRING_VIEW - #error "PHMAP_HAVE_STD_STRING_VIEW cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<string_view>) && __cplusplus >= 201703L && \ - (!defined(_MSC_VER) || _MSC_VER >= 1920) // vs2019 - #define PHMAP_HAVE_STD_STRING_VIEW 1 - #endif -#endif - -// #pragma message(PHMAP_VAR_NAME_VALUE(_MSVC_LANG)) - -#if defined(_MSC_VER) && _MSC_VER >= 1910 && PHMAP_HAVE_CC17 - // #define PHMAP_HAVE_STD_ANY 1 - #define PHMAP_HAVE_STD_OPTIONAL 1 - #define PHMAP_HAVE_STD_VARIANT 1 - #if !defined(PHMAP_HAVE_STD_STRING_VIEW) && _MSC_VER >= 1920 - #define PHMAP_HAVE_STD_STRING_VIEW 1 - #endif -#endif - -#if PHMAP_HAVE_CC17 - #define PHMAP_HAVE_SHARED_MUTEX 1 -#endif - -#ifndef PHMAP_HAVE_STD_STRING_VIEW - #define PHMAP_HAVE_STD_STRING_VIEW 0 -#endif - -// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION -// SEH exception from emplace for variant<SomeStruct> when constructing the -// struct can throw. This defeats some of variant_test and -// variant_exception_safety_test. -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG) - #define PHMAP_INTERNAL_MSVC_2017_DBG_MODE -#endif - -// --------------------------------------------------------------------------- -// Checks whether wchar_t is treated as a native type -// (MSVC: /Zc:wchar_t- treats wchar_t as unsigned short) -// --------------------------------------------------------------------------- -#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) -#define PHMAP_HAS_NATIVE_WCHAR_T -#endif - -// ----------------------------------------------------------------------------- -// Sanitizer Attributes -// ----------------------------------------------------------------------------- -// -// Sanitizer-related attributes are not "defined" in this file (and indeed -// are not defined as such in any file). To utilize the following -// sanitizer-related attributes within your builds, define the following macros -// within your build using a `-D` flag, along with the given value for -// `-fsanitize`: -// -// * `ADDRESS_SANITIZER` + `-fsanitize=address` (Clang, GCC 4.8) -// * `MEMORY_SANITIZER` + `-fsanitize=memory` (Clang-only) -// * `THREAD_SANITIZER + `-fsanitize=thread` (Clang, GCC 4.8+) -// * `UNDEFINED_BEHAVIOR_SANITIZER` + `-fsanitize=undefined` (Clang, GCC 4.9+) -// * `CONTROL_FLOW_INTEGRITY` + -fsanitize=cfi (Clang-only) -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// A function-like feature checking macro that is a wrapper around -// `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a -// nonzero constant integer if the attribute is supported or 0 if not. -// -// It evaluates to zero if `__has_attribute` is not defined by the compiler. -// ----------------------------------------------------------------------------- -#ifdef __has_attribute - #define PHMAP_HAVE_ATTRIBUTE(x) __has_attribute(x) -#else - #define PHMAP_HAVE_ATTRIBUTE(x) 0 -#endif - -// ----------------------------------------------------------------------------- -// A function-like feature checking macro that accepts C++11 style attributes. -// It's a wrapper around `__has_cpp_attribute`, defined by ISO C++ SD-6 -// (https://en.cppreference.com/w/cpp/experimental/feature_test). If we don't -// find `__has_cpp_attribute`, will evaluate to 0. -// ----------------------------------------------------------------------------- -#if defined(__cplusplus) && defined(__has_cpp_attribute) - #define PHMAP_HAVE_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) -#else - #define PHMAP_HAVE_CPP_ATTRIBUTE(x) 0 -#endif - -// ----------------------------------------------------------------------------- -// Function Attributes -// ----------------------------------------------------------------------------- -#if PHMAP_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(__printf__, string_index, first_to_check))) - #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(__scanf__, string_index, first_to_check))) -#else - #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) - #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(always_inline) || \ - (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) - #define PHMAP_HAVE_ATTRIBUTE_ALWAYS_INLINE 1 -#else - #define PHMAP_ATTRIBUTE_ALWAYS_INLINE -#endif - -#if !defined(__INTEL_COMPILER) && (PHMAP_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__))) - #define PHMAP_ATTRIBUTE_NOINLINE __attribute__((noinline)) - #define PHMAP_HAVE_ATTRIBUTE_NOINLINE 1 -#else - #define PHMAP_ATTRIBUTE_NOINLINE -#endif - -#if PHMAP_HAVE_ATTRIBUTE(disable_tail_calls) - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL __attribute__((disable_tail_calls)) -#elif defined(__GNUC__) && !defined(__clang__) - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL \ - __attribute__((optimize("no-optimize-sibling-calls"))) -#else - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 0 -#endif - -#if (PHMAP_HAVE_ATTRIBUTE(weak) || \ - (defined(__GNUC__) && !defined(__clang__))) && \ - !(defined(__llvm__) && defined(_WIN32)) - #undef PHMAP_ATTRIBUTE_WEAK - #define PHMAP_ATTRIBUTE_WEAK __attribute__((weak)) - #define PHMAP_HAVE_ATTRIBUTE_WEAK 1 -#else - #define PHMAP_ATTRIBUTE_WEAK - #define PHMAP_HAVE_ATTRIBUTE_WEAK 0 -#endif - -#if PHMAP_HAVE_ATTRIBUTE(nonnull) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_NONNULL(arg_index) __attribute__((nonnull(arg_index))) -#else - #define PHMAP_ATTRIBUTE_NONNULL(...) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_NORETURN __attribute__((noreturn)) -#elif defined(_MSC_VER) - #define PHMAP_ATTRIBUTE_NORETURN __declspec(noreturn) -#else - #define PHMAP_ATTRIBUTE_NORETURN -#endif - -#if defined(__GNUC__) && defined(ADDRESS_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS -#endif - -#if defined(__GNUC__) && defined(MEMORY_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY -#endif - -#if defined(__GNUC__) && defined(THREAD_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD -#endif - -#if defined(__GNUC__) && \ - (defined(UNDEFINED_BEHAVIOR_SANITIZER) || defined(ADDRESS_SANITIZER)) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED \ - __attribute__((no_sanitize("undefined"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED -#endif - -#if defined(__GNUC__) && defined(CONTROL_FLOW_INTEGRITY) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI __attribute__((no_sanitize("cfi"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI -#endif - -#if defined(__GNUC__) && defined(SAFESTACK_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK \ - __attribute__((no_sanitize("safe-stack"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK -#endif - -#if PHMAP_HAVE_ATTRIBUTE(returns_nonnull) || \ - (defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ - !defined(__clang__)) - #define PHMAP_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) -#else - #define PHMAP_ATTRIBUTE_RETURNS_NONNULL -#endif - -#ifdef PHMAP_HAVE_ATTRIBUTE_SECTION - #error PHMAP_HAVE_ATTRIBUTE_SECTION cannot be directly set -#elif (PHMAP_HAVE_ATTRIBUTE(section) || \ - (defined(__GNUC__) && !defined(__clang__))) && \ - !defined(__APPLE__) && PHMAP_HAVE_ATTRIBUTE_WEAK - #define PHMAP_HAVE_ATTRIBUTE_SECTION 1 - #ifndef PHMAP_ATTRIBUTE_SECTION - #define PHMAP_ATTRIBUTE_SECTION(name) \ - __attribute__((section(#name))) __attribute__((noinline)) - #endif - #ifndef PHMAP_ATTRIBUTE_SECTION_VARIABLE - #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) - #endif - #ifndef PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS - #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ - extern char __start_##name[] PHMAP_ATTRIBUTE_WEAK; \ - extern char __stop_##name[] PHMAP_ATTRIBUTE_WEAK - #endif - #ifndef PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS - #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) - #endif - #define PHMAP_ATTRIBUTE_SECTION_START(name) \ - (reinterpret_cast<void *>(__start_##name)) - #define PHMAP_ATTRIBUTE_SECTION_STOP(name) \ - (reinterpret_cast<void *>(__stop_##name)) -#else // !PHMAP_HAVE_ATTRIBUTE_SECTION - #define PHMAP_HAVE_ATTRIBUTE_SECTION 0 - #define PHMAP_ATTRIBUTE_SECTION(name) - #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) - #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0)) - #define PHMAP_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0)) -#endif // PHMAP_ATTRIBUTE_SECTION - -#if PHMAP_HAVE_ATTRIBUTE(force_align_arg_pointer) || \ - (defined(__GNUC__) && !defined(__clang__)) - #if defined(__i386__) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC \ - __attribute__((force_align_arg_pointer)) - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) - #elif defined(__x86_64__) - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (1) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #else // !__i386__ && !__x86_64 - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #endif // __i386__ -#else - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(nodiscard) - #define PHMAP_MUST_USE_RESULT [[nodiscard]] -#elif defined(__clang__) && PHMAP_HAVE_ATTRIBUTE(warn_unused_result) - #define PHMAP_MUST_USE_RESULT __attribute__((warn_unused_result)) -#else - #define PHMAP_MUST_USE_RESULT -#endif - -#if PHMAP_HAVE_ATTRIBUTE(hot) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_HOT __attribute__((hot)) -#else - #define PHMAP_ATTRIBUTE_HOT -#endif - -#if PHMAP_HAVE_ATTRIBUTE(cold) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_COLD __attribute__((cold)) -#else - #define PHMAP_ATTRIBUTE_COLD -#endif - -#if defined(__clang__) - #if PHMAP_HAVE_CPP_ATTRIBUTE(clang::reinitializes) - #define PHMAP_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] - #else - #define PHMAP_ATTRIBUTE_REINITIALIZES - #endif -#else - #define PHMAP_ATTRIBUTE_REINITIALIZES -#endif - -#if PHMAP_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) - #undef PHMAP_ATTRIBUTE_UNUSED - #define PHMAP_ATTRIBUTE_UNUSED __attribute__((__unused__)) -#else - #define PHMAP_ATTRIBUTE_UNUSED -#endif - -#if PHMAP_HAVE_ATTRIBUTE(tls_model) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_INITIAL_EXEC __attribute__((tls_model("initial-exec"))) -#else - #define PHMAP_ATTRIBUTE_INITIAL_EXEC -#endif - -#if PHMAP_HAVE_ATTRIBUTE(packed) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_PACKED __attribute__((__packed__)) -#else - #define PHMAP_ATTRIBUTE_PACKED -#endif - -#if PHMAP_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes))) -#else - #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) -#endif - -// ---------------------------------------------------------------------- -// Figure out SSE support -// ---------------------------------------------------------------------- -#ifndef PHMAP_HAVE_SSE2 - #if defined(__SSE2__) || \ - (defined(_MSC_VER) && \ - (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) - #define PHMAP_HAVE_SSE2 1 - #else - #define PHMAP_HAVE_SSE2 0 - #endif -#endif - -#ifndef PHMAP_HAVE_SSSE3 - #if defined(__SSSE3__) || defined(__AVX2__) - #define PHMAP_HAVE_SSSE3 1 - #else - #define PHMAP_HAVE_SSSE3 0 - #endif -#endif - -#if PHMAP_HAVE_SSSE3 && !PHMAP_HAVE_SSE2 - #error "Bad configuration!" -#endif - -#if PHMAP_HAVE_SSE2 - #include <emmintrin.h> -#endif - -#if PHMAP_HAVE_SSSE3 - #include <tmmintrin.h> -#endif - - -// ---------------------------------------------------------------------- -// constexpr if -// ---------------------------------------------------------------------- -#if PHMAP_HAVE_CC17 - #define PHMAP_IF_CONSTEXPR(expr) if constexpr ((expr)) -#else - #define PHMAP_IF_CONSTEXPR(expr) if ((expr)) -#endif - -// ---------------------------------------------------------------------- -// base/macros.h -// ---------------------------------------------------------------------- - -// PHMAP_ARRAYSIZE() -// -// Returns the number of elements in an array as a compile-time constant, which -// can be used in defining new arrays. If you use this macro on a pointer by -// mistake, you will get a compile-time error. -#define PHMAP_ARRAYSIZE(array) \ - (sizeof(::phmap::macros_internal::ArraySizeHelper(array))) - -namespace phmap { -namespace macros_internal { - // Note: this internal template function declaration is used by PHMAP_ARRAYSIZE. - // The function doesn't need a definition, as we only use its type. - template <typename T, size_t N> - auto ArraySizeHelper(const T (&array)[N]) -> char (&)[N]; -} // namespace macros_internal -} // namespace phmap - -// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported. -#if defined(__clang__) && defined(__has_warning) - #if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") - #define PHMAP_FALLTHROUGH_INTENDED [[clang::fallthrough]] - #endif -#elif defined(__GNUC__) && __GNUC__ >= 7 - #define PHMAP_FALLTHROUGH_INTENDED [[gnu::fallthrough]] -#endif - -#ifndef PHMAP_FALLTHROUGH_INTENDED - #define PHMAP_FALLTHROUGH_INTENDED \ - do { } while (0) -#endif - -// PHMAP_DEPRECATED() -// -// Marks a deprecated class, struct, enum, function, method and variable -// declarations. The macro argument is used as a custom diagnostic message (e.g. -// suggestion of a better alternative). -// -// Example: -// -// class PHMAP_DEPRECATED("Use Bar instead") Foo {...}; -// PHMAP_DEPRECATED("Use Baz instead") void Bar() {...} -// -// Every usage of a deprecated entity will trigger a warning when compiled with -// clang's `-Wdeprecated-declarations` option. This option is turned off by -// default, but the warnings will be reported by clang-tidy. -#if defined(__clang__) && __cplusplus >= 201103L - #define PHMAP_DEPRECATED(message) __attribute__((deprecated(message))) -#endif - -#ifndef PHMAP_DEPRECATED - #define PHMAP_DEPRECATED(message) -#endif - -// PHMAP_BAD_CALL_IF() -// -// Used on a function overload to trap bad calls: any call that matches the -// overload will cause a compile-time error. This macro uses a clang-specific -// "enable_if" attribute, as described at -// http://clang.llvm.org/docs/AttributeReference.html#enable-if -// -// Overloads which use this macro should be bracketed by -// `#ifdef PHMAP_BAD_CALL_IF`. -// -// Example: -// -// int isdigit(int c); -// #ifdef PHMAP_BAD_CALL_IF -// int isdigit(int c) -// PHMAP_BAD_CALL_IF(c <= -1 || c > 255, -// "'c' must have the value of an unsigned char or EOF"); -// #endif // PHMAP_BAD_CALL_IF - -#if defined(__clang__) - #if __has_attribute(enable_if) - #define PHMAP_BAD_CALL_IF(expr, msg) \ - __attribute__((enable_if(expr, "Bad call trap"), unavailable(msg))) - #endif -#endif - -// PHMAP_ASSERT() -// -// In C++11, `assert` can't be used portably within constexpr functions. -// PHMAP_ASSERT functions as a runtime assert but works in C++11 constexpr -// functions. Example: -// -// constexpr double Divide(double a, double b) { -// return PHMAP_ASSERT(b != 0), a / b; -// } -// -// This macro is inspired by -// https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ -#if defined(NDEBUG) - #define PHMAP_ASSERT(expr) (false ? (void)(expr) : (void)0) -#else - #define PHMAP_ASSERT(expr) \ - (PHMAP_PREDICT_TRUE((expr)) ? (void)0 \ - : [] { assert(false && #expr); }()) // NOLINT -#endif - -#ifdef PHMAP_HAVE_EXCEPTIONS - #define PHMAP_INTERNAL_TRY try - #define PHMAP_INTERNAL_CATCH_ANY catch (...) - #define PHMAP_INTERNAL_RETHROW do { throw; } while (false) -#else // PHMAP_HAVE_EXCEPTIONS - #define PHMAP_INTERNAL_TRY if (true) - #define PHMAP_INTERNAL_CATCH_ANY else if (false) - #define PHMAP_INTERNAL_RETHROW do {} while (false) -#endif // PHMAP_HAVE_EXCEPTIONS - - -#endif // phmap_config_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_dump.h b/benchmarks/external/parallel_hashmap/phmap_dump.h deleted file mode 100644 index 0f2018ef..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_dump.h +++ /dev/null @@ -1,227 +0,0 @@ -#if !defined(phmap_dump_h_guard_) -#define phmap_dump_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// providing dump/load/mmap_load -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#include <iostream> -#include <fstream> -#include <sstream> -#include "phmap.h" -namespace phmap -{ - -namespace type_traits_internal { - -#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 - template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {}; -#else - template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {}; -#endif - -template <class T1, class T2> -struct IsTriviallyCopyable<std::pair<T1, T2>> { - static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value; -}; -} - -namespace priv { - -// ------------------------------------------------------------------------ -// dump/load for raw_hash_set -// ------------------------------------------------------------------------ -template <class Policy, class Hash, class Eq, class Alloc> -template<typename OutputArchive> -bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - if (!ar.dump(size_)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast<char*>(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast<char*>(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; -} - -template <class Policy, class Hash, class Eq, class Alloc> -template<typename InputArchive> -bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content - if (!ar.load(&size_)) { - std::cerr << "Failed to load size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_)) { - std::cerr << "Failed to load capacity_" << std::endl; - return false; - } - - // allocate memory for ctrl_ and slots_ - initialize_slots(); - if (!ar.load(reinterpret_cast<char*>(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - std::cerr << "Failed to load ctrl" << std::endl; - return false; - } - if (!ar.load(reinterpret_cast<char*>(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to load slot" << std::endl; - return false; - } - return true; -} - -// ------------------------------------------------------------------------ -// dump/load for parallel_hash_set -// ------------------------------------------------------------------------ -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template<typename OutputArchive> -bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - if (! ar.dump(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); - if (!inner.set_.dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; -} - -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template<typename InputArchive> -bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - size_t submap_count = 0; - if (!ar.load(&submap_count)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } - - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } - - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); - if (!inner.set_.load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; -} -} // namespace priv - - - -// ------------------------------------------------------------------------ -// BinaryArchive -// File is closed when archive object is destroyed -// ------------------------------------------------------------------------ - -// ------------------------------------------------------------------------ -// ------------------------------------------------------------------------ -class BinaryOutputArchive { -public: - BinaryOutputArchive(const char *file_path) { - ofs_.open(file_path, std::ios_base::binary); - } - - bool dump(const char *p, size_t sz) { - ofs_.write(p, sz); - return true; - } - - template<typename V> - typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type - dump(const V& v) { - ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V)); - return true; - } - -private: - std::ofstream ofs_; -}; - - -class BinaryInputArchive { -public: - BinaryInputArchive(const char * file_path) { - ifs_.open(file_path, std::ios_base::binary); - } - - bool load(char* p, size_t sz) { - ifs_.read(p, sz); - return true; - } - - template<typename V> - typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type - load(V* v) { - ifs_.read(reinterpret_cast<char *>(v), sizeof(V)); - return true; - } - -private: - std::ifstream ifs_; -}; - -} // namespace phmap - -#endif // phmap_dump_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h b/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h deleted file mode 100644 index a7719c49..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h +++ /dev/null @@ -1,154 +0,0 @@ -#if !defined(phmap_fwd_decl_h_guard_) -#define phmap_fwd_decl_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion -#endif - -#include <memory> -#include <utility> - -#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_) - namespace absl { template <class T> struct Hash; }; -#endif - -namespace phmap { - -#if defined(PHMAP_USE_ABSL_HASH) - template <class T> using Hash = ::absl::Hash<T>; -#else - template <class T> struct Hash; -#endif - - template <class T> struct EqualTo; - template <class T> struct Less; - template <class T> using Allocator = typename std::allocator<T>; - template<class T1, class T2> using Pair = typename std::pair<T1, T2>; - - class NullMutex; - - namespace priv { - - // The hash of an object of type T is computed by using phmap::Hash. - template <class T, class E = void> - struct HashEq - { - using Hash = phmap::Hash<T>; - using Eq = phmap::EqualTo<T>; - }; - - template <class T> - using hash_default_hash = typename priv::HashEq<T>::Hash; - - template <class T> - using hash_default_eq = typename priv::HashEq<T>::Eq; - - // type alias for std::allocator so we can forward declare without including other headers - template <class T> - using Allocator = typename phmap::Allocator<T>; - - // type alias for std::pair so we can forward declare without including other headers - template<class T1, class T2> - using Pair = typename phmap::Pair<T1, T2>; - - } // namespace priv - - // ------------- forward declarations for hash containers ---------------------------------- - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator - class flat_hash_set; - - template <class K, class V, - class Hash = phmap::priv::hash_default_hash<K>, - class Eq = phmap::priv::hash_default_eq<K>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const K, V>>> // alias for std::allocator - class flat_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator - class node_hash_set; - - template <class Key, class Value, - class Hash = phmap::priv::hash_default_hash<Key>, - class Eq = phmap::priv::hash_default_eq<Key>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const Key, Value>>> // alias for std::allocator - class node_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_flat_hash_set; - - template <class K, class V, - class Hash = phmap::priv::hash_default_hash<K>, - class Eq = phmap::priv::hash_default_eq<K>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const K, V>>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_flat_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_node_hash_set; - - template <class Key, class Value, - class Hash = phmap::priv::hash_default_hash<Key>, - class Eq = phmap::priv::hash_default_eq<Key>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const Key, Value>>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_node_hash_map; - - // ------------- forward declarations for btree containers ---------------------------------- - template <typename Key, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<Key>> - class btree_set; - - template <typename Key, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<Key>> - class btree_multiset; - - template <typename Key, typename Value, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> - class btree_map; - - template <typename Key, typename Value, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> - class btree_multimap; - -} // namespace phmap - - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_fwd_decl_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_utils.h b/benchmarks/external/parallel_hashmap/phmap_utils.h deleted file mode 100644 index 1d0c4728..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_utils.h +++ /dev/null @@ -1,378 +0,0 @@ -#if !defined(phmap_utils_h_guard_) -#define phmap_utils_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// minimal header providing phmap::HashState -// -// use as: phmap::HashState().combine(0, _first_name, _last_name, _age); -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion -#endif - -#include <cstdint> -#include <functional> -#include <tuple> -#include "phmap_bits.h" - -// --------------------------------------------------------------- -// Absl forward declaration requires global scope. -// --------------------------------------------------------------- -#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) && !defined(ABSL_HASH_HASH_H_) - namespace absl { template <class T> struct Hash; }; -#endif - -namespace phmap -{ - -// --------------------------------------------------------------- -// --------------------------------------------------------------- -template<int n> -struct phmap_mix -{ - inline size_t operator()(size_t) const; -}; - -template<> -struct phmap_mix<4> -{ - inline size_t operator()(size_t a) const - { - static constexpr uint64_t kmul = 0xcc9e2d51UL; - // static constexpr uint64_t kmul = 0x3B9ACB93UL; // [greg] my own random prime - uint64_t l = a * kmul; - return static_cast<size_t>(l ^ (l >> 32)); - } -}; - -#if defined(PHMAP_HAS_UMUL128) - template<> - struct phmap_mix<8> - { - // Very fast mixing (similar to Abseil) - inline size_t operator()(size_t a) const - { - static constexpr uint64_t k = 0xde5fb9d2630458e9ULL; - // static constexpr uint64_t k = 0x7C9D0BF0567102A5ULL; // [greg] my own random prime - uint64_t h; - uint64_t l = umul128(a, k, &h); - return static_cast<size_t>(h + l); - } - }; -#else - template<> - struct phmap_mix<8> - { - inline size_t operator()(size_t a) const - { - a = (~a) + (a << 21); // a = (a << 21) - a - 1; - a = a ^ (a >> 24); - a = (a + (a << 3)) + (a << 8); // a * 265 - a = a ^ (a >> 14); - a = (a + (a << 2)) + (a << 4); // a * 21 - a = a ^ (a >> 28); - a = a + (a << 31); - return static_cast<size_t>(a); - } - }; -#endif - -// -------------------------------------------- -template<int n> -struct fold_if_needed -{ - inline size_t operator()(uint64_t) const; -}; - -template<> -struct fold_if_needed<4> -{ - inline size_t operator()(uint64_t a) const - { - return static_cast<size_t>(a ^ (a >> 32)); - } -}; - -template<> -struct fold_if_needed<8> -{ - inline size_t operator()(uint64_t a) const - { - return static_cast<size_t>(a); - } -}; - -// --------------------------------------------------------------- -// see if class T has a hash_value() friend method -// --------------------------------------------------------------- -template<typename T> -struct has_hash_value -{ -private: - typedef std::true_type yes; - typedef std::false_type no; - - template<typename U> static auto test(int) -> decltype(hash_value(std::declval<const U&>()) == 1, yes()); - - template<typename> static no test(...); - -public: - static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; -}; - -#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) - template <class T> using Hash = ::absl::Hash<T>; -#elif !defined(PHMAP_USE_ABSL_HASH) -// --------------------------------------------------------------- -// phmap::Hash -// --------------------------------------------------------------- -template <class T> -struct Hash -{ - template <class U, typename std::enable_if<has_hash_value<U>::value, int>::type = 0> - size_t _hash(const T& val) const - { - return hash_value(val); - } - - template <class U, typename std::enable_if<!has_hash_value<U>::value, int>::type = 0> - size_t _hash(const T& val) const - { - return std::hash<T>()(val); - } - - inline size_t operator()(const T& val) const - { - return _hash<T>(val); - } -}; - -template <class T> -struct Hash<T *> -{ - inline size_t operator()(const T *val) const noexcept - { - return static_cast<size_t>(reinterpret_cast<const uintptr_t>(val)); - } -}; - -template<class ArgumentType, class ResultType> -struct phmap_unary_function -{ - typedef ArgumentType argument_type; - typedef ResultType result_type; -}; - -template <> -struct Hash<bool> : public phmap_unary_function<bool, size_t> -{ - inline size_t operator()(bool val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<char> : public phmap_unary_function<char, size_t> -{ - inline size_t operator()(char val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<signed char> : public phmap_unary_function<signed char, size_t> -{ - inline size_t operator()(signed char val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<unsigned char> : public phmap_unary_function<unsigned char, size_t> -{ - inline size_t operator()(unsigned char val) const noexcept - { return static_cast<size_t>(val); } -}; - -#ifdef PHMAP_HAS_NATIVE_WCHAR_T -template <> -struct Hash<wchar_t> : public phmap_unary_function<wchar_t, size_t> -{ - inline size_t operator()(wchar_t val) const noexcept - { return static_cast<size_t>(val); } -}; -#endif - -template <> -struct Hash<int16_t> : public phmap_unary_function<int16_t, size_t> -{ - inline size_t operator()(int16_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<uint16_t> : public phmap_unary_function<uint16_t, size_t> -{ - inline size_t operator()(uint16_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<int32_t> : public phmap_unary_function<int32_t, size_t> -{ - inline size_t operator()(int32_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<uint32_t> : public phmap_unary_function<uint32_t, size_t> -{ - inline size_t operator()(uint32_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<int64_t> : public phmap_unary_function<int64_t, size_t> -{ - inline size_t operator()(int64_t val) const noexcept - { return fold_if_needed<sizeof(size_t)>()(static_cast<uint64_t>(val)); } -}; - -template <> -struct Hash<uint64_t> : public phmap_unary_function<uint64_t, size_t> -{ - inline size_t operator()(uint64_t val) const noexcept - { return fold_if_needed<sizeof(size_t)>()(val); } -}; - -template <> -struct Hash<float> : public phmap_unary_function<float, size_t> -{ - inline size_t operator()(float val) const noexcept - { - // -0.0 and 0.0 should return same hash - uint32_t *as_int = reinterpret_cast<uint32_t *>(&val); - return (val == 0) ? static_cast<size_t>(0) : - static_cast<size_t>(*as_int); - } -}; - -template <> -struct Hash<double> : public phmap_unary_function<double, size_t> -{ - inline size_t operator()(double val) const noexcept - { - // -0.0 and 0.0 should return same hash - uint64_t *as_int = reinterpret_cast<uint64_t *>(&val); - return (val == 0) ? static_cast<size_t>(0) : - fold_if_needed<sizeof(size_t)>()(*as_int); - } -}; - -#endif - -template <class H, int sz> struct Combiner -{ - H operator()(H seed, size_t value); -}; - -template <class H> struct Combiner<H, 4> -{ - H operator()(H seed, size_t value) - { - return seed ^ (value + 0x9e3779b9 + (seed << 6) + (seed >> 2)); - } -}; - -template <class H> struct Combiner<H, 8> -{ - H operator()(H seed, size_t value) - { - return seed ^ (value + size_t(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2)); - } -}; - -// define HashState to combine member hashes... see example below -// ----------------------------------------------------------------------------- -template <typename H> -class HashStateBase { -public: - template <typename T, typename... Ts> - static H combine(H state, const T& value, const Ts&... values); - - static H combine(H state) { return state; } -}; - -template <typename H> -template <typename T, typename... Ts> -H HashStateBase<H>::combine(H seed, const T& v, const Ts&... vs) -{ - return HashStateBase<H>::combine(Combiner<H, sizeof(H)>()( - seed, phmap::Hash<T>()(v)), - vs...); -} - -using HashState = HashStateBase<size_t>; - -// ----------------------------------------------------------------------------- - -#if !defined(PHMAP_USE_ABSL_HASH) - -// define Hash for std::pair -// ------------------------- -template<class T1, class T2> -struct Hash<std::pair<T1, T2>> { - size_t operator()(std::pair<T1, T2> const& p) const noexcept { - return phmap::HashState().combine(phmap::Hash<T1>()(p.first), p.second); - } -}; - -// define Hash for std::tuple -// -------------------------- -template<class... T> -struct Hash<std::tuple<T...>> { - size_t operator()(std::tuple<T...> const& t) const noexcept { - return _hash_helper(t); - } - -private: - template<size_t I = 0, class ...P> - typename std::enable_if<I == sizeof...(P), size_t>::type - _hash_helper(const std::tuple<P...> &) const noexcept { return 0; } - - template<size_t I = 0, class ...P> - typename std::enable_if<I < sizeof...(P), size_t>::type - _hash_helper(const std::tuple<P...> &t) const noexcept { - const auto &el = std::get<I>(t); - using el_type = typename std::remove_cv<typename std::remove_reference<decltype(el)>::type>::type; - return Combiner<size_t, sizeof(size_t)>()( - phmap::Hash<el_type>()(el), _hash_helper<I + 1>(t)); - } -}; - - -#endif - - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_utils_h_guard_ |