From 9aafd5ce068da6f87c2f3ac8c9cf0721976f2bbc Mon Sep 17 00:00:00 2001 From: Tyge Løvset Date: Thu, 7 Jul 2022 10:47:14 +0200 Subject: Added emhash by ktprime to external benchmarks. Very fast iteration and lookups. --- benchmarks/external/emhash/hash_table7.hpp | 1887 +++++++ benchmarks/external/emhash/wyhash.h | 272 + benchmarks/external/parallel_hashmap/btree.h | 4050 --------------- benchmarks/external/parallel_hashmap/conanfile.py | 36 - benchmarks/external/parallel_hashmap/meminfo.h | 195 - benchmarks/external/parallel_hashmap/phmap.h | 4788 ------------------ benchmarks/external/parallel_hashmap/phmap_base.h | 5171 -------------------- benchmarks/external/parallel_hashmap/phmap_bits.h | 663 --- .../external/parallel_hashmap/phmap_config.h | 771 --- benchmarks/external/parallel_hashmap/phmap_dump.h | 227 - .../external/parallel_hashmap/phmap_fwd_decl.h | 154 - benchmarks/external/parallel_hashmap/phmap_utils.h | 378 -- benchmarks/external/skarupke/bytell_hash_map.hpp | 1260 ----- benchmarks/external/tsl/hopscotch_growth_policy.h | 404 -- benchmarks/external/tsl/hopscotch_hash.h | 1894 ------- benchmarks/external/tsl/hopscotch_map.h | 735 --- benchmarks/external/update.sh | 70 +- 17 files changed, 2197 insertions(+), 20758 deletions(-) create mode 100644 benchmarks/external/emhash/hash_table7.hpp create mode 100644 benchmarks/external/emhash/wyhash.h delete mode 100644 benchmarks/external/parallel_hashmap/btree.h delete mode 100644 benchmarks/external/parallel_hashmap/conanfile.py delete mode 100644 benchmarks/external/parallel_hashmap/meminfo.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_base.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_bits.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_config.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_dump.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_fwd_decl.h delete mode 100644 benchmarks/external/parallel_hashmap/phmap_utils.h delete mode 100644 benchmarks/external/skarupke/bytell_hash_map.hpp delete mode 100644 benchmarks/external/tsl/hopscotch_growth_policy.h delete mode 100644 benchmarks/external/tsl/hopscotch_hash.h delete mode 100644 benchmarks/external/tsl/hopscotch_map.h diff --git a/benchmarks/external/emhash/hash_table7.hpp b/benchmarks/external/emhash/hash_table7.hpp new file mode 100644 index 00000000..dbf90c86 --- /dev/null +++ b/benchmarks/external/emhash/hash_table7.hpp @@ -0,0 +1,1887 @@ +// emhash7::HashMap for C++11/14/17 +// version 2.1.2 +// https://github.com/ktprime/ktprime/blob/master/hash_table7.hpp +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2019-2022 Huang Yuanbing & bailuzhou AT 163.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE + +// From +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// separator chain resolution 1 + L / 2 exp(-L) + L + +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 +// SEPARATE CHAN RES. +// probes/successful lookup 1.05 1.25 1.3 1.25 1.4 1.45 1.50 +// probes/unsuccessful lookup 1.00 1.11 1.15 1.22 1.25 1.31 1.37 +// clacul/unsuccessful lookup 1.01 1.25 1.36, 1.56, 1.64, 1.81, 1.97 + +/**************** + under random hashCodes, the frequency of nodes in bins follows a Poisson +distribution(http://en.wikipedia.org/wiki/Poisson_distribution) with a parameter of about 0.5 +on average for the default resizing threshold of 0.75, although with a large variance because +of resizing granularity. Ignoring variance, the expected occurrences of list size k are +(exp(-0.5) * pow(0.5, k)/factorial(k)). The first values are: +0: 0.60653066 +1: 0.30326533 +2: 0.07581633 +3: 0.01263606 +4: 0.00157952 +5: 0.00015795 +6: 0.00001316 +7: 0.00000094 +8: 0.00000006 + + ============== buckets size ration ======== + 1 1543981 0.36884964|0.36787944 36.885 + 2 768655 0.36725597|0.36787944 73.611 + 3 256236 0.18364065|0.18393972 91.975 + 4 64126 0.06127757|0.06131324 98.102 + 5 12907 0.01541710|0.01532831 99.644 + 6 2050 0.00293841|0.00306566 99.938 + 7 310 0.00051840|0.00051094 99.990 + 8 49 0.00009365|0.00007299 99.999 + 9 4 0.00000860|0.00000913 100.000 +========== collision miss ration =========== + _num_filled aver_size k.v size_kv = 4185936, 1.58, x.x 24 + collision,possion,cache_miss hit_find|hit_miss, load_factor = 36.73%,36.74%,31.31% 1.50|2.00, 1.00 +============== buckets size ration ======== +*******************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __has_include + #if __has_include("wyhash.h") + #include "wyhash.h" + #endif +#elif EMH_WY_HASH + #include "wyhash.h" +#endif + +#ifdef EMH_KEY + #undef EMH_KEY + #undef EMH_VAL + #undef EMH_PKV + #undef EMH_NEW + #undef EMH_SET + #undef EMH_BUCKET + #undef EMH_EMPTY +#endif + +// likely/unlikely +#if (__GNUC__ >= 4 || __clang__) +# define EMH_LIKELY(condition) __builtin_expect(condition, 1) +# define EMH_UNLIKELY(condition) __builtin_expect(condition, 0) +#else +# define EMH_LIKELY(condition) condition +# define EMH_UNLIKELY(condition) condition +#endif + +#ifndef EMH_BUCKET_INDEX + #define EMH_BUCKET_INDEX 1 +#endif + +#ifndef EMH_DEFAULT_LOAD_FACTOR +#define EMH_DEFAULT_LOAD_FACTOR 0.80f +#endif + +#if EMH_BUCKET_INDEX == 0 + #define EMH_KEY(p,n) p[n].second.first + #define EMH_VAL(p,n) p[n].second.second + #define EMH_BUCKET(p,n) p[n].first + #define EMH_PKV(p,n) p[n].second + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(bucket, value_type(key, val));\ + _num_filled ++; EMH_SET(bucket) +#elif EMH_BUCKET_INDEX == 2 + #define EMH_KEY(p,n) p[n].first.first + #define EMH_VAL(p,n) p[n].first.second + #define EMH_BUCKET(p,n) p[n].second + #define EMH_PKV(p,n) p[n].first + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(value_type(key, val), bucket);\ + _num_filled ++; EMH_SET(bucket) +#else + #define EMH_KEY(p,n) p[n].first + #define EMH_VAL(p,n) p[n].second + #define EMH_BUCKET(p,n) p[n].bucket + #define EMH_PKV(p,n) p[n] + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(key, val, bucket);\ + _num_filled ++; EMH_SET(bucket) +#endif + +#define EMH_MASK(bucket) 1 << (bucket % MASK_BIT) +#define EMH_SET(bucket) _bitmask[bucket / MASK_BIT] &= ~(EMH_MASK(bucket)) +#define EMH_CLS(bucket) _bitmask[bucket / MASK_BIT] |= EMH_MASK(bucket) +#define EMH_EMPTY(bitmask, bucket) (_bitmask[bucket / MASK_BIT] & (EMH_MASK(bucket))) != 0 + +#if _WIN32 + #include +#if _WIN64 + #pragma intrinsic(_umul128) +#endif +#endif + +namespace emhash7 { + +#ifdef EMH_SIZE_TYPE_16BIT + typedef uint16_t size_type; + static constexpr size_type INACTIVE = 0xFFFE; +#elif EMH_SIZE_TYPE_64BIT + typedef uint64_t size_type; + static constexpr size_type INACTIVE = 0 - 0x1ull; +#else + typedef uint32_t size_type; + static constexpr size_type INACTIVE = 0 - 0x1u; +#endif + +#ifndef EMH_SIZE_TYPE_16BIT +static_assert((int)INACTIVE < 0, "INACTIVE must negative (to int)"); +#endif + +//count the leading zero bit +inline static size_type CTZ(size_t n) +{ +#if defined(__x86_64__) || defined(_WIN32) || (__BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + +#elif __BIG_ENDIAN__ || (__BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + n = __builtin_bswap64(n); +#else + static uint32_t endianness = 0x12345678; + const auto is_big = *(const char *)&endianness == 0x12; + if (is_big) + n = __builtin_bswap64(n); +#endif + +#if _WIN32 + unsigned long index; + #if defined(_WIN64) + _BitScanForward64(&index, n); + #else + _BitScanForward(&index, n); + #endif +#elif defined (__LP64__) || (SIZE_MAX == UINT64_MAX) || defined (__x86_64__) + auto index = __builtin_ctzll(n); +#elif 1 + auto index = __builtin_ctzl(n); +#else + #if defined (__LP64__) || (SIZE_MAX == UINT64_MAX) || defined (__x86_64__) + size_type index; + __asm__("bsfq %1, %0\n" : "=r" (index) : "rm" (n) : "cc"); + #else + size_type index; + __asm__("bsf %1, %0\n" : "=r" (index) : "rm" (n) : "cc"); + #endif +#endif + + return (size_type)index; +} + +template +struct entry { + using first_type = First; + using second_type = Second; + entry(const First& key, const Second& val, size_type ibucket) + :second(val),first(key) + { + bucket = ibucket; + } + + entry(First&& key, Second&& val, size_type ibucket) + :second(std::move(val)), first(std::move(key)) + { + bucket = ibucket; + } + + template + entry(K&& key, V&& val, size_type ibucket) + :second(std::forward(val)), first(std::forward(key)) + { + bucket = ibucket; + } + + entry(const std::pair& pair) + :second(pair.second),first(pair.first) + { + bucket = INACTIVE; + } + + entry(std::pair&& pair) + :second(std::move(pair.second)), first(std::move(pair.first)) + { + bucket = INACTIVE; + } + + entry(std::tuple&& tup) + :second(std::move(std::get<2>(tup))), first(std::move(std::get<1>(tup))) + { + bucket = INACTIVE; + } + + entry(const entry& rhs) + :second(rhs.second),first(rhs.first) + { + bucket = rhs.bucket; + } + + entry(entry&& rhs) noexcept + :second(std::move(rhs.second)),first(std::move(rhs.first)) + { + bucket = rhs.bucket; + } + + entry& operator = (entry&& rhs) + { + second = std::move(rhs.second); + bucket = rhs.bucket; + first = std::move(rhs.first); + return *this; + } + + entry& operator = (const entry& rhs) + { + second = rhs.second; + bucket = rhs.bucket; + first = rhs.first; + return *this; + } + + bool operator == (const entry& p) const + { + return first == p.first && second == p.second; + } + + bool operator == (const std::pair& p) const + { + return first == p.first && second == p.second; + } + + void swap(entry& o) + { + std::swap(second, o.second); + std::swap(first, o.first); + } + +#ifndef EMH_ORDER_KV + Second second;//int + size_type bucket; + First first; //long +#else + First first; //long + size_type bucket; + Second second;//int +#endif +};// __attribute__ ((packed)); + +/// A cache-friendly hash table with open addressing, linear/qua probing and power-of-two capacity +template , typename EqT = std::equal_to> +class HashMap +{ +public: + typedef HashMap htype; + typedef std::pair value_type; + +#if EMH_BUCKET_INDEX == 0 + typedef value_type value_pair; + typedef std::pair PairT; +#elif EMH_BUCKET_INDEX == 2 + typedef value_type value_pair; + typedef std::pair PairT; +#else + typedef entry value_pair; + typedef entry PairT; +#endif + + typedef KeyT key_type; + typedef ValueT val_type; + typedef ValueT mapped_type; + typedef HashT hasher; + typedef EqT key_equal; + typedef PairT& reference; + typedef const PairT& const_reference; + + class const_iterator; + class iterator + { + public: + typedef std::forward_iterator_tag iterator_category; + typedef std::ptrdiff_t difference_type; + typedef value_pair value_type; + + typedef value_pair* pointer; + typedef value_pair& reference; + + iterator() : _map(nullptr) { } + iterator(const const_iterator& it) : _map(it._map), _bucket(it._bucket), _from(it._from), _bmask(it._bmask) { } + iterator(const htype* hash_map, size_type bucket, bool) : _map(hash_map), _bucket(bucket) { init(); } + iterator(const htype* hash_map, size_type bucket) : _map(hash_map), _bucket(bucket) { _bmask = _from = 0; } + + void init() + { + _from = (_bucket / SIZE_BIT) * SIZE_BIT; + if (_bucket < _map->bucket_count()) { + _bmask = *(size_t*)((size_t*)_map->_bitmask + _from / SIZE_BIT); + _bmask |= (1ull << _bucket % SIZE_BIT) - 1; + _bmask = ~_bmask; + } else { + _bmask = 0; + } + } + + size_t bucket() const + { + return _bucket; + } + + void clear(size_type bucket) + { + if (_bucket / SIZE_BIT == bucket / SIZE_BIT) + _bmask &= ~(1ull << (bucket % SIZE_BIT)); + } + + iterator& next() + { + goto_next_element(); + return *this; + } + + iterator& operator++() + { + _bmask &= _bmask - 1; + goto_next_element(); + return *this; + } + + iterator operator++(int) + { + iterator old = *this; + _bmask &= _bmask - 1; + goto_next_element(); + return old; + } + + reference operator*() const + { + return _map->EMH_PKV(_pairs, _bucket); + } + + pointer operator->() const + { + return &(_map->EMH_PKV(_pairs, _bucket)); + } + + bool operator==(const iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const iterator& rhs) const { return _bucket != rhs._bucket; } + bool operator==(const const_iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const const_iterator& rhs) const { return _bucket != rhs._bucket; } + + private: + void goto_next_element() + { + if (EMH_LIKELY(_bmask != 0)) { + _bucket = _from + CTZ(_bmask); + return; + } + + do { + _bmask = ~*(size_t*)((size_t*)_map->_bitmask + (_from += SIZE_BIT) / SIZE_BIT); + } while (_bmask == 0); + + _bucket = _from + CTZ(_bmask); + } + + public: + const htype* _map; + size_type _bucket; + size_type _from; + size_t _bmask; + }; + + class const_iterator + { + public: + typedef std::forward_iterator_tag iterator_category; + typedef std::ptrdiff_t difference_type; + typedef value_pair value_type; + + typedef const value_pair* pointer; + typedef const value_pair& reference; + + const_iterator(const iterator& it) : _map(it._map), _bucket(it._bucket), _from(it._from), _bmask(it._bmask) { } + const_iterator(const htype* hash_map, size_type bucket, bool) : _map(hash_map), _bucket(bucket) { init(); } + const_iterator(const htype* hash_map, size_type bucket) : _map(hash_map), _bucket(bucket) { _bmask = _from = 0; } + + void init() + { + _from = (_bucket / SIZE_BIT) * SIZE_BIT; + if (_bucket < _map->bucket_count()) { + _bmask = *(size_t*)((size_t*)_map->_bitmask + _from / SIZE_BIT); + _bmask |= (1ull << _bucket % SIZE_BIT) - 1; + _bmask = ~_bmask; + } else { + _bmask = 0; + } + } + + size_t bucket() const + { + return _bucket; + } + + const_iterator& operator++() + { + goto_next_element(); + return *this; + } + + const_iterator operator++(int) + { + const_iterator old(*this); + goto_next_element(); + return old; + } + + reference operator*() const + { + return _map->EMH_PKV(_pairs, _bucket); + } + + pointer operator->() const + { + return &(_map->EMH_PKV(_pairs, _bucket)); + } + + bool operator==(const const_iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const const_iterator& rhs) const { return _bucket != rhs._bucket; } + + private: + void goto_next_element() + { + _bmask &= _bmask - 1; + if (EMH_LIKELY(_bmask != 0)) { + _bucket = _from + CTZ(_bmask); + return; + } + + do { + _bmask = ~*(size_t*)((size_t*)_map->_bitmask + (_from += SIZE_BIT) / SIZE_BIT); + } while (_bmask == 0); + + _bucket = _from + CTZ(_bmask); + } + + public: + const htype* _map; + size_type _bucket; + size_type _from; + size_t _bmask; + }; + + void init(size_type bucket, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + _pairs = nullptr; + _bitmask = nullptr; + _num_buckets = _num_filled = 0; + max_load_factor(mlf); + reserve(bucket); + } + + HashMap(size_type bucket = 2, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + init(bucket, mlf); + } + + size_type AllocSize(size_type num_buckets) const + { + return num_buckets * sizeof(PairT) + EPACK_SIZE * sizeof(PairT) + (num_buckets + 7) / 8 + BIT_PACK; + } + + HashMap(const HashMap& rhs) + { + _pairs = (PairT*)malloc(AllocSize(rhs._num_buckets)); + clone(rhs); + } + + HashMap(HashMap&& rhs) + { +#ifndef EMH_ZERO_MOVE + init(4); +#else + _num_buckets = _num_filled = _mask = 0; + _pairs = nullptr; +#endif + swap(rhs); + } + + HashMap(std::initializer_list ilist) + { + init((size_type)ilist.size()); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template + HashMap(InputIt first, InputIt last, size_type bucket_count=4) + { + init(std::distance(first, last) + bucket_count); + for (; first != last; ++first) + emplace(*first); + } + + HashMap& operator=(const HashMap& rhs) + { + if (this == &rhs) + return *this; + + if (_num_filled) + clearkv(); + + if (_num_buckets != rhs._num_buckets) { + free(_pairs); + _pairs = (PairT*)malloc(AllocSize(rhs._num_buckets)); + } + + clone(rhs); + return *this; + } + + HashMap& operator=(HashMap&& rhs) + { + if (this != &rhs) { + swap(rhs); + rhs.clear(); + } + return *this; + } + + template + bool operator == (const Con& rhs) const + { + if (size() != rhs.size()) + return false; + + for (auto it = begin(), last = end(); it != last; ++it) { + auto oi = rhs.find(it->first); + if (oi == rhs.end() || it->second != oi->second) + return false; + } + return true; + } + + template + bool operator != (const Con& rhs) const { return !(*this == rhs); } + + ~HashMap() + { + if (is_triviall_destructable() && _num_filled) { + for (auto it = cbegin(); _num_filled; ++it) { + _num_filled --; + it->~value_pair(); + } + } + free(_pairs); + } + + void clone(const HashMap& rhs) + { + _hasher = rhs._hasher; +// _eq = rhs._eq; + + _num_filled = rhs._num_filled; + _mask = rhs._mask; + _mlf = rhs._mlf; + _num_buckets = rhs._num_buckets; + + _bitmask = decltype(_bitmask)(_pairs + EPACK_SIZE + _num_buckets); + auto* opairs = rhs._pairs; + + if (is_copy_trivially()) + memcpy(_pairs, opairs, AllocSize(_num_buckets)); + else { + memcpy(_pairs + _num_buckets, opairs + _num_buckets, EPACK_SIZE * sizeof(PairT) + (_num_buckets + 7) / 8 + BIT_PACK); + for (auto it = rhs.cbegin(); it.bucket() <= _mask; ++it) { + const auto bucket = it.bucket(); + EMH_BUCKET(_pairs, bucket) = EMH_BUCKET(opairs, bucket); + new(_pairs + bucket) PairT(opairs[bucket]); + } + } + } + + void swap(HashMap& rhs) + { + std::swap(_hasher, rhs._hasher); + // std::swap(_eq, rhs._eq); + std::swap(_pairs, rhs._pairs); + std::swap(_num_buckets, rhs._num_buckets); + std::swap(_num_filled, rhs._num_filled); + std::swap(_mask, rhs._mask); + std::swap(_mlf, rhs._mlf); + std::swap(_bitmask, rhs._bitmask); + //std::swap(EMH_BUCKET(_pairs, _num_buckets), EMH_BUCKET(rhs._pairs, rhs._num_buckets)); + } + + // ------------------------------------------------------------- + iterator begin() + { +#ifdef EMH_ZERO_MOVE + if (0 == _num_filled) + return {this, _num_buckets}; +#endif + + const auto bmask = ~(*(size_t*)_bitmask); + if (bmask != 0) + return {this, CTZ(bmask), true}; + + iterator it(this, sizeof(bmask) * 8 - 1); + return it.next(); + } + + const_iterator cbegin() const + { +#ifdef EMH_ZERO_MOVE + if (0 == _num_filled) + return {this, _num_buckets}; +#endif + + const auto bmask = ~(*(size_t*)_bitmask); + if (bmask != 0) + return {this, CTZ(bmask), true}; + + iterator it(this, sizeof(bmask) * 8 - 1); + return it.next(); + } + + iterator last() const + { + if (_num_filled == 0) + return end(); + + auto bucket = _mask; + while (EMH_EMPTY(_pairs, bucket)) bucket--; + return {this, bucket, true}; + } + + const_iterator begin() const { return cbegin(); } + + iterator end() { return {this, _num_buckets}; } + const_iterator cend() const { return {this, _num_buckets}; } + const_iterator end() const { return cend(); } + + size_type size() const { return _num_filled; } + bool empty() const { return _num_filled == 0; } + + size_type bucket_count() const { return _num_buckets; } + float load_factor() const { return static_cast(_num_filled) / (_mask + 1); } + + HashT& hash_function() const { return _hasher; } + EqT& key_eq() const { return _eq; } + + void max_load_factor(float mlf) + { + if (mlf < 0.9999f && mlf > 0.2f) + _mlf = (uint32_t)((1 << 27) / mlf); + } + + constexpr float max_load_factor() const { return (1 << 27) / (float)_mlf; } + constexpr size_type max_size() const { return (1ull << (sizeof(size_type) * 8 - 2)); } + constexpr size_type max_bucket_count() const { return max_size(); } + +#if EMH_STATIS + size_type bucket_main() const + { + auto main_size = 0; + for (size_type bucket = 0; bucket < _num_buckets; ++bucket) { + if (EMH_BUCKET(_pairs, bucket) == bucket) + main_size ++; + } + return main_size; + } + + //Returns the bucket number where the element with key k is located. + size_type bucket(const KeyT& key) const + { + const auto bucket = hash_key(key) & _mask; + const auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (EMH_EMPTY(_pairs, bucket)) + return 0; + else if (bucket == next_bucket) + return bucket + 1; + + const auto& bucket_key = EMH_KEY(_pairs, bucket); + return (hash_key(bucket_key) & _mask) + 1; + } + + //Returns the number of elements in bucket n. + size_type bucket_size(const size_type bucket) const + { + if (EMH_EMPTY(_pairs, bucket)) + return 0; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + next_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + size_type bucket_size = 1; + + //iterator each item in current main bucket + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) { + break; + } + bucket_size++; + next_bucket = nbucket; + } + return bucket_size; + } + + size_type get_main_bucket(const size_type bucket) const + { + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto& bucket_key = EMH_KEY(_pairs, bucket); + const auto main_bucket = hash_key(bucket_key) & _mask; + return main_bucket; + } + + size_type get_diss(size_type bucket, size_type next_bucket, const size_type slots) const + { + const int cahe_line_size = 64; + auto pbucket = reinterpret_cast(&_pairs[bucket]); + auto pnext = reinterpret_cast(&_pairs[next_bucket]); + if (pbucket / cahe_line_size == pnext / cahe_line_size) + return 0; + size_type diff = pbucket > pnext ? (pbucket - pnext) : (pnext - pbucket); + if (diff / cahe_line_size + 1 < slots) + return (diff / cahe_line_size + 1); + return slots - 1; + } + + int get_bucket_info(const size_type bucket, size_type steps[], const size_type slots) const + { + if (EMH_EMPTY(_pairs, bucket)) + return -1; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if ((hash_key(EMH_KEY(_pairs, bucket)) & _mask) != bucket) + return 0; + else if (next_bucket == bucket) + return 1; + + steps[get_diss(bucket, next_bucket, slots)] ++; + size_type bucket_size = 2; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + + steps[get_diss(nbucket, next_bucket, slots)] ++; + bucket_size ++; + next_bucket = nbucket; + } + + return bucket_size; + } + + void dump_statics(bool show_cache) const + { + const int slots = 128; + size_type buckets[slots + 1] = {0}; + size_type steps[slots + 1] = {0}; + char buff[1024 * 8]; + for (size_type bucket = 0; bucket < _num_buckets; ++bucket) { + auto bsize = get_bucket_info(bucket, steps, slots); + if (bsize >= 0) + buckets[bsize] ++; + } + + size_type sumb = 0, sums = 0, sumn = 0; + size_type miss = 0, finds = 0, bucket_coll = 0; + double lf = load_factor(), fk = 1.0 / exp(lf), sum_poisson = 0; + int bsize = sprintf (buff, "============== buckets size ration ========\n"); + + miss += _num_buckets - _num_filled; + for (int i = 1, factorial = 1; i < sizeof(buckets) / sizeof(buckets[0]); i++) { + double poisson = fk / factorial; factorial *= i; fk *= lf; + if (poisson > 1e-13 && i < 20) + sum_poisson += poisson * 100.0 * (i - 1) / i; + + const int64_t bucketsi = buckets[i]; + if (bucketsi == 0) + continue; + + sumb += bucketsi; + sumn += bucketsi * i; + bucket_coll += bucketsi * (i - 1); + finds += bucketsi * i * (i + 1) / 2; + miss += bucketsi * i * i; + auto errs = (bucketsi * 1.0 * i / _num_filled - poisson) * 100 / poisson; + bsize += sprintf(buff + bsize, " %2d %8ld %0.8lf|%0.2lf%% %2.3lf\n", + i, bucketsi, bucketsi * 1.0 * i / _num_filled, errs, sumn * 100.0 / _num_filled); + if (sumn >= _num_filled) + break; + } + + bsize += sprintf(buff + bsize, "========== collision miss ration ===========\n"); + for (size_type i = 0; show_cache && i < sizeof(steps) / sizeof(steps[0]); i++) { + sums += steps[i]; + if (steps[i] == 0) + continue; + if (steps[i] > 10) + bsize += sprintf(buff + bsize, " %2d %8u %0.2lf %.2lf\n", (int)i, steps[i], steps[i] * 100.0 / bucket_coll, sums * 100.0 / bucket_coll); + } + + if (sumb == 0) return; + + bsize += sprintf(buff + bsize, " _num_filled aver_size k.v size_kv = %u, %.2lf, %s.%s %zd\n", + _num_filled, _num_filled * 1.0 / sumb, typeid(KeyT).name(), typeid(ValueT).name(), sizeof(PairT)); + + bsize += sprintf(buff + bsize, " collision,poisson,cache_miss hit_find|hit_miss, load_factor = %.2lf%%,%.2lf%%,%.2lf%% %.2lf|%.2lf, %.2lf\n", + (bucket_coll * 100.0 / _num_filled), sum_poisson, (bucket_coll - steps[0]) * 100.0 / _num_filled, + finds * 1.0 / _num_filled, miss * 1.0 / _num_buckets, _num_filled * 1.0 / _num_buckets); + + bsize += sprintf(buff + bsize, "============== buckets size end =============\n"); + buff[bsize + 1] = 0; + +#ifdef EMH_LOG + EMH_LOG << __FUNCTION__ << "|" << buff << endl; +#else + puts(buff); +#endif + assert(sumn == _num_filled); + assert(sums == bucket_coll || !show_cache); + assert(bucket_coll == buckets[0]); + } +#endif + + // ------------------------------------------------------------ + template + iterator find(const Key& key, size_t key_hash) noexcept + { + return {this, find_filled_hash(key, key_hash)}; + } + + template + const_iterator find(const Key& key, size_t key_hash) const noexcept + { + return {this, find_filled_hash(key, key_hash)}; + } + + template + iterator find(const Key& key) noexcept + { + return {this, find_filled_bucket(key)}; + } + + template + const_iterator find(const Key& key) const noexcept + { + return {this, find_filled_bucket(key)}; + } + + template + ValueT& at(const KeyT& key) + { + const auto bucket = find_filled_bucket(key); + //throw + return EMH_VAL(_pairs, bucket); + } + + template + const ValueT& at(const KeyT& key) const + { + const auto bucket = find_filled_bucket(key); + //throw + return EMH_VAL(_pairs, bucket); + } + + template + bool contains(const Key& key) const noexcept + { + return find_filled_bucket(key) != _num_buckets; + } + + template + size_type count(const Key& key) const noexcept + { + return find_filled_bucket(key) != _num_buckets ? 1 : 0; + } + + template + std::pair equal_range(const Key& key) const noexcept + { + const auto found = {this, find_filled_bucket(key), true}; + if (found.bucket() == _num_buckets) + return { found, found }; + else + return { found, std::next(found) }; + } + + template + std::pair equal_range(const K& key) const + { + const auto found = {this, find_filled_bucket(key), true}; + if (found.bucket() == _num_buckets) + return { found, found }; + else + return { found, std::next(found) }; + } + + void merge(HashMap& rhs) + { + if (empty()) { + *this = std::move(rhs); + return; + } + + for (auto rit = rhs.begin(); rit != rhs.end(); ) { + auto fit = find(rit->first); + if (fit.bucket() == _num_buckets) { + insert_unique(rit->first, std::move(rit->second)); + rit = rhs.erase(rit); + } else { + ++rit; + } + } + } + + /// Returns false if key isn't found. + bool try_get(const KeyT& key, ValueT& val) const noexcept + { + const auto bucket = find_filled_bucket(key); + const auto found = bucket != _num_buckets; + if (found) { + val = EMH_VAL(_pairs, bucket); + } + return found; + } + + /// Returns the matching ValueT or nullptr if k isn't found. + ValueT* try_get(const KeyT& key) noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? nullptr : &EMH_VAL(_pairs, bucket); + } + + /// Const version of the above + ValueT* try_get(const KeyT& key) const noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? nullptr : &EMH_VAL(_pairs, bucket); + } + + /// Convenience function. + ValueT get_or_return_default(const KeyT& key) const noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? ValueT() : EMH_VAL(_pairs, bucket); + } + + // ----------------------------------------------------- + template + std::pair do_assign(K&& key, V&& val) + { + reserve(_num_filled); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::forward(key), std::forward(val), bucket); + } else { + EMH_VAL(_pairs, bucket) = std::move(val); + } + return { {this, bucket}, isempty }; + } + + std::pair do_insert(const value_type& value) + { + bool isempty; + const auto bucket = find_or_allocate(value.first, isempty); + if (isempty) { + EMH_NEW(value.first, value.second, bucket); + } + return { {this, bucket}, isempty }; + } + + std::pair do_insert(value_type&& value) + { + bool isempty; + const auto bucket = find_or_allocate(value.first, isempty); + if (isempty) { + EMH_NEW(std::forward(value.first), std::forward(value.second), bucket); + } + return { {this, bucket}, isempty }; + } + + template + std::pair do_insert(K&& key, V&& val) + { + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::forward(key), std::forward(val), bucket); + } + return { {this, bucket}, isempty }; + } + + std::pair insert(const value_type& value) + { + check_expand_need(); + return do_insert(value); + } + + std::pair insert(value_type && value) + { + check_expand_need(); + return do_insert(std::move(value)); + } + + void insert(std::initializer_list ilist) + { + reserve(ilist.size() + _num_filled); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template + void insert(Iter first, Iter last) + { + reserve(std::distance(first, last) + _num_filled); + for (auto it = first; it != last; ++it) + do_insert(it->first, it->second); + } + +#if 0 + template + void insert2(Iter begin, Iter end) + { + Iter citbeg = begin; + Iter citend = begin; + reserve(std::distance(begin, end) + _num_filled); + for (; begin != end; ++begin) { + if (try_insert_mainbucket(begin->first, begin->second) == INACTIVE) { + std::swap(*begin, *citend++); + } + } + + for (; citbeg != citend; ++citbeg) + insert(*citbeg); + } + + size_type try_insert_mainbucket(const KeyT& key, const ValueT& val) + { + const auto bucket = hash_key(key) & _mask; + if (!EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + EMH_NEW(key, val, bucket); + return bucket; + } +#endif + + template + void insert_unique(Iter begin, Iter end) + { + reserve(std::distance(begin, end) + _num_filled); + for (; begin != end; ++begin) + do_insert_unqiue(*begin); + } + + /// Same as above, but contains(key) MUST be false + size_type insert_unique(KeyT&& key, ValueT&& val) + { + return do_insert_unqiue(std::move(key), std::forward(val)); + } + + size_type insert_unique(const KeyT& key, ValueT&& val) + { + return do_insert_unqiue(key, std::forward(val)); + } + + size_type insert_unique(value_type&& value) + { + return do_insert_unqiue(std::move(value.first), std::move(value.second)); + } + + size_type insert_unique(const value_type& value) + { + return do_insert_unqiue(value.first, value.second); + } + + template + inline size_type do_insert_unqiue(K&& key, V&& val) + { + check_expand_need(); + auto bucket = find_unique_bucket(key); + EMH_NEW(std::forward(key), std::forward(val), bucket); + return bucket; + } + + std::pair insert_or_assign(const KeyT& key, ValueT&& val) { return do_assign(key, std::forward(val)); } + std::pair insert_or_assign(KeyT&& key, ValueT&& val) { return do_assign(std::move(key), std::forward(val)); } + + template + inline std::pair emplace(Args&&... args) + { + check_expand_need(); + return do_insert(std::forward(args)...); + } + + template + iterator emplace_hint(const_iterator hint, Args&&... args) + { + (void)hint; + check_expand_need(); + return do_insert(std::forward(args)...).first; + } + + template + std::pair try_emplace(const KeyT& key, Args&&... args) + { + check_expand_need(); + return do_insert(key, std::forward(args)...).first; + } + + template + std::pair try_emplace(KeyT&& key, Args&&... args) + { + check_expand_need(); + return do_insert(std::forward(key), std::forward(args)...).first; + } + + template + inline size_type emplace_unique(Args&&... args) + { + return insert_unique(std::forward(args)...); + } + + /* Check if inserting a new value rather than overwriting an old entry */ + ValueT& operator[](const KeyT& key) + { + check_expand_need(); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(key, std::move(ValueT()), bucket); + } + + return EMH_VAL(_pairs, bucket); + } + + ValueT& operator[](KeyT&& key) + { + check_expand_need(); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::move(key), std::move(ValueT()), bucket); + } + + return EMH_VAL(_pairs, bucket); + } + + // ------------------------------------------------------- + /// Erase an element from the hash table. + /// return 0 if element was not found + template + size_type erase(const Key& key) + { + const auto bucket = erase_key(key); + if (bucket == INACTIVE) + return 0; + + clear_bucket(bucket); + return 1; + } + + //iterator erase const_iterator + iterator erase(const_iterator cit) + { + iterator it(cit); + return erase(it); + } + + /// Erase an element typedef an iterator. + /// Returns an iterator to the next element (or end()). + iterator erase(iterator it) + { + const auto bucket = erase_bucket(it._bucket); + clear_bucket(bucket); + if (bucket == it._bucket) { + return ++it; + } else { + //erase main bucket as next + it.clear(bucket); + return it; + } + } + + /// Erase an element typedef an iterator without return next iterator + void _erase(const_iterator it) + { + const auto bucket = erase_bucket(it._bucket); + clear_bucket(bucket); + } + + iterator erase(const_iterator first, const_iterator last) + { + auto iend = cend(); + auto next = first; + for (; next.bucket() < last.bucket() && next != iend; ) + next = erase(next); + + return {this, next.bucket()}; + } + + template + size_type erase_if(Pred pred) + { + auto old_size = size(); + for (auto it = begin(), last = end(); it != last; ) { + if (pred(*it)) + it = erase(it); + else + ++it; + } + return old_size - size(); + } + + static constexpr bool is_triviall_destructable() + { +#if __cplusplus >= 201402L || _MSC_VER > 1600 + return !(std::is_trivially_destructible::value && std::is_trivially_destructible::value); +#else + return !(std::is_pod::value && std::is_pod::value); +#endif + } + + static constexpr bool is_copy_trivially() + { +#if __cplusplus >= 201402L || _MSC_VER > 1600 + return (std::is_trivially_copyable::value && std::is_trivially_copyable::value); +#else + return (std::is_pod::value && std::is_pod::value); +#endif + } + + void clearkv() + { + if (is_triviall_destructable()) { + for (auto it = cbegin(); _num_filled; ++it) + clear_bucket(it.bucket()); + } + } + + /// Remove all elements, keeping full capacity. + void clear() + { + if (!is_triviall_destructable() && _num_filled) + memset(_bitmask, 0xFFFFFFFF, (_num_buckets + 7) / 8); + else if (_num_filled) + clearkv(); + + //EMH_BUCKET(_pairs, _num_buckets) = 0; //_last + _num_filled = 0; + } + + void shrink_to_fit() + { + rehash(_num_filled); + } + + /// Make room for this many elements + bool reserve(uint64_t num_elems) + { + const auto required_buckets = (num_elems * _mlf >> 27); + if (EMH_LIKELY(required_buckets < _num_buckets)) + return false; + +#if EMH_HIGH_LOAD + if (required_buckets < 64 && _num_filled < _num_buckets) + return false; +#endif + +#if EMH_STATIS + if (_num_filled > EMH_STATIS) dump_statics(true); +#endif + rehash(required_buckets + 2); + return true; + } + + void rehash(uint64_t required_buckets) + { + if (required_buckets < _num_filled) + return; + + auto num_buckets = _num_filled > (1u << 16) ? (1u << 16) : 2u; + while (num_buckets < required_buckets) { num_buckets *= 2; } + + //TODO: throwOverflowError + auto old_num_filled = _num_filled; + auto old_mask = _num_buckets - 1; + auto old_pairs = _pairs; + auto* obmask = _bitmask; + + _num_filled = 0; + _num_buckets = num_buckets; + _mask = num_buckets - 1; + + _pairs = (PairT*)malloc(AllocSize(_num_buckets)); + memset((char*)(_pairs + _num_buckets), 0, sizeof(PairT) * EPACK_SIZE); + + _bitmask = decltype(_bitmask)(_pairs + EPACK_SIZE + num_buckets); + + const auto mask_byte = (num_buckets + 7) / 8; + memset(_bitmask, 0xFFFFFFFF, mask_byte); + memset(((char*)_bitmask) + mask_byte, 0, BIT_PACK); + if (num_buckets < 8) + _bitmask[0] = (1 << num_buckets) - 1; + + //for (size_type src_bucket = 0; _num_filled < old_num_filled; src_bucket++) { + for (size_type src_bucket = old_mask; _num_filled < old_num_filled; src_bucket --) { + if (obmask[src_bucket / MASK_BIT] & (EMH_MASK(src_bucket))) + continue; + + auto& key = EMH_KEY(old_pairs, src_bucket); + const auto bucket = find_unique_bucket(key); + EMH_NEW(std::move(key), std::move(EMH_VAL(old_pairs, src_bucket)), bucket); + if (is_triviall_destructable()) + old_pairs[src_bucket].~PairT(); + } + +#if EMH_REHASH_LOG + if (_num_filled > EMH_REHASH_LOG) { + auto mbucket = bucket_main(); + char buff[255] = {0}; + sprintf(buff, " _num_filled/aver_size/K.V/pack/ = %u/%2.lf/%s.%s/%zd", + _num_filled, double (_num_filled) / mbucket, typeid(KeyT).name(), typeid(ValueT).name(), sizeof(_pairs[0])); +#ifdef EMH_LOG + static size_t ihashs = 0; + EMH_LOG << "rhash_nums = " << ihashs ++ << "|" <<__FUNCTION__ << "|" << buff << endl; +#else + puts(buff); +#endif + } +#endif + + free(old_pairs); + assert(old_num_filled == _num_filled); + } + +private: + // Can we fit another element? + inline bool check_expand_need() + { + return reserve(_num_filled); + } + + void clear_bucket(size_type bucket) + { + EMH_CLS(bucket); + if (is_triviall_destructable()) + _pairs[bucket].~PairT(); + _num_filled--; + } + +#if 1 + //template::value, size_type>::type = 0> + template + size_type erase_key(const UType& key) + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto eqkey = _eq(key, EMH_KEY(_pairs, bucket)); + if (eqkey) { + if (next_bucket == bucket) + return bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (is_copy_trivially()) + EMH_PKV(_pairs, bucket) = EMH_PKV(_pairs, next_bucket); + else + EMH_PKV(_pairs, bucket).swap(EMH_PKV(_pairs, next_bucket)); + + EMH_BUCKET(_pairs, bucket) = (nbucket == next_bucket) ? bucket : nbucket; + return next_bucket; + } else if (next_bucket == bucket) + return INACTIVE; + /* else if (EMH_UNLIKELY(bucket != hash_key(EMH_KEY(_pairs, bucket)) & _mask)) + return INACTIVE; + */ + + auto prev_bucket = bucket; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (_eq(key, EMH_KEY(_pairs, next_bucket))) { + EMH_BUCKET(_pairs, prev_bucket) = (nbucket == next_bucket) ? prev_bucket : nbucket; + return next_bucket; + } + + if (nbucket == next_bucket) + break; + prev_bucket = next_bucket; + next_bucket = nbucket; + } + + return INACTIVE; + } +#else + template::value, size_type>::type = 0> + size_type erase_key(const UType& key) + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _eq(key, EMH_KEY(_pairs, bucket)) ? bucket : INACTIVE; +// else if (bucket != hash_key(EMH_KEY(_pairs, bucket))) +// return INACTIVE; + + //find erase key and swap to last bucket + size_type prev_bucket = bucket, find_bucket = INACTIVE; + next_bucket = bucket; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (_eq(key, EMH_KEY(_pairs, next_bucket))) { + find_bucket = next_bucket; + if (nbucket == next_bucket) { + EMH_BUCKET(_pairs, prev_bucket) = prev_bucket; + break; + } + } + if (nbucket == next_bucket) { + if (find_bucket != INACTIVE) { + EMH_PKV(_pairs, find_bucket).swap(EMH_PKV(_pairs, nbucket)); +// EMH_PKV(_pairs, find_bucket) = EMH_PKV(_pairs, nbucket); + EMH_BUCKET(_pairs, prev_bucket) = prev_bucket; + find_bucket = nbucket; + } + break; + } + prev_bucket = next_bucket; + next_bucket = nbucket; + } + + return find_bucket; + } +#endif + + size_type erase_bucket(const size_type bucket) + { + const auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto main_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + if (bucket == main_bucket) { + if (bucket != next_bucket) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (is_copy_trivially()) + EMH_PKV(_pairs, bucket) = EMH_PKV(_pairs, next_bucket); + else + EMH_PKV(_pairs, bucket).swap(EMH_PKV(_pairs, next_bucket)); + EMH_BUCKET(_pairs, bucket) = (nbucket == next_bucket) ? bucket : nbucket; + } + return next_bucket; + } + + const auto prev_bucket = find_prev_bucket(main_bucket, bucket); + EMH_BUCKET(_pairs, prev_bucket) = (bucket == next_bucket) ? prev_bucket : next_bucket; + return bucket; + } + + // Find the bucket with this key, or return bucket size + template + size_type find_filled_hash(const K& key, const size_t key_hash) const + { + const auto bucket = key_hash & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + else if (next_bucket == bucket) + return _num_buckets; + + while (true) { + if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + next_bucket = nbucket; + } + + return _num_buckets; + } + + // Find the bucket with this key, or return bucket size + template + size_type find_filled_bucket(const K& key) const + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + +#if 1 + if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _num_buckets; +#elif 0 + else if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + else if (next_bucket == bucket) + return _num_buckets; + + else if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return _num_buckets; + next_bucket = nbucket; +#elif 0 + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + else if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _num_buckets; +#endif +// else if (bucket != (hash_key(bucket_key) & _mask)) +// return _num_buckets; + + while (true) { + if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return _num_buckets; + next_bucket = nbucket; + } + + return 0; + } + + //kick out bucket and find empty to occpuy + //it will break the orgin link and relnik again. + //before: main_bucket-->prev_bucket --> bucket --> next_bucket + //atfer : main_bucket-->prev_bucket --> (removed)--> new_bucket--> next_bucket + size_type kickout_bucket(const size_type kmain, const size_type kbucket) + { + const auto next_bucket = EMH_BUCKET(_pairs, kbucket); + const auto new_bucket = find_empty_bucket(next_bucket, kbucket); + const auto prev_bucket = find_prev_bucket(kmain, kbucket); + new(_pairs + new_bucket) PairT(std::move(_pairs[kbucket])); + if (is_triviall_destructable()) + _pairs[kbucket].~PairT(); + + if (next_bucket == kbucket) + EMH_BUCKET(_pairs, new_bucket) = new_bucket; + EMH_BUCKET(_pairs, prev_bucket) = new_bucket; + + //set new bucket bit + EMH_SET(new_bucket); + + //clear kickout bit + //EMH_CLS(kbucket); + return kbucket; + } + +/* +** inserts a new key into a hash table; first check whether key's main +** bucket/position is free. If not, check whether colliding node/bucket is in its main +** position or not: if it is not, move colliding bucket to an empty place and +** put new key in its main position; otherwise (colliding bucket is in its main +** position), new key goes to an empty position. ***/ + + template + size_type find_or_allocate(const Key& key, bool& isempty) + { + const auto bucket = hash_key(key) & _mask; + const auto& bucket_key = EMH_KEY(_pairs, bucket); + if (EMH_EMPTY(_pairs, bucket)) { + isempty = true; + return bucket; + } + else if (_eq(key, bucket_key)) { + isempty = false; + return bucket; + } + + isempty = true; + auto next_bucket = EMH_BUCKET(_pairs, bucket); + //check current bucket_key is in main bucket or not + const auto kmain_bucket = hash_key(bucket_key) & _mask; + if (kmain_bucket != bucket) + return kickout_bucket(kmain_bucket, bucket); + else if (next_bucket == bucket) + return EMH_BUCKET(_pairs, next_bucket) = find_empty_bucket(next_bucket, bucket); + +#if EMH_LRU_SET + auto prev_bucket = bucket; +#endif + //find next linked bucket and check key, if lru is set then swap current key with prev_bucket + while (true) { + if (EMH_UNLIKELY(_eq(key, EMH_KEY(_pairs, next_bucket)))) { + isempty = false; +#if EMH_LRU_SET + EMH_PKV(_pairs, next_bucket).swap(EMH_PKV(_pairs, prev_bucket)); + return prev_bucket; +#else + return next_bucket; +#endif + } + +#if EMH_LRU_SET + prev_bucket = next_bucket; +#endif + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + next_bucket = nbucket; + } + + //find a new empty and link it to tail, TODO link after main bucket? + const auto new_bucket = find_empty_bucket(next_bucket, bucket);// : find_empty_bucket(next_bucket); + return EMH_BUCKET(_pairs, next_bucket) = new_bucket; + } + + // key is not in this map. Find a place to put it. + size_type find_empty_bucket(const size_type bucket_from, const size_t main_bucket) + { +#if 1 || __arm64__ || __aarch64__ + const auto boset = bucket_from % MASK_BIT; + auto* const align = _bitmask + bucket_from / MASK_BIT; + const auto bmask = ((size_t)align[1] << (MASK_BIT - boset)) | (align[0] >> boset); +#else + const auto boset = bucket_from % 8; + auto* const align = (uint8_t*)_bitmask + bucket_from / 8; + const auto bmask = *(size_t*)(align) >> boset; +#endif + if (EMH_LIKELY(bmask != 0)) + return bucket_from + CTZ(bmask); + + const auto qmask = _mask / SIZE_BIT; + if (1) { + const auto step = (bucket_from - SIZE_BIT / 2) & qmask; + const auto bmask3 = *((size_t*)_bitmask + step); + if (bmask3 != 0) + return step * SIZE_BIT + CTZ(bmask3); + } + + auto& _last = EMH_BUCKET(_pairs, _num_buckets); + for (; ; ) { //2.4.7 + const auto bmask2 = *((size_t*)_bitmask + _last); + if (bmask2 != 0) + return _last * SIZE_BIT + CTZ(bmask2); + + const auto next1 = (qmask / 2 + _last) & qmask; + const auto bmask1 = *((size_t*)_bitmask + next1); + if (bmask1 != 0) { + //_last = next1; + return next1 * SIZE_BIT + CTZ(bmask1); + } + _last = (_last + 1) & qmask; + } + return 0; + } + + size_type find_last_bucket(size_type main_bucket) const + { + auto next_bucket = EMH_BUCKET(_pairs, main_bucket); + if (next_bucket == main_bucket) + return main_bucket; + + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type find_prev_bucket(size_type main_bucket, const size_type bucket) const + { + auto next_bucket = EMH_BUCKET(_pairs, main_bucket); + if (next_bucket == bucket) + return main_bucket; + + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type find_unique_bucket(const KeyT& key) + { + const size_type bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return bucket; + + //check current bucket_key is in main bucket or not + const auto kmain_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + if (EMH_UNLIKELY(kmain_bucket != bucket)) + return kickout_bucket(kmain_bucket, bucket); + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket != bucket) + next_bucket = find_last_bucket(next_bucket); + + //find a new empty and link it to tail + return EMH_BUCKET(_pairs, next_bucket) = find_empty_bucket(next_bucket, bucket); + } + + static constexpr uint64_t KC = UINT64_C(11400714819323198485); + static inline uint64_t hash64(uint64_t key) + { +#if __SIZEOF_INT128__ && EMH_FIBONACCI_HASH == 1 + __uint128_t r = key; r *= KC; + return (uint64_t)(r >> 64) + (uint64_t)r; +#elif EMH_FIBONACCI_HASH == 2 + //MurmurHash3Mixer + uint64_t h = key; + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + return h; +#elif _WIN64 && EMH_FIBONACCI_HASH == 1 + uint64_t high; + return _umul128(key, KC, &high) + high; +#elif EMH_FIBONACCI_HASH == 3 + auto ror = (key >> 32) | (key << 32); + auto low = key * 0xA24BAED4963EE407ull; + auto high = ror * 0x9FB21C651E98DF25ull; + auto mix = low + high; + return mix; +#elif EMH_FIBONACCI_HASH == 1 + uint64_t r = key * UINT64_C(0xca4bcaa75ec3f625); + return (r >> 32) + r; +#else + uint64_t x = key; + x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); + x = x ^ (x >> 31); + return x; +#endif + } + + template::value, size_type>::type = 0> + inline size_type hash_key(const UType key) const + { +#ifdef EMH_FIBONACCI_HASH + return hash64(key); +#elif EMH_IDENTITY_HASH + return key + (key >> (sizeof(UType) * 4)); +#elif EMH_WYHASH64 + return wyhash64(key, KC); +#else + return (size_type)_hasher(key); +#endif + } + + template::value, size_type>::type = 0> + inline size_type hash_key(const UType& key) const + { +#ifdef WYHASH_LITTLE_ENDIAN + return wyhash(key.data(), key.size(), key.size()); +#else + return (size_type)_hasher(key); +#endif + } + + template::value && !std::is_same::value, size_type>::type = 0> + inline size_type hash_key(const UType& key) const + { +#ifdef EMH_FIBONACCI_HASH + return _hasher(key) * KC; +#else + return (size_type)_hasher(key); +#endif + } + +private: + uint32_t* _bitmask; + PairT* _pairs; + HashT _hasher; + EqT _eq; + size_type _mask; + size_type _num_buckets; + + size_type _num_filled; + uint32_t _mlf; + +private: + static constexpr uint32_t BIT_PACK = sizeof(_bitmask[0]) * 2; + static constexpr uint32_t MASK_BIT = sizeof(_bitmask[0]) * 8; + static constexpr uint32_t SIZE_BIT = sizeof(size_t) * 8; + static constexpr uint32_t EPACK_SIZE = sizeof(PairT) >= sizeof(size_t) == 0 ? 1 : 2; // > 1 +}; +} // namespace emhash +#if __cplusplus >= 201103L +//template using emhash7 = emhash7::HashMap, std::equal_to>; +#endif + +//TODO +//2. improve rehash and find miss performance(reduce peak memory) +//3. dump or Serialization interface +//4. node hash map support +//5. support load_factor > 1.0 +//6. add grow ration +//8. ... https://godbolt.org/ diff --git a/benchmarks/external/emhash/wyhash.h b/benchmarks/external/emhash/wyhash.h new file mode 100644 index 00000000..b89352a7 --- /dev/null +++ b/benchmarks/external/emhash/wyhash.h @@ -0,0 +1,272 @@ +// This is free and unencumbered software released into the public domain under The Unlicense (http://unlicense.org/) +// main repo: https://github.com/wangyi-fudan/wyhash +// author: 王一 Wang Yi +// contributors: Reini Urban, Dietrich Epp, Joshua Haberman, Tommy Ettinger, Daniel Lemire, Otmar Ertl, cocowalla, leo-yuriev, Diego Barrios Romero, paulie-g, dumblob, Yann Collet, ivte-ms, hyb, James Z.M. Gao, easyaspi314 (Devin), TheOneric + +/* quick example: + uint64_t _wyp[4]; + make_secret(time(NULL),_wyp); + string s="fjsakfdsjkf"; + uint64_t hash=wyhash(s.c_str(), s.size(), 0, _wyp); +*/ + +#ifndef wyhash_final_version_3 +#define wyhash_final_version_3 + +#ifndef WYHASH_CONDOM +//protections that produce different results: +//1: normal valid behavior +//2: extra protection against entropy loss (probability=2^-63), aka. "blind multiplication" +#define WYHASH_CONDOM 1 +#endif + +#ifndef WYHASH_32BIT_MUM +//0: normal version, slow on 32 bit systems +//1: faster on 32 bit systems but produces different results, incompatible with wy2u0k function +#define WYHASH_32BIT_MUM 0 +#endif + +//includes +#include +#include +#if defined(_MSC_VER) && defined(_M_X64) + #include + #pragma intrinsic(_umul128) +#endif + +//likely and unlikely macros +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) + #define _likely_(x) __builtin_expect(x,1) + #define _unlikely_(x) __builtin_expect(x,0) +#else + #define _likely_(x) (x) + #define _unlikely_(x) (x) +#endif + +//128bit multiply function +static inline uint64_t _wyrot(uint64_t x) { return (x>>32)|(x<<32); } +static inline void _wymum(uint64_t *A, uint64_t *B){ +#if(WYHASH_32BIT_MUM) + uint64_t hh=(*A>>32)*(*B>>32), hl=(*A>>32)*(uint32_t)*B, lh=(uint32_t)*A*(*B>>32), ll=(uint64_t)(uint32_t)*A*(uint32_t)*B; + #if(WYHASH_CONDOM>1) + *A^=_wyrot(hl)^hh; *B^=_wyrot(lh)^ll; + #else + *A=_wyrot(hl)^hh; *B=_wyrot(lh)^ll; + #endif +#elif defined(__SIZEOF_INT128__) + __uint128_t r=*A; r*=*B; + #if(WYHASH_CONDOM>1) + *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); + #else + *A=(uint64_t)r; *B=(uint64_t)(r>>64); + #endif +#elif defined(_MSC_VER) && defined(_M_X64) + #if(WYHASH_CONDOM>1) + uint64_t a, b; + a=_umul128(*A,*B,&b); + *A^=a; *B^=b; + #else + *A=_umul128(*A,*B,B); + #endif +#else + uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; + uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; + #if(WYHASH_CONDOM>1) + *A^=lo; *B^=hi; + #else + *A=lo; *B=hi; + #endif +#endif +} + +//multiply and xor mix function, aka MUM +static inline uint64_t _wymix(uint64_t A, uint64_t B){ _wymum(&A,&B); return A^B; } + +//endian macros +#ifndef WYHASH_LITTLE_ENDIAN + #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 1 + #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 0 + #else + #warning could not determine endianness! Falling back to little endian. + #define WYHASH_LITTLE_ENDIAN 1 + #endif +#endif + +//read functions +#if (WYHASH_LITTLE_ENDIAN) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v;} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return v;} +#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return __builtin_bswap32(v);} +#elif defined(_MSC_VER) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return _byteswap_ulong(v);} +#else +static inline uint64_t _wyr8(const uint8_t *p) { + uint64_t v; memcpy(&v, p, 8); + return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); +} +static inline uint64_t _wyr4(const uint8_t *p) { + uint32_t v; memcpy(&v, p, 4); + return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); +} +#endif +static inline uint64_t _wyr3(const uint8_t *p, size_t k) { return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1];} +//wyhash main function +static inline uint64_t wyhash(const void *key, size_t len, uint64_t seed, const uint64_t *secret){ + const uint8_t *p=(const uint8_t *)key; seed^=*secret; uint64_t a, b; + if(_likely_(len<=16)){ + if(_likely_(len>=4)){ a=(_wyr4(p)<<32)|_wyr4(p+((len>>3)<<2)); b=(_wyr4(p+len-4)<<32)|_wyr4(p+len-4-((len>>3)<<2)); } + else if(_likely_(len>0)){ a=_wyr3(p,len); b=0;} + else a=b=0; + } + else{ + size_t i=len; + if(_unlikely_(i>48)){ + uint64_t see1=seed, see2=seed; + do{ + seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); + see1=_wymix(_wyr8(p+16)^secret[2],_wyr8(p+24)^see1); + see2=_wymix(_wyr8(p+32)^secret[3],_wyr8(p+40)^see2); + p+=48; i-=48; + }while(_likely_(i>48)); + seed^=see1^see2; + } + while(_unlikely_(i>16)){ seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); i-=16; p+=16; } + a=_wyr8(p+i-16); b=_wyr8(p+i-8); + } + return _wymix(secret[1]^len,_wymix(a^secret[1],b^seed)); +} + +//the default secret parameters +static const uint64_t _wyp[4] = {0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x8ebc6af09c88c6e3ull, 0x589965cc75374cc3ull}; + +static inline uint64_t wyhash(const void *key, size_t len, uint64_t seed) { return wyhash(key, len, seed, _wyp); } + +//a useful 64bit-64bit mix function to produce deterministic pseudo random numbers that can pass BigCrush and PractRand +static inline uint64_t wyhash64(uint64_t A, uint64_t B){ A^=0xa0761d6478bd642full; B^=0xe7037ed1a0b428dbull; _wymum(&A,&B); return _wymix(A^0xa0761d6478bd642full,B^0xe7037ed1a0b428dbull);} + +//The wyrand PRNG that pass BigCrush and PractRand +static inline uint64_t wyrand(uint64_t *seed){ *seed+=0xa0761d6478bd642full; return _wymix(*seed,*seed^0xe7037ed1a0b428dbull);} + +//convert any 64 bit pseudo random numbers to uniform distribution [0,1). It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>12)*_wynorm;} + +//convert any 64 bit pseudo random numbers to APPROXIMATE Gaussian distribution. It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0;} + +#if(!WYHASH_32BIT_MUM) +//fast range integer random number generation on [0,k) credit to Daniel Lemire. May not work when WYHASH_32BIT_MUM=1. It can be combined with wyrand, wyhash64 or wyhash. +static inline uint64_t wy2u0k(uint64_t r, uint64_t k){ _wymum(&r,&k); return k; } +#endif + +//make your own secret +static inline void make_secret(uint64_t seed, uint64_t *secret){ + uint8_t c[] = {15, 23, 27, 29, 30, 39, 43, 45, 46, 51, 53, 54, 57, 58, 60, 71, 75, 77, 78, 83, 85, 86, 89, 90, 92, 99, 101, 102, 105, 106, 108, 113, 114, 116, 120, 135, 139, 141, 142, 147, 149, 150, 153, 154, 156, 163, 165, 166, 169, 170, 172, 177, 178, 180, 184, 195, 197, 198, 201, 202, 204, 209, 210, 212, 216, 225, 226, 228, 232, 240 }; + for(size_t i=0;i<4;i++){ + uint8_t ok; + do{ + ok=1; secret[i]=0; + for(size_t j=0;j<64;j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<> 1) & 0x5555555555555555; + x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); + x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; + x = (x * 0x0101010101010101) >> 56; + if(x!=32){ ok=0; break; } +#endif + } + }while(!ok); + } +} + +/* This is world's fastest hash map: 2x faster than bytell_hash_map. + It does not store the keys, but only the hash/signature of keys. + First we use pos=hash1(key) to approximately locate the bucket. + Then we search signature=hash2(key) from pos linearly. + If we find a bucket with matched signature we report the bucket + Or if we meet a bucket whose signature=0, we report a new position to insert + The signature collision probability is very low as we usually searched N~10 buckets. + By combining hash1 and hash2, we acturally have 128 bit anti-collision strength. + hash1 and hash2 can be the same function, resulting lower collision resistance but faster. + The signature is 64 bit, but can be modified to 32 bit if necessary for save space. + The above two can be activated by define WYHASHMAP_WEAK_SMALL_FAST + simple examples: + const size_t size=213432; + vector idx(size); // allocate the index of fixed size. idx MUST be zeroed. + vector value(size); // we only care about the index, user should maintain his own value vectors. + string key="dhskfhdsj" // the object to be inserted into idx + size_t pos=wyhashmap(idx.data(), idx.size(), key.c_str(), key.size(), 1); // get the position and insert + if(pos +*/ diff --git a/benchmarks/external/parallel_hashmap/btree.h b/benchmarks/external/parallel_hashmap/btree.h deleted file mode 100644 index b8c95433..00000000 --- a/benchmarks/external/parallel_hashmap/btree.h +++ /dev/null @@ -1,4050 +0,0 @@ -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_ -#define PHMAP_BTREE_BTREE_CONTAINER_H_ - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4355) // 'this': used in base member initializer list - #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - - -#include -#include -#include -#include -#include - -#include "phmap_fwd_decl.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include -#endif - -// MSVC constructibility traits do not detect destructor properties and so our -// implementations should not use them as a source-of-truth. -#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) - #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 -#endif - -namespace phmap { - - // Defined and documented later on in this file. - template - struct is_trivially_destructible; - - // Defined and documented later on in this file. - template - struct is_trivially_move_assignable; - - namespace type_traits_internal { - - // Silence MSVC warnings about the destructor being defined as deleted. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(push) - #pragma warning(disable : 4624) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template - union SingleMemberUnion { - T t; - }; - - // Restore the state of the destructor warning that was silenced above. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(pop) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template - struct IsTriviallyMoveConstructibleObject - : std::integral_constant< - bool, std::is_move_constructible< - type_traits_internal::SingleMemberUnion>::value && - phmap::is_trivially_destructible::value> {}; - - template - struct IsTriviallyCopyConstructibleObject - : std::integral_constant< - bool, std::is_copy_constructible< - type_traits_internal::SingleMemberUnion>::value && - phmap::is_trivially_destructible::value> {}; - - template - struct IsTriviallyMoveAssignableReference : std::false_type {}; - - template - struct IsTriviallyMoveAssignableReference - : phmap::is_trivially_move_assignable::type {}; - - template - struct IsTriviallyMoveAssignableReference - : phmap::is_trivially_move_assignable::type {}; - - } // namespace type_traits_internal - - - template - using void_t = typename type_traits_internal::VoidTImpl::type; - - - template - struct is_function - : std::integral_constant< - bool, !(std::is_reference::value || - std::is_const::type>::value)> {}; - - - namespace type_traits_internal { - - template - class is_trivially_copyable_impl { - using ExtentsRemoved = typename std::remove_all_extents::type; - static constexpr bool kIsCopyOrMoveConstructible = - std::is_copy_constructible::value || - std::is_move_constructible::value; - static constexpr bool kIsCopyOrMoveAssignable = - phmap::is_copy_assignable::value || - phmap::is_move_assignable::value; - - public: - static constexpr bool kValue = - (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && - (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && - (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && - is_trivially_destructible::value && - // We need to check for this explicitly because otherwise we'll say - // references are trivial copyable when compiled by MSVC. - !std::is_reference::value; - }; - - template - struct is_trivially_copyable - : std::integral_constant< - bool, type_traits_internal::is_trivially_copyable_impl::kValue> {}; - } // namespace type_traits_internal - - namespace swap_internal { - - // Necessary for the traits. - using std::swap; - - // This declaration prevents global `swap` and `phmap::swap` overloads from being - // considered unless ADL picks them up. - void swap(); - - template - using IsSwappableImpl = decltype(swap(std::declval(), std::declval())); - - // NOTE: This dance with the default template parameter is for MSVC. - template (), std::declval()))>> - using IsNothrowSwappableImpl = typename std::enable_if::type; - - template - struct IsSwappable - : phmap::type_traits_internal::is_detected {}; - - template - struct IsNothrowSwappable - : phmap::type_traits_internal::is_detected {}; - - template ::value, int> = 0> - void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable::value) { - swap(lhs, rhs); - } - - using StdSwapIsUnconstrained = IsSwappable; - - } // namespace swap_internal - - namespace type_traits_internal { - - // Make the swap-related traits/function accessible from this namespace. - using swap_internal::IsNothrowSwappable; - using swap_internal::IsSwappable; - using swap_internal::Swap; - using swap_internal::StdSwapIsUnconstrained; - - } // namespace type_traits_internal - - namespace compare_internal { - - using value_type = int8_t; - - template - struct Fail { - static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); - }; - - template - struct OnlyLiteralZero { - constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT - - template < - typename T, - typename = typename std::enable_if< - std::is_same::value || - (std::is_integral::value && !std::is_same::value)>::type, - typename = typename Fail::type> - OnlyLiteralZero(T); // NOLINT - }; - - enum class eq : value_type { - equal = 0, - equivalent = equal, - nonequal = 1, - nonequivalent = nonequal, - }; - - enum class ord : value_type { less = -1, greater = 1 }; - - enum class ncmp : value_type { unordered = -127 }; - -#if defined(__cpp_inline_variables) && !defined(_MSC_VER) - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ - static const type name; - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - inline constexpr type type::name(init) - -#else // __cpp_inline_variables - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ - static const T name; - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - template \ - const T compare_internal::type##_base::name(init) - -#endif // __cpp_inline_variables - - // These template base classes allow for defining the values of the constants - // in the header file (for performance) without using inline variables (which - // aren't available in C++11). - template - struct weak_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template - struct strong_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template - struct partial_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) - }; - - template - struct weak_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - template - struct strong_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - } // namespace compare_internal - - class weak_equality - : public compare_internal::weak_equality_base { - explicit constexpr weak_equality(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::weak_equality_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) - - // Comparisons - friend constexpr bool operator==( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class strong_equality - : public compare_internal::strong_equality_base { - explicit constexpr strong_equality(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::strong_equality_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, - compare_internal::eq::nonequal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class partial_ordering - : public compare_internal::partial_ordering_base { - explicit constexpr partial_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr partial_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::partial_ordering_base; - - constexpr bool is_ordered() const noexcept { - return value_ != - compare_internal::value_type(compare_internal::ncmp::unordered); - } - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ == 0; - } - friend constexpr bool operator!=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return !v.is_ordered() || v.value_ != 0; - } - friend constexpr bool operator<( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ < 0; - } - friend constexpr bool operator<=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ <= 0; - } - friend constexpr bool operator>( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ > 0; - } - friend constexpr bool operator>=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return !v.is_ordered() || 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, - compare_internal::ord::greater); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, - compare_internal::ncmp::unordered); - - class weak_ordering - : public compare_internal::weak_ordering_base { - explicit constexpr weak_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr weak_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::weak_ordering_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, - compare_internal::ord::greater); - - class strong_ordering - : public compare_internal::strong_ordering_base { - explicit constexpr strong_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr strong_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::strong_ordering_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator strong_equality() const noexcept { // NOLINT - return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - constexpr operator weak_ordering() const noexcept { // NOLINT - return value_ == 0 - ? weak_ordering::equivalent - : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, - compare_internal::ord::greater); - -#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL -#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL -#undef PHMAP_COMPARE_INLINE_INIT - - namespace compare_internal { - // We also provide these comparator adapter functions for internal phmap use. - - // Helper functions to do a boolean comparison of two keys given a boolean - // or three-way comparator. - // SFINAE prevents implicit conversions to bool (such as from int). - template ::value, int> = 0> - constexpr bool compare_result_as_less_than(const BoolType r) { return r; } - constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { - return r < 0; - } - - template - constexpr bool do_less_than_comparison(const Compare &compare, const K &x, - const LK &y) { - return compare_result_as_less_than(compare(x, y)); - } - - // Helper functions to do a three-way comparison of two keys given a boolean or - // three-way comparator. - // SFINAE prevents implicit conversions to int (such as from bool). - template ::value, int> = 0> - constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { - return c < 0 ? phmap::weak_ordering::less - : c == 0 ? phmap::weak_ordering::equivalent - : phmap::weak_ordering::greater; - } - constexpr phmap::weak_ordering compare_result_as_ordering( - const phmap::weak_ordering c) { - return c; - } - - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare_result_as_ordering(compare(x, y)); - } - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare(x, y) ? phmap::weak_ordering::less - : compare(y, x) ? phmap::weak_ordering::greater - : phmap::weak_ordering::equivalent; - } - - } // namespace compare_internal -} - - -namespace phmap { - -namespace priv { - - // A helper class that indicates if the Compare parameter is a key-compare-to - // comparator. - template - using btree_is_key_compare_to = - std::is_convertible, - phmap::weak_ordering>; - - struct StringBtreeDefaultLess { - using is_transparent = void; - - StringBtreeDefaultLess() = default; - - // Compatibility constructor. - StringBtreeDefaultLess(std::less) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultLess(std::less) {} // NOLINT - StringBtreeDefaultLess(phmap::Less) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#endif - }; - - struct StringBtreeDefaultGreater { - using is_transparent = void; - - StringBtreeDefaultGreater() = default; - - StringBtreeDefaultGreater(std::greater) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultGreater(std::greater) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#endif - }; - - // A helper class to convert a boolean comparison into a three-way "compare-to" - // comparison that returns a negative value to indicate less-than, zero to - // indicate equality and a positive value to indicate greater-than. This helper - // class is specialized for less, greater, - // less, and greater. - // - // key_compare_to_adapter is provided so that btree users - // automatically get the more efficient compare-to code when using common - // google string types with common comparison functors. - // These string-like specializations also turn on heterogeneous lookup by - // default. - template - struct key_compare_to_adapter { - using type = Compare; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultGreater; - }; - -#if PHMAP_HAVE_STD_STRING_VIEW - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultGreater; - }; -#endif - - template - struct common_params { - // If Compare is a common comparator for a std::string-like type, then we adapt it - // to use heterogeneous lookup and to be a key-compare-to comparator. - using key_compare = typename key_compare_to_adapter::type; - // A type which indicates if we have a key-compare-to functor or a plain old - // key-compare functor. - using is_key_compare_to = btree_is_key_compare_to; - - using allocator_type = Alloc; - using key_type = Key; - using size_type = std::size_t ; - using difference_type = ptrdiff_t; - - // True if this is a multiset or multimap. - using is_multi_container = std::integral_constant; - - using slot_policy = SlotPolicy; - using slot_type = typename slot_policy::slot_type; - using value_type = typename slot_policy::value_type; - using init_type = typename slot_policy::mutable_value_type; - using pointer = value_type *; - using const_pointer = const value_type *; - using reference = value_type &; - using const_reference = const value_type &; - - enum { - kTargetNodeSize = TargetNodeSize, - - // Upper bound for the available space for values. This is largest for leaf - // nodes, which have overhead of at least a pointer + 4 bytes (for storing - // 3 field_types and an enum). - kNodeValueSpace = - TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), - }; - - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. - using node_count_type = - phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > - (std::numeric_limits::max)()), - uint16_t, uint8_t>; // NOLINT - - // The following methods are necessary for passing this struct as PolicyTraits - // for node_handle and/or are used within btree. - static value_type &element(slot_type *slot) { - return slot_policy::element(slot); - } - static const value_type &element(const slot_type *slot) { - return slot_policy::element(slot); - } - template - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - slot_policy::construct(alloc, slot, std::forward(args)...); - } - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - slot_policy::construct(alloc, slot, other); - } - static void destroy(Alloc *alloc, slot_type *slot) { - slot_policy::destroy(alloc, slot); - } - static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { - construct(alloc, new_slot, old_slot); - destroy(alloc, old_slot); - } - static void swap(Alloc *alloc, slot_type *a, slot_type *b) { - slot_policy::swap(alloc, a, b); - } - static void move(Alloc *alloc, slot_type *src, slot_type *dest) { - slot_policy::move(alloc, src, dest); - } - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - slot_policy::move(alloc, first, last, result); - } - }; - - // A parameters structure for holding the type parameters for a btree_map. - // Compare and Alloc should be nothrow copy-constructible. - template - struct map_params : common_params> { - using super_type = typename map_params::common_params; - using mapped_type = Data; - // This type allows us to move keys when it is safe to do so. It is safe - // for maps in which value_type and mutable_value_type are layout compatible. - using slot_policy = typename super_type::slot_policy; - using slot_type = typename super_type::slot_type; - using value_type = typename super_type::value_type; - using init_type = typename super_type::init_type; - - using key_compare = typename super_type::key_compare; - // Inherit from key_compare for empty base class optimization. - struct value_compare : private key_compare { - value_compare() = default; - explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} - - template - auto operator()(const T &left, const U &right) const - -> decltype(std::declval()(left.first, right.first)) { - return key_compare::operator()(left.first, right.first); - } - }; - using is_map_container = std::true_type; - - static const Key &key(const value_type &x) { return x.first; } - static const Key &key(const init_type &x) { return x.first; } - static const Key &key(const slot_type *x) { return slot_policy::key(x); } - static mapped_type &value(value_type *value) { return value->second; } - }; - - // This type implements the necessary functions from the - // btree::priv::slot_type interface. - template - struct set_slot_policy { - using slot_type = Key; - using value_type = Key; - using mutable_value_type = Key; - - static value_type &element(slot_type *slot) { return *slot; } - static const value_type &element(const slot_type *slot) { return *slot; } - - template - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - phmap::allocator_traits::construct(*alloc, slot, - std::forward(args)...); - } - - template - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - phmap::allocator_traits::construct(*alloc, slot, std::move(*other)); - } - - template - static void destroy(Alloc *alloc, slot_type *slot) { - phmap::allocator_traits::destroy(*alloc, slot); - } - - template - static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { - using std::swap; - swap(*a, *b); - } - - template - static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { - *dest = std::move(*src); - } - - template - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } - }; - - // A parameters structure for holding the type parameters for a btree_set. - // Compare and Alloc should be nothrow copy-constructible. - template - struct set_params : common_params> { - using value_type = Key; - using slot_type = typename set_params::common_params::slot_type; - using value_compare = typename set_params::common_params::key_compare; - using is_map_container = std::false_type; - - static const Key &key(const value_type &x) { return x; } - static const Key &key(const slot_type *x) { return *x; } - }; - - // An adapter class that converts a lower-bound compare into an upper-bound - // compare. Note: there is no need to make a version of this adapter specialized - // for key-compare-to functors because the upper-bound (the first value greater - // than the input) is never an exact match. - template - struct upper_bound_adapter { - explicit upper_bound_adapter(const Compare &c) : comp(c) {} - template - bool operator()(const K &a, const LK &b) const { - // Returns true when a is not greater than b. - return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); - } - - private: - Compare comp; - }; - - enum class MatchKind : uint8_t { kEq, kNe }; - - template - struct SearchResult { - V value; - MatchKind match; - - static constexpr bool HasMatch() { return true; } - bool IsEq() const { return match == MatchKind::kEq; } - }; - - // When we don't use CompareTo, `match` is not present. - // This ensures that callers can't use it accidentally when it provides no - // useful information. - template - struct SearchResult { - V value; - - static constexpr bool HasMatch() { return false; } - static constexpr bool IsEq() { return false; } - }; - - // A node in the btree holding. The same node type is used for both internal - // and leaf nodes in the btree, though the nodes are allocated in such a way - // that the children array is only valid in internal nodes. - template - class btree_node { - using is_key_compare_to = typename Params::is_key_compare_to; - using is_multi_container = typename Params::is_multi_container; - using field_type = typename Params::node_count_type; - using allocator_type = typename Params::allocator_type; - using slot_type = typename Params::slot_type; - - public: - using params_type = Params; - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using key_compare = typename Params::key_compare; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - - // Btree decides whether to use linear node search as follows: - // - If the key is arithmetic and the comparator is std::less or - // std::greater, choose linear. - // - Otherwise, choose binary. - // TODO(ezb): Might make sense to add condition(s) based on node-size. - using use_linear_search = std::integral_constant< - bool, - std::is_arithmetic::value && - (std::is_same, key_compare>::value || - std::is_same, key_compare>::value || - std::is_same, key_compare>::value)>; - - - ~btree_node() = default; - btree_node(btree_node const &) = delete; - btree_node &operator=(btree_node const &) = delete; - - // Public for EmptyNodeType. - constexpr static size_type Alignment() { - static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), - "Alignment of all nodes must be equal."); - return (size_type)InternalLayout().Alignment(); - } - - protected: - btree_node() = default; - - private: - using layout_type = phmap::priv::Layout; - constexpr static size_type SizeWithNValues(size_type n) { - return (size_type)layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)n, - /*children*/ 0) - .AllocSize(); - } - // A lower bound for the overhead of fields other than values in a leaf node. - constexpr static size_type MinimumOverhead() { - return (size_type)(SizeWithNValues(1) - sizeof(value_type)); - } - - // Compute how many values we can fit onto a leaf node taking into account - // padding. - constexpr static size_type NodeTargetValues(const int begin, const int end) { - return begin == end ? begin - : SizeWithNValues((begin + end) / 2 + 1) > - params_type::kTargetNodeSize - ? NodeTargetValues(begin, (begin + end) / 2) - : NodeTargetValues((begin + end) / 2 + 1, end); - } - - enum { - kTargetNodeSize = params_type::kTargetNodeSize, - kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), - - // We need a minimum of 3 values per internal node in order to perform - // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, - - // The node is internal (i.e. is not a leaf node) if and only if `max_count` - // has this value. - kInternalNodeMaxCount = 0, - }; - - // Leaves can have less than kNodeValues values. - constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)max_values, - /*children*/ 0); - } - constexpr static layout_type InternalLayout() { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ kNodeValues, - /*children*/ kNodeValues + 1); - } - constexpr static size_type LeafSize(const int max_values = kNodeValues) { - return (size_type)LeafLayout(max_values).AllocSize(); - } - constexpr static size_type InternalSize() { - return (size_type)InternalLayout().AllocSize(); - } - - // N is the index of the type in the Layout definition. - // ElementType is the Nth type in the Layout definition. - template - inline typename layout_type::template ElementType *GetField() { - // We assert that we don't read from values that aren't there. - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer(reinterpret_cast(this)); - } - - template - inline const typename layout_type::template ElementType *GetField() const { - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer( - reinterpret_cast(this)); - } - - void set_parent(btree_node *p) { *GetField<0>() = p; } - field_type &mutable_count() { return GetField<1>()[2]; } - slot_type *slot(size_type i) { return &GetField<2>()[i]; } - const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } - void set_position(field_type v) { GetField<1>()[0] = v; } - void set_start(field_type v) { GetField<1>()[1] = v; } - void set_count(field_type v) { GetField<1>()[2] = v; } - void set_max_count(field_type v) { GetField<1>()[3] = v; } - - public: - // Whether this is a leaf node or not. This value doesn't change after the - // node is created. - bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } - - // Getter for the position of this node in its parent. - field_type position() const { return GetField<1>()[0]; } - - // Getter for the offset of the first value in the `values` array. - field_type start() const { return GetField<1>()[1]; } - - // Getters for the number of values stored in this node. - field_type count() const { return GetField<1>()[2]; } - field_type max_count() const { - // Internal nodes have max_count==kInternalNodeMaxCount. - // Leaf nodes have max_count in [1, kNodeValues]. - const field_type max_cnt = GetField<1>()[3]; - return max_cnt == field_type{kInternalNodeMaxCount} - ? field_type{kNodeValues} - : max_cnt; - } - - // Getter for the parent of this node. - btree_node *parent() const { return *GetField<0>(); } - // Getter for whether the node is the root of the tree. The parent of the - // root of the tree is the leftmost node in the tree which is guaranteed to - // be a leaf. - bool is_root() const { return parent()->leaf(); } - void make_root() { - assert(parent()->is_root()); - set_parent(parent()->parent()); - } - - // Getters for the key/value at position i in the node. - const key_type &key(size_type i) const { return params_type::key(slot(i)); } - reference value(size_type i) { return params_type::element(slot(i)); } - const_reference value(size_type i) const { return params_type::element(slot(i)); } - - // Getters/setter for the child at position i in the node. - btree_node *child(size_type i) const { return GetField<3>()[i]; } - btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } - void clear_child(size_type i) { - phmap::priv::SanitizerPoisonObject(&mutable_child(i)); - } - void set_child(size_type i, btree_node *c) { - phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); - mutable_child(i) = c; - c->set_position((field_type)i); - } - void init_child(int i, btree_node *c) { - set_child(i, c); - c->set_parent(this); - } - - // Returns the position of the first value whose key is not less than k. - template - SearchResult lower_bound( - const K &k, const key_compare &comp) const { - return use_linear_search::value ? linear_search(k, comp) - : binary_search(k, comp); - } - // Returns the position of the first value whose key is greater than k. - template - int upper_bound(const K &k, const key_compare &comp) const { - auto upper_compare = upper_bound_adapter(comp); - return use_linear_search::value ? linear_search(k, upper_compare).value - : binary_search(k, upper_compare).value; - } - - template - SearchResult::value> - linear_search(const K &k, const Compare &comp) const { - return linear_search_impl(k, 0, count(), comp, - btree_is_key_compare_to()); - } - - template - SearchResult::value> - binary_search(const K &k, const Compare &comp) const { - return binary_search_impl(k, 0, count(), comp, - btree_is_key_compare_to()); - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using plain compare. - template - SearchResult linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s < e) { - if (!comp(key(s), k)) { - break; - } - ++s; - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using compare-to. - template - SearchResult linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::true_type /* IsCompareTo */) const { - while (s < e) { - const phmap::weak_ordering c = comp(key(s), k); - if (c == 0) { - return {s, MatchKind::kEq}; - } else if (c > 0) { - break; - } - ++s; - } - return {s, MatchKind::kNe}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using plain compare. - template - SearchResult binary_search_impl( - const K &k, int s, int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s != e) { - const int mid = (s + e) >> 1; - if (comp(key(mid), k)) { - s = mid + 1; - } else { - e = mid; - } - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using compare-to. - template - SearchResult binary_search_impl( - const K &k, int s, int e, const CompareTo &comp, - std::true_type /* IsCompareTo */) const { - if (is_multi_container::value) { - MatchKind exact_match = MatchKind::kNe; - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else { - e = mid; - if (c == 0) { - // Need to return the first value whose key is not less than k, - // which requires continuing the binary search if this is a - // multi-container. - exact_match = MatchKind::kEq; - } - } - } - return {s, exact_match}; - } else { // Not a multi-container. - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else if (c > 0) { - e = mid; - } else { - return {mid, MatchKind::kEq}; - } - } - return {s, MatchKind::kNe}; - } - } - - // Emplaces a value at position i, shifting all existing values and - // children at positions >= i to the right by 1. - template - void emplace_value(size_type i, allocator_type *alloc, Args &&... args); - - // Removes the value at position i, shifting all existing values and children - // at positions > i to the left by 1. - void remove_value(int i, allocator_type *alloc); - - // Removes the values at positions [i, i + to_erase), shifting all values - // after that range to the left by to_erase. Does not change children at all. - void remove_values_ignore_children(int i, size_type to_erase, - allocator_type *alloc); - - // Rebalances a node with its right sibling. - void rebalance_right_to_left(int to_move, btree_node *right, - allocator_type *alloc); - void rebalance_left_to_right(int to_move, btree_node *right, - allocator_type *alloc); - - // Splits a node, moving a portion of the node's values to its right sibling. - void split(int insert_position, btree_node *dest, allocator_type *alloc); - - // Merges a node with its right sibling, moving all of the values and the - // delimiting key in the parent node onto itself. - void merge(btree_node *sibling, allocator_type *alloc); - - // Swap the contents of "this" and "src". - void swap(btree_node *src, allocator_type *alloc); - - // Node allocation/deletion routines. - static btree_node *init_leaf(btree_node *n, btree_node *parent, - int max_cnt) { - n->set_parent(parent); - n->set_position(0); - n->set_start(0); - n->set_count(0); - n->set_max_count((field_type)max_cnt); - phmap::priv::SanitizerPoisonMemoryRegion( - n->slot(0), max_cnt * sizeof(slot_type)); - return n; - } - static btree_node *init_internal(btree_node *n, btree_node *parent) { - init_leaf(n, parent, kNodeValues); - // Set `max_count` to a sentinel value to indicate that this node is - // internal. - n->set_max_count(kInternalNodeMaxCount); - phmap::priv::SanitizerPoisonMemoryRegion( - &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); - return n; - } - void destroy(allocator_type *alloc) { - for (int i = 0; i < count(); ++i) { - value_destroy(i, alloc); - } - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return use_linear_search::value; - } - - private: - template - void value_init(const size_type i, allocator_type *alloc, Args &&... args) { - phmap::priv::SanitizerUnpoisonObject(slot(i)); - params_type::construct(alloc, slot(i), std::forward(args)...); - } - void value_destroy(const size_type i, allocator_type *alloc) { - params_type::destroy(alloc, slot(i)); - phmap::priv::SanitizerPoisonObject(slot(i)); - } - - // Move n values starting at value i in this node into the values starting at - // value j in node x. - void uninitialized_move_n(const size_type n, const size_type i, - const size_type j, btree_node *x, - allocator_type *alloc) { - phmap::priv::SanitizerUnpoisonMemoryRegion( - x->slot(j), n * sizeof(slot_type)); - for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); - src != end; ++src, ++dest) { - params_type::construct(alloc, dest, src); - } - } - - // Destroys a range of n values, starting at index i. - void value_destroy_n(const size_type i, const size_type n, - allocator_type *alloc) { - for (int j = 0; j < n; ++j) { - value_destroy(i + j, alloc); - } - } - - template - friend class btree; - template - friend struct btree_iterator; - friend class BtreeNodePeer; - }; - - template - struct btree_iterator { - private: - using key_type = typename Node::key_type; - using size_type = typename Node::size_type; - using params_type = typename Node::params_type; - - using node_type = Node; - using normal_node = typename std::remove_const::type; - using const_node = const Node; - using normal_pointer = typename params_type::pointer; - using normal_reference = typename params_type::reference; - using const_pointer = typename params_type::const_pointer; - using const_reference = typename params_type::const_reference; - using slot_type = typename params_type::slot_type; - - using iterator = - btree_iterator; - using const_iterator = - btree_iterator; - - public: - // These aliases are public for std::iterator_traits. - using difference_type = typename Node::difference_type; - using value_type = typename params_type::value_type; - using pointer = Pointer; - using reference = Reference; - using iterator_category = std::bidirectional_iterator_tag; - - btree_iterator() : node(nullptr), position(-1) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} - - // NOTE: this SFINAE allows for implicit conversions from iterator to - // const_iterator, but it specifically avoids defining copy constructors so - // that btree_iterator can be trivially copyable. This is for performance and - // binary size reasons. - template , iterator>::value && - std::is_same::value, - int> = 0> - btree_iterator(const btree_iterator &x) // NOLINT - : node(x.node), position(x.position) {} - - private: - // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids defining a copy constructor. - // NOTE: the const_cast is safe because this constructor is only called by - // non-const methods and the container owns the nodes. - template , const_iterator>::value && - std::is_same::value, - int> = 0> - explicit btree_iterator(const btree_iterator &x) - : node(const_cast(x.node)), position(x.position) {} - - // Increment/decrement the iterator. - void increment() { - if (node->leaf() && ++position < node->count()) { - return; - } - increment_slow(); - } - void increment_slow(); - - void decrement() { - if (node->leaf() && --position >= 0) { - return; - } - decrement_slow(); - } - void decrement_slow(); - - public: - bool operator==(const const_iterator &x) const { - return node == x.node && position == x.position; - } - bool operator!=(const const_iterator &x) const { - return node != x.node || position != x.position; - } - - // Accessors for the key/value the iterator is pointing at. - reference operator*() const { - return node->value(position); - } - pointer operator->() const { - return &node->value(position); - } - - btree_iterator& operator++() { - increment(); - return *this; - } - btree_iterator& operator--() { - decrement(); - return *this; - } - btree_iterator operator++(int) { - btree_iterator tmp = *this; - ++*this; - return tmp; - } - btree_iterator operator--(int) { - btree_iterator tmp = *this; - --*this; - return tmp; - } - - private: - template - friend class btree; - template - friend class btree_container; - template - friend class btree_set_container; - template - friend class btree_map_container; - template - friend class btree_multiset_container; - template - friend struct btree_iterator; - template - friend class base_checker; - - const key_type &key() const { return node->key(position); } - slot_type *slot() { return node->slot(position); } - - // The node in the tree the iterator is pointing at. - Node *node; - // The position within the node of the tree the iterator is pointing at. - // TODO(ezb): make this a field_type - int position; - }; - - template - class btree { - using node_type = btree_node; - using is_key_compare_to = typename Params::is_key_compare_to; - - // We use a static empty node for the root/leftmost/rightmost of empty btrees - // in order to avoid branching in begin()/end(). - struct alignas(node_type::Alignment()) EmptyNodeType : node_type { - using field_type = typename node_type::field_type; - node_type *parent; - field_type position = 0; - field_type start = 0; - field_type count = 0; - // max_count must be != kInternalNodeMaxCount (so that this node is regarded - // as a leaf node). max_count() is never called when the tree is empty. - field_type max_count = node_type::kInternalNodeMaxCount + 1; - -#ifdef _MSC_VER - // MSVC has constexpr code generations bugs here. - EmptyNodeType() : parent(this) {} -#else - constexpr EmptyNodeType(node_type *p) : parent(p) {} -#endif - }; - - static node_type *EmptyNode() { -#ifdef _MSC_VER - static EmptyNodeType empty_node; - // This assert fails on some other construction methods. - assert(empty_node.parent == &empty_node); - return &empty_node; -#else - static constexpr EmptyNodeType empty_node( - const_cast(&empty_node)); - return const_cast(&empty_node); -#endif - } - - enum { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, - }; - - struct node_stats { - using size_type = typename Params::size_type; - - node_stats(size_type l, size_type i) - : leaf_nodes(l), - internal_nodes(i) { - } - - node_stats& operator+=(const node_stats &x) { - leaf_nodes += x.leaf_nodes; - internal_nodes += x.internal_nodes; - return *this; - } - - size_type leaf_nodes; - size_type internal_nodes; - }; - - public: - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - using key_compare = typename Params::key_compare; - using value_compare = typename Params::value_compare; - using allocator_type = typename Params::allocator_type; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using iterator = btree_iterator; - using const_iterator = typename iterator::const_iterator; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - using node_handle_type = node_handle; - - // Internal types made public for use by btree_container types. - using params_type = Params; - using slot_type = typename Params::slot_type; - - private: - // For use in copy_or_move_values_in_order. - const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } - value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } - - // Copies or moves (depending on the template parameter) the values in - // x into this btree in their order in x. This btree must be empty before this - // method is called. This method is used in copy construction, copy - // assignment, and move assignment. - template - void copy_or_move_values_in_order(Btree *x); - - // Validates that various assumptions/requirements are true at compile time. - constexpr static bool static_assert_validation(); - - public: - btree(const key_compare &comp, const allocator_type &alloc); - - btree(const btree &x); - btree(btree &&x) noexcept - : root_(std::move(x.root_)), - rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), - size_(phmap::exchange(x.size_, 0)) { - x.mutable_root() = EmptyNode(); - } - - ~btree() { - // Put static_asserts in destructor to avoid triggering them before the type - // is complete. - static_assert(static_assert_validation(), "This call must be elided."); - clear(); - } - - // Assign the contents of x to *this. - btree &operator=(const btree &x); - btree &operator=(btree &&x) noexcept; - - iterator begin() { - return iterator(leftmost(), 0); - } - const_iterator begin() const { - return const_iterator(leftmost(), 0); - } - iterator end() { return iterator(rightmost_, rightmost_->count()); } - const_iterator end() const { - return const_iterator(rightmost_, rightmost_->count()); - } - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - reverse_iterator rend() { - return reverse_iterator(begin()); - } - const_reverse_iterator rend() const { - return const_reverse_iterator(begin()); - } - - // Finds the first element whose key is not less than key. - template - iterator lower_bound(const K &key) { - return internal_end(internal_lower_bound(key)); - } - template - const_iterator lower_bound(const K &key) const { - return internal_end(internal_lower_bound(key)); - } - - // Finds the first element whose key is greater than key. - template - iterator upper_bound(const K &key) { - return internal_end(internal_upper_bound(key)); - } - template - const_iterator upper_bound(const K &key) const { - return internal_end(internal_upper_bound(key)); - } - - // Finds the range of values which compare equal to key. The first member of - // the returned pair is equal to lower_bound(key). The second member pair of - // the pair is equal to upper_bound(key). - template - std::pair equal_range(const K &key) { - return {lower_bound(key), upper_bound(key)}; - } - template - std::pair equal_range(const K &key) const { - return {lower_bound(key), upper_bound(key)}; - } - - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template - std::pair insert_unique(const key_type &key, Args &&... args); - - // Inserts with hint. Checks to see if the value should be placed immediately - // before `position` in the tree. If so, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_unique() were made. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template - std::pair insert_hint_unique(iterator position, - const key_type &key, - Args &&... args); - - // Insert a range of values into the btree. - template - void insert_iterator_unique(InputIterator b, InputIterator e); - - // Inserts a value into the btree. - template - iterator insert_multi(const key_type &key, ValueType &&v); - - // Inserts a value into the btree. - template - iterator insert_multi(ValueType &&v) { - return insert_multi(params_type::key(v), std::forward(v)); - } - - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_multi(v) were made. - template - iterator insert_hint_multi(iterator position, ValueType &&v); - - // Insert a range of values into the btree. - template - void insert_iterator_multi(InputIterator b, InputIterator e); - - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - // Requirement: does not read the value at `*iter`. - iterator erase(iterator iter); - - // Erases range. Returns the number of keys erased and an iterator pointing - // to the element after the last erased element. - std::pair erase(iterator begin, iterator end); - - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - template - size_type erase_unique(const K &key); - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - template - size_type erase_multi(const K &key); - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. - template - iterator find(const K &key) { - return internal_end(internal_find(key)); - } - template - const_iterator find(const K &key) const { - return internal_end(internal_find(key)); - } - - // Returns a count of the number of times the key appears in the btree. - template - size_type count_unique(const K &key) const { - const iterator beg = internal_find(key); - if (beg.node == nullptr) { - // The key doesn't exist in the tree. - return 0; - } - return 1; - } - // Returns a count of the number of times the key appears in the btree. - template - size_type count_multi(const K &key) const { - const auto range = equal_range(key); - return std::distance(range.first, range.second); - } - - // Clear the btree, deleting all of the values it contains. - void clear(); - - // Swap the contents of *this and x. - void swap(btree &x); - - const key_compare &key_comp() const noexcept { - return root_.template get<0>(); - } - template - bool compare_keys(const K &x, const LK &y) const { - return compare_internal::compare_result_as_less_than(key_comp()(x, y)); - } - - value_compare value_comp() const { return value_compare(key_comp()); } - - // Verifies the structure of the btree. - void verify() const; - - // Size routines. - size_type size() const { return size_; } - size_type max_size() const { return (std::numeric_limits::max)(); } - bool empty() const { return size_ == 0; } - - // The height of the btree. An empty tree will have height 0. - size_type height() const { - size_type h = 0; - if (!empty()) { - // Count the length of the chain from the leftmost node up to the - // root. We actually count from the root back around to the level below - // the root, but the calculation is the same because of the circularity - // of that traversal. - const node_type *n = root(); - do { - ++h; - n = n->parent(); - } while (n != root()); - } - return h; - } - - // The number of internal, leaf and total nodes used by the btree. - size_type leaf_nodes() const { - return internal_stats(root()).leaf_nodes; - } - size_type internal_nodes() const { - return internal_stats(root()).internal_nodes; - } - size_type nodes() const { - node_stats stats = internal_stats(root()); - return stats.leaf_nodes + stats.internal_nodes; - } - - // The total number of bytes used by the btree. - size_type bytes_used() const { - node_stats stats = internal_stats(root()); - if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { - return sizeof(*this) + - node_type::LeafSize(root()->max_count()); - } else { - return sizeof(*this) + - stats.leaf_nodes * node_type::LeafSize() + - stats.internal_nodes * node_type::InternalSize(); - } - } - - // The average number of bytes used per value stored in the btree. - static double average_bytes_per_value() { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return node_type::LeafSize() / (kNodeValues * 0.75); - } - - // The fullness of the btree. Computed as the number of elements in the btree - // divided by the maximum number of elements a tree with the current number - // of nodes could hold. A value of 1 indicates perfect space - // utilization. Smaller values indicate space wastage. - // Returns 0 for empty trees. - double fullness() const { - if (empty()) return 0.0; - return static_cast(size()) / (nodes() * kNodeValues); - } - // The overhead of the btree structure in bytes per node. Computed as the - // total number of bytes used by the btree minus the number of bytes used for - // storing elements divided by the number of elements. - // Returns 0 for empty trees. - double overhead() const { - if (empty()) return 0.0; - return (bytes_used() - size() * sizeof(value_type)) / - static_cast(size()); - } - - // The allocator used by the btree. - allocator_type get_allocator() const { - return allocator(); - } - - private: - // Internal accessor routines. - node_type *root() { return root_.template get<2>(); } - const node_type *root() const { return root_.template get<2>(); } - node_type *&mutable_root() noexcept { return root_.template get<2>(); } - key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } - - // The leftmost node is stored as the parent of the root node. - node_type *leftmost() { return root()->parent(); } - const node_type *leftmost() const { return root()->parent(); } - - // Allocator routines. - allocator_type *mutable_allocator() noexcept { - return &root_.template get<1>(); - } - const allocator_type &allocator() const noexcept { - return root_.template get<1>(); - } - - // Allocates a correctly aligned node of at least size bytes using the - // allocator. - node_type *allocate(const size_type sz) { - return reinterpret_cast( - phmap::priv::Allocate( - mutable_allocator(), (size_t)sz)); - } - - // Node creation/deletion routines. - node_type* new_internal_node(node_type *parent) { - node_type *p = allocate(node_type::InternalSize()); - return node_type::init_internal(p, parent); - } - node_type* new_leaf_node(node_type *parent) { - node_type *p = allocate(node_type::LeafSize()); - return node_type::init_leaf(p, parent, kNodeValues); - } - node_type *new_leaf_root_node(const int max_count) { - node_type *p = allocate(node_type::LeafSize(max_count)); - return node_type::init_leaf(p, p, max_count); - } - - // Deletion helper routines. - void erase_same_node(iterator begin, iterator end); - iterator erase_from_leaf_node(iterator begin, size_type to_erase); - iterator rebalance_after_delete(iterator iter); - - // Deallocates a node of a certain size in bytes using the allocator. - void deallocate(const size_type sz, node_type *node) { - phmap::priv::Deallocate( - mutable_allocator(), node, (size_t)sz); - } - - void delete_internal_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::InternalSize(), node); - } - void delete_leaf_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::LeafSize(node->max_count()), node); - } - - // Rebalances or splits the node iter points to. - void rebalance_or_split(iterator *iter); - - // Merges the values of left, right and the delimiting key on their parent - // onto left, removing the delimiting key and deleting right. - void merge_nodes(node_type *left, node_type *right); - - // Tries to merge node with its left or right sibling, and failing that, - // rebalance with its left or right sibling. Returns true if a merge - // occurred, at which point it is no longer valid to access node. Returns - // false if no merging took place. - bool try_merge_or_rebalance(iterator *iter); - - // Tries to shrink the height of the tree by 1. - void try_shrink(); - - iterator internal_end(iterator iter) { - return iter.node != nullptr ? iter : end(); - } - const_iterator internal_end(const_iterator iter) const { - return iter.node != nullptr ? iter : end(); - } - - // Emplaces a value into the btree immediately before iter. Requires that - // key(v) <= iter.key() and (--iter).key() <= key(v). - template - iterator internal_emplace(iterator iter, Args &&... args); - - // Returns an iterator pointing to the first value >= the value "iter" is - // pointing at. Note that "iter" might be pointing to an invalid location as - // iter.position == iter.node->count(). This routine simply moves iter up in - // the tree to a valid location. - // Requires: iter.node is non-null. - template - static IterType internal_last(IterType iter); - - // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version uses a less-than comparator and is incapable of distinguishing when - // there is an exact match. The second version is for the key-compare-to - // specialization and distinguishes exact matches. The key-compare-to - // specialization allows the caller to avoid a subsequent comparison to - // determine if an exact match was made, which is important for keys with - // expensive comparison, such as strings. - template - SearchResult internal_locate( - const K &key) const; - - template - SearchResult internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const; - - template - SearchResult internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const; - - // Internal routine which implements lower_bound(). - template - iterator internal_lower_bound(const K &key) const; - - // Internal routine which implements upper_bound(). - template - iterator internal_upper_bound(const K &key) const; - - // Internal routine which implements find(). - template - iterator internal_find(const K &key) const; - - // Deletes a node and all of its children. - void internal_clear(node_type *node); - - // Verifies the tree structure of node. - int internal_verify(const node_type *node, - const key_type *lo, const key_type *hi) const; - - node_stats internal_stats(const node_type *node) const { - // The root can be a static empty node. - if (node == nullptr || (node == root() && empty())) { - return node_stats(0, 0); - } - if (node->leaf()) { - return node_stats(1, 0); - } - node_stats res(0, 1); - for (int i = 0; i <= node->count(); ++i) { - res += internal_stats(node->child(i)); - } - return res; - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return node_type::testonly_uses_linear_node_search(); - } - - private: - // We use compressed tuple in order to save space because key_compare and - // allocator_type are usually empty. - phmap::priv::CompressedTuple - root_; - - // A pointer to the rightmost node. Note that the leftmost node is stored as - // the root's parent. - node_type *rightmost_; - - // Number of values. - size_type size_; - }; - - //// - // btree_node methods - template - template - inline void btree_node

::emplace_value(const size_type i, - allocator_type *alloc, - Args &&... args) { - assert(i <= count()); - // Shift old values to create space for new value and then construct it in - // place. - if (i < count()) { - value_init(count(), alloc, slot(count() - 1)); - for (size_type j = count() - 1; j > i; --j) - params_type::move(alloc, slot(j - 1), slot(j)); - value_destroy(i, alloc); - } - value_init(i, alloc, std::forward(args)...); - set_count((field_type)(count() + 1)); - - if (!leaf() && count() > i + 1) { - for (int j = count(); j > i + 1; --j) { - set_child(j, child(j - 1)); - } - clear_child(i + 1); - } - } - - template - inline void btree_node

::remove_value(const int i, allocator_type *alloc) { - if (!leaf() && count() > i + 1) { - assert(child(i + 1)->count() == 0); - for (size_type j = i + 1; j < count(); ++j) { - set_child(j, child(j + 1)); - } - clear_child(count()); - } - - remove_values_ignore_children(i, /*to_erase=*/1, alloc); - } - - template - inline void btree_node

::remove_values_ignore_children( - int i, size_type to_erase, allocator_type *alloc) { - params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); - value_destroy_n(count() - to_erase, to_erase, alloc); - set_count((field_type)(count() - to_erase)); - } - - template - void btree_node

::rebalance_right_to_left(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(right->count() >= count()); - assert(to_move >= 1); - assert(to_move <= right->count()); - - // 1) Move the delimiting value in the parent to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // 2) Move the (to_move - 1) values from the right node to the left node. - right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); - - // 3) Move the new delimiting value to the parent from the right node. - params_type::move(alloc, right->slot(to_move - 1), - parent()->slot(position())); - - // 4) Shift the values in the right node to their correct position. - params_type::move(alloc, right->slot(to_move), right->slot(right->count()), - right->slot(0)); - - // 5) Destroy the now-empty to_move entries in the right node. - right->value_destroy_n(right->count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i < to_move; ++i) { - init_child(count() + i + 1, right->child(i)); - } - for (int i = 0; i <= right->count() - to_move; ++i) { - assert(i + to_move <= right->max_count()); - right->init_child(i, right->child(i + to_move)); - right->clear_child(i + to_move); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() + to_move)); - right->set_count((field_type)(right->count() - to_move)); - } - - template - void btree_node

::rebalance_left_to_right(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(count() >= right->count()); - assert(to_move >= 1); - assert(to_move <= count()); - - // Values in the right node are shifted to the right to make room for the - // new to_move values. Then, the delimiting value in the parent and the - // other (to_move - 1) values in the left node are moved into the right node. - // Lastly, a new delimiting value is moved from the left node into the - // parent, and the remaining empty left node entries are destroyed. - - if (right->count() >= to_move) { - // The original location of the right->count() values are sufficient to hold - // the new to_move entries from the parent and left node. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(to_move, right->count() - to_move, - right->count(), right, alloc); - for (slot_type *src = right->slot(right->count() - to_move - 1), - *dest = right->slot(right->count() - 1), - *end = right->slot(0); - src >= end; --src, --dest) { - params_type::move(alloc, src, dest); - } - - // 2) Move the delimiting value in the parent to the right node. - params_type::move(alloc, parent()->slot(position()), - right->slot(to_move - 1)); - - // 3) Move the (to_move - 1) values from the left node to the right node. - params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), - right->slot(0)); - } else { - // The right node does not have enough initialized space to hold the new - // to_move entries, so part of them will move to uninitialized space. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); - - // 2) Move the delimiting value in the parent to the right node. - right->value_init(to_move - 1, alloc, parent()->slot(position())); - - // 3) Move the (to_move - 1) values from the left node to the right node. - const size_type uninitialized_remaining = to_move - right->count() - 1; - uninitialized_move_n(uninitialized_remaining, - count() - uninitialized_remaining, right->count(), - right, alloc); - params_type::move(alloc, slot(count() - (to_move - 1)), - slot(count() - uninitialized_remaining), right->slot(0)); - } - - // 4) Move the new delimiting value to the parent from the left node. - params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); - - // 5) Destroy the now-empty to_move entries in the left node. - value_destroy_n(count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the left to the right node. - for (int i = right->count(); i >= 0; --i) { - right->init_child(i + to_move, right->child(i)); - right->clear_child(i); - } - for (int i = 1; i <= to_move; ++i) { - right->init_child(i - 1, child(count() - to_move + i)); - clear_child(count() - to_move + i); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() - to_move)); - right->set_count((field_type)(right->count() + to_move)); - } - - template - void btree_node

::split(const int insert_position, btree_node *dest, - allocator_type *alloc) { - assert(dest->count() == 0); - assert(max_count() == kNodeValues); - - // We bias the split based on the position being inserted. If we're - // inserting at the beginning of the left node then bias the split to put - // more values on the right node. If we're inserting at the end of the - // right node then bias the split to put more values on the left node. - if (insert_position == 0) { - dest->set_count((field_type)(count() - 1)); - } else if (insert_position == kNodeValues) { - dest->set_count(0); - } else { - dest->set_count((field_type)(count() / 2)); - } - set_count((field_type)(count() - dest->count())); - assert(count() >= 1); - - // Move values from the left sibling to the right sibling. - uninitialized_move_n(dest->count(), count(), 0, dest, alloc); - - // Destroy the now-empty entries in the left node. - value_destroy_n(count(), dest->count(), alloc); - - // The split key is the largest value in the left sibling. - set_count((field_type)(count() - 1)); - parent()->emplace_value(position(), alloc, slot(count())); - value_destroy(count(), alloc); - parent()->init_child(position() + 1, dest); - - if (!leaf()) { - for (int i = 0; i <= dest->count(); ++i) { - assert(child(count() + i + 1) != nullptr); - dest->init_child(i, child(count() + i + 1)); - clear_child(count() + i + 1); - } - } - } - - template - void btree_node

::merge(btree_node *src, allocator_type *alloc) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); - - // Move the delimiting value to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // Move the values from the right to the left node. - src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); - - // Destroy the now-empty entries in the right node. - src->value_destroy_n(0, src->count(), alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i <= src->count(); ++i) { - init_child(count() + i + 1, src->child(i)); - src->clear_child(i); - } - } - - // Fixup the counts on the src and dest nodes. - set_count((field_type)(1 + count() + src->count())); - src->set_count(0); - - // Remove the value on the parent node. - parent()->remove_value(position(), alloc); - } - - template - void btree_node

::swap(btree_node *x, allocator_type *alloc) { - using std::swap; - assert(leaf() == x->leaf()); - - // Determine which is the smaller/larger node. - btree_node *smaller = this, *larger = x; - if (smaller->count() > larger->count()) { - swap(smaller, larger); - } - - // Swap the values. - for (slot_type *a = smaller->slot(0), *b = larger->slot(0), - *end = a + smaller->count(); - a != end; ++a, ++b) { - params_type::swap(alloc, a, b); - } - - // Move values that can't be swapped. - const size_type to_move = larger->count() - smaller->count(); - larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), - smaller, alloc); - larger->value_destroy_n(smaller->count(), to_move, alloc); - - if (!leaf()) { - // Swap the child pointers. - std::swap_ranges(&smaller->mutable_child(0), - &smaller->mutable_child(smaller->count() + 1), - &larger->mutable_child(0)); - // Update swapped children's parent pointers. - int i = 0; - for (; i <= smaller->count(); ++i) { - smaller->child(i)->set_parent(smaller); - larger->child(i)->set_parent(larger); - } - // Move the child pointers that couldn't be swapped. - for (; i <= larger->count(); ++i) { - smaller->init_child(i, larger->child(i)); - larger->clear_child(i); - } - } - - // Swap the counts. - swap(mutable_count(), x->mutable_count()); - } - - //// - // btree_iterator methods - template - void btree_iterator::increment_slow() { - if (node->leaf()) { - assert(position >= node->count()); - btree_iterator save(*this); - while (position == node->count() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); - } - if (position == node->count()) { - *this = save; - } - } else { - assert(position < node->count()); - node = node->child(position + 1); - while (!node->leaf()) { - node = node->child(0); - } - position = 0; - } - } - - template - void btree_iterator::decrement_slow() { - if (node->leaf()) { - assert(position <= -1); - btree_iterator save(*this); - while (position < 0 && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); - } - if (position < 0) { - *this = save; - } - } else { - assert(position >= 0); - node = node->child(position); - while (!node->leaf()) { - node = node->child(node->count()); - } - position = node->count() - 1; - } - } - - //// - // btree methods - template - template - void btree

::copy_or_move_values_in_order(Btree *x) { - static_assert(std::is_same::value || - std::is_same::value, - "Btree type must be same or const."); - assert(empty()); - - // We can avoid key comparisons because we know the order of the - // values is the same order we'll store them in. - auto iter = x->begin(); - if (iter == x->end()) return; - insert_multi(maybe_move_from_iterator(iter)); - ++iter; - for (; iter != x->end(); ++iter) { - // If the btree is not empty, we can just insert the new value at the end - // of the tree. - internal_emplace(end(), maybe_move_from_iterator(iter)); - } - } - - template - constexpr bool btree

::static_assert_validation() { - static_assert(std::is_nothrow_copy_constructible::value, - "Key comparison must be nothrow copy constructible"); - static_assert(std::is_nothrow_copy_constructible::value, - "Allocator must be nothrow copy constructible"); - static_assert(type_traits_internal::is_trivially_copyable::value, - "iterator not trivially copyable."); - - // Note: We assert that kTargetValues, which is computed from - // Params::kTargetNodeSize, must fit the node_type::field_type. - static_assert( - kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), - "target node size too large"); - - // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. - using compare_result_type = - phmap::invoke_result; - static_assert( - std::is_same::value || - std::is_convertible::value, - "key comparison function must return phmap::{weak,strong}_ordering or " - "bool."); - - // Test the assumption made in setting kNodeValueSpace. - static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, - "node space assumption incorrect"); - - return true; - } - - template - btree

::btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - - template - btree

::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { - copy_or_move_values_in_order(&x); - } - - template - template - auto btree

::insert_unique(const key_type &key, Args &&... args) - -> std::pair { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - auto res = internal_locate(key); - iterator &iter = res.value; - - if (res.HasMatch()) { - if (res.IsEq()) { - // The key already exists in the tree, do nothing. - return {iter, false}; - } - } else { - iterator last = internal_last(iter); - if (last.node && !compare_keys(key, last.key())) { - // The key already exists in the tree, do nothing. - return {last, false}; - } - } - return {internal_emplace(iter, std::forward(args)...), true}; - } - - template - template - inline auto btree

::insert_hint_unique(iterator position, const key_type &key, - Args &&... args) - -> std::pair { - if (!empty()) { - if (position == end() || compare_keys(key, position.key())) { - iterator prev = position; - if (position == begin() || compare_keys((--prev).key(), key)) { - // prev.key() < key < position.key() - return {internal_emplace(position, std::forward(args)...), true}; - } - } else if (compare_keys(position.key(), key)) { - ++position; - if (position == end() || compare_keys(key, position.key())) { - // {original `position`}.key() < key < {current `position`}.key() - return {internal_emplace(position, std::forward(args)...), true}; - } - } else { - // position.key() == key - return {position, false}; - } - } - return insert_unique(key, std::forward(args)...); - } - - template - template - void btree

::insert_iterator_unique(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_unique(end(), params_type::key(*b), *b); - } - } - - template - template - auto btree

::insert_multi(const key_type &key, ValueType &&v) -> iterator { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - iterator iter = internal_upper_bound(key); - if (iter.node == nullptr) { - iter = end(); - } - return internal_emplace(iter, std::forward(v)); - } - - template - template - auto btree

::insert_hint_multi(iterator position, ValueType &&v) -> iterator { - if (!empty()) { - const key_type &key = params_type::key(v); - if (position == end() || !compare_keys(position.key(), key)) { - iterator prev = position; - if (position == begin() || !compare_keys(key, (--prev).key())) { - // prev.key() <= key <= position.key() - return internal_emplace(position, std::forward(v)); - } - } else { - iterator next = position; - ++next; - if (next == end() || !compare_keys(next.key(), key)) { - // position.key() < key <= next.key() - return internal_emplace(next, std::forward(v)); - } - } - } - return insert_multi(std::forward(v)); - } - - template - template - void btree

::insert_iterator_multi(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_multi(end(), *b); - } - } - - template - auto btree

::operator=(const btree &x) -> btree & { - if (this != &x) { - clear(); - - *mutable_key_comp() = x.key_comp(); - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - *mutable_allocator() = x.allocator(); - } - - copy_or_move_values_in_order(&x); - } - return *this; - } - - template - auto btree

::operator=(btree &&x) noexcept -> btree & { - if (this != &x) { - clear(); - - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - if (allocator() == x.allocator()) { - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - // We aren't allowed to propagate the allocator and the allocator is - // different so we can't take over its memory. We must move each element - // individually. We need both `x` and `this` to have `x`s key comparator - // while moving the values so we can't swap the key comparators. - *mutable_key_comp() = x.key_comp(); - copy_or_move_values_in_order(&x); - } - } - } - return *this; - } - - template - auto btree

::erase(iterator iter) -> iterator { - bool internal_delete = false; - if (!iter.node->leaf()) { - // Deletion of a value on an internal node. First, move the largest value - // from our left child here, then delete that position (in remove_value() - // below). We can get to the largest value from our left child by - // decrementing iter. - iterator internal_iter(iter); - --iter; - assert(iter.node->leaf()); - params_type::move(mutable_allocator(), iter.node->slot(iter.position), - internal_iter.node->slot(internal_iter.position)); - internal_delete = true; - } - - // Delete the key from the leaf. - iter.node->remove_value(iter.position, mutable_allocator()); - --size_; - - // We want to return the next value after the one we just erased. If we - // erased from an internal node (internal_delete == true), then the next - // value is ++(++iter). If we erased from a leaf node (internal_delete == - // false) then the next value is ++iter. Note that ++iter may point to an - // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. - - iterator res = rebalance_after_delete(iter); - - // If we erased from an internal node, advance the iterator. - if (internal_delete) { - ++res; - } - return res; - } - - template - auto btree

::rebalance_after_delete(iterator iter) -> iterator { - // Merge/rebalance as we walk back up the tree. - iterator res(iter); - bool first_iteration = true; - for (;;) { - if (iter.node == root()) { - try_shrink(); - if (empty()) { - return end(); - } - break; - } - if (iter.node->count() >= kMinNodeValues) { - break; - } - bool merged = try_merge_or_rebalance(&iter); - // On the first iteration, we should update `res` with `iter` because `res` - // may have been invalidated. - if (first_iteration) { - res = iter; - first_iteration = false; - } - if (!merged) { - break; - } - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - } - - // Adjust our return value. If we're pointing at the end of a node, advance - // the iterator. - if (res.position == res.node->count()) { - res.position = res.node->count() - 1; - ++res; - } - - return res; - } - - template - auto btree

::erase(iterator _begin, iterator _end) - -> std::pair { - difference_type count = std::distance(_begin, _end); - assert(count >= 0); - - if (count == 0) { - return {0, _begin}; - } - - if (count == (difference_type)size_) { - clear(); - return {count, this->end()}; - } - - if (_begin.node == _end.node) { - erase_same_node(_begin, _end); - size_ -= count; - return {count, rebalance_after_delete(_begin)}; - } - - const size_type target_size = size_ - count; - while (size_ > target_size) { - if (_begin.node->leaf()) { - const size_type remaining_to_erase = size_ - target_size; - const size_type remaining_in_node = _begin.node->count() - _begin.position; - _begin = erase_from_leaf_node( - _begin, (std::min)(remaining_to_erase, remaining_in_node)); - } else { - _begin = erase(_begin); - } - } - return {count, _begin}; - } - - template - void btree

::erase_same_node(iterator _begin, iterator _end) { - assert(_begin.node == _end.node); - assert(_end.position > _begin.position); - - node_type *node = _begin.node; - size_type to_erase = _end.position - _begin.position; - if (!node->leaf()) { - // Delete all children between _begin and _end. - for (size_type i = 0; i < to_erase; ++i) { - internal_clear(node->child(_begin.position + i + 1)); - } - // Rotate children after _end into new positions. - for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { - node->set_child(i - to_erase, node->child(i)); - node->clear_child(i); - } - } - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - // Do not need to update rightmost_, because - // * either _end == this->end(), and therefore node == rightmost_, and still - // exists - // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since - // it wasn't covered in [_begin, _end) - } - - template - auto btree

::erase_from_leaf_node(iterator _begin, size_type to_erase) - -> iterator { - node_type *node = _begin.node; - assert(node->leaf()); - assert(node->count() > _begin.position); - assert(_begin.position + to_erase <= node->count()); - - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - size_ -= to_erase; - - return rebalance_after_delete(_begin); - } - - template - template - auto btree

::erase_unique(const K &key) -> size_type { - const iterator iter = internal_find(key); - if (iter.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - erase(iter); - return 1; - } - - template - template - auto btree

::erase_multi(const K &key) -> size_type { - const iterator _begin = internal_lower_bound(key); - if (_begin.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - // Delete all of the keys between _begin and upper_bound(key). - const iterator _end = internal_end(internal_upper_bound(key)); - return erase(_begin, _end).first; - } - - template - void btree

::clear() { - if (!empty()) { - internal_clear(root()); - } - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - size_ = 0; - } - - template - void btree

::swap(btree &x) { - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_swap::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - } else { - // It's undefined behavior if the allocators are unequal here. - assert(allocator() == x.allocator()); - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - } - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } - - template - void btree

::verify() const { - assert(root() != nullptr); - assert(leftmost() != nullptr); - assert(rightmost_ != nullptr); - assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost_ == (--const_iterator(root(), root()->count())).node); - assert(leftmost()->leaf()); - assert(rightmost_->leaf()); - } - - template - void btree

::rebalance_or_split(iterator *iter) { - node_type *&node = iter->node; - int &insert_position = iter->position; - assert(node->count() == node->max_count()); - assert(kNodeValues == node->max_count()); - - // First try to make room on the node by rebalancing. - node_type *parent = node->parent(); - if (node != root()) { - if (node->position() > 0) { - // Try rebalancing with our left sibling. - node_type *left = parent->child(node->position() - 1); - assert(left->max_count() == kNodeValues); - if (left->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the end of the right node then we bias rebalancing to - // fill up the left node. - int to_move = (kNodeValues - left->count()) / - (1 + (insert_position < kNodeValues)); - to_move = (std::max)(1, to_move); - - if (((insert_position - to_move) >= 0) || - ((left->count() + to_move) < kNodeValues)) { - left->rebalance_right_to_left(to_move, node, mutable_allocator()); - - assert(node->max_count() - node->count() == to_move); - insert_position = insert_position - to_move; - if (insert_position < 0) { - insert_position = insert_position + left->count() + 1; - node = left; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - if (node->position() < parent->count()) { - // Try rebalancing with our right sibling. - node_type *right = parent->child(node->position() + 1); - assert(right->max_count() == kNodeValues); - if (right->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the _beginning of the left node then we bias rebalancing - // to fill up the right node. - int to_move = - (kNodeValues - right->count()) / (1 + (insert_position > 0)); - to_move = (std::max)(1, to_move); - - if ((insert_position <= (node->count() - to_move)) || - ((right->count() + to_move) < kNodeValues)) { - node->rebalance_left_to_right(to_move, right, mutable_allocator()); - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = right; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - // Rebalancing failed, make sure there is room on the parent node for a new - // value. - assert(parent->max_count() == kNodeValues); - if (parent->count() == kNodeValues) { - iterator parent_iter(node->parent(), node->position()); - rebalance_or_split(&parent_iter); - } - } else { - // Rebalancing not possible because this is the root node. - // Create a new root node and set the current root node as the child of the - // new root. - parent = new_internal_node(parent); - parent->init_child(0, root()); - mutable_root() = parent; - // If the former root was a leaf node, then it's now the rightmost node. - assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); - } - - // Split the node. - node_type *split_node; - if (node->leaf()) { - split_node = new_leaf_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - if (rightmost_ == node) rightmost_ = split_node; - } else { - split_node = new_internal_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - } - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = split_node; - } - } - - template - void btree

::merge_nodes(node_type *left, node_type *right) { - left->merge(right, mutable_allocator()); - if (right->leaf()) { - if (rightmost_ == right) rightmost_ = left; - delete_leaf_node(right); - } else { - delete_internal_node(right); - } - } - - template - bool btree

::try_merge_or_rebalance(iterator *iter) { - node_type *parent = iter->node->parent(); - if (iter->node->position() > 0) { - // Try merging with our left sibling. - node_type *left = parent->child(iter->node->position() - 1); - assert(left->max_count() == kNodeValues); - if ((1 + left->count() + iter->node->count()) <= kNodeValues) { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; - return true; - } - } - if (iter->node->position() < parent->count()) { - // Try merging with our right sibling. - node_type *right = parent->child(iter->node->position() + 1); - assert(right->max_count() == kNodeValues); - if ((1 + iter->node->count() + right->count()) <= kNodeValues) { - merge_nodes(iter->node, right); - return true; - } - // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the front of the tree. - if ((right->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position > 0))) { - int to_move = (right->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); - return false; - } - } - if (iter->node->position() > 0) { - // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the back of the tree. - node_type *left = parent->child(iter->node->position() - 1); - if ((left->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position < iter->node->count()))) { - int to_move = (left->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, left->count() - 1); - left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); - iter->position += to_move; - return false; - } - } - return false; - } - - template - void btree

::try_shrink() { - if (root()->count() > 0) { - return; - } - // Deleted the last item on the root node, shrink the height of the tree. - if (root()->leaf()) { - assert(size() == 0); - delete_leaf_node(root()); - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - } else { - node_type *child = root()->child(0); - child->make_root(); - delete_internal_node(root()); - mutable_root() = child; - } - } - - template - template - inline IterType btree

::internal_last(IterType iter) { - assert(iter.node != nullptr); - while (iter.position == iter.node->count()) { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - if (iter.node->leaf()) { - iter.node = nullptr; - break; - } - } - return iter; - } - - template - template - inline auto btree

::internal_emplace(iterator iter, Args &&... args) - -> iterator { - if (!iter.node->leaf()) { - // We can't insert on an internal node. Instead, we'll insert after the - // previous value which is guaranteed to be on a leaf node. - --iter; - ++iter.position; - } - const int max_count = iter.node->max_count(); - if (iter.node->count() == max_count) { - // Make room in the leaf for the new item. - if (max_count < kNodeValues) { - // Insertion into the root where the root is smaller than the full node - // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = - new_leaf_root_node((std::min)(kNodeValues, 2 * max_count)); - iter.node->swap(root(), mutable_allocator()); - delete_leaf_node(root()); - mutable_root() = iter.node; - rightmost_ = iter.node; - } else { - rebalance_or_split(&iter); - } - } - iter.node->emplace_value(iter.position, mutable_allocator(), - std::forward(args)...); - ++size_; - return iter; - } - - template - template - inline auto btree

::internal_locate(const K &key) const - -> SearchResult { - return internal_locate_impl(key, is_key_compare_to()); - } - - template - template - inline auto btree

::internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const - -> SearchResult { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - // NOTE: we don't need to walk all the way down the tree if the keys are - // equal, but determining equality would require doing an extra comparison - // on each node on the way down, and we will need to go all the way to the - // leaf node in the expected case. - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter}; - } - - template - template - inline auto btree

::internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const - -> SearchResult { - iterator iter(const_cast(root()), 0); - for (;;) { - SearchResult res = iter.node->lower_bound(key, key_comp()); - iter.position = res.value; - if (res.match == MatchKind::kEq) { - return {iter, MatchKind::kEq}; - } - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter, MatchKind::kNe}; - } - - template - template - auto btree

::internal_lower_bound(const K &key) const -> iterator { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template - template - auto btree

::internal_upper_bound(const K &key) const -> iterator { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->upper_bound(key, key_comp()); - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template - template - auto btree

::internal_find(const K &key) const -> iterator { - auto res = internal_locate(key); - if (res.HasMatch()) { - if (res.IsEq()) { - return res.value; - } - } else { - const iterator iter = internal_last(res.value); - if (iter.node != nullptr && !compare_keys(key, iter.key())) { - return iter; - } - } - return {nullptr, 0}; - } - - template - void btree

::internal_clear(node_type *node) { - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - internal_clear(node->child(i)); - } - delete_internal_node(node); - } else { - delete_leaf_node(node); - } - } - - template - int btree

::internal_verify( - const node_type *node, const key_type *lo, const key_type *hi) const { - assert(node->count() > 0); - assert(node->count() <= node->max_count()); - if (lo) { - assert(!compare_keys(node->key(0), *lo)); - } - if (hi) { - assert(!compare_keys(*hi, node->key(node->count() - 1))); - } - for (int i = 1; i < node->count(); ++i) { - assert(!compare_keys(node->key(i), node->key(i - 1))); - } - int count = node->count(); - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - assert(node->child(i) != nullptr); - assert(node->child(i)->parent() == node); - assert(node->child(i)->position() == i); - count += internal_verify( - node->child(i), - (i == 0) ? lo : &node->key(i - 1), - (i == node->count()) ? hi : &node->key(i)); - } - } - return count; - } - - // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. - // --------------------------------------------------------------------------------- - template - class btree_container { - using params_type = typename Tree::params_type; - - protected: - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template - using key_arg = - typename KeyArg::value>:: - template type; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using difference_type = typename Tree::difference_type; - using key_compare = typename Tree::key_compare; - using value_compare = typename Tree::value_compare; - using allocator_type = typename Tree::allocator_type; - using reference = typename Tree::reference; - using const_reference = typename Tree::const_reference; - using pointer = typename Tree::pointer; - using const_pointer = typename Tree::const_pointer; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using reverse_iterator = typename Tree::reverse_iterator; - using const_reverse_iterator = typename Tree::const_reverse_iterator; - using node_type = typename Tree::node_handle_type; - - // Constructors/assignments. - btree_container() : tree_(key_compare(), allocator_type()) {} - explicit btree_container(const key_compare &comp, - const allocator_type &alloc = allocator_type()) - : tree_(comp, alloc) {} - btree_container(const btree_container &x) = default; - btree_container(btree_container &&x) noexcept = default; - btree_container &operator=(const btree_container &x) = default; - btree_container &operator=(btree_container &&x) noexcept( - std::is_nothrow_move_assignable::value) = default; - - // Iterator routines. - iterator begin() { return tree_.begin(); } - const_iterator begin() const { return tree_.begin(); } - const_iterator cbegin() const { return tree_.begin(); } - iterator end() { return tree_.end(); } - const_iterator end() const { return tree_.end(); } - const_iterator cend() const { return tree_.end(); } - reverse_iterator rbegin() { return tree_.rbegin(); } - const_reverse_iterator rbegin() const { return tree_.rbegin(); } - const_reverse_iterator crbegin() const { return tree_.rbegin(); } - reverse_iterator rend() { return tree_.rend(); } - const_reverse_iterator rend() const { return tree_.rend(); } - const_reverse_iterator crend() const { return tree_.rend(); } - - // Lookup routines. - template - iterator find(const key_arg &key) { - return tree_.find(key); - } - template - const_iterator find(const key_arg &key) const { return tree_.find(key); } - - template - bool contains(const key_arg &key) const { return find(key) != end(); } - - template - iterator lower_bound(const key_arg &key) { return tree_.lower_bound(key); } - - template - const_iterator lower_bound(const key_arg &key) const { return tree_.lower_bound(key); } - - template - iterator upper_bound(const key_arg &key) { return tree_.upper_bound(key); } - - template - const_iterator upper_bound(const key_arg &key) const { return tree_.upper_bound(key); } - - template - std::pair equal_range(const key_arg &key) { return tree_.equal_range(key); } - - template - std::pair equal_range( - const key_arg &key) const { - return tree_.equal_range(key); - } - - iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } - iterator erase(iterator iter) { return tree_.erase(iter); } - iterator erase(const_iterator first, const_iterator last) { - return tree_.erase(iterator(first), iterator(last)).second; - } - - node_type extract(iterator position) { - // Use Move instead of Transfer, because the rebalancing code expects to - // have a valid object to scribble metadata bits on top of. - auto node = CommonAccess::Move(get_allocator(), position.slot()); - erase(position); - return node; - } - - node_type extract(const_iterator position) { - return extract(iterator(position)); - } - - public: - void clear() { tree_.clear(); } - void swap(btree_container &x) { tree_.swap(x.tree_); } - void verify() const { tree_.verify(); } - - size_type size() const { return tree_.size(); } - size_type max_size() const { return tree_.max_size(); } - bool empty() const { return tree_.empty(); } - - friend bool operator==(const btree_container &x, const btree_container &y) { - if (x.size() != y.size()) return false; - return std::equal(x.begin(), x.end(), y.begin()); - } - - friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } - - friend bool operator<(const btree_container &x, const btree_container &y) { - return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); - } - - friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } - - friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } - - friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } - - // The allocator used by the btree. - allocator_type get_allocator() const { return tree_.get_allocator(); } - - // The key comparator used by the btree. - key_compare key_comp() const { return tree_.key_comp(); } - value_compare value_comp() const { return tree_.value_comp(); } - - // Support absl::Hash. - template - friend State AbslHashValue(State h, const btree_container &b) { - for (const auto &v : b) { - h = State::combine(std::move(h), v); - } - return State::combine(std::move(h), b.size()); - } - - protected: - Tree tree_; - }; - - // A common base class for btree_set and btree_map. - // ----------------------------------------------- - template - class btree_set_container : public btree_container { - using super_type = btree_container; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - friend class BtreeNodePeer; - - protected: - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - using insert_return_type = InsertReturnType; - using super_type::super_type; - btree_set_container() {} - - template - btree_set_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - btree_set_container(std::initializer_list init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_set_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template - size_type count(const key_arg &key) const { - return this->tree_.count_unique(key); - } - - // Insertion routines. - std::pair insert(const value_type &x) { - return this->tree_.insert_unique(params_type::key(x), x); - } - std::pair insert(value_type &&x) { - return this->tree_.insert_unique(params_type::key(x), std::move(x)); - } - template - std::pair emplace(Args &&... args) { - init_type v(std::forward(args)...); - return this->tree_.insert_unique(params_type::key(v), std::move(v)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), x) - .first; - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), - std::move(x)) - .first; - } - - template - iterator emplace_hint(const_iterator position, Args &&... args) { - init_type v(std::forward(args)...); - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(v), - std::move(v)) - .first; - } - - template - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_unique(b, e); - } - - void insert(std::initializer_list init) { - this->tree_.insert_iterator_unique(init.begin(), init.end()); - } - - insert_return_type insert(node_type &&node) { - if (!node) return {this->end(), false, node_type()}; - std::pair res = - this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) { - CommonAccess::Destroy(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - std::pair res = this->tree_.insert_hint_unique( - iterator(hint), params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) CommonAccess::Destroy(&node); - return res.first; - } - - template - size_type erase(const key_arg &key) { return this->tree_.erase_unique(key); } - using super_type::erase; - - template - node_type extract(const key_arg &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - - using super_type::extract; - - // Merge routines. - // Moves elements from `src` into `this`. If the element already exists in - // `this`, it is left unmodified in `src`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same, - std::is_same, - std::is_same>::value, - int> = 0> - void merge(btree_container &src) { // NOLINT - for (auto src_it = src.begin(); src_it != src.end();) { - if (insert(std::move(*src_it)).second) { - src_it = src.erase(src_it); - } else { - ++src_it; - } - } - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same, - std::is_same, - std::is_same>::value, - int> = 0> - void merge(btree_container &&src) { - merge(src); - } - }; - - // Base class for btree_map. - // ------------------------- - template - class btree_map_container : public btree_set_container { - using super_type = btree_set_container; - using params_type = typename Tree::params_type; - - protected: - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using mapped_type = typename params_type::mapped_type; - using value_type = typename Tree::value_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - - // Inherit constructors. - using super_type::super_type; - btree_map_container() {} - - // Insertion routines. - template - std::pair try_emplace(const key_type &k, Args &&... args) { - return this->tree_.insert_unique( - k, std::piecewise_construct, std::forward_as_tuple(k), - std::forward_as_tuple(std::forward(args)...)); - } - template - std::pair try_emplace(key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_unique guarantees that `key` is never - // referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_.insert_unique( - key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward(args)...)); - } - template - iterator try_emplace(const_iterator hint, const key_type &k, - Args &&... args) { - return this->tree_ - .insert_hint_unique(iterator(hint), k, std::piecewise_construct, - std::forward_as_tuple(k), - std::forward_as_tuple(std::forward(args)...)) - .first; - } - template - iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_hint_unique guarantees that `key` is - // never referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_ - .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, - std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward(args)...)) - .first; - } - mapped_type &operator[](const key_type &k) { - return try_emplace(k).first->second; - } - mapped_type &operator[](key_type &&k) { - return try_emplace(std::move(k)).first->second; - } - - template - mapped_type &at(const key_arg &key) { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - template - const mapped_type &at(const key_arg &key) const { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - }; - - // A common base class for btree_multiset and btree_multimap. - template - class btree_multiset_container : public btree_container { - using super_type = btree_container; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - - // Inherit constructors. - using super_type::super_type; - btree_multiset_container() {} - - // Range constructor. - template - btree_multiset_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - // Initializer list constructor. - btree_multiset_container(std::initializer_list init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template - size_type count(const key_arg &key) const { - return this->tree_.count_multi(key); - } - - // Insertion routines. - iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } - iterator insert(value_type &&x) { - return this->tree_.insert_multi(std::move(x)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_.insert_hint_multi(iterator(position), x); - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_.insert_hint_multi(iterator(position), std::move(x)); - } - template - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_multi(b, e); - } - void insert(std::initializer_list init) { - this->tree_.insert_iterator_multi(init.begin(), init.end()); - } - template - iterator emplace(Args &&... args) { - return this->tree_.insert_multi(init_type(std::forward(args)...)); - } - template - iterator emplace_hint(const_iterator position, Args &&... args) { - return this->tree_.insert_hint_multi( - iterator(position), init_type(std::forward(args)...)); - } - iterator insert(node_type &&node) { - if (!node) return this->end(); - iterator res = - this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - CommonAccess::Destroy(&node); - return res; - } - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - iterator res = this->tree_.insert_hint_multi( - iterator(hint), - std::move(params_type::element(CommonAccess::GetSlot(node)))); - CommonAccess::Destroy(&node); - return res; - } - - // Deletion routines. - template - size_type erase(const key_arg &key) { - return this->tree_.erase_multi(key); - } - using super_type::erase; - - // Node extraction routines. - template - node_type extract(const key_arg &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - using super_type::extract; - - // Merge routines. - // Moves all elements from `src` into `this`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same, - std::is_same, - std::is_same>::value, - int> = 0> - void merge(btree_container &src) { // NOLINT - insert(std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); - src.clear(); - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same, - std::is_same, - std::is_same>::value, - int> = 0> - void merge(btree_container &&src) { - merge(src); - } - }; - - // A base class for btree_multimap. - template - class btree_multimap_container : public btree_multiset_container { - using super_type = btree_multiset_container; - using params_type = typename Tree::params_type; - - public: - using mapped_type = typename params_type::mapped_type; - - // Inherit constructors. - using super_type::super_type; - btree_multimap_container() {} - }; - -} // namespace priv - - - - // ---------------------------------------------------------------------- - // btree_set - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_set : public priv::btree_set_container< - priv::btree>> - { - using Base = typename btree_set::btree_set_container; - - public: - btree_set() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_set` containers. - // ------------------------------------------------------- - template - void swap(btree_set &x, btree_set &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_set &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multiset - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_multiset : public priv::btree_multiset_container< - priv::btree>> - { - using Base = typename btree_multiset::btree_multiset_container; - - public: - btree_multiset() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multiset` containers. - // ------------------------------------------------------------ - template - void swap(btree_multiset &x, btree_multiset &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_multiset &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - - // ---------------------------------------------------------------------- - // btree_map - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_map : public priv::btree_map_container< - priv::btree>> - { - using Base = typename btree_map::btree_map_container; - - public: - btree_map() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_map` containers. - // ------------------------------------------------------- - template - void swap(btree_map &x, btree_map &y) { - return x.swap(y); - } - - // ---------------------------------------------------------------------- - template - void erase_if(btree_map &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multimap - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_multimap : public priv::btree_multimap_container< - priv::btree>> - { - using Base = typename btree_multimap::btree_multimap_container; - - public: - btree_multimap() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multimap` containers. - // ------------------------------------------------------------ - template - void swap(btree_multimap &x, btree_multimap &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_multimap &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - -} // namespace btree - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/benchmarks/external/parallel_hashmap/conanfile.py b/benchmarks/external/parallel_hashmap/conanfile.py deleted file mode 100644 index c046377d..00000000 --- a/benchmarks/external/parallel_hashmap/conanfile.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from conans import ConanFile, tools -import os - -class SparseppConan(ConanFile): - name = "parallel_hashmap" - version = "1.27" - description = "A header-only, very fast and memory-friendly hash map" - - # Indicates License type of the packaged library - license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" - - # Packages the license for the conanfile.py - exports = ["LICENSE"] - - # Custom attributes for Bincrafters recipe conventions - source_subfolder = "source_subfolder" - - def source(self): - source_url = "https://github.com/greg7mdp/parallel-hashmap" - tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) - extracted_dir = self.name + "-" + self.version - - #Rename to "source_folder" is a convention to simplify later steps - os.rename(extracted_dir, self.source_subfolder) - - - def package(self): - include_folder = os.path.join(self.source_subfolder, "parallel_hashmap") - self.copy(pattern="LICENSE") - self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder) - - def package_id(self): - self.info.header_only() diff --git a/benchmarks/external/parallel_hashmap/meminfo.h b/benchmarks/external/parallel_hashmap/meminfo.h deleted file mode 100644 index 872f3c69..00000000 --- a/benchmarks/external/parallel_hashmap/meminfo.h +++ /dev/null @@ -1,195 +0,0 @@ -#if !defined(spp_memory_h_guard) -#define spp_memory_h_guard - -#include -#include -#include - -#if defined(_WIN32) || defined( __CYGWIN__) - #define SPP_WIN -#endif - -#ifdef SPP_WIN - #include - #include - #undef min - #undef max -#elif defined(__linux__) - #include - #include -#elif defined(__FreeBSD__) - #include - #include - #include - #include - #include - #include -#endif - -namespace spp -{ - uint64_t GetSystemMemory(); - uint64_t GetTotalMemoryUsed(); - uint64_t GetProcessMemoryUsed(); - uint64_t GetPhysicalMemory(); - - uint64_t GetSystemMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo (&memInfo); - auto totalVirtualMem = memInfo.totalram; - - totalVirtualMem += memInfo.totalswap; - totalVirtualMem *= memInfo.mem_unit; - return static_cast(totalVirtualMem); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageCnt; - size_t pageCntLen = sizeof(pageCnt); - u_int pageSize; - struct kvm_swap kswap; - uint64_t totalVirtualMem; - - pageSize = static_cast(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - totalVirtualMem = pageCnt * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - totalVirtualMem += kswap.ksw_total * pageSize; - - return totalVirtualMem; -#else - return 0; -#endif - } - - uint64_t GetTotalMemoryUsed() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - auto virtualMemUsed = memInfo.totalram - memInfo.freeram; - - virtualMemUsed += memInfo.totalswap - memInfo.freeswap; - virtualMemUsed *= memInfo.mem_unit; - - return static_cast(virtualMemUsed); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageSize; - u_int pageCnt, freeCnt; - size_t pageCntLen = sizeof(pageCnt); - size_t freeCntLen = sizeof(freeCnt); - struct kvm_swap kswap; - uint64_t virtualMemUsed; - - pageSize = static_cast(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); - virtualMemUsed = (pageCnt - freeCnt) * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - virtualMemUsed += kswap.ksw_used * pageSize; - - return virtualMemUsed; -#else - return 0; -#endif - } - - uint64_t GetProcessMemoryUsed() - { -#ifdef SPP_WIN - PROCESS_MEMORY_COUNTERS_EX pmc; - GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast(&pmc), sizeof(pmc)); - return static_cast(pmc.PrivateUsage); -#elif defined(__linux__) - auto parseLine = - [](char* line)->int - { - auto i = strlen(line); - - while(*line < '0' || *line > '9') - { - line++; - } - - line[i-3] = '\0'; - i = atoi(line); - return i; - }; - - auto file = fopen("/proc/self/status", "r"); - auto result = -1; - char line[128]; - - while(fgets(line, 128, file) != nullptr) - { - if(strncmp(line, "VmSize:", 7) == 0) - { - result = parseLine(line); - break; - } - } - - fclose(file); - return static_cast(result) * 1024; -#elif defined(__FreeBSD__) - struct kinfo_proc info; - size_t infoLen = sizeof(info); - int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); - return static_cast(info.ki_rssize * getpagesize()); -#else - return 0; -#endif - } - - uint64_t GetPhysicalMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPhys); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - - auto totalPhysMem = memInfo.totalram; - - totalPhysMem *= memInfo.mem_unit; - return static_cast(totalPhysMem); -#elif defined(__FreeBSD__) - u_long physMem; - size_t physMemLen = sizeof(physMem); - int mib[] = { CTL_HW, HW_PHYSMEM }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); - return physMem; -#else - return 0; -#endif - } - -} - -#endif // spp_memory_h_guard diff --git a/benchmarks/external/parallel_hashmap/phmap.h b/benchmarks/external/parallel_hashmap/phmap.h deleted file mode 100644 index 653ae5ea..00000000 --- a/benchmarks/external/parallel_hashmap/phmap.h +++ /dev/null @@ -1,4788 +0,0 @@ -#if !defined(phmap_h_guard_) -#define phmap_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "phmap_fwd_decl.h" -#include "phmap_utils.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include -#endif - -namespace phmap { - -namespace priv { - -// -------------------------------------------------------------------------- -template -class probe_seq -{ -public: - probe_seq(size_t hashval, size_t mask) { - assert(((mask + 1) & mask) == 0 && "not a mask"); - mask_ = mask; - offset_ = hashval & mask_; - } - size_t offset() const { return offset_; } - size_t offset(size_t i) const { return (offset_ + i) & mask_; } - - void next() { - index_ += Width; - offset_ += index_; - offset_ &= mask_; - } - // 0-based probe index. The i-th probe in the probe sequence. - size_t getindex() const { return index_; } - -private: - size_t mask_; - size_t offset_; - size_t index_ = 0; -}; - -// -------------------------------------------------------------------------- -template -struct RequireUsableKey -{ - template - std::pair< - decltype(std::declval()(std::declval())), - decltype(std::declval()(std::declval(), - std::declval()))>* - operator()(const PassedKey&, const Args&...) const; -}; - -// -------------------------------------------------------------------------- -template -struct IsDecomposable : std::false_type {}; - -template -struct IsDecomposable< - phmap::void_t(), - std::declval()...))>, - Policy, Hash, Eq, Ts...> : std::true_type {}; - -// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. -// -------------------------------------------------------------------------- -template -constexpr bool IsNoThrowSwappable() { - using std::swap; - return noexcept(swap(std::declval(), std::declval())); -} - -// -------------------------------------------------------------------------- -template -int TrailingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountTrailingZerosNonZero64(static_cast(x)); - else - return base_internal::CountTrailingZerosNonZero32(static_cast(x)); -} - -// -------------------------------------------------------------------------- -template -int LeadingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountLeadingZeros64(static_cast(x)); - else - return base_internal::CountLeadingZeros32(static_cast(x)); -} - -// -------------------------------------------------------------------------- -// An abstraction over a bitmask. It provides an easy way to iterate through the -// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), -// this is a true bitmask. On non-SSE, platforms the arithematic used to -// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as -// either 0x00 or 0x80. -// -// For example: -// for (int i : BitMask(0x5)) -> yields 0, 2 -// for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 -// -------------------------------------------------------------------------- -template -class BitMask -{ - static_assert(std::is_unsigned::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - -public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } - int LowestBitSet() const { - return priv::TrailingZeros(mask_) >> Shift; - } - int HighestBitSet() const { - return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - - 1) >> - Shift; - } - - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - - int TrailingZeros() const { - return priv::TrailingZeros(mask_) >> Shift; - } - - int LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return priv::LeadingZeros(mask_ << extra_bits) >> Shift; - } - -private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } - - T mask_; -}; - -// -------------------------------------------------------------------------- -using ctrl_t = signed char; -using h2_t = uint8_t; - -// -------------------------------------------------------------------------- -// The values here are selected for maximum performance. See the static asserts -// below for details. -// -------------------------------------------------------------------------- -enum Ctrl : ctrl_t -{ - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 -}; - -static_assert( - kEmpty & kDeleted & kSentinel & 0x80, - "Special markers need to have the MSB to make checking for them efficient"); -static_assert(kEmpty < kSentinel && kDeleted < kSentinel, - "kEmpty and kDeleted must be smaller than kSentinel to make the " - "SIMD test of IsEmptyOrDeleted() efficient"); -static_assert(kSentinel == -1, - "kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(kEmpty == -128, - "kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); -static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, - "kEmpty and kDeleted must share an unset bit that is not shared " - "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " - "efficient"); -static_assert(kDeleted == -2, - "kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - -// -------------------------------------------------------------------------- -// A single block of empty control bytes for tables without any slots allocated. -// This enables removing a branch in the hot path of find(). -// -------------------------------------------------------------------------- -inline ctrl_t* EmptyGroup() { - alignas(16) static constexpr ctrl_t empty_group[] = { - kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, - kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; - return const_cast(empty_group); -} - -// -------------------------------------------------------------------------- -inline size_t HashSeed(const ctrl_t* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return reinterpret_cast(ctrl) >> 12; -} - -#ifdef PHMAP_NON_DETERMINISTIC - -inline size_t H1(size_t hashval, const ctrl_t* ctrl) { - // use ctrl_ pointer to add entropy to ensure - // non-deterministic iteration order. - return (hashval >> 7) ^ HashSeed(ctrl); -} - -#else - -inline size_t H1(size_t hashval, const ctrl_t* ) { - return (hashval >> 7); -} - -#endif - - -inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } - -inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= 0; } -inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } -inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } - -#if PHMAP_HAVE_SSE2 - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch -#endif - -// -------------------------------------------------------------------------- -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -// -------------------------------------------------------------------------- -inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { -#if defined(__GNUC__) && !defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Woverflow" - - if (std::is_unsigned::value) { - const __m128i mask = _mm_set1_epi8(static_cast(0x80)); - const __m128i diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } - - #pragma GCC diagnostic pop -#endif - return _mm_cmpgt_epi8(a, b); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupSse2Impl -{ - enum { kWidth = 16 }; // the number of slots per group - - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast(pos)); - } - - // Returns a bitmask representing the positions of slots that match hash. - // ---------------------------------------------------------------------- - BitMask Match(h2_t hash) const { - auto match = _mm_set1_epi8((char)hash); - return BitMask( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); - } - - // Returns a bitmask representing the positions of empty slots. - // ------------------------------------------------------------ - BitMask MatchEmpty() const { -#if PHMAP_HAVE_SSSE3 - // This only works because kEmpty is -128. - return BitMask( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); -#else - return Match(static_cast(kEmpty)); -#endif - } - - // Returns a bitmask representing the positions of empty or deleted slots. - // ----------------------------------------------------------------------- - BitMask MatchEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return BitMask( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); - } - - // Returns the number of trailing empty or deleted elements in the group. - // ---------------------------------------------------------------------- - uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return TrailingZeros( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); - } - - // ---------------------------------------------------------------------- - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast(-128)); - auto x126 = _mm_set1_epi8(126); -#if PHMAP_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); -#else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); -#endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } - - __m128i ctrl; -}; - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // PHMAP_HAVE_SSE2 - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupPortableImpl -{ - enum { kWidth = 8 }; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on kEmpty, kDeleted, kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask((x - lsbs) & ~x & msbs); - } - - BitMask MatchEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask((ctrl & (~ctrl << 6)) & msbs); - } - - BitMask MatchEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask((ctrl & (~ctrl << 7)) & msbs); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } - - uint64_t ctrl; -}; - -#if PHMAP_HAVE_SSE2 - using Group = GroupSse2Impl; -#else - using Group = GroupPortableImpl; -#endif - -template -class raw_hash_set; - -inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } - -// -------------------------------------------------------------------------- -// PRECONDITION: -// IsValidCapacity(capacity) -// ctrl[capacity] == kSentinel -// ctrl[i] != kSentinel for all i < capacity -// Applies mapping for every byte in ctrl: -// DELETED -> EMPTY -// EMPTY -> EMPTY -// FULL -> DELETED -// -------------------------------------------------------------------------- -inline void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) -{ - assert(ctrl[capacity] == kSentinel); - assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { - Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); - } - // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; -} - -// -------------------------------------------------------------------------- -// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. -// -------------------------------------------------------------------------- -inline size_t NormalizeCapacity(size_t n) -{ - return n ? ~size_t{} >> LeadingZeros(n) : 1; -} - -// -------------------------------------------------------------------------- -// We use 7/8th as maximum load factor. -// For 16-wide groups, that gives an average of two empty slots per group. -// -------------------------------------------------------------------------- -inline size_t CapacityToGrowth(size_t capacity) -{ - assert(IsValidCapacity(capacity)); - // `capacity*7/8` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (capacity == 7) - { - // x-x/8 does not work when x==7. - return 6; - } - } - return capacity - capacity / 8; -} - -// -------------------------------------------------------------------------- -// From desired "growth" to a lowerbound of the necessary capacity. -// Might not be a valid one and required NormalizeCapacity(). -// -------------------------------------------------------------------------- -inline size_t GrowthToLowerboundCapacity(size_t growth) -{ - // `growth*8/7` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (growth == 7) - { - // x+(x-1)/7 does not work when x==7. - return 8; - } - } - return growth + static_cast((static_cast(growth) - 1) / 7); -} - -namespace hashtable_debug_internal { - -// If it is a map, call get<0>(). -using std::get; -template -auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { - return get<0>(pair); -} - -// If it is not a map, return the value directly. -template -const typename T::key_type& GetKey(const typename T::key_type& key, char) { - return key; -} - -// -------------------------------------------------------------------------- -// Containers should specialize this to provide debug information for that -// container. -// -------------------------------------------------------------------------- -template -struct HashtableDebugAccess -{ - // Returns the number of probes required to find `key` in `c`. The "number of - // probes" is a concept that can vary by container. Implementations should - // return 0 when `key` was found in the minimum number of operations and - // should increment the result for each non-trivial operation required to find - // `key`. - // - // The default implementation uses the bucket api from the standard and thus - // works for `std::unordered_*` containers. - // -------------------------------------------------------------------------- - static size_t GetNumProbes(const Container& c, - const typename Container::key_type& key) { - if (!c.bucket_count()) return {}; - size_t num_probes = 0; - size_t bucket = c.bucket(key); - for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { - if (it == e) return num_probes; - if (c.key_eq()(key, GetKey(*it, 0))) return num_probes; - } - } -}; - -} // namespace hashtable_debug_internal - -// ---------------------------------------------------------------------------- -// I N F O Z S T U B S -// ---------------------------------------------------------------------------- -struct HashtablezInfo -{ - void PrepareForSampling() {} -}; - -inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} - -static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} - -static inline void RecordEraseSlow(HashtablezInfo*) {} - -static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } -static inline void UnsampleSlow(HashtablezInfo* ) {} - -class HashtablezInfoHandle -{ -public: - inline void RecordStorageChanged(size_t , size_t ) {} - inline void RecordRehash(size_t ) {} - inline void RecordInsert(size_t , size_t ) {} - inline void RecordErase() {} - friend inline void swap(HashtablezInfoHandle& , - HashtablezInfoHandle& ) noexcept {} -}; - -static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } - -class HashtablezSampler -{ -public: - // Returns a global Sampler. - static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } - HashtablezInfo* Register() { static HashtablezInfo info; return &info; } - void Unregister(HashtablezInfo* ) {} - - using DisposeCallback = void (*)(const HashtablezInfo&); - DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } - int64_t Iterate(const std::function& ) { return 0; } -}; - -static inline void SetHashtablezEnabled(bool ) {} -static inline void SetHashtablezSampleParameter(int32_t ) {} -static inline void SetHashtablezMaxSamples(int32_t ) {} - - -namespace memory_internal { - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template -void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, - phmap::index_sequence) { - phmap::allocator_traits::construct( - *alloc, ptr, std::get(std::forward(t))...); -} - -template -struct WithConstructedImplF { - template - decltype(std::declval()(std::declval())) operator()( - Args&&... args) const { - return std::forward(f)(T(std::forward(args)...)); - } - F&& f; -}; - -template -decltype(std::declval()(std::declval())) WithConstructedImpl( - Tuple&& t, phmap::index_sequence, F&& f) { - return WithConstructedImplF{std::forward(f)}( - std::get(std::forward(t))...); -} - -template -auto TupleRefImpl(T&& t, phmap::index_sequence) - -> decltype(std::forward_as_tuple(std::get(std::forward(t))...)) { - return std::forward_as_tuple(std::get(std::forward(t))...); -} - -// Returns a tuple of references to the elements of the input tuple. T must be a -// tuple. -// ---------------------------------------------------------------------------- -template -auto TupleRef(T&& t) -> decltype( - TupleRefImpl(std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>())) { - return TupleRefImpl( - std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>()); -} - -template -decltype(std::declval()(std::declval(), std::piecewise_construct, - std::declval>(), std::declval())) -DecomposePairImpl(F&& f, std::pair, V> p) { - const auto& key = std::get<0>(p.first); - return std::forward(f)(key, std::piecewise_construct, std::move(p.first), - std::move(p.second)); -} - -} // namespace memory_internal - - -// ---------------------------------------------------------------------------- -// R A W _ H A S H _ S E T -// ---------------------------------------------------------------------------- -// An open-addressing -// hashtable with quadratic probing. -// -// This is a low level hashtable on top of which different interfaces can be -// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. -// -// The table interface is similar to that of std::unordered_set. Notable -// differences are that most member functions support heterogeneous keys when -// BOTH the hash and eq functions are marked as transparent. They do so by -// providing a typedef called `is_transparent`. -// -// When heterogeneous lookup is enabled, functions that take key_type act as if -// they have an overload set like: -// -// iterator find(const key_type& key); -// template -// iterator find(const K& key); -// -// size_type erase(const key_type& key); -// template -// size_type erase(const K& key); -// -// std::pair equal_range(const key_type& key); -// template -// std::pair equal_range(const K& key); -// -// When heterogeneous lookup is disabled, only the explicit `key_type` overloads -// exist. -// -// find() also supports passing the hash explicitly: -// -// iterator find(const key_type& key, size_t hash); -// template -// iterator find(const U& key, size_t hash); -// -// In addition the pointer to element and iterator stability guarantees are -// weaker: all iterators and pointers are invalidated after a new element is -// inserted. -// -// IMPLEMENTATION DETAILS -// -// The table stores elements inline in a slot array. In addition to the slot -// array the table maintains some control state per slot. The extra state is one -// byte per slot and stores empty or deleted marks, or alternatively 7 bits from -// the hash of an occupied slot. The table is split into logical groups of -// slots, like so: -// -// Group 1 Group 2 Group 3 -// +---------------+---------------+---------------+ -// | | | | | | | | | | | | | | | | | | | | | | | | | -// +---------------+---------------+---------------+ -// -// On lookup the hash is split into two parts: -// - H2: 7 bits (those stored in the control bytes) -// - H1: the rest of the bits -// The groups are probed using H1. For each group the slots are matched to H2 in -// parallel. Because H2 is 7 bits (128 states) and the number of slots per group -// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. -// -// On insert, once the right group is found (as in lookup), its slots are -// filled in order. -// -// On erase a slot is cleared. In case the group did not have any empty slots -// before the erase, the erased slot is marked as deleted. -// -// Groups without empty slots (but maybe with deleted slots) extend the probe -// sequence. The probing algorithm is quadratic. Given N the number of groups, -// the probing function for the i'th probe is: -// -// P(0) = H1 % N -// -// P(i) = (P(i - 1) + i) % N -// -// This probing function guarantees that after N probes, all the groups of the -// table will be probed exactly once. -// ---------------------------------------------------------------------------- -template -class raw_hash_set -{ - using PolicyTraits = hash_policy_traits; - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - -public: - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template - using key_arg = typename KeyArgImpl::template type; - -private: - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using Layout = phmap::priv::Layout; - - static Layout MakeLayout(size_t capacity) { - assert(IsValidCapacity(capacity)); - return Layout(capacity + Group::kWidth + 1, capacity); - } - - using AllocTraits = phmap::allocator_traits; - using SlotAlloc = typename phmap::allocator_traits< - allocator_type>::template rebind_alloc; - using SlotAllocTraits = typename phmap::allocator_traits< - allocator_type>::template rebind_traits; - - static_assert(std::is_lvalue_reference::value, - "Policy::element() must return a reference"); - - template - struct SameAsElementReference - : std::is_same::type>::type, - typename std::remove_cv< - typename std::remove_reference::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - template - using RequiresInsertable = typename std::enable_if< - phmap::disjunction, - SameAsElementReference>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template - using RequiresNotInit = - typename std::enable_if::value, int>::type; - - template - using IsDecomposable = IsDecomposable; - -public: - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - - class iterator - { - friend class raw_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename raw_hash_set::value_type; - using reference = - phmap::conditional_t; - using pointer = phmap::remove_reference_t*; - using difference_type = typename raw_hash_set::difference_type; - - iterator() {} - - // PRECONDITION: not an end() iterator. - reference operator*() const { return PolicyTraits::element(slot_); } - - // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } - - // PRECONDITION: not an end() iterator. - iterator& operator++() { - ++ctrl_; - ++slot_; - skip_empty_or_deleted(); - return *this; - } - // PRECONDITION: not an end() iterator. - iterator operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } - -#if PHMAP_BIDIRECTIONAL - // PRECONDITION: not a begin() iterator. - iterator& operator--() { - assert(ctrl_); - do { - --ctrl_; - --slot_; - } while (IsEmptyOrDeleted(*ctrl_)); - return *this; - } - - // PRECONDITION: not a begin() iterator. - iterator operator--(int) { - auto tmp = *this; - --*this; - return tmp; - } -#endif - - friend bool operator==(const iterator& a, const iterator& b) { - return a.ctrl_ == b.ctrl_; - } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() - iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} - - void skip_empty_or_deleted() { - while (IsEmptyOrDeleted(*ctrl_)) { - // ctrl is not necessarily aligned to Group::kWidth. It is also likely - // to read past the space for ctrl bytes and into slots. This is ok - // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there - // is no way to read outside the combined slot array. - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); - ctrl_ += shift; - slot_ += shift; - } - } - - ctrl_t* ctrl_ = nullptr; - // To avoid uninitialized member warnings, put slot_ in an anonymous union. - // The member is not initialized on singleton and end iterators. - union { - slot_type* slot_; - }; - }; - - class const_iterator - { - friend class raw_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename raw_hash_set::value_type; - using reference = typename raw_hash_set::const_reference; - using pointer = typename raw_hash_set::const_pointer; - using difference_type = typename raw_hash_set::difference_type; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} - - reference operator*() const { return *inner_; } - pointer operator->() const { return inner_.operator->(); } - - const_iterator& operator++() { - ++inner_; - return *this; - } - const_iterator operator++(int) { return inner_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.inner_ == b.inner_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast(ctrl), const_cast(slot)) {} - - iterator inner_; - }; - - using node_type = node_handle, Alloc>; - using insert_return_type = InsertReturnType; - - raw_hash_set() noexcept( - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value) {} - - explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { - if (bucket_cnt) { - capacity_ = NormalizeCapacity(bucket_cnt); - reset_growth_left(); - initialize_slots(); - } - } - - raw_hash_set(size_t bucket_cnt, const hasher& hashfn, - const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit raw_hash_set(const allocator_type& alloc) - : raw_hash_set(0, hasher(), key_equal(), alloc) {} - - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { - insert(first, last); - } - - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} - - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template - raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list as the first - // argument like std::unordered_set does, we have two overloads - // that accept std::initializer_list and std::initializer_list. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // std::unordered_set s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // phmap::flat_hash_set s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set a, b{a}; - // - // RequiresNotInit is a workaround for gcc prior to 7.1. - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, - const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(const raw_hash_set& that) - : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - reserve(that.size()); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(hashval); - set_ctrl(target.offset, H2(hashval)); - emplace_at(target.offset, v); - infoz_.RecordInsert(hashval, target.probe_length); - } - size_ = that.size(); - growth_left() -= that.size(); - } - - raw_hash_set(raw_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value) - : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), - slots_(phmap::exchange(that.slots_, nullptr)), - size_(phmap::exchange(that.size_, 0)), - capacity_(phmap::exchange(that.capacity_, 0)), - infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function, moving it - // would create a nullptr functor that cannot be called. - settings_(that.settings_) { - // growth_left was copied above, reset the one from `that`. - that.growth_left() = 0; - } - - raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), - settings_(0, that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); - std::swap(infoz_, that.infoz_); - } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); - } - } - - raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; - } - - raw_hash_set& operator=(raw_hash_set&& that) noexcept( - phmap::allocator_traits::is_always_equal::value&& - std::is_nothrow_move_assignable::value&& - std::is_nothrow_move_assignable::value) { - // TODO(sbenza): We should only use the operations from the noexcept clause - // to make sure we actually adhere to that contract. - return move_assign( - std::move(that), - typename AllocTraits::propagate_on_container_move_assignment()); - } - - ~raw_hash_set() { destroy_slots(); } - - iterator begin() { - auto it = iterator_at(0); - it.skip_empty_or_deleted(); - return it; - } - iterator end() - { -#if PHMAP_BIDIRECTIONAL - return iterator_at(capacity_); -#else - return {ctrl_ + capacity_}; -#endif - } - - const_iterator begin() const { - return const_cast(this)->begin(); - } - const_iterator end() const { return const_cast(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } - size_t max_size() const { return (std::numeric_limits::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (empty()) - return; - if (capacity_ > 127) { - destroy_slots(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - size_ = 0; - reset_ctrl(); - reset_growth_left(); - } - assert(empty()); - infoz_.RecordStorageChanged(0, capacity_); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map m; - // m.insert(std::make_pair("abc", 42)); - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - std::pair insert(T&& value) { - return emplace(std::forward(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set s; - // s.insert(n); - // - // flat_hash_set s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template = 0, - typename std::enable_if::value, int>::type = 0> - std::pair insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set s; - // s.insert({"abc", 42}); - std::pair insert(init_type&& value) { - return emplace(std::move(value)); - } - - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template = 0, - typename std::enable_if::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template - using IsRandomAccess = std::is_same::iterator_category, - std::random_access_iterator_tag>; - - - template - struct has_difference_operator - { - private: - using yes = std::true_type; - using no = std::false_type; - - template static auto test(int) -> decltype(std::declval() - std::declval() == 1, yes()); - template static no test(...); - - public: - static constexpr bool value = std::is_same(0)), yes>::value; - }; - - template ::value, int> = 0> - void insert(InputIt first, InputIt last) { - this->reserve(this->size() + (last - first)); - for (; first != last; ++first) - emplace(*first); - } - - template ::value, int> = 0> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) - emplace(*first); - } - - template = 0, RequiresInsertable = 0> - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlot{*this, std::move(*CommonAccess::GetSlot(node))}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - insert_return_type insert(node_type&& node, size_t hashval) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlotWithHash{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward(args)...); - } - - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - typename std::aligned_storage::type - raw; - slot_type* slot = reinterpret_cast(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlot{*this, std::move(*slot)}, elem); - } - - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlotWithHash{*this, std::move(*slot), hashval}, elem); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward(args)...).first; - } - - template - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward(args)...).first; - } - - // Extension API: support for lazy emplace. - // - // Looks up key in the table. If found, returns the iterator to the element. - // Otherwise calls f with one argument of type raw_hash_set::constructor. f - // MUST call raw_hash_set::constructor with arguments as if a - // raw_hash_set::value_type is constructed, otherwise the behavior is - // undefined. - // - // For example: - // - // std::unordered_set s; - // // Makes ArenaStr even if "abc" is in the map. - // s.insert(ArenaString(&arena, "abc")); - // - // flat_hash_set s; - // // Makes ArenaStr only if "abc" is not in the map. - // s.lazy_emplace("abc", [&](const constructor& ctor) { - // ctor(&arena, "abc"); - // }); - // - // WARNING: This API is currently experimental. If there is a way to implement - // the same thing with the rest of the API, prefer that. - class constructor - { - friend class raw_hash_set; - - public: - template - void operator()(Args&&... args) const { - assert(*slot_); - PolicyTraits::construct(alloc_, *slot_, std::forward(args)...); - *slot_ = nullptr; - } - - private: - constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - - allocator_type* alloc_; - slot_type** slot_; - }; - - template - iterator lazy_emplace(const key_arg& key, F&& f) { - auto res = find_or_prepare_insert(key); - if (res.second) { - lazy_emplace_at(res.first, std::forward(f)); - } - return iterator_at(res.first); - } - - template - iterator lazy_emplace_with_hash(const key_arg& key, size_t &hashval, F&& f) { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - lazy_emplace_at(res.first, std::forward(f)); - } - return iterator_at(res.first); - } - - template - void lazy_emplace_at(size_t& idx, F&& f) { - slot_type* slot = slots_ + idx; - std::forward(f)(constructor(&alloc_ref(), &slot)); - assert(!slot); - } - - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - template - size_type erase(const key_arg& key) { - auto it = find(key); - if (it == end()) return 0; - _erase(it); - return 1; - } - - - iterator erase(const_iterator cit) { return erase(cit.inner_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if () { - // m._erase(it++); - // } else { - // ++it; - // } - // } - void _erase(iterator it) { - assert(it != end()); - PolicyTraits::destroy(&alloc_ref(), it.slot_); - erase_meta_only(it); - } - void _erase(const_iterator cit) { _erase(cit.inner_); } - - // This overload is necessary because otherwise erase(const K&) would be - // a better match if non-const iterator is passed as an argument. - iterator erase(iterator it) { - auto res = it; - ++res; - _erase(it); - return res; - } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.inner_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - template - void merge(raw_hash_set& src) { // NOLINT - assert(this != &src); - for (auto it = src.begin(), e = src.end(); it != e; ++it) { - if (PolicyTraits::apply(InsertSlot{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) - .second) { - src.erase_meta_only(it); - } - } - } - - template - void merge(raw_hash_set&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - auto node = - CommonAccess::Make(alloc_ref(), position.inner_.slot_); - erase_meta_only(position); - return node; - } - - template < - class K = key_type, - typename std::enable_if::value, int>::type = 0> - node_type extract(const key_arg& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); - swap(hash_ref(), that.hash_ref()); - swap(eq_ref(), that.eq_ref()); - swap(infoz_, that.infoz_); - if (AllocTraits::propagate_on_container_swap::value) { - swap(alloc_ref(), that.alloc_ref()); - } else { - // If the allocators do not compare equal it is officially undefined - // behavior. We choose to do nothing. - } - } - -#ifndef PHMAP_NON_DETERMINISTIC - template - bool dump(OutputArchive&) const; - - template - bool load(InputArchive&); -#endif - - void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); - infoz_.RecordStorageChanged(0, 0); - return; - } - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity((std::max)(n, size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { - resize(m); - } - } - - void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - template - size_t count(const key_arg& key) const { - return find(key) == end() ? size_t(0) : size_t(1); - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - void prefetch_hash(size_t hashval) const { - (void)hashval; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - auto seq = probe(hashval); - _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); - _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); -#elif defined(__GNUC__) - auto seq = probe(hashval); - __builtin_prefetch(static_cast(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast(slots_ + seq.offset())); -#endif // __GNUC__ - } - - template - void prefetch(const key_arg& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - template - iterator find(const key_arg& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return iterator_at(offset); - else - return end(); - } - - template - pointer find_ptr(const key_arg& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return &PolicyTraits::element(slots_ + offset); - else - return nullptr; - } - - template - iterator find(const key_arg& key) { - return find(key, this->hash(key)); - } - - template - const_iterator find(const key_arg& key, size_t hashval) const { - return const_cast(this)->find(key, hashval); - } - template - const_iterator find(const key_arg& key) const { - return find(key, this->hash(key)); - } - - template - bool contains(const key_arg& key) const { - return find(key) != end(); - } - - template - bool contains(const key_arg& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template - std::pair equal_range(const key_arg& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - template - std::pair equal_range( - const key_arg& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { return capacity_; } - float load_factor() const { - return capacity_ ? static_cast(size()) / capacity_ : 0.0; - } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { - if (a.size() != b.size()) return false; - const raw_hash_set* outer = &a; - const raw_hash_set* inner = &b; - if (outer->capacity() > inner->capacity()) - std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; - return true; - } - - friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { - return !(a == b); - } - - friend void swap(raw_hash_set& a, - raw_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -private: - template - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - template - bool find_impl(const key_arg& key, size_t hashval, size_t& offset) { - auto seq = probe(hashval); - while (true) { - Group g{ ctrl_ + seq.offset() }; - for (int i : g.Match((h2_t)H2(hashval))) { - offset = seq.offset((size_t)i); - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slots_ + offset)))) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) - return false; - seq.next(); - } - } - - struct FindElement - { - template - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const raw_hash_set& s; - }; - - struct HashElement - { - template - size_t operator()(const K& key, Args&&...) const { - return phmap_mix()(h(key)); - } - const hasher& h; - }; - - template - struct EqualElement - { - template - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - template - std::pair emplace_decomposable(const K& key, size_t hashval, - Args&&... args) - { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - emplace_at(res.first, std::forward(args)...); - } - return {iterator_at(res.first), res.second}; - } - - struct EmplaceDecomposable - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, s.hash(key), std::forward(args)...); - } - raw_hash_set& s; - }; - - struct EmplaceDecomposableHashval { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, hashval, std::forward(args)...); - } - raw_hash_set& s; - size_t hashval; - }; - - template - struct InsertSlot - { - template - std::pair operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - }; - - template - struct InsertSlotWithHash - { - template - std::pair operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key, hashval); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - size_t &hashval; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - - set_ctrl(index, was_never_full ? kEmpty : kDeleted); - growth_left() += was_never_full; - infoz_.RecordErase(); - } - - void initialize_slots() { - assert(capacity_); - if (std::is_same>::value && - slots_ == nullptr) { - infoz_ = Sample(); - } - - auto layout = MakeLayout(capacity_); - char* mem = static_cast( - Allocate(&alloc_ref(), layout.AllocSize())); - ctrl_ = reinterpret_cast(layout.template Pointer<0>(mem)); - slots_ = layout.template Pointer<1>(mem); - reset_ctrl(); - reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); - } - - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - auto layout = MakeLayout(capacity_); - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate(&alloc_ref(), ctrl_, layout.AllocSize()); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; - } - - void resize(size_t new_capacity) { - assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - capacity_ = new_capacity; - initialize_slots(); - - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); - } - } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); - auto layout = MakeLayout(old_capacity); - Deallocate(&alloc_ref(), old_ctrl, - layout.AllocSize()); - } - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); - assert(!is_small()); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); - typename std::aligned_storage::type - raw; - slot_type* slot = reinterpret_cast(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - - // Verify if the old and new i fall within the same group wrt the hashval. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const auto probe_index = [&](size_t pos) { - return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - set_ctrl(i, H2(hashval)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. - // set_ctrl poisons/unpoisons the slots so we have to call it at the - // right time. - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - set_ctrl(i, kEmpty); - } else { - assert(IsDeleted(ctrl_[new_i])); - set_ctrl(new_i, H2(hashval)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(); - } - - void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); - } else if (size() <= CapacityToGrowth(capacity()) / 2) { - // Squash DELETED without growing if there is enough capacity. - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(capacity_ * 2 + 1); - } - } - - bool has_element(const value_type& elem, size_t hashval) const { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == - elem)) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; - seq.next(); - assert(seq.getindex() < capacity_ && "full table!"); - } - return false; - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - return has_element(elem, hashval); - } - - // Probes the raw_hash_set with the probe sequence for hash and returns the - // pointer to the first empty or deleted slot. - // NOTE: this function must work with tables having both kEmpty and kDelete - // in one group. Such tables appears during drop_deletes_without_resize. - // - // This function is very useful when insertions happen and: - // - the input is already a set - // - there are enough slots - // - the element with the hash is not in the table - struct FindInfo - { - size_t offset; - size_t probe_length; - }; - FindInfo find_first_non_full(size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); - if (mask) { - return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; - } - assert(seq.getindex() < capacity_ && "full table!"); - seq.next(); - } - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template - std::pair find_or_prepare_insert(const K& key, size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset((size_t)i))))) - return {seq.offset((size_t)i), false}; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; - seq.next(); - } - return {prepare_insert(hashval), true}; - } - - template - std::pair find_or_prepare_insert(const K& key) { - return find_or_prepare_insert(key, this->hash(key)); - } - - size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(hashval); - if (PHMAP_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { - rehash_and_grow_if_necessary(); - target = find_first_non_full(hashval); - } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); - set_ctrl(target.offset, H2(hashval)); - infoz_.RecordInsert(hashval, target.probe_length); - return target.offset; - } - - // Constructs the value in the space pointed by the iterator. This only works - // after an unsuccessful find_or_prepare_insert() and before any other - // modifications happen in the raw_hash_set. - // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. - // POSTCONDITION: *m.iterator_at(i) == value_type(forward(args)...). - template - void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, - std::forward(args)...); - - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && - "constructed value does not match the lookup key"); - } - - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } - -private: - friend struct RawHashSetTestOnlyAccess; - - probe_seq probe(size_t hashval) const { - return probe_seq(H1(hashval, ctrl_), capacity_); - } - - // Reset all ctrl bytes back to kEmpty, except the sentinel. - void reset_ctrl() { - std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); - ctrl_[capacity_] = kSentinel; - SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - } - - void reset_growth_left() { - growth_left() = CapacityToGrowth(capacity()) - size_; - } - - // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at - // the end too. - void set_ctrl(size_t i, ctrl_t h) { - assert(i < capacity_); - - if (IsFull(h)) { - SanitizerUnpoisonObject(slots_ + i); - } else { - SanitizerPoisonObject(slots_ + i); - } - - ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + 1 + - ((Group::kWidth - 1) & capacity_)] = h; - } - - size_t& growth_left() { return settings_.template get<0>(); } - - template class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_set; - - template class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_map; - - // The representation of the object has two modes: - // - small: For capacities < kWidth-1 - // - large: For the rest. - // - // Differences: - // - In small mode we are able to use the whole capacity. The extra control - // bytes give us at least one "empty" control byte to stop the iteration. - // This is important to make 1 a valid capacity. - // - // - In small mode only the first `capacity()` control bytes after the - // sentinel are valid. The rest contain dummy kEmpty values that do not - // represent a real slot. This is important to take into account on - // find_first_non_full(), where we never try ShouldInsertBackwards() for - // small tables. - bool is_small() const { return capacity_ < Group::kWidth - 1; } - - hasher& hash_ref() { return settings_.template get<1>(); } - const hasher& hash_ref() const { return settings_.template get<1>(); } - key_equal& eq_ref() { return settings_.template get<2>(); } - const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } - const allocator_type& alloc_ref() const { - return settings_.template get<3>(); - } - - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array - ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] - slot_type* slots_ = nullptr; // [capacity * slot_type] - size_t size_ = 0; // number of full slots - size_t capacity_ = 0; // total number of slots - HashtablezInfoHandle infoz_; - phmap::priv::CompressedTuple - settings_{0, hasher{}, key_equal{}, allocator_type{}}; -}; - - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template -class raw_hash_map : public raw_hash_set -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map. - // MappedReference<> may be a non-reference type. - template - using MappedReference = decltype(P::value( - std::addressof(std::declval()))); - - // MappedConstReference<> may be a non-reference type. - template - using MappedConstReference = decltype(P::value( - std::addressof(std::declval()))); - - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - using Base = raw_hash_set; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template - using key_arg = typename KeyArgImpl::template type; - - static_assert(!std::is_reference::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference::value, ""); - - using iterator = typename raw_hash_map::raw_hash_set::iterator; - using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; - - raw_hash_map() {} - using Base::raw_hash_set; // use raw_hash_set constructor - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map m; - // m.insert_or_assign(n, n); - template - std::pair insert_or_assign(key_arg&& k, V&& v) { - return insert_or_assign_impl(std::forward(k), std::forward(v)); - } - - template - std::pair insert_or_assign(key_arg&& k, const V& v) { - return insert_or_assign_impl(std::forward(k), v); - } - - template - std::pair insert_or_assign(const key_arg& k, V&& v) { - return insert_or_assign_impl(k, std::forward(v)); - } - - template - std::pair insert_or_assign(const key_arg& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { - return insert_or_assign(std::forward(k), std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { - return insert_or_assign(std::forward(k), v).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { - return insert_or_assign(k, std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace(key_arg&& k, Args&&... args) { - return try_emplace_impl(std::forward(k), std::forward(args)...); - } - - template ::value, int>::type = 0> - std::pair try_emplace(const key_arg& k, Args&&... args) { - return try_emplace_impl(k, std::forward(args)...); - } - - template - iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { - return try_emplace(std::forward(k), std::forward(args)...).first; - } - - template - iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { - return try_emplace(k, std::forward(args)...).first; - } - - template - MappedReference

at(const key_arg& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template - MappedConstReference

at(const key_arg& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template - MappedReference

operator[](key_arg&& key) { - return Policy::value(&*try_emplace(std::forward(key)).first); - } - - template - MappedReference

operator[](const key_arg& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template - std::pair insert_or_assign_impl(K&& k, V&& v) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::forward(k), std::forward(v)); - else - Policy::value(&*this->iterator_at(res.first)) = std::forward(v); - return {this->iterator_at(res.first), res.second}; - } - - template - std::pair try_emplace_impl(K&& k, Args&&... args) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(res.first), res.second}; - } -}; - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -// Returns "random" seed. -inline size_t RandomSeed() -{ -#if PHMAP_HAVE_THREAD_LOCAL - static thread_local size_t counter = 0; - size_t value = ++counter; -#else // PHMAP_HAVE_THREAD_LOCAL - static std::atomic counter(0); - size_t value = counter.fetch_add(1, std::memory_order_relaxed); -#endif // PHMAP_HAVE_THREAD_LOCAL - return value ^ static_cast(reinterpret_cast(&counter)); -} - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_set -{ - using PolicyTraits = hash_policy_traits; - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - static_assert(N <= 12, "N = 12 means 4096 hash tables!"); - constexpr static size_t num_tables = 1 << N; - constexpr static size_t mask = num_tables - 1; - -public: - using EmbeddedSet = RefSet; - using EmbeddedIterator= typename EmbeddedSet::iterator; - using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; - using constructor = typename EmbeddedSet::constructor; - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - // -------------------------------------------------------------------- - template - using key_arg = typename KeyArgImpl::template type; - -protected: - using Lockable = phmap::LockableImpl; - - // -------------------------------------------------------------------- - struct Inner : public Lockable - { - bool operator==(const Inner& o) const - { - typename Lockable::SharedLocks l(const_cast(*this), const_cast(o)); - return set_ == o.set_; - } - - EmbeddedSet set_; - }; - -private: - // Give an early error when key_type is not hashable/eq. - // -------------------------------------------------------------------- - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using AllocTraits = phmap::allocator_traits; - - static_assert(std::is_lvalue_reference::value, - "Policy::element() must return a reference"); - - template - struct SameAsElementReference : std::is_same< - typename std::remove_cv::type>::type, - typename std::remove_cv::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - // -------------------------------------------------------------------- - template - using RequiresInsertable = typename std::enable_if< - phmap::disjunction, - SameAsElementReference>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template - using RequiresNotInit = - typename std::enable_if::value, int>::type; - - template - using IsDecomposable = IsDecomposable; - -public: - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - - // --------------------- i t e r a t o r ------------------------------ - class iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename parallel_hash_set::value_type; - using reference = - phmap::conditional_t; - using pointer = phmap::remove_reference_t*; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; - using EmbeddedIterator = typename EmbeddedSet::iterator; - - iterator() {} - - reference operator*() const { return *it_; } - pointer operator->() const { return &operator*(); } - - iterator& operator++() { - assert(inner_); // null inner means we are already at the end - ++it_; - skip_empty(); - return *this; - } - - iterator operator++(int) { - assert(inner_); // null inner means we are already at the end - auto tmp = *this; - ++*this; - return tmp; - } - - friend bool operator==(const iterator& a, const iterator& b) { - return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); - } - - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : - inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() - if (inner) - it_end_ = inner->set_.end(); - } - - void skip_empty() { - while (it_ == it_end_) { - ++inner_; - if (inner_ == inner_end_) { - inner_ = nullptr; // marks end() - break; - } - else { - it_ = inner_->set_.begin(); - it_end_ = inner_->set_.end(); - } - } - } - - Inner *inner_ = nullptr; - Inner *inner_end_ = nullptr; - EmbeddedIterator it_, it_end_; - }; - - // --------------------- c o n s t i t e r a t o r ----------------- - class const_iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename parallel_hash_set::value_type; - using reference = typename parallel_hash_set::const_reference; - using pointer = typename parallel_hash_set::const_pointer; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : iter_(std::move(i)) {} - - reference operator*() const { return *(iter_); } - pointer operator->() const { return iter_.operator->(); } - - const_iterator& operator++() { - ++iter_; - return *this; - } - const_iterator operator++(int) { return iter_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.iter_ == b.iter_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) - : iter_(const_cast(inner), - const_cast(inner_end), - const_cast(it)) {} - - iterator iter_; - }; - - using node_type = node_handle, Alloc>; - using insert_return_type = InsertReturnType; - - // ------------------------- c o n s t r u c t o r s ------------------ - - parallel_hash_set() noexcept( - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value) {} - - explicit parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) { - for (auto& inner : sets_) - inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); - } - - parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param, - const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit parallel_hash_set(const allocator_type& alloc) - : parallel_hash_set(0, hasher(), key_equal(), alloc) {} - - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { - insert(first, last); - } - - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} - - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template - parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list as the first - // argument like std::unordered_set does, we have two overloads - // that accept std::initializer_list and std::initializer_list. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list, then copies - // // the strings into the set. - // std::unordered_set s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // phmap::flat_hash_set s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set a, b{a}; - // - // RequiresNotInit is a workaround for gcc prior to 7.1. - // -------------------------------------------------------------------- - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, - const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(const parallel_hash_set& that) - : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) - : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - for (size_t i=0; i::value&& - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value) - : parallel_hash_set(std::move(that), that.alloc_ref()) { - } - - parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) - { - for (size_t i=0; i::is_always_equal::value && - std::is_nothrow_move_assignable::value && - std::is_nothrow_move_assignable::value) { - for (size_t i=0; i(this)->begin(); } - const_iterator end() const { return const_cast(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - - size_t size() const { - size_t sz = 0; - for (const auto& inner : sets_) - sz += inner.set_.size(); - return sz; - } - - size_t capacity() const { - size_t c = 0; - for (const auto& inner : sets_) - c += inner.set_.capacity(); - return c; - } - - size_t max_size() const { return (std::numeric_limits::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - } - - // extension - clears only soecified submap - // ---------------------------------------- - void clear(std::size_t submap_index) { - Inner& inner = sets_[submap_index]; - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map m; - // m.insert(std::make_pair("abc", 42)); - // -------------------------------------------------------------------- - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - std::pair insert(T&& value) { - return emplace(std::forward(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set s; - // s.insert(n); - // - // flat_hash_set s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable = 0, - typename std::enable_if::value, int>::type = 0> - std::pair insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set> s; - // s.insert({"abc", 42}); - // -------------------------------------------------------------------- - std::pair insert(init_type&& value) { - return emplace(std::move(value)); - } - - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable = 0, - typename std::enable_if::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) insert(*first); - } - - template = 0, RequiresInsertable = 0> - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) - return {end(), false, node_type()}; - auto& key = node.key(); - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - - typename Lockable::UniqueLock m(inner); - auto res = set.insert(std::move(node), hashval); - return { make_iterator(&inner, res.position), - res.inserted, - res.inserted ? node_type() : std::move(res.node) }; - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - struct ReturnKey_ - { - template - Key operator()(Key&& k, const Args&...) const { - return std::forward(k); - } - }; - - // -------------------------------------------------------------------- - // phmap expension: emplace_with_hash - // ---------------------------------- - // same as emplace, but hashval is provided - // -------------------------------------------------------------------- - template - std::pair emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); - } - - struct EmplaceDecomposableHashval - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable_with_hash(key, hashval, std::forward(args)...); - } - parallel_hash_set& s; - size_t hashval; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, - std::forward(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward(args)...).first; - } - - template - iterator lazy_emplace_with_hash(size_t hashval, const key_arg& key, F&& f) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); - } - - // -------------------------------------------------------------------- - // end of phmap expension - // -------------------------------------------------------------------- - - template - std::pair emplace_decomposable(const K& key, Args&&... args) - { - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); - } - - struct EmplaceDecomposable - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, std::forward(args)...); - } - parallel_hash_set& s; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); - size_t hashval = this->hash(PolicyTraits::key(slot)); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward(args)...).first; - } - - iterator make_iterator(Inner* inner, const EmbeddedIterator it) - { - if (it == inner->set_.end()) - return iterator(); - return iterator(inner, &sets_[0] + num_tables, it); - } - - std::pair make_rv(Inner* inner, - const std::pair& res) - { - return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; - } - - template - iterator lazy_emplace(const key_arg& key, F&& f) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); - } - - template - bool lazy_emplace_l(const key_arg& key, FExists&& fExists, FEmplace&& fEmplace) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(key, m); - Inner* inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.lazy_emplace_at(std::get<1>(res), std::forward(fEmplace)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward(fExists)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - // -------------------------------------------------------------------- - template - size_type erase(const key_arg& key) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UpgradeLock m(inner); - auto it = set.find(key, hashval); - if (it == set.end()) - return 0; - - typename Lockable::UpgradeToUnique unique(m); - set._erase(it); - return 1; - } - - // -------------------------------------------------------------------- - iterator erase(const_iterator cit) { return erase(cit.iter_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if () { - // m._erase(it++); - // } else { - // ++it; - // } - // } - // -------------------------------------------------------------------- - void _erase(iterator it) { - assert(it.inner_ != nullptr); - it.inner_->set_._erase(it.it_); - } - void _erase(const_iterator cit) { _erase(cit.iter_); } - - // This overload is necessary because otherwise erase(const K&) would be - // a better match if non-const iterator is passed as an argument. - // -------------------------------------------------------------------- - iterator erase(iterator it) { _erase(it++); return it; } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.iter_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - // -------------------------------------------------------------------- - template - void merge(parallel_hash_set& src) { // NOLINT - assert(this != &src); - if (this != &src) - { - for (size_t i=0; i - void merge(parallel_hash_set&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); - } - - template < - class K = key_type, - typename std::enable_if::value, int>::type = 0> - node_type extract(const key_arg& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(parallel_hash_set& that) noexcept( - IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - using std::swap; - for (size_t i=0; i target ? normalized : target); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - // -------------------------------------------------------------------- - template - size_t count(const key_arg& key) const { - return find(key) == end() ? 0 : 1; - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - // -------------------------------------------------------------------- - void prefetch_hash(size_t hashval) const { - const Inner& inner = sets_[subidx(hashval)]; - const auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast(inner)); - set.prefetch_hash(hashval); - } - - template - void prefetch(const key_arg& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - // -------------------------------------------------------------------- - template - iterator find(const key_arg& key, size_t hashval) { - typename Lockable::SharedLock m; - return find(key, hashval, m); - } - - template - iterator find(const key_arg& key) { - return find(key, this->hash(key)); - } - - template - const_iterator find(const key_arg& key, size_t hashval) const { - return const_cast(this)->find(key, hashval); - } - - template - const_iterator find(const key_arg& key) const { - return find(key, this->hash(key)); - } - - template - bool contains(const key_arg& key) const { - return find(key) != end(); - } - - template - bool contains(const key_arg& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template - std::pair equal_range(const key_arg& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - template - std::pair equal_range( - const key_arg& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { - size_t sz = 0; - for (const auto& inner : sets_) - { - typename Lockable::SharedLock m(const_cast(inner)); - sz += inner.set_.bucket_count(); - } - return sz; - } - - float load_factor() const { - size_t _capacity = bucket_count(); - return _capacity ? static_cast(static_cast(size()) / _capacity) : 0; - } - - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { - return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); - } - - friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { - return !(a == b); - } - - friend void swap(parallel_hash_set& a, - parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -#ifndef PHMAP_NON_DETERMINISTIC - template - bool dump(OutputArchive& ar) const; - - template - bool load(InputArchive& ar); -#endif - -private: - template - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - struct FindElement - { - template - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const parallel_hash_set& s; - }; - - struct HashElement - { - template - size_t operator()(const K& key, Args&&...) const { - return phmap_mix()(h(key)); - } - const hasher& h; - }; - - template - struct EqualElement - { - template - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - // -------------------------------------------------------------------- - void erase_meta_only(const_iterator cit) { - auto &it = cit.iter_; - assert(it.set_ != nullptr); - it.set_.erase_meta_only(const_iterator(it.it_)); - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.drop_deletes_without_resize(); - } - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast(inner)); - return set.has_element(elem, hashval); - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - // -------------------------------------------------------------------- - parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { - parallel_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - - parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { - parallel_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template - pointer find_ptr(const key_arg& key, size_t hashval, L& mutexlock) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return set.find_ptr(key, hashval); - } - - template - iterator find(const key_arg& key, size_t hashval, L& mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return make_iterator(&inner, set.find(key, hashval)); - } - - template - std::tuple - find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(typename Lockable::UniqueLock(inner)); - auto p = set.find_or_prepare_insert(key, hashval); // std::pair - return std::make_tuple(&inner, p.first, p.second); - } - - template - std::tuple - find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { - return find_or_prepare_insert_with_hash(this->hash(key), key, mutexlock); - } - - iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) { - return {inner, &sets_[0] + num_tables, it}; - } - const_iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) const { - return {inner, &sets_[0] + num_tables, it}; - } - - static size_t subidx(size_t hashval) { - return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; - } - - static size_t subcnt() { - return num_tables; - } - -private: - friend struct RawHashSetTestOnlyAccess; - - size_t growth_left() { - size_t sz = 0; - for (const auto& set : sets_) - sz += set.growth_left(); - return sz; - } - - hasher& hash_ref() { return sets_[0].set_.hash_ref(); } - const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } - key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } - const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } - allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } - const allocator_type& alloc_ref() const { - return sets_[0].set_.alloc_ref(); - } - -protected: // protected in case users want to derive fromm this - std::array sets_; -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_map : public parallel_hash_set -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map. - // MappedReference<> may be a non-reference type. - template - using MappedReference = decltype(P::value( - std::addressof(std::declval()))); - - // MappedConstReference<> may be a non-reference type. - template - using MappedConstReference = decltype(P::value( - std::addressof(std::declval()))); - - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - using Base = typename parallel_hash_map::parallel_hash_set; - using Lockable = phmap::LockableImpl; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template - using key_arg = typename KeyArgImpl::template type; - - static_assert(!std::is_reference::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference::value, ""); - - using iterator = typename parallel_hash_map::parallel_hash_set::iterator; - using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; - - parallel_hash_map() {} - -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using parallel_hash_map::parallel_hash_set::parallel_hash_set; -#endif - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map m; - // m.insert_or_assign(n, n); - template - std::pair insert_or_assign(key_arg&& k, V&& v) { - return insert_or_assign_impl(std::forward(k), std::forward(v)); - } - - template - std::pair insert_or_assign(key_arg&& k, const V& v) { - return insert_or_assign_impl(std::forward(k), v); - } - - template - std::pair insert_or_assign(const key_arg& k, V&& v) { - return insert_or_assign_impl(k, std::forward(v)); - } - - template - std::pair insert_or_assign(const key_arg& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { - return insert_or_assign(std::forward(k), std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { - return insert_or_assign(std::forward(k), v).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { - return insert_or_assign(k, std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace(key_arg&& k, Args&&... args) { - return try_emplace_impl(std::forward(k), std::forward(args)...); - } - - template ::value, int>::type = 0> - std::pair try_emplace(const key_arg& k, Args&&... args) { - return try_emplace_impl(k, std::forward(args)...); - } - - template - iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { - return try_emplace(std::forward(k), std::forward(args)...).first; - } - - template - iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { - return try_emplace(k, std::forward(args)...).first; - } - - template - MappedReference

at(const key_arg& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template - MappedConstReference

at(const key_arg& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - // ----------- phmap extensions -------------------------- - - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace_with_hash(size_t hashval, key_arg&& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, std::forward(k), std::forward(args)...); - } - - template ::value, int>::type = 0> - std::pair try_emplace_with_hash(size_t hashval, const key_arg& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, k, std::forward(args)...); - } - - template - iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg&& k, Args&&... args) { - return try_emplace_with_hash(hashval, std::forward(k), std::forward(args)...).first; - } - - template - iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg& k, Args&&... args) { - return try_emplace_with_hash(hashval, k, std::forward(args)...).first; - } - - // if map contains key, lambda is called with the mapped value (under read lock protection), - // and if_contains returns true. This is a const API and lambda should not modify the value - // ----------------------------------------------------------------------------------------- - template - bool if_contains(const key_arg& key, F&& f) const { - return const_cast(this)->template - modify_if_impl(key, std::forward(f)); - } - - // if map contains key, lambda is called with the mapped value without read lock protection, - // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value - // This should be used only if we know that no other thread may be mutating the map at the time. - // ----------------------------------------------------------------------------------------- - template - bool if_contains_unsafe(const key_arg& key, F&& f) const { - return const_cast(this)->template - modify_if_impl::DoNothing>(key, std::forward(f)); - } - - // if map contains key, lambda is called with the mapped value (under write lock protection), - // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value - // ---------------------------------------------------------------------------------------------------- - template - bool modify_if(const key_arg& key, F&& f) { - return modify_if_impl(key, std::forward(f)); - } - - - // if map contains key, lambda is called with the mapped value (under write lock protection). - // If the lambda returns true, the key is subsequently erased from the map (the write lock - // is only released after erase). - // returns true if key was erased, false otherwise. - // ---------------------------------------------------------------------------------------------------- - template - bool erase_if(const key_arg& key, F&& f) { - return erase_if_impl(key, std::forward(f)); - } - - // if map does not contains key, it is inserted and the mapped value is value-constructed - // with the provided arguments (if any), as with try_emplace. - // if map already contains key, then the lambda is called with the mapped value (under - // write lock protection) and can update the mapped value. - // returns true if key was not already present, false otherwise. - // --------------------------------------------------------------------------------------- - template - bool try_emplace_l(K&& k, F&& f, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward(f)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // ----------- end of phmap extensions -------------------------- - - template - MappedReference

operator[](key_arg&& key) { - return Policy::value(&*try_emplace(std::forward(key)).first); - } - - template - MappedReference

operator[](const key_arg& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template - bool modify_if_impl(const key_arg& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable::value); -#endif - L m; - auto ptr = this->template find_ptr(key, this->hash(key), m); - if (ptr == nullptr) - return false; - std::forward(f)(Policy::value(ptr)); - return true; - } - - template - bool erase_if_impl(const key_arg& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable::value); -#endif - L m; - auto it = this->template find(key, this->hash(key), m); - if (it == this->end()) return false; - if (std::forward(f)(Policy::value(&*it))) - { - this->erase(it); - return true; - } - return false; - } - - - template - std::pair insert_or_assign_impl(K&& k, V&& v) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::forward(k), std::forward(v)); - else - Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward(v); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template - std::pair try_emplace_impl(K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template - std::pair try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - -}; - - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template -void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { - memory_internal::ConstructFromTupleImpl( - alloc, ptr, std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>()); -} - -// Constructs T using the args specified in the tuple and calls F with the -// constructed value. -// ---------------------------------------------------------------------------- -template -decltype(std::declval()(std::declval())) WithConstructed( - Tuple&& t, F&& f) { - return memory_internal::WithConstructedImpl( - std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>(), - std::forward(f)); -} - -// ---------------------------------------------------------------------------- -// Given arguments of an std::pair's consructor, PairArgs() returns a pair of -// tuples with references to the passed arguments. The tuples contain -// constructor arguments for the first and the second elements of the pair. -// -// The following two snippets are equivalent. -// -// 1. std::pair p(args...); -// -// 2. auto a = PairArgs(args...); -// std::pair p(std::piecewise_construct, -// std::move(p.first), std::move(p.second)); -// ---------------------------------------------------------------------------- -inline std::pair, std::tuple<>> PairArgs() { return {}; } - -template -std::pair, std::tuple> PairArgs(F&& f, S&& s) { - return {std::piecewise_construct, std::forward_as_tuple(std::forward(f)), - std::forward_as_tuple(std::forward(s))}; -} - -template -std::pair, std::tuple> PairArgs( - const std::pair& p) { - return PairArgs(p.first, p.second); -} - -template -std::pair, std::tuple> PairArgs(std::pair&& p) { - return PairArgs(std::forward(p.first), std::forward(p.second)); -} - -template -auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) - -> decltype(std::make_pair(memory_internal::TupleRef(std::forward(f)), - memory_internal::TupleRef(std::forward(s)))) { - return std::make_pair(memory_internal::TupleRef(std::forward(f)), - memory_internal::TupleRef(std::forward(s))); -} - -// A helper function for implementing apply() in map policies. -// ---------------------------------------------------------------------------- -template -auto DecomposePair(F&& f, Args&&... args) - -> decltype(memory_internal::DecomposePairImpl( - std::forward(f), PairArgs(std::forward(args)...))) { - return memory_internal::DecomposePairImpl( - std::forward(f), PairArgs(std::forward(args)...)); -} - -// A helper function for implementing apply() in set policies. -// ---------------------------------------------------------------------------- -template -decltype(std::declval()(std::declval(), std::declval())) -DecomposeValue(F&& f, Arg&& arg) { - const auto& key = arg; - return std::forward(f)(key, std::forward(arg)); -} - - -// -------------------------------------------------------------------------- -// Policy: a policy defines how to perform different operations on -// the slots of the hashtable (see hash_policy_traits.h for the full interface -// of policy). -// -// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The -// functor should accept a key and return size_t as hash. For best performance -// it is important that the hash function provides high entropy across all bits -// of the hash. -// -// Eq: a (possibly polymorphic) functor that compares two keys for equality. It -// should accept two (of possibly different type) keys and return a bool: true -// if they are equal, false if they are not. If two keys compare equal, then -// their hash values as defined by Hash MUST be equal. -// -// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which -// the storage of the hashtable will be allocated and the elements will be -// constructed and destroyed. -// -------------------------------------------------------------------------- -template -struct FlatHashSetPolicy -{ - using slot_type = T; - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - phmap::allocator_traits::construct(*alloc, slot, - std::forward(args)...); - } - - template - static void destroy(Allocator* alloc, slot_type* slot) { - phmap::allocator_traits::destroy(*alloc, slot); - } - - template - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - construct(alloc, new_slot, std::move(*old_slot)); - destroy(alloc, old_slot); - } - - static T& element(slot_type* slot) { return *slot; } - - template - static decltype(phmap::priv::DecomposeValue( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward(f), std::forward(args)...); - } - - static size_t space_used(const T*) { return 0; } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template -struct FlatHashMapPolicy -{ - using slot_policy = priv::map_slot_policy; - using slot_type = typename slot_policy::slot_type; - using key_type = K; - using mapped_type = V; - using init_type = std::pair; - - template - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - slot_policy::construct(alloc, slot, std::forward(args)...); - } - - template - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); - } - - template - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - slot_policy::transfer(alloc, new_slot, old_slot); - } - - template - static decltype(phmap::priv::DecomposePair( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward(f), - std::forward(args)...); - } - - static size_t space_used(const slot_type*) { return 0; } - - static std::pair& element(slot_type* slot) { return slot->value; } - - static V& value(std::pair* kv) { return kv->second; } - static const V& value(const std::pair* kv) { return kv->second; } -}; - -template -struct node_hash_policy { - static_assert(std::is_lvalue_reference::value, ""); - - using slot_type = typename std::remove_cv< - typename std::remove_reference::type>::type*; - - template - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - *slot = Policy::new_element(alloc, std::forward(args)...); - } - - template - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::delete_element(alloc, *slot); - } - - template - static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { - *new_slot = *old_slot; - } - - static size_t space_used(const slot_type* slot) { - if (slot == nullptr) return Policy::element_space_used(nullptr); - return Policy::element_space_used(*slot); - } - - static Reference element(slot_type* slot) { return **slot; } - - template - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - - template - static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward(ts)...)) { - return P::apply(std::forward(ts)...); - } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template -struct NodeHashSetPolicy - : phmap::priv::node_hash_policy> -{ - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template - static T* new_element(Allocator* alloc, Args&&... args) { - using ValueAlloc = - typename phmap::allocator_traits::template rebind_alloc; - ValueAlloc value_alloc(*alloc); - T* res = phmap::allocator_traits::allocate(value_alloc, 1); - phmap::allocator_traits::construct(value_alloc, res, - std::forward(args)...); - return res; - } - - template - static void delete_element(Allocator* alloc, T* elem) { - using ValueAlloc = - typename phmap::allocator_traits::template rebind_alloc; - ValueAlloc value_alloc(*alloc); - phmap::allocator_traits::destroy(value_alloc, elem); - phmap::allocator_traits::deallocate(value_alloc, elem, 1); - } - - template - static decltype(phmap::priv::DecomposeValue( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward(f), std::forward(args)...); - } - - static size_t element_space_used(const T*) { return sizeof(T); } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template -class NodeHashMapPolicy - : public phmap::priv::node_hash_policy< - std::pair&, NodeHashMapPolicy> -{ - using value_type = std::pair; - -public: - using key_type = Key; - using mapped_type = Value; - using init_type = std::pair; - - template - static value_type* new_element(Allocator* alloc, Args&&... args) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc; - PairAlloc pair_alloc(*alloc); - value_type* res = - phmap::allocator_traits::allocate(pair_alloc, 1); - phmap::allocator_traits::construct(pair_alloc, res, - std::forward(args)...); - return res; - } - - template - static void delete_element(Allocator* alloc, value_type* pair) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc; - PairAlloc pair_alloc(*alloc); - phmap::allocator_traits::destroy(pair_alloc, pair); - phmap::allocator_traits::deallocate(pair_alloc, pair, 1); - } - - template - static decltype(phmap::priv::DecomposePair( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward(f), - std::forward(args)...); - } - - static size_t element_space_used(const value_type*) { - return sizeof(value_type); - } - - static Value& value(value_type* elem) { return elem->second; } - static const Value& value(const value_type* elem) { return elem->second; } -}; - - -// -------------------------------------------------------------------------- -// hash_default -// -------------------------------------------------------------------------- - -#if PHMAP_HAVE_STD_STRING_VIEW - -// support char16_t wchar_t .... -template -struct StringHashT -{ - using is_transparent = void; - - size_t operator()(std::basic_string_view v) const { - std::string_view bv{reinterpret_cast(v.data()), v.size() * sizeof(CharT)}; - return std::hash()(bv); - } -}; - -// Supports heterogeneous lookup for basic_string-like elements. -template -struct StringHashEqT -{ - using Hash = StringHashT; - - struct Eq { - using is_transparent = void; - - bool operator()(std::basic_string_view lhs, std::basic_string_view rhs) const { - return lhs == rhs; - } - }; -}; - -template <> -struct HashEq : StringHashEqT {}; - -template <> -struct HashEq : StringHashEqT {}; - -// char16_t -template <> -struct HashEq : StringHashEqT {}; - -template <> -struct HashEq : StringHashEqT {}; - -// wchar_t -template <> -struct HashEq : StringHashEqT {}; - -template <> -struct HashEq : StringHashEqT {}; - -#endif - -// Supports heterogeneous lookup for pointers and smart pointers. -// ------------------------------------------------------------- -template -struct HashEq -{ - struct Hash { - using is_transparent = void; - template - size_t operator()(const U& ptr) const { - return phmap::Hash{}(HashEq::ToPtr(ptr)); - } - }; - - struct Eq { - using is_transparent = void; - template - bool operator()(const A& a, const B& b) const { - return HashEq::ToPtr(a) == HashEq::ToPtr(b); - } - }; - -private: - static const T* ToPtr(const T* ptr) { return ptr; } - - template - static const T* ToPtr(const std::unique_ptr& ptr) { - return ptr.get(); - } - - template - static const T* ToPtr(const std::shared_ptr& ptr) { - return ptr.get(); - } -}; - -template -struct HashEq> : HashEq {}; - -template -struct HashEq> : HashEq {}; - -namespace hashtable_debug_internal { - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template -struct HashtableDebugAccess> -{ - using Traits = typename Set::PolicyTraits; - using Slot = typename Traits::slot_type; - - static size_t GetNumProbes(const Set& set, - const typename Set::key_type& key) { - size_t num_probes = 0; - size_t hashval = set.hash(key); - auto seq = set.probe(hashval); - while (true) { - priv::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(priv::H2(hashval))) { - if (Traits::apply( - typename Set::template EqualElement{ - key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset((size_t)i)))) - return num_probes; - ++num_probes; - } - if (g.MatchEmpty()) return num_probes; - seq.next(); - ++num_probes; - } - } - - static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); - size_t m = layout.AllocSize(); - - size_t per_slot = Traits::space_used(static_cast(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * c.size(); - } else { - for (size_t i = 0; i != capacity; ++i) { - if (priv::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); - } - } - } - return m; - } - - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); - size_t m = layout.AllocSize(); - size_t per_slot = Traits::space_used(static_cast(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; - } -}; - -} // namespace hashtable_debug_internal -} // namespace priv - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::flat_hash_set` is an unordered associative container which has -// been optimized for both speed and memory footprint in most common use cases. -// Its interface is similar to that of `std::unordered_set` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the set is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class flat_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::FlatHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename flat_hash_set::raw_hash_set; - -public: - flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::flat_hash_map` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class flat_hash_map : public phmap::priv::raw_hash_map< - phmap::priv::FlatHashMapPolicy, - Hash, Eq, Alloc> { - using Base = typename flat_hash_map::raw_hash_map; - -public: - flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::node_hash_set` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_set` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class node_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename node_hash_set::raw_hash_set; - -public: - node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::node_hash_map` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class node_hash_map - : public phmap::priv::raw_hash_map< - phmap::priv::NodeHashMapPolicy, Hash, Eq, - Alloc> -{ - using Base = typename node_hash_map::raw_hash_map; - -public: - node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_set -// ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class parallel_flat_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashSetPolicy, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_set::parallel_hash_set; - -public: - parallel_flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h -// ----------------------------------------------------------------------------- -template -class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashMapPolicy, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_map::parallel_hash_map; - -public: - parallel_flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_set -// ----------------------------------------------------------------------------- -template -class parallel_node_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename parallel_node_hash_set::parallel_hash_set; - -public: - parallel_node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_map -// ----------------------------------------------------------------------------- -template -class parallel_node_hash_map - : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashMapPolicy, Hash, Eq, - Alloc> -{ - using Base = typename parallel_node_hash_map::parallel_hash_map; - -public: - parallel_node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // phmap_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_base.h b/benchmarks/external/parallel_hashmap/phmap_base.h deleted file mode 100644 index d0c6f3ce..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_base.h +++ /dev/null @@ -1,5171 +0,0 @@ -#if !defined(phmap_base_h_guard_) -#define phmap_base_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // for std::lock - -#include "phmap_config.h" - -#ifdef PHMAP_HAVE_SHARED_MUTEX - #include // after "phmap_config.h" -#endif - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4582) // constructor is not implicitly called - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member -#endif // _MSC_VER - -namespace phmap { - -template using Allocator = typename std::allocator; - -template using Pair = typename std::pair; - -template -struct EqualTo -{ - inline bool operator()(const T& a, const T& b) const - { - return std::equal_to()(a, b); - } -}; - -template -struct Less -{ - inline bool operator()(const T& a, const T& b) const - { - return std::less()(a, b); - } -}; - -namespace type_traits_internal { - -template -struct VoidTImpl { - using type = void; -}; - -// This trick to retrieve a default alignment is necessary for our -// implementation of aligned_storage_t to be consistent with any implementation -// of std::aligned_storage. -// --------------------------------------------------------------------------- -template > -struct default_alignment_of_aligned_storage; - -template -struct default_alignment_of_aligned_storage> { - static constexpr size_t value = Align; -}; - -// NOTE: The `is_detected` family of templates here differ from the library -// fundamentals specification in that for library fundamentals, `Op` is -// evaluated as soon as the type `is_detected` undergoes -// substitution, regardless of whether or not the `::value` is accessed. That -// is inconsistent with all other standard traits and prevents lazy evaluation -// in larger contexts (such as if the `is_detected` check is a trailing argument -// of a `conjunction`. This implementation opts to instead be lazy in the same -// way that the standard traits are (this "defect" of the detection idiom -// specifications has been reported). -// --------------------------------------------------------------------------- - -template class Op, class... Args> -struct is_detected_impl { - using type = std::false_type; -}; - -template