diff options
| author | Tyge Løvset <[email protected]> | 2022-07-07 10:47:14 +0200 |
|---|---|---|
| committer | Tyge Løvset <[email protected]> | 2022-07-07 10:47:14 +0200 |
| commit | 9aafd5ce068da6f87c2f3ac8c9cf0721976f2bbc (patch) | |
| tree | b0a3a29177ac1d891c606aaee51fe438a917401b | |
| parent | d108c705ae0b51874f1d18b8916466bdc5830415 (diff) | |
| download | STC-modified-9aafd5ce068da6f87c2f3ac8c9cf0721976f2bbc.tar.gz STC-modified-9aafd5ce068da6f87c2f3ac8c9cf0721976f2bbc.zip | |
Added emhash by ktprime to external benchmarks. Very fast iteration and lookups.
17 files changed, 2197 insertions, 20758 deletions
diff --git a/benchmarks/external/emhash/hash_table7.hpp b/benchmarks/external/emhash/hash_table7.hpp new file mode 100644 index 00000000..dbf90c86 --- /dev/null +++ b/benchmarks/external/emhash/hash_table7.hpp @@ -0,0 +1,1887 @@ +// emhash7::HashMap for C++11/14/17 +// version 2.1.2 +// https://github.com/ktprime/ktprime/blob/master/hash_table7.hpp +// +// Licensed under the MIT License <http://opensource.org/licenses/MIT>. +// SPDX-License-Identifier: MIT +// Copyright (c) 2019-2022 Huang Yuanbing & bailuzhou AT 163.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE + +// From +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// separator chain resolution 1 + L / 2 exp(-L) + L + +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 +// SEPARATE CHAN RES. +// probes/successful lookup 1.05 1.25 1.3 1.25 1.4 1.45 1.50 +// probes/unsuccessful lookup 1.00 1.11 1.15 1.22 1.25 1.31 1.37 +// clacul/unsuccessful lookup 1.01 1.25 1.36, 1.56, 1.64, 1.81, 1.97 + +/**************** + under random hashCodes, the frequency of nodes in bins follows a Poisson +distribution(http://en.wikipedia.org/wiki/Poisson_distribution) with a parameter of about 0.5 +on average for the default resizing threshold of 0.75, although with a large variance because +of resizing granularity. Ignoring variance, the expected occurrences of list size k are +(exp(-0.5) * pow(0.5, k)/factorial(k)). The first values are: +0: 0.60653066 +1: 0.30326533 +2: 0.07581633 +3: 0.01263606 +4: 0.00157952 +5: 0.00015795 +6: 0.00001316 +7: 0.00000094 +8: 0.00000006 + + ============== buckets size ration ======== + 1 1543981 0.36884964|0.36787944 36.885 + 2 768655 0.36725597|0.36787944 73.611 + 3 256236 0.18364065|0.18393972 91.975 + 4 64126 0.06127757|0.06131324 98.102 + 5 12907 0.01541710|0.01532831 99.644 + 6 2050 0.00293841|0.00306566 99.938 + 7 310 0.00051840|0.00051094 99.990 + 8 49 0.00009365|0.00007299 99.999 + 9 4 0.00000860|0.00000913 100.000 +========== collision miss ration =========== + _num_filled aver_size k.v size_kv = 4185936, 1.58, x.x 24 + collision,possion,cache_miss hit_find|hit_miss, load_factor = 36.73%,36.74%,31.31% 1.50|2.00, 1.00 +============== buckets size ration ======== +*******************************************************/ + +#pragma once + +#include <cstring> +#include <string> +#include <cmath> +#include <cstdlib> +#include <type_traits> +#include <cassert> +#include <utility> +#include <cstdint> +#include <functional> +#include <iterator> +#include <algorithm> + +#ifdef __has_include + #if __has_include("wyhash.h") + #include "wyhash.h" + #endif +#elif EMH_WY_HASH + #include "wyhash.h" +#endif + +#ifdef EMH_KEY + #undef EMH_KEY + #undef EMH_VAL + #undef EMH_PKV + #undef EMH_NEW + #undef EMH_SET + #undef EMH_BUCKET + #undef EMH_EMPTY +#endif + +// likely/unlikely +#if (__GNUC__ >= 4 || __clang__) +# define EMH_LIKELY(condition) __builtin_expect(condition, 1) +# define EMH_UNLIKELY(condition) __builtin_expect(condition, 0) +#else +# define EMH_LIKELY(condition) condition +# define EMH_UNLIKELY(condition) condition +#endif + +#ifndef EMH_BUCKET_INDEX + #define EMH_BUCKET_INDEX 1 +#endif + +#ifndef EMH_DEFAULT_LOAD_FACTOR +#define EMH_DEFAULT_LOAD_FACTOR 0.80f +#endif + +#if EMH_BUCKET_INDEX == 0 + #define EMH_KEY(p,n) p[n].second.first + #define EMH_VAL(p,n) p[n].second.second + #define EMH_BUCKET(p,n) p[n].first + #define EMH_PKV(p,n) p[n].second + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(bucket, value_type(key, val));\ + _num_filled ++; EMH_SET(bucket) +#elif EMH_BUCKET_INDEX == 2 + #define EMH_KEY(p,n) p[n].first.first + #define EMH_VAL(p,n) p[n].first.second + #define EMH_BUCKET(p,n) p[n].second + #define EMH_PKV(p,n) p[n].first + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(value_type(key, val), bucket);\ + _num_filled ++; EMH_SET(bucket) +#else + #define EMH_KEY(p,n) p[n].first + #define EMH_VAL(p,n) p[n].second + #define EMH_BUCKET(p,n) p[n].bucket + #define EMH_PKV(p,n) p[n] + #define EMH_NEW(key, val, bucket)\ + new(_pairs + bucket) PairT(key, val, bucket);\ + _num_filled ++; EMH_SET(bucket) +#endif + +#define EMH_MASK(bucket) 1 << (bucket % MASK_BIT) +#define EMH_SET(bucket) _bitmask[bucket / MASK_BIT] &= ~(EMH_MASK(bucket)) +#define EMH_CLS(bucket) _bitmask[bucket / MASK_BIT] |= EMH_MASK(bucket) +#define EMH_EMPTY(bitmask, bucket) (_bitmask[bucket / MASK_BIT] & (EMH_MASK(bucket))) != 0 + +#if _WIN32 + #include <intrin.h> +#if _WIN64 + #pragma intrinsic(_umul128) +#endif +#endif + +namespace emhash7 { + +#ifdef EMH_SIZE_TYPE_16BIT + typedef uint16_t size_type; + static constexpr size_type INACTIVE = 0xFFFE; +#elif EMH_SIZE_TYPE_64BIT + typedef uint64_t size_type; + static constexpr size_type INACTIVE = 0 - 0x1ull; +#else + typedef uint32_t size_type; + static constexpr size_type INACTIVE = 0 - 0x1u; +#endif + +#ifndef EMH_SIZE_TYPE_16BIT +static_assert((int)INACTIVE < 0, "INACTIVE must negative (to int)"); +#endif + +//count the leading zero bit +inline static size_type CTZ(size_t n) +{ +#if defined(__x86_64__) || defined(_WIN32) || (__BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + +#elif __BIG_ENDIAN__ || (__BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + n = __builtin_bswap64(n); +#else + static uint32_t endianness = 0x12345678; + const auto is_big = *(const char *)&endianness == 0x12; + if (is_big) + n = __builtin_bswap64(n); +#endif + +#if _WIN32 + unsigned long index; + #if defined(_WIN64) + _BitScanForward64(&index, n); + #else + _BitScanForward(&index, n); + #endif +#elif defined (__LP64__) || (SIZE_MAX == UINT64_MAX) || defined (__x86_64__) + auto index = __builtin_ctzll(n); +#elif 1 + auto index = __builtin_ctzl(n); +#else + #if defined (__LP64__) || (SIZE_MAX == UINT64_MAX) || defined (__x86_64__) + size_type index; + __asm__("bsfq %1, %0\n" : "=r" (index) : "rm" (n) : "cc"); + #else + size_type index; + __asm__("bsf %1, %0\n" : "=r" (index) : "rm" (n) : "cc"); + #endif +#endif + + return (size_type)index; +} + +template <typename First, typename Second> +struct entry { + using first_type = First; + using second_type = Second; + entry(const First& key, const Second& val, size_type ibucket) + :second(val),first(key) + { + bucket = ibucket; + } + + entry(First&& key, Second&& val, size_type ibucket) + :second(std::move(val)), first(std::move(key)) + { + bucket = ibucket; + } + + template<typename K, typename V> + entry(K&& key, V&& val, size_type ibucket) + :second(std::forward<V>(val)), first(std::forward<K>(key)) + { + bucket = ibucket; + } + + entry(const std::pair<First,Second>& pair) + :second(pair.second),first(pair.first) + { + bucket = INACTIVE; + } + + entry(std::pair<First, Second>&& pair) + :second(std::move(pair.second)), first(std::move(pair.first)) + { + bucket = INACTIVE; + } + + entry(std::tuple<First, Second>&& tup) + :second(std::move(std::get<2>(tup))), first(std::move(std::get<1>(tup))) + { + bucket = INACTIVE; + } + + entry(const entry& rhs) + :second(rhs.second),first(rhs.first) + { + bucket = rhs.bucket; + } + + entry(entry&& rhs) noexcept + :second(std::move(rhs.second)),first(std::move(rhs.first)) + { + bucket = rhs.bucket; + } + + entry& operator = (entry&& rhs) + { + second = std::move(rhs.second); + bucket = rhs.bucket; + first = std::move(rhs.first); + return *this; + } + + entry& operator = (const entry& rhs) + { + second = rhs.second; + bucket = rhs.bucket; + first = rhs.first; + return *this; + } + + bool operator == (const entry<First, Second>& p) const + { + return first == p.first && second == p.second; + } + + bool operator == (const std::pair<First, Second>& p) const + { + return first == p.first && second == p.second; + } + + void swap(entry<First, Second>& o) + { + std::swap(second, o.second); + std::swap(first, o.first); + } + +#ifndef EMH_ORDER_KV + Second second;//int + size_type bucket; + First first; //long +#else + First first; //long + size_type bucket; + Second second;//int +#endif +};// __attribute__ ((packed)); + +/// A cache-friendly hash table with open addressing, linear/qua probing and power-of-two capacity +template <typename KeyT, typename ValueT, typename HashT = std::hash<KeyT>, typename EqT = std::equal_to<KeyT>> +class HashMap +{ +public: + typedef HashMap<KeyT, ValueT, HashT, EqT> htype; + typedef std::pair<KeyT,ValueT> value_type; + +#if EMH_BUCKET_INDEX == 0 + typedef value_type value_pair; + typedef std::pair<size_type, value_type> PairT; +#elif EMH_BUCKET_INDEX == 2 + typedef value_type value_pair; + typedef std::pair<value_type, size_type> PairT; +#else + typedef entry<KeyT, ValueT> value_pair; + typedef entry<KeyT, ValueT> PairT; +#endif + + typedef KeyT key_type; + typedef ValueT val_type; + typedef ValueT mapped_type; + typedef HashT hasher; + typedef EqT key_equal; + typedef PairT& reference; + typedef const PairT& const_reference; + + class const_iterator; + class iterator + { + public: + typedef std::forward_iterator_tag iterator_category; + typedef std::ptrdiff_t difference_type; + typedef value_pair value_type; + + typedef value_pair* pointer; + typedef value_pair& reference; + + iterator() : _map(nullptr) { } + iterator(const const_iterator& it) : _map(it._map), _bucket(it._bucket), _from(it._from), _bmask(it._bmask) { } + iterator(const htype* hash_map, size_type bucket, bool) : _map(hash_map), _bucket(bucket) { init(); } + iterator(const htype* hash_map, size_type bucket) : _map(hash_map), _bucket(bucket) { _bmask = _from = 0; } + + void init() + { + _from = (_bucket / SIZE_BIT) * SIZE_BIT; + if (_bucket < _map->bucket_count()) { + _bmask = *(size_t*)((size_t*)_map->_bitmask + _from / SIZE_BIT); + _bmask |= (1ull << _bucket % SIZE_BIT) - 1; + _bmask = ~_bmask; + } else { + _bmask = 0; + } + } + + size_t bucket() const + { + return _bucket; + } + + void clear(size_type bucket) + { + if (_bucket / SIZE_BIT == bucket / SIZE_BIT) + _bmask &= ~(1ull << (bucket % SIZE_BIT)); + } + + iterator& next() + { + goto_next_element(); + return *this; + } + + iterator& operator++() + { + _bmask &= _bmask - 1; + goto_next_element(); + return *this; + } + + iterator operator++(int) + { + iterator old = *this; + _bmask &= _bmask - 1; + goto_next_element(); + return old; + } + + reference operator*() const + { + return _map->EMH_PKV(_pairs, _bucket); + } + + pointer operator->() const + { + return &(_map->EMH_PKV(_pairs, _bucket)); + } + + bool operator==(const iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const iterator& rhs) const { return _bucket != rhs._bucket; } + bool operator==(const const_iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const const_iterator& rhs) const { return _bucket != rhs._bucket; } + + private: + void goto_next_element() + { + if (EMH_LIKELY(_bmask != 0)) { + _bucket = _from + CTZ(_bmask); + return; + } + + do { + _bmask = ~*(size_t*)((size_t*)_map->_bitmask + (_from += SIZE_BIT) / SIZE_BIT); + } while (_bmask == 0); + + _bucket = _from + CTZ(_bmask); + } + + public: + const htype* _map; + size_type _bucket; + size_type _from; + size_t _bmask; + }; + + class const_iterator + { + public: + typedef std::forward_iterator_tag iterator_category; + typedef std::ptrdiff_t difference_type; + typedef value_pair value_type; + + typedef const value_pair* pointer; + typedef const value_pair& reference; + + const_iterator(const iterator& it) : _map(it._map), _bucket(it._bucket), _from(it._from), _bmask(it._bmask) { } + const_iterator(const htype* hash_map, size_type bucket, bool) : _map(hash_map), _bucket(bucket) { init(); } + const_iterator(const htype* hash_map, size_type bucket) : _map(hash_map), _bucket(bucket) { _bmask = _from = 0; } + + void init() + { + _from = (_bucket / SIZE_BIT) * SIZE_BIT; + if (_bucket < _map->bucket_count()) { + _bmask = *(size_t*)((size_t*)_map->_bitmask + _from / SIZE_BIT); + _bmask |= (1ull << _bucket % SIZE_BIT) - 1; + _bmask = ~_bmask; + } else { + _bmask = 0; + } + } + + size_t bucket() const + { + return _bucket; + } + + const_iterator& operator++() + { + goto_next_element(); + return *this; + } + + const_iterator operator++(int) + { + const_iterator old(*this); + goto_next_element(); + return old; + } + + reference operator*() const + { + return _map->EMH_PKV(_pairs, _bucket); + } + + pointer operator->() const + { + return &(_map->EMH_PKV(_pairs, _bucket)); + } + + bool operator==(const const_iterator& rhs) const { return _bucket == rhs._bucket; } + bool operator!=(const const_iterator& rhs) const { return _bucket != rhs._bucket; } + + private: + void goto_next_element() + { + _bmask &= _bmask - 1; + if (EMH_LIKELY(_bmask != 0)) { + _bucket = _from + CTZ(_bmask); + return; + } + + do { + _bmask = ~*(size_t*)((size_t*)_map->_bitmask + (_from += SIZE_BIT) / SIZE_BIT); + } while (_bmask == 0); + + _bucket = _from + CTZ(_bmask); + } + + public: + const htype* _map; + size_type _bucket; + size_type _from; + size_t _bmask; + }; + + void init(size_type bucket, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + _pairs = nullptr; + _bitmask = nullptr; + _num_buckets = _num_filled = 0; + max_load_factor(mlf); + reserve(bucket); + } + + HashMap(size_type bucket = 2, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + init(bucket, mlf); + } + + size_type AllocSize(size_type num_buckets) const + { + return num_buckets * sizeof(PairT) + EPACK_SIZE * sizeof(PairT) + (num_buckets + 7) / 8 + BIT_PACK; + } + + HashMap(const HashMap& rhs) + { + _pairs = (PairT*)malloc(AllocSize(rhs._num_buckets)); + clone(rhs); + } + + HashMap(HashMap&& rhs) + { +#ifndef EMH_ZERO_MOVE + init(4); +#else + _num_buckets = _num_filled = _mask = 0; + _pairs = nullptr; +#endif + swap(rhs); + } + + HashMap(std::initializer_list<value_type> ilist) + { + init((size_type)ilist.size()); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template<class InputIt> + HashMap(InputIt first, InputIt last, size_type bucket_count=4) + { + init(std::distance(first, last) + bucket_count); + for (; first != last; ++first) + emplace(*first); + } + + HashMap& operator=(const HashMap& rhs) + { + if (this == &rhs) + return *this; + + if (_num_filled) + clearkv(); + + if (_num_buckets != rhs._num_buckets) { + free(_pairs); + _pairs = (PairT*)malloc(AllocSize(rhs._num_buckets)); + } + + clone(rhs); + return *this; + } + + HashMap& operator=(HashMap&& rhs) + { + if (this != &rhs) { + swap(rhs); + rhs.clear(); + } + return *this; + } + + template<typename Con> + bool operator == (const Con& rhs) const + { + if (size() != rhs.size()) + return false; + + for (auto it = begin(), last = end(); it != last; ++it) { + auto oi = rhs.find(it->first); + if (oi == rhs.end() || it->second != oi->second) + return false; + } + return true; + } + + template<typename Con> + bool operator != (const Con& rhs) const { return !(*this == rhs); } + + ~HashMap() + { + if (is_triviall_destructable() && _num_filled) { + for (auto it = cbegin(); _num_filled; ++it) { + _num_filled --; + it->~value_pair(); + } + } + free(_pairs); + } + + void clone(const HashMap& rhs) + { + _hasher = rhs._hasher; +// _eq = rhs._eq; + + _num_filled = rhs._num_filled; + _mask = rhs._mask; + _mlf = rhs._mlf; + _num_buckets = rhs._num_buckets; + + _bitmask = decltype(_bitmask)(_pairs + EPACK_SIZE + _num_buckets); + auto* opairs = rhs._pairs; + + if (is_copy_trivially()) + memcpy(_pairs, opairs, AllocSize(_num_buckets)); + else { + memcpy(_pairs + _num_buckets, opairs + _num_buckets, EPACK_SIZE * sizeof(PairT) + (_num_buckets + 7) / 8 + BIT_PACK); + for (auto it = rhs.cbegin(); it.bucket() <= _mask; ++it) { + const auto bucket = it.bucket(); + EMH_BUCKET(_pairs, bucket) = EMH_BUCKET(opairs, bucket); + new(_pairs + bucket) PairT(opairs[bucket]); + } + } + } + + void swap(HashMap& rhs) + { + std::swap(_hasher, rhs._hasher); + // std::swap(_eq, rhs._eq); + std::swap(_pairs, rhs._pairs); + std::swap(_num_buckets, rhs._num_buckets); + std::swap(_num_filled, rhs._num_filled); + std::swap(_mask, rhs._mask); + std::swap(_mlf, rhs._mlf); + std::swap(_bitmask, rhs._bitmask); + //std::swap(EMH_BUCKET(_pairs, _num_buckets), EMH_BUCKET(rhs._pairs, rhs._num_buckets)); + } + + // ------------------------------------------------------------- + iterator begin() + { +#ifdef EMH_ZERO_MOVE + if (0 == _num_filled) + return {this, _num_buckets}; +#endif + + const auto bmask = ~(*(size_t*)_bitmask); + if (bmask != 0) + return {this, CTZ(bmask), true}; + + iterator it(this, sizeof(bmask) * 8 - 1); + return it.next(); + } + + const_iterator cbegin() const + { +#ifdef EMH_ZERO_MOVE + if (0 == _num_filled) + return {this, _num_buckets}; +#endif + + const auto bmask = ~(*(size_t*)_bitmask); + if (bmask != 0) + return {this, CTZ(bmask), true}; + + iterator it(this, sizeof(bmask) * 8 - 1); + return it.next(); + } + + iterator last() const + { + if (_num_filled == 0) + return end(); + + auto bucket = _mask; + while (EMH_EMPTY(_pairs, bucket)) bucket--; + return {this, bucket, true}; + } + + const_iterator begin() const { return cbegin(); } + + iterator end() { return {this, _num_buckets}; } + const_iterator cend() const { return {this, _num_buckets}; } + const_iterator end() const { return cend(); } + + size_type size() const { return _num_filled; } + bool empty() const { return _num_filled == 0; } + + size_type bucket_count() const { return _num_buckets; } + float load_factor() const { return static_cast<float>(_num_filled) / (_mask + 1); } + + HashT& hash_function() const { return _hasher; } + EqT& key_eq() const { return _eq; } + + void max_load_factor(float mlf) + { + if (mlf < 0.9999f && mlf > 0.2f) + _mlf = (uint32_t)((1 << 27) / mlf); + } + + constexpr float max_load_factor() const { return (1 << 27) / (float)_mlf; } + constexpr size_type max_size() const { return (1ull << (sizeof(size_type) * 8 - 2)); } + constexpr size_type max_bucket_count() const { return max_size(); } + +#if EMH_STATIS + size_type bucket_main() const + { + auto main_size = 0; + for (size_type bucket = 0; bucket < _num_buckets; ++bucket) { + if (EMH_BUCKET(_pairs, bucket) == bucket) + main_size ++; + } + return main_size; + } + + //Returns the bucket number where the element with key k is located. + size_type bucket(const KeyT& key) const + { + const auto bucket = hash_key(key) & _mask; + const auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (EMH_EMPTY(_pairs, bucket)) + return 0; + else if (bucket == next_bucket) + return bucket + 1; + + const auto& bucket_key = EMH_KEY(_pairs, bucket); + return (hash_key(bucket_key) & _mask) + 1; + } + + //Returns the number of elements in bucket n. + size_type bucket_size(const size_type bucket) const + { + if (EMH_EMPTY(_pairs, bucket)) + return 0; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + next_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + size_type bucket_size = 1; + + //iterator each item in current main bucket + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) { + break; + } + bucket_size++; + next_bucket = nbucket; + } + return bucket_size; + } + + size_type get_main_bucket(const size_type bucket) const + { + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto& bucket_key = EMH_KEY(_pairs, bucket); + const auto main_bucket = hash_key(bucket_key) & _mask; + return main_bucket; + } + + size_type get_diss(size_type bucket, size_type next_bucket, const size_type slots) const + { + const int cahe_line_size = 64; + auto pbucket = reinterpret_cast<uint64_t>(&_pairs[bucket]); + auto pnext = reinterpret_cast<uint64_t>(&_pairs[next_bucket]); + if (pbucket / cahe_line_size == pnext / cahe_line_size) + return 0; + size_type diff = pbucket > pnext ? (pbucket - pnext) : (pnext - pbucket); + if (diff / cahe_line_size + 1 < slots) + return (diff / cahe_line_size + 1); + return slots - 1; + } + + int get_bucket_info(const size_type bucket, size_type steps[], const size_type slots) const + { + if (EMH_EMPTY(_pairs, bucket)) + return -1; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if ((hash_key(EMH_KEY(_pairs, bucket)) & _mask) != bucket) + return 0; + else if (next_bucket == bucket) + return 1; + + steps[get_diss(bucket, next_bucket, slots)] ++; + size_type bucket_size = 2; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + + steps[get_diss(nbucket, next_bucket, slots)] ++; + bucket_size ++; + next_bucket = nbucket; + } + + return bucket_size; + } + + void dump_statics(bool show_cache) const + { + const int slots = 128; + size_type buckets[slots + 1] = {0}; + size_type steps[slots + 1] = {0}; + char buff[1024 * 8]; + for (size_type bucket = 0; bucket < _num_buckets; ++bucket) { + auto bsize = get_bucket_info(bucket, steps, slots); + if (bsize >= 0) + buckets[bsize] ++; + } + + size_type sumb = 0, sums = 0, sumn = 0; + size_type miss = 0, finds = 0, bucket_coll = 0; + double lf = load_factor(), fk = 1.0 / exp(lf), sum_poisson = 0; + int bsize = sprintf (buff, "============== buckets size ration ========\n"); + + miss += _num_buckets - _num_filled; + for (int i = 1, factorial = 1; i < sizeof(buckets) / sizeof(buckets[0]); i++) { + double poisson = fk / factorial; factorial *= i; fk *= lf; + if (poisson > 1e-13 && i < 20) + sum_poisson += poisson * 100.0 * (i - 1) / i; + + const int64_t bucketsi = buckets[i]; + if (bucketsi == 0) + continue; + + sumb += bucketsi; + sumn += bucketsi * i; + bucket_coll += bucketsi * (i - 1); + finds += bucketsi * i * (i + 1) / 2; + miss += bucketsi * i * i; + auto errs = (bucketsi * 1.0 * i / _num_filled - poisson) * 100 / poisson; + bsize += sprintf(buff + bsize, " %2d %8ld %0.8lf|%0.2lf%% %2.3lf\n", + i, bucketsi, bucketsi * 1.0 * i / _num_filled, errs, sumn * 100.0 / _num_filled); + if (sumn >= _num_filled) + break; + } + + bsize += sprintf(buff + bsize, "========== collision miss ration ===========\n"); + for (size_type i = 0; show_cache && i < sizeof(steps) / sizeof(steps[0]); i++) { + sums += steps[i]; + if (steps[i] == 0) + continue; + if (steps[i] > 10) + bsize += sprintf(buff + bsize, " %2d %8u %0.2lf %.2lf\n", (int)i, steps[i], steps[i] * 100.0 / bucket_coll, sums * 100.0 / bucket_coll); + } + + if (sumb == 0) return; + + bsize += sprintf(buff + bsize, " _num_filled aver_size k.v size_kv = %u, %.2lf, %s.%s %zd\n", + _num_filled, _num_filled * 1.0 / sumb, typeid(KeyT).name(), typeid(ValueT).name(), sizeof(PairT)); + + bsize += sprintf(buff + bsize, " collision,poisson,cache_miss hit_find|hit_miss, load_factor = %.2lf%%,%.2lf%%,%.2lf%% %.2lf|%.2lf, %.2lf\n", + (bucket_coll * 100.0 / _num_filled), sum_poisson, (bucket_coll - steps[0]) * 100.0 / _num_filled, + finds * 1.0 / _num_filled, miss * 1.0 / _num_buckets, _num_filled * 1.0 / _num_buckets); + + bsize += sprintf(buff + bsize, "============== buckets size end =============\n"); + buff[bsize + 1] = 0; + +#ifdef EMH_LOG + EMH_LOG << __FUNCTION__ << "|" << buff << endl; +#else + puts(buff); +#endif + assert(sumn == _num_filled); + assert(sums == bucket_coll || !show_cache); + assert(bucket_coll == buckets[0]); + } +#endif + + // ------------------------------------------------------------ + template<typename Key = KeyT> + iterator find(const Key& key, size_t key_hash) noexcept + { + return {this, find_filled_hash(key, key_hash)}; + } + + template<typename Key = KeyT> + const_iterator find(const Key& key, size_t key_hash) const noexcept + { + return {this, find_filled_hash(key, key_hash)}; + } + + template<typename Key=KeyT> + iterator find(const Key& key) noexcept + { + return {this, find_filled_bucket(key)}; + } + + template<typename Key = KeyT> + const_iterator find(const Key& key) const noexcept + { + return {this, find_filled_bucket(key)}; + } + + template<typename Key = KeyT> + ValueT& at(const KeyT& key) + { + const auto bucket = find_filled_bucket(key); + //throw + return EMH_VAL(_pairs, bucket); + } + + template<typename Key = KeyT> + const ValueT& at(const KeyT& key) const + { + const auto bucket = find_filled_bucket(key); + //throw + return EMH_VAL(_pairs, bucket); + } + + template<typename Key = KeyT> + bool contains(const Key& key) const noexcept + { + return find_filled_bucket(key) != _num_buckets; + } + + template<typename Key = KeyT> + size_type count(const Key& key) const noexcept + { + return find_filled_bucket(key) != _num_buckets ? 1 : 0; + } + + template<typename Key = KeyT> + std::pair<iterator, iterator> equal_range(const Key& key) const noexcept + { + const auto found = {this, find_filled_bucket(key), true}; + if (found.bucket() == _num_buckets) + return { found, found }; + else + return { found, std::next(found) }; + } + + template<typename K=KeyT> + std::pair<const_iterator, const_iterator> equal_range(const K& key) const + { + const auto found = {this, find_filled_bucket(key), true}; + if (found.bucket() == _num_buckets) + return { found, found }; + else + return { found, std::next(found) }; + } + + void merge(HashMap& rhs) + { + if (empty()) { + *this = std::move(rhs); + return; + } + + for (auto rit = rhs.begin(); rit != rhs.end(); ) { + auto fit = find(rit->first); + if (fit.bucket() == _num_buckets) { + insert_unique(rit->first, std::move(rit->second)); + rit = rhs.erase(rit); + } else { + ++rit; + } + } + } + + /// Returns false if key isn't found. + bool try_get(const KeyT& key, ValueT& val) const noexcept + { + const auto bucket = find_filled_bucket(key); + const auto found = bucket != _num_buckets; + if (found) { + val = EMH_VAL(_pairs, bucket); + } + return found; + } + + /// Returns the matching ValueT or nullptr if k isn't found. + ValueT* try_get(const KeyT& key) noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? nullptr : &EMH_VAL(_pairs, bucket); + } + + /// Const version of the above + ValueT* try_get(const KeyT& key) const noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? nullptr : &EMH_VAL(_pairs, bucket); + } + + /// Convenience function. + ValueT get_or_return_default(const KeyT& key) const noexcept + { + const auto bucket = find_filled_bucket(key); + return bucket == _num_buckets ? ValueT() : EMH_VAL(_pairs, bucket); + } + + // ----------------------------------------------------- + template<typename K = KeyT, typename V = ValueT> + std::pair<iterator, bool> do_assign(K&& key, V&& val) + { + reserve(_num_filled); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::forward<K>(key), std::forward<V>(val), bucket); + } else { + EMH_VAL(_pairs, bucket) = std::move(val); + } + return { {this, bucket}, isempty }; + } + + std::pair<iterator, bool> do_insert(const value_type& value) + { + bool isempty; + const auto bucket = find_or_allocate(value.first, isempty); + if (isempty) { + EMH_NEW(value.first, value.second, bucket); + } + return { {this, bucket}, isempty }; + } + + std::pair<iterator, bool> do_insert(value_type&& value) + { + bool isempty; + const auto bucket = find_or_allocate(value.first, isempty); + if (isempty) { + EMH_NEW(std::forward<KeyT>(value.first), std::forward<ValueT>(value.second), bucket); + } + return { {this, bucket}, isempty }; + } + + template<typename K = KeyT, typename V = ValueT> + std::pair<iterator, bool> do_insert(K&& key, V&& val) + { + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::forward<K>(key), std::forward<V>(val), bucket); + } + return { {this, bucket}, isempty }; + } + + std::pair<iterator, bool> insert(const value_type& value) + { + check_expand_need(); + return do_insert(value); + } + + std::pair<iterator, bool> insert(value_type && value) + { + check_expand_need(); + return do_insert(std::move(value)); + } + + void insert(std::initializer_list<value_type> ilist) + { + reserve(ilist.size() + _num_filled); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template <typename Iter> + void insert(Iter first, Iter last) + { + reserve(std::distance(first, last) + _num_filled); + for (auto it = first; it != last; ++it) + do_insert(it->first, it->second); + } + +#if 0 + template <typename Iter> + void insert2(Iter begin, Iter end) + { + Iter citbeg = begin; + Iter citend = begin; + reserve(std::distance(begin, end) + _num_filled); + for (; begin != end; ++begin) { + if (try_insert_mainbucket(begin->first, begin->second) == INACTIVE) { + std::swap(*begin, *citend++); + } + } + + for (; citbeg != citend; ++citbeg) + insert(*citbeg); + } + + size_type try_insert_mainbucket(const KeyT& key, const ValueT& val) + { + const auto bucket = hash_key(key) & _mask; + if (!EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + EMH_NEW(key, val, bucket); + return bucket; + } +#endif + + template <typename Iter> + void insert_unique(Iter begin, Iter end) + { + reserve(std::distance(begin, end) + _num_filled); + for (; begin != end; ++begin) + do_insert_unqiue(*begin); + } + + /// Same as above, but contains(key) MUST be false + size_type insert_unique(KeyT&& key, ValueT&& val) + { + return do_insert_unqiue(std::move(key), std::forward<ValueT>(val)); + } + + size_type insert_unique(const KeyT& key, ValueT&& val) + { + return do_insert_unqiue(key, std::forward<ValueT>(val)); + } + + size_type insert_unique(value_type&& value) + { + return do_insert_unqiue(std::move(value.first), std::move(value.second)); + } + + size_type insert_unique(const value_type& value) + { + return do_insert_unqiue(value.first, value.second); + } + + template<typename K, typename V> + inline size_type do_insert_unqiue(K&& key, V&& val) + { + check_expand_need(); + auto bucket = find_unique_bucket(key); + EMH_NEW(std::forward<K>(key), std::forward<V>(val), bucket); + return bucket; + } + + std::pair<iterator, bool> insert_or_assign(const KeyT& key, ValueT&& val) { return do_assign(key, std::forward<ValueT>(val)); } + std::pair<iterator, bool> insert_or_assign(KeyT&& key, ValueT&& val) { return do_assign(std::move(key), std::forward<ValueT>(val)); } + + template <typename... Args> + inline std::pair<iterator, bool> emplace(Args&&... args) + { + check_expand_need(); + return do_insert(std::forward<Args>(args)...); + } + + template <class... Args> + iterator emplace_hint(const_iterator hint, Args&&... args) + { + (void)hint; + check_expand_need(); + return do_insert(std::forward<Args>(args)...).first; + } + + template<class... Args> + std::pair<iterator, bool> try_emplace(const KeyT& key, Args&&... args) + { + check_expand_need(); + return do_insert(key, std::forward<Args>(args)...).first; + } + + template<class... Args> + std::pair<iterator, bool> try_emplace(KeyT&& key, Args&&... args) + { + check_expand_need(); + return do_insert(std::forward<KeyT>(key), std::forward<Args>(args)...).first; + } + + template <class... Args> + inline size_type emplace_unique(Args&&... args) + { + return insert_unique(std::forward<Args>(args)...); + } + + /* Check if inserting a new value rather than overwriting an old entry */ + ValueT& operator[](const KeyT& key) + { + check_expand_need(); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(key, std::move(ValueT()), bucket); + } + + return EMH_VAL(_pairs, bucket); + } + + ValueT& operator[](KeyT&& key) + { + check_expand_need(); + + bool isempty; + const auto bucket = find_or_allocate(key, isempty); + if (isempty) { + EMH_NEW(std::move(key), std::move(ValueT()), bucket); + } + + return EMH_VAL(_pairs, bucket); + } + + // ------------------------------------------------------- + /// Erase an element from the hash table. + /// return 0 if element was not found + template<typename Key = KeyT> + size_type erase(const Key& key) + { + const auto bucket = erase_key(key); + if (bucket == INACTIVE) + return 0; + + clear_bucket(bucket); + return 1; + } + + //iterator erase const_iterator + iterator erase(const_iterator cit) + { + iterator it(cit); + return erase(it); + } + + /// Erase an element typedef an iterator. + /// Returns an iterator to the next element (or end()). + iterator erase(iterator it) + { + const auto bucket = erase_bucket(it._bucket); + clear_bucket(bucket); + if (bucket == it._bucket) { + return ++it; + } else { + //erase main bucket as next + it.clear(bucket); + return it; + } + } + + /// Erase an element typedef an iterator without return next iterator + void _erase(const_iterator it) + { + const auto bucket = erase_bucket(it._bucket); + clear_bucket(bucket); + } + + iterator erase(const_iterator first, const_iterator last) + { + auto iend = cend(); + auto next = first; + for (; next.bucket() < last.bucket() && next != iend; ) + next = erase(next); + + return {this, next.bucket()}; + } + + template<typename Pred> + size_type erase_if(Pred pred) + { + auto old_size = size(); + for (auto it = begin(), last = end(); it != last; ) { + if (pred(*it)) + it = erase(it); + else + ++it; + } + return old_size - size(); + } + + static constexpr bool is_triviall_destructable() + { +#if __cplusplus >= 201402L || _MSC_VER > 1600 + return !(std::is_trivially_destructible<KeyT>::value && std::is_trivially_destructible<ValueT>::value); +#else + return !(std::is_pod<KeyT>::value && std::is_pod<ValueT>::value); +#endif + } + + static constexpr bool is_copy_trivially() + { +#if __cplusplus >= 201402L || _MSC_VER > 1600 + return (std::is_trivially_copyable<KeyT>::value && std::is_trivially_copyable<ValueT>::value); +#else + return (std::is_pod<KeyT>::value && std::is_pod<ValueT>::value); +#endif + } + + void clearkv() + { + if (is_triviall_destructable()) { + for (auto it = cbegin(); _num_filled; ++it) + clear_bucket(it.bucket()); + } + } + + /// Remove all elements, keeping full capacity. + void clear() + { + if (!is_triviall_destructable() && _num_filled) + memset(_bitmask, 0xFFFFFFFF, (_num_buckets + 7) / 8); + else if (_num_filled) + clearkv(); + + //EMH_BUCKET(_pairs, _num_buckets) = 0; //_last + _num_filled = 0; + } + + void shrink_to_fit() + { + rehash(_num_filled); + } + + /// Make room for this many elements + bool reserve(uint64_t num_elems) + { + const auto required_buckets = (num_elems * _mlf >> 27); + if (EMH_LIKELY(required_buckets < _num_buckets)) + return false; + +#if EMH_HIGH_LOAD + if (required_buckets < 64 && _num_filled < _num_buckets) + return false; +#endif + +#if EMH_STATIS + if (_num_filled > EMH_STATIS) dump_statics(true); +#endif + rehash(required_buckets + 2); + return true; + } + + void rehash(uint64_t required_buckets) + { + if (required_buckets < _num_filled) + return; + + auto num_buckets = _num_filled > (1u << 16) ? (1u << 16) : 2u; + while (num_buckets < required_buckets) { num_buckets *= 2; } + + //TODO: throwOverflowError + auto old_num_filled = _num_filled; + auto old_mask = _num_buckets - 1; + auto old_pairs = _pairs; + auto* obmask = _bitmask; + + _num_filled = 0; + _num_buckets = num_buckets; + _mask = num_buckets - 1; + + _pairs = (PairT*)malloc(AllocSize(_num_buckets)); + memset((char*)(_pairs + _num_buckets), 0, sizeof(PairT) * EPACK_SIZE); + + _bitmask = decltype(_bitmask)(_pairs + EPACK_SIZE + num_buckets); + + const auto mask_byte = (num_buckets + 7) / 8; + memset(_bitmask, 0xFFFFFFFF, mask_byte); + memset(((char*)_bitmask) + mask_byte, 0, BIT_PACK); + if (num_buckets < 8) + _bitmask[0] = (1 << num_buckets) - 1; + + //for (size_type src_bucket = 0; _num_filled < old_num_filled; src_bucket++) { + for (size_type src_bucket = old_mask; _num_filled < old_num_filled; src_bucket --) { + if (obmask[src_bucket / MASK_BIT] & (EMH_MASK(src_bucket))) + continue; + + auto& key = EMH_KEY(old_pairs, src_bucket); + const auto bucket = find_unique_bucket(key); + EMH_NEW(std::move(key), std::move(EMH_VAL(old_pairs, src_bucket)), bucket); + if (is_triviall_destructable()) + old_pairs[src_bucket].~PairT(); + } + +#if EMH_REHASH_LOG + if (_num_filled > EMH_REHASH_LOG) { + auto mbucket = bucket_main(); + char buff[255] = {0}; + sprintf(buff, " _num_filled/aver_size/K.V/pack/ = %u/%2.lf/%s.%s/%zd", + _num_filled, double (_num_filled) / mbucket, typeid(KeyT).name(), typeid(ValueT).name(), sizeof(_pairs[0])); +#ifdef EMH_LOG + static size_t ihashs = 0; + EMH_LOG << "rhash_nums = " << ihashs ++ << "|" <<__FUNCTION__ << "|" << buff << endl; +#else + puts(buff); +#endif + } +#endif + + free(old_pairs); + assert(old_num_filled == _num_filled); + } + +private: + // Can we fit another element? + inline bool check_expand_need() + { + return reserve(_num_filled); + } + + void clear_bucket(size_type bucket) + { + EMH_CLS(bucket); + if (is_triviall_destructable()) + _pairs[bucket].~PairT(); + _num_filled--; + } + +#if 1 + //template<typename UType, typename std::enable_if<std::is_integral<UType>::value, size_type>::type = 0> + template<typename UType> + size_type erase_key(const UType& key) + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto eqkey = _eq(key, EMH_KEY(_pairs, bucket)); + if (eqkey) { + if (next_bucket == bucket) + return bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (is_copy_trivially()) + EMH_PKV(_pairs, bucket) = EMH_PKV(_pairs, next_bucket); + else + EMH_PKV(_pairs, bucket).swap(EMH_PKV(_pairs, next_bucket)); + + EMH_BUCKET(_pairs, bucket) = (nbucket == next_bucket) ? bucket : nbucket; + return next_bucket; + } else if (next_bucket == bucket) + return INACTIVE; + /* else if (EMH_UNLIKELY(bucket != hash_key(EMH_KEY(_pairs, bucket)) & _mask)) + return INACTIVE; + */ + + auto prev_bucket = bucket; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (_eq(key, EMH_KEY(_pairs, next_bucket))) { + EMH_BUCKET(_pairs, prev_bucket) = (nbucket == next_bucket) ? prev_bucket : nbucket; + return next_bucket; + } + + if (nbucket == next_bucket) + break; + prev_bucket = next_bucket; + next_bucket = nbucket; + } + + return INACTIVE; + } +#else + template<typename UType, typename std::enable_if<!std::is_integral<UType>::value, size_type>::type = 0> + size_type erase_key(const UType& key) + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return INACTIVE; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _eq(key, EMH_KEY(_pairs, bucket)) ? bucket : INACTIVE; +// else if (bucket != hash_key(EMH_KEY(_pairs, bucket))) +// return INACTIVE; + + //find erase key and swap to last bucket + size_type prev_bucket = bucket, find_bucket = INACTIVE; + next_bucket = bucket; + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (_eq(key, EMH_KEY(_pairs, next_bucket))) { + find_bucket = next_bucket; + if (nbucket == next_bucket) { + EMH_BUCKET(_pairs, prev_bucket) = prev_bucket; + break; + } + } + if (nbucket == next_bucket) { + if (find_bucket != INACTIVE) { + EMH_PKV(_pairs, find_bucket).swap(EMH_PKV(_pairs, nbucket)); +// EMH_PKV(_pairs, find_bucket) = EMH_PKV(_pairs, nbucket); + EMH_BUCKET(_pairs, prev_bucket) = prev_bucket; + find_bucket = nbucket; + } + break; + } + prev_bucket = next_bucket; + next_bucket = nbucket; + } + + return find_bucket; + } +#endif + + size_type erase_bucket(const size_type bucket) + { + const auto next_bucket = EMH_BUCKET(_pairs, bucket); + const auto main_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + if (bucket == main_bucket) { + if (bucket != next_bucket) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (is_copy_trivially()) + EMH_PKV(_pairs, bucket) = EMH_PKV(_pairs, next_bucket); + else + EMH_PKV(_pairs, bucket).swap(EMH_PKV(_pairs, next_bucket)); + EMH_BUCKET(_pairs, bucket) = (nbucket == next_bucket) ? bucket : nbucket; + } + return next_bucket; + } + + const auto prev_bucket = find_prev_bucket(main_bucket, bucket); + EMH_BUCKET(_pairs, prev_bucket) = (bucket == next_bucket) ? prev_bucket : next_bucket; + return bucket; + } + + // Find the bucket with this key, or return bucket size + template<typename K = KeyT> + size_type find_filled_hash(const K& key, const size_t key_hash) const + { + const auto bucket = key_hash & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + else if (next_bucket == bucket) + return _num_buckets; + + while (true) { + if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + next_bucket = nbucket; + } + + return _num_buckets; + } + + // Find the bucket with this key, or return bucket size + template<typename K = KeyT> + size_type find_filled_bucket(const K& key) const + { + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + +#if 1 + if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _num_buckets; +#elif 0 + else if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + else if (next_bucket == bucket) + return _num_buckets; + + else if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return _num_buckets; + next_bucket = nbucket; +#elif 0 + const auto bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return _num_buckets; + else if (_eq(key, EMH_KEY(_pairs, bucket))) + return bucket; + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket == bucket) + return _num_buckets; +#endif +// else if (bucket != (hash_key(bucket_key) & _mask)) +// return _num_buckets; + + while (true) { + if (_eq(key, EMH_KEY(_pairs, next_bucket))) + return next_bucket; + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return _num_buckets; + next_bucket = nbucket; + } + + return 0; + } + + //kick out bucket and find empty to occpuy + //it will break the orgin link and relnik again. + //before: main_bucket-->prev_bucket --> bucket --> next_bucket + //atfer : main_bucket-->prev_bucket --> (removed)--> new_bucket--> next_bucket + size_type kickout_bucket(const size_type kmain, const size_type kbucket) + { + const auto next_bucket = EMH_BUCKET(_pairs, kbucket); + const auto new_bucket = find_empty_bucket(next_bucket, kbucket); + const auto prev_bucket = find_prev_bucket(kmain, kbucket); + new(_pairs + new_bucket) PairT(std::move(_pairs[kbucket])); + if (is_triviall_destructable()) + _pairs[kbucket].~PairT(); + + if (next_bucket == kbucket) + EMH_BUCKET(_pairs, new_bucket) = new_bucket; + EMH_BUCKET(_pairs, prev_bucket) = new_bucket; + + //set new bucket bit + EMH_SET(new_bucket); + + //clear kickout bit + //EMH_CLS(kbucket); + return kbucket; + } + +/* +** inserts a new key into a hash table; first check whether key's main +** bucket/position is free. If not, check whether colliding node/bucket is in its main +** position or not: if it is not, move colliding bucket to an empty place and +** put new key in its main position; otherwise (colliding bucket is in its main +** position), new key goes to an empty position. ***/ + + template<typename Key=KeyT> + size_type find_or_allocate(const Key& key, bool& isempty) + { + const auto bucket = hash_key(key) & _mask; + const auto& bucket_key = EMH_KEY(_pairs, bucket); + if (EMH_EMPTY(_pairs, bucket)) { + isempty = true; + return bucket; + } + else if (_eq(key, bucket_key)) { + isempty = false; + return bucket; + } + + isempty = true; + auto next_bucket = EMH_BUCKET(_pairs, bucket); + //check current bucket_key is in main bucket or not + const auto kmain_bucket = hash_key(bucket_key) & _mask; + if (kmain_bucket != bucket) + return kickout_bucket(kmain_bucket, bucket); + else if (next_bucket == bucket) + return EMH_BUCKET(_pairs, next_bucket) = find_empty_bucket(next_bucket, bucket); + +#if EMH_LRU_SET + auto prev_bucket = bucket; +#endif + //find next linked bucket and check key, if lru is set then swap current key with prev_bucket + while (true) { + if (EMH_UNLIKELY(_eq(key, EMH_KEY(_pairs, next_bucket)))) { + isempty = false; +#if EMH_LRU_SET + EMH_PKV(_pairs, next_bucket).swap(EMH_PKV(_pairs, prev_bucket)); + return prev_bucket; +#else + return next_bucket; +#endif + } + +#if EMH_LRU_SET + prev_bucket = next_bucket; +#endif + + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + break; + next_bucket = nbucket; + } + + //find a new empty and link it to tail, TODO link after main bucket? + const auto new_bucket = find_empty_bucket(next_bucket, bucket);// : find_empty_bucket(next_bucket); + return EMH_BUCKET(_pairs, next_bucket) = new_bucket; + } + + // key is not in this map. Find a place to put it. + size_type find_empty_bucket(const size_type bucket_from, const size_t main_bucket) + { +#if 1 || __arm64__ || __aarch64__ + const auto boset = bucket_from % MASK_BIT; + auto* const align = _bitmask + bucket_from / MASK_BIT; + const auto bmask = ((size_t)align[1] << (MASK_BIT - boset)) | (align[0] >> boset); +#else + const auto boset = bucket_from % 8; + auto* const align = (uint8_t*)_bitmask + bucket_from / 8; + const auto bmask = *(size_t*)(align) >> boset; +#endif + if (EMH_LIKELY(bmask != 0)) + return bucket_from + CTZ(bmask); + + const auto qmask = _mask / SIZE_BIT; + if (1) { + const auto step = (bucket_from - SIZE_BIT / 2) & qmask; + const auto bmask3 = *((size_t*)_bitmask + step); + if (bmask3 != 0) + return step * SIZE_BIT + CTZ(bmask3); + } + + auto& _last = EMH_BUCKET(_pairs, _num_buckets); + for (; ; ) { //2.4.7 + const auto bmask2 = *((size_t*)_bitmask + _last); + if (bmask2 != 0) + return _last * SIZE_BIT + CTZ(bmask2); + + const auto next1 = (qmask / 2 + _last) & qmask; + const auto bmask1 = *((size_t*)_bitmask + next1); + if (bmask1 != 0) { + //_last = next1; + return next1 * SIZE_BIT + CTZ(bmask1); + } + _last = (_last + 1) & qmask; + } + return 0; + } + + size_type find_last_bucket(size_type main_bucket) const + { + auto next_bucket = EMH_BUCKET(_pairs, main_bucket); + if (next_bucket == main_bucket) + return main_bucket; + + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == next_bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type find_prev_bucket(size_type main_bucket, const size_type bucket) const + { + auto next_bucket = EMH_BUCKET(_pairs, main_bucket); + if (next_bucket == bucket) + return main_bucket; + + while (true) { + const auto nbucket = EMH_BUCKET(_pairs, next_bucket); + if (nbucket == bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type find_unique_bucket(const KeyT& key) + { + const size_type bucket = hash_key(key) & _mask; + if (EMH_EMPTY(_pairs, bucket)) + return bucket; + + //check current bucket_key is in main bucket or not + const auto kmain_bucket = hash_key(EMH_KEY(_pairs, bucket)) & _mask; + if (EMH_UNLIKELY(kmain_bucket != bucket)) + return kickout_bucket(kmain_bucket, bucket); + + auto next_bucket = EMH_BUCKET(_pairs, bucket); + if (next_bucket != bucket) + next_bucket = find_last_bucket(next_bucket); + + //find a new empty and link it to tail + return EMH_BUCKET(_pairs, next_bucket) = find_empty_bucket(next_bucket, bucket); + } + + static constexpr uint64_t KC = UINT64_C(11400714819323198485); + static inline uint64_t hash64(uint64_t key) + { +#if __SIZEOF_INT128__ && EMH_FIBONACCI_HASH == 1 + __uint128_t r = key; r *= KC; + return (uint64_t)(r >> 64) + (uint64_t)r; +#elif EMH_FIBONACCI_HASH == 2 + //MurmurHash3Mixer + uint64_t h = key; + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + return h; +#elif _WIN64 && EMH_FIBONACCI_HASH == 1 + uint64_t high; + return _umul128(key, KC, &high) + high; +#elif EMH_FIBONACCI_HASH == 3 + auto ror = (key >> 32) | (key << 32); + auto low = key * 0xA24BAED4963EE407ull; + auto high = ror * 0x9FB21C651E98DF25ull; + auto mix = low + high; + return mix; +#elif EMH_FIBONACCI_HASH == 1 + uint64_t r = key * UINT64_C(0xca4bcaa75ec3f625); + return (r >> 32) + r; +#else + uint64_t x = key; + x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); + x = x ^ (x >> 31); + return x; +#endif + } + + template<typename UType, typename std::enable_if<std::is_integral<UType>::value, size_type>::type = 0> + inline size_type hash_key(const UType key) const + { +#ifdef EMH_FIBONACCI_HASH + return hash64(key); +#elif EMH_IDENTITY_HASH + return key + (key >> (sizeof(UType) * 4)); +#elif EMH_WYHASH64 + return wyhash64(key, KC); +#else + return (size_type)_hasher(key); +#endif + } + + template<typename UType, typename std::enable_if<std::is_same<UType, std::string>::value, size_type>::type = 0> + inline size_type hash_key(const UType& key) const + { +#ifdef WYHASH_LITTLE_ENDIAN + return wyhash(key.data(), key.size(), key.size()); +#else + return (size_type)_hasher(key); +#endif + } + + template<typename UType, typename std::enable_if<!std::is_integral<UType>::value && !std::is_same<UType, std::string>::value, size_type>::type = 0> + inline size_type hash_key(const UType& key) const + { +#ifdef EMH_FIBONACCI_HASH + return _hasher(key) * KC; +#else + return (size_type)_hasher(key); +#endif + } + +private: + uint32_t* _bitmask; + PairT* _pairs; + HashT _hasher; + EqT _eq; + size_type _mask; + size_type _num_buckets; + + size_type _num_filled; + uint32_t _mlf; + +private: + static constexpr uint32_t BIT_PACK = sizeof(_bitmask[0]) * 2; + static constexpr uint32_t MASK_BIT = sizeof(_bitmask[0]) * 8; + static constexpr uint32_t SIZE_BIT = sizeof(size_t) * 8; + static constexpr uint32_t EPACK_SIZE = sizeof(PairT) >= sizeof(size_t) == 0 ? 1 : 2; // > 1 +}; +} // namespace emhash +#if __cplusplus >= 201103L +//template <class Key, class Val> using emhash7 = emhash7::HashMap<Key, Val, std::hash<Key>, std::equal_to<Key>>; +#endif + +//TODO +//2. improve rehash and find miss performance(reduce peak memory) +//3. dump or Serialization interface +//4. node hash map support +//5. support load_factor > 1.0 +//6. add grow ration +//8. ... https://godbolt.org/ diff --git a/benchmarks/external/emhash/wyhash.h b/benchmarks/external/emhash/wyhash.h new file mode 100644 index 00000000..b89352a7 --- /dev/null +++ b/benchmarks/external/emhash/wyhash.h @@ -0,0 +1,272 @@ +// This is free and unencumbered software released into the public domain under The Unlicense (http://unlicense.org/) +// main repo: https://github.com/wangyi-fudan/wyhash +// author: 王一 Wang Yi <[email protected]> +// contributors: Reini Urban, Dietrich Epp, Joshua Haberman, Tommy Ettinger, Daniel Lemire, Otmar Ertl, cocowalla, leo-yuriev, Diego Barrios Romero, paulie-g, dumblob, Yann Collet, ivte-ms, hyb, James Z.M. Gao, easyaspi314 (Devin), TheOneric + +/* quick example: + uint64_t _wyp[4]; + make_secret(time(NULL),_wyp); + string s="fjsakfdsjkf"; + uint64_t hash=wyhash(s.c_str(), s.size(), 0, _wyp); +*/ + +#ifndef wyhash_final_version_3 +#define wyhash_final_version_3 + +#ifndef WYHASH_CONDOM +//protections that produce different results: +//1: normal valid behavior +//2: extra protection against entropy loss (probability=2^-63), aka. "blind multiplication" +#define WYHASH_CONDOM 1 +#endif + +#ifndef WYHASH_32BIT_MUM +//0: normal version, slow on 32 bit systems +//1: faster on 32 bit systems but produces different results, incompatible with wy2u0k function +#define WYHASH_32BIT_MUM 0 +#endif + +//includes +#include <stdint.h> +#include <string.h> +#if defined(_MSC_VER) && defined(_M_X64) + #include <intrin.h> + #pragma intrinsic(_umul128) +#endif + +//likely and unlikely macros +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) + #define _likely_(x) __builtin_expect(x,1) + #define _unlikely_(x) __builtin_expect(x,0) +#else + #define _likely_(x) (x) + #define _unlikely_(x) (x) +#endif + +//128bit multiply function +static inline uint64_t _wyrot(uint64_t x) { return (x>>32)|(x<<32); } +static inline void _wymum(uint64_t *A, uint64_t *B){ +#if(WYHASH_32BIT_MUM) + uint64_t hh=(*A>>32)*(*B>>32), hl=(*A>>32)*(uint32_t)*B, lh=(uint32_t)*A*(*B>>32), ll=(uint64_t)(uint32_t)*A*(uint32_t)*B; + #if(WYHASH_CONDOM>1) + *A^=_wyrot(hl)^hh; *B^=_wyrot(lh)^ll; + #else + *A=_wyrot(hl)^hh; *B=_wyrot(lh)^ll; + #endif +#elif defined(__SIZEOF_INT128__) + __uint128_t r=*A; r*=*B; + #if(WYHASH_CONDOM>1) + *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); + #else + *A=(uint64_t)r; *B=(uint64_t)(r>>64); + #endif +#elif defined(_MSC_VER) && defined(_M_X64) + #if(WYHASH_CONDOM>1) + uint64_t a, b; + a=_umul128(*A,*B,&b); + *A^=a; *B^=b; + #else + *A=_umul128(*A,*B,B); + #endif +#else + uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; + uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl; + lo=t+(rm1<<32); c+=lo<t; hi=rh+(rm0>>32)+(rm1>>32)+c; + #if(WYHASH_CONDOM>1) + *A^=lo; *B^=hi; + #else + *A=lo; *B=hi; + #endif +#endif +} + +//multiply and xor mix function, aka MUM +static inline uint64_t _wymix(uint64_t A, uint64_t B){ _wymum(&A,&B); return A^B; } + +//endian macros +#ifndef WYHASH_LITTLE_ENDIAN + #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 1 + #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 0 + #else + #warning could not determine endianness! Falling back to little endian. + #define WYHASH_LITTLE_ENDIAN 1 + #endif +#endif + +//read functions +#if (WYHASH_LITTLE_ENDIAN) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v;} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return v;} +#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return __builtin_bswap32(v);} +#elif defined(_MSC_VER) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return _byteswap_ulong(v);} +#else +static inline uint64_t _wyr8(const uint8_t *p) { + uint64_t v; memcpy(&v, p, 8); + return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); +} +static inline uint64_t _wyr4(const uint8_t *p) { + uint32_t v; memcpy(&v, p, 4); + return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); +} +#endif +static inline uint64_t _wyr3(const uint8_t *p, size_t k) { return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1];} +//wyhash main function +static inline uint64_t wyhash(const void *key, size_t len, uint64_t seed, const uint64_t *secret){ + const uint8_t *p=(const uint8_t *)key; seed^=*secret; uint64_t a, b; + if(_likely_(len<=16)){ + if(_likely_(len>=4)){ a=(_wyr4(p)<<32)|_wyr4(p+((len>>3)<<2)); b=(_wyr4(p+len-4)<<32)|_wyr4(p+len-4-((len>>3)<<2)); } + else if(_likely_(len>0)){ a=_wyr3(p,len); b=0;} + else a=b=0; + } + else{ + size_t i=len; + if(_unlikely_(i>48)){ + uint64_t see1=seed, see2=seed; + do{ + seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); + see1=_wymix(_wyr8(p+16)^secret[2],_wyr8(p+24)^see1); + see2=_wymix(_wyr8(p+32)^secret[3],_wyr8(p+40)^see2); + p+=48; i-=48; + }while(_likely_(i>48)); + seed^=see1^see2; + } + while(_unlikely_(i>16)){ seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); i-=16; p+=16; } + a=_wyr8(p+i-16); b=_wyr8(p+i-8); + } + return _wymix(secret[1]^len,_wymix(a^secret[1],b^seed)); +} + +//the default secret parameters +static const uint64_t _wyp[4] = {0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x8ebc6af09c88c6e3ull, 0x589965cc75374cc3ull}; + +static inline uint64_t wyhash(const void *key, size_t len, uint64_t seed) { return wyhash(key, len, seed, _wyp); } + +//a useful 64bit-64bit mix function to produce deterministic pseudo random numbers that can pass BigCrush and PractRand +static inline uint64_t wyhash64(uint64_t A, uint64_t B){ A^=0xa0761d6478bd642full; B^=0xe7037ed1a0b428dbull; _wymum(&A,&B); return _wymix(A^0xa0761d6478bd642full,B^0xe7037ed1a0b428dbull);} + +//The wyrand PRNG that pass BigCrush and PractRand +static inline uint64_t wyrand(uint64_t *seed){ *seed+=0xa0761d6478bd642full; return _wymix(*seed,*seed^0xe7037ed1a0b428dbull);} + +//convert any 64 bit pseudo random numbers to uniform distribution [0,1). It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>12)*_wynorm;} + +//convert any 64 bit pseudo random numbers to APPROXIMATE Gaussian distribution. It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0;} + +#if(!WYHASH_32BIT_MUM) +//fast range integer random number generation on [0,k) credit to Daniel Lemire. May not work when WYHASH_32BIT_MUM=1. It can be combined with wyrand, wyhash64 or wyhash. +static inline uint64_t wy2u0k(uint64_t r, uint64_t k){ _wymum(&r,&k); return k; } +#endif + +//make your own secret +static inline void make_secret(uint64_t seed, uint64_t *secret){ + uint8_t c[] = {15, 23, 27, 29, 30, 39, 43, 45, 46, 51, 53, 54, 57, 58, 60, 71, 75, 77, 78, 83, 85, 86, 89, 90, 92, 99, 101, 102, 105, 106, 108, 113, 114, 116, 120, 135, 139, 141, 142, 147, 149, 150, 153, 154, 156, 163, 165, 166, 169, 170, 172, 177, 178, 180, 184, 195, 197, 198, 201, 202, 204, 209, 210, 212, 216, 225, 226, 228, 232, 240 }; + for(size_t i=0;i<4;i++){ + uint8_t ok; + do{ + ok=1; secret[i]=0; + for(size_t j=0;j<64;j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<<j; + if(secret[i]%2==0){ ok=0; continue; } + for(size_t j=0;j<i;j++) { +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) + if(__builtin_popcountll(secret[j]^secret[i])!=32){ ok=0; break; } +#elif defined(_MSC_VER) && defined(_M_X64) + if(_mm_popcnt_u64(secret[j]^secret[i])!=32){ ok=0; break; } +#else + //manual popcount + uint64_t x = secret[j]^secret[i]; + x -= (x >> 1) & 0x5555555555555555; + x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); + x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; + x = (x * 0x0101010101010101) >> 56; + if(x!=32){ ok=0; break; } +#endif + } + }while(!ok); + } +} + +/* This is world's fastest hash map: 2x faster than bytell_hash_map. + It does not store the keys, but only the hash/signature of keys. + First we use pos=hash1(key) to approximately locate the bucket. + Then we search signature=hash2(key) from pos linearly. + If we find a bucket with matched signature we report the bucket + Or if we meet a bucket whose signature=0, we report a new position to insert + The signature collision probability is very low as we usually searched N~10 buckets. + By combining hash1 and hash2, we acturally have 128 bit anti-collision strength. + hash1 and hash2 can be the same function, resulting lower collision resistance but faster. + The signature is 64 bit, but can be modified to 32 bit if necessary for save space. + The above two can be activated by define WYHASHMAP_WEAK_SMALL_FAST + simple examples: + const size_t size=213432; + vector<wyhashmap_t> idx(size); // allocate the index of fixed size. idx MUST be zeroed. + vector<value_class> value(size); // we only care about the index, user should maintain his own value vectors. + string key="dhskfhdsj" // the object to be inserted into idx + size_t pos=wyhashmap(idx.data(), idx.size(), key.c_str(), key.size(), 1); // get the position and insert + if(pos<size) value[pos]++; // we process the vallue + else cerr<<"map is full\n"; + pos=wyhashmap(idx.data(), idx.size(), key.c_str(), key.size(), 0); // just lookup by setting insert=0 + if(pos<size) value[pos]++; // we process the vallue + else cerr<<"the key does not exist\n"; +*/ +/* +#ifdef WYHASHMAP_WEAK_SMALL_FAST // for small hashmaps whose size < 2^24 and acceptable collision +typedef uint32_t wyhashmap_t; +#else +typedef uint64_t wyhashmap_t; +#endif + +static inline size_t wyhashmap(wyhashmap_t *idx, size_t idx_size, const void *key, size_t key_size, uint8_t insert, uint64_t *secret){ + size_t i=1; uint64_t h2; wyhashmap_t sig; + do{ sig=h2=wyhash(key,key_size,i,secret); i++; }while(_unlikely_(!sig)); +#ifdef WYHASHMAP_WEAK_SMALL_FAST + size_t i0=wy2u0k(h2,idx_size); +#else + size_t i0=wy2u0k(wyhash(key,key_size,0,secret),idx_size); +#endif + for(i=i0; i<idx_size&&idx[i]&&idx[i]!=sig; i++); + if(_unlikely_(i==idx_size)){ + for(i=0; i<i0&&idx[i]&&idx[i]!=sig; i++); + if(i==i0) return idx_size; + } + if(!idx[i]){ + if(insert) idx[i]=sig; + else return idx_size; + } + return i; +} +*/ +#endif + +/* The Unlicense +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to <http://unlicense.org/> +*/ diff --git a/benchmarks/external/parallel_hashmap/btree.h b/benchmarks/external/parallel_hashmap/btree.h deleted file mode 100644 index b8c95433..00000000 --- a/benchmarks/external/parallel_hashmap/btree.h +++ /dev/null @@ -1,4050 +0,0 @@ -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_ -#define PHMAP_BTREE_BTREE_CONTAINER_H_ - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4355) // 'this': used in base member initializer list - #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - - -#include <cstdint> -#include <cstdlib> -#include <cstring> -#include <limits> -#include <new> - -#include "phmap_fwd_decl.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include <string_view> -#endif - -// MSVC constructibility traits do not detect destructor properties and so our -// implementations should not use them as a source-of-truth. -#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) - #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 -#endif - -namespace phmap { - - // Defined and documented later on in this file. - template <typename T> - struct is_trivially_destructible; - - // Defined and documented later on in this file. - template <typename T> - struct is_trivially_move_assignable; - - namespace type_traits_internal { - - // Silence MSVC warnings about the destructor being defined as deleted. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(push) - #pragma warning(disable : 4624) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template <class T> - union SingleMemberUnion { - T t; - }; - - // Restore the state of the destructor warning that was silenced above. -#if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(pop) -#endif // defined(_MSC_VER) && !defined(__GNUC__) - - template <class T> - struct IsTriviallyMoveConstructibleObject - : std::integral_constant< - bool, std::is_move_constructible< - type_traits_internal::SingleMemberUnion<T>>::value && - phmap::is_trivially_destructible<T>::value> {}; - - template <class T> - struct IsTriviallyCopyConstructibleObject - : std::integral_constant< - bool, std::is_copy_constructible< - type_traits_internal::SingleMemberUnion<T>>::value && - phmap::is_trivially_destructible<T>::value> {}; - - template <class T> - struct IsTriviallyMoveAssignableReference : std::false_type {}; - - template <class T> - struct IsTriviallyMoveAssignableReference<T&> - : phmap::is_trivially_move_assignable<T>::type {}; - - template <class T> - struct IsTriviallyMoveAssignableReference<T&&> - : phmap::is_trivially_move_assignable<T>::type {}; - - } // namespace type_traits_internal - - - template <typename... Ts> - using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; - - - template <typename T> - struct is_function - : std::integral_constant< - bool, !(std::is_reference<T>::value || - std::is_const<typename std::add_const<T>::type>::value)> {}; - - - namespace type_traits_internal { - - template <typename T> - class is_trivially_copyable_impl { - using ExtentsRemoved = typename std::remove_all_extents<T>::type; - static constexpr bool kIsCopyOrMoveConstructible = - std::is_copy_constructible<ExtentsRemoved>::value || - std::is_move_constructible<ExtentsRemoved>::value; - static constexpr bool kIsCopyOrMoveAssignable = - phmap::is_copy_assignable<ExtentsRemoved>::value || - phmap::is_move_assignable<ExtentsRemoved>::value; - - public: - static constexpr bool kValue = - (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && - (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && - (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && - is_trivially_destructible<ExtentsRemoved>::value && - // We need to check for this explicitly because otherwise we'll say - // references are trivial copyable when compiled by MSVC. - !std::is_reference<ExtentsRemoved>::value; - }; - - template <typename T> - struct is_trivially_copyable - : std::integral_constant< - bool, type_traits_internal::is_trivially_copyable_impl<T>::kValue> {}; - } // namespace type_traits_internal - - namespace swap_internal { - - // Necessary for the traits. - using std::swap; - - // This declaration prevents global `swap` and `phmap::swap` overloads from being - // considered unless ADL picks them up. - void swap(); - - template <class T> - using IsSwappableImpl = decltype(swap(std::declval<T&>(), std::declval<T&>())); - - // NOTE: This dance with the default template parameter is for MSVC. - template <class T, - class IsNoexcept = std::integral_constant< - bool, noexcept(swap(std::declval<T&>(), std::declval<T&>()))>> - using IsNothrowSwappableImpl = typename std::enable_if<IsNoexcept::value>::type; - - template <class T> - struct IsSwappable - : phmap::type_traits_internal::is_detected<IsSwappableImpl, T> {}; - - template <class T> - struct IsNothrowSwappable - : phmap::type_traits_internal::is_detected<IsNothrowSwappableImpl, T> {}; - - template <class T, phmap::enable_if_t<IsSwappable<T>::value, int> = 0> - void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable<T>::value) { - swap(lhs, rhs); - } - - using StdSwapIsUnconstrained = IsSwappable<void()>; - - } // namespace swap_internal - - namespace type_traits_internal { - - // Make the swap-related traits/function accessible from this namespace. - using swap_internal::IsNothrowSwappable; - using swap_internal::IsSwappable; - using swap_internal::Swap; - using swap_internal::StdSwapIsUnconstrained; - - } // namespace type_traits_internal - - namespace compare_internal { - - using value_type = int8_t; - - template <typename T> - struct Fail { - static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); - }; - - template <typename NullPtrT = std::nullptr_t> - struct OnlyLiteralZero { - constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT - - template < - typename T, - typename = typename std::enable_if< - std::is_same<T, std::nullptr_t>::value || - (std::is_integral<T>::value && !std::is_same<T, int>::value)>::type, - typename = typename Fail<T>::type> - OnlyLiteralZero(T); // NOLINT - }; - - enum class eq : value_type { - equal = 0, - equivalent = equal, - nonequal = 1, - nonequivalent = nonequal, - }; - - enum class ord : value_type { less = -1, greater = 1 }; - - enum class ncmp : value_type { unordered = -127 }; - -#if defined(__cpp_inline_variables) && !defined(_MSC_VER) - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ - static const type name; - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - inline constexpr type type::name(init) - -#else // __cpp_inline_variables - -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ - static const T name; - -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) - -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - template <typename T> \ - const T compare_internal::type##_base<T>::name(init) - -#endif // __cpp_inline_variables - - // These template base classes allow for defining the values of the constants - // in the header file (for performance) without using inline variables (which - // aren't available in C++11). - template <typename T> - struct weak_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template <typename T> - struct strong_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template <typename T> - struct partial_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) - }; - - template <typename T> - struct weak_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - template <typename T> - struct strong_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - } // namespace compare_internal - - class weak_equality - : public compare_internal::weak_equality_base<weak_equality> { - explicit constexpr weak_equality(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::weak_equality_base<weak_equality>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) - - // Comparisons - friend constexpr bool operator==( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class strong_equality - : public compare_internal::strong_equality_base<strong_equality> { - explicit constexpr strong_equality(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::strong_equality_base<strong_equality>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, - compare_internal::eq::nonequal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class partial_ordering - : public compare_internal::partial_ordering_base<partial_ordering> { - explicit constexpr partial_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr partial_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::partial_ordering_base<partial_ordering>; - - constexpr bool is_ordered() const noexcept { - return value_ != - compare_internal::value_type(compare_internal::ncmp::unordered); - } - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ == 0; - } - friend constexpr bool operator!=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return !v.is_ordered() || v.value_ != 0; - } - friend constexpr bool operator<( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ < 0; - } - friend constexpr bool operator<=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ <= 0; - } - friend constexpr bool operator>( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ > 0; - } - friend constexpr bool operator>=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return !v.is_ordered() || 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, - compare_internal::ord::greater); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, - compare_internal::ncmp::unordered); - - class weak_ordering - : public compare_internal::weak_ordering_base<weak_ordering> { - explicit constexpr weak_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr weak_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::weak_ordering_base<weak_ordering>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, - compare_internal::ord::greater); - - class strong_ordering - : public compare_internal::strong_ordering_base<strong_ordering> { - explicit constexpr strong_ordering(compare_internal::eq v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - explicit constexpr strong_ordering(compare_internal::ord v) noexcept - : value_(static_cast<compare_internal::value_type>(v)) {} - friend struct compare_internal::strong_ordering_base<strong_ordering>; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator strong_equality() const noexcept { // NOLINT - return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - constexpr operator weak_ordering() const noexcept { // NOLINT - return value_ == 0 - ? weak_ordering::equivalent - : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, - compare_internal::ord::greater); - -#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL -#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL -#undef PHMAP_COMPARE_INLINE_INIT - - namespace compare_internal { - // We also provide these comparator adapter functions for internal phmap use. - - // Helper functions to do a boolean comparison of two keys given a boolean - // or three-way comparator. - // SFINAE prevents implicit conversions to bool (such as from int). - template <typename BoolType, - phmap::enable_if_t<std::is_same<bool, BoolType>::value, int> = 0> - constexpr bool compare_result_as_less_than(const BoolType r) { return r; } - constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { - return r < 0; - } - - template <typename Compare, typename K, typename LK> - constexpr bool do_less_than_comparison(const Compare &compare, const K &x, - const LK &y) { - return compare_result_as_less_than(compare(x, y)); - } - - // Helper functions to do a three-way comparison of two keys given a boolean or - // three-way comparator. - // SFINAE prevents implicit conversions to int (such as from bool). - template <typename Int, - phmap::enable_if_t<std::is_same<int, Int>::value, int> = 0> - constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { - return c < 0 ? phmap::weak_ordering::less - : c == 0 ? phmap::weak_ordering::equivalent - : phmap::weak_ordering::greater; - } - constexpr phmap::weak_ordering compare_result_as_ordering( - const phmap::weak_ordering c) { - return c; - } - - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t<!std::is_same<bool, phmap::invoke_result< - Compare, const K &, const LK &>>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare_result_as_ordering(compare(x, y)); - } - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t<std::is_same<bool, phmap::invoke_result<Compare, - const K &, const LK &>>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare(x, y) ? phmap::weak_ordering::less - : compare(y, x) ? phmap::weak_ordering::greater - : phmap::weak_ordering::equivalent; - } - - } // namespace compare_internal -} - - -namespace phmap { - -namespace priv { - - // A helper class that indicates if the Compare parameter is a key-compare-to - // comparator. - template <typename Compare, typename T> - using btree_is_key_compare_to = - std::is_convertible<phmap::invoke_result<Compare, const T &, const T &>, - phmap::weak_ordering>; - - struct StringBtreeDefaultLess { - using is_transparent = void; - - StringBtreeDefaultLess() = default; - - // Compatibility constructor. - StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultLess(std::less<std::string_view>) {} // NOLINT - StringBtreeDefaultLess(phmap::Less<std::string_view>) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } -#endif - }; - - struct StringBtreeDefaultGreater { - using is_transparent = void; - - StringBtreeDefaultGreater() = default; - - StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT -#if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultGreater(std::greater<std::string_view>) {} // NOLINT - - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } -#endif - }; - - // A helper class to convert a boolean comparison into a three-way "compare-to" - // comparison that returns a negative value to indicate less-than, zero to - // indicate equality and a positive value to indicate greater-than. This helper - // class is specialized for less<std::string>, greater<std::string>, - // less<std::string_view>, and greater<std::string_view>. - // - // key_compare_to_adapter is provided so that btree users - // automatically get the more efficient compare-to code when using common - // google string types with common comparison functors. - // These string-like specializations also turn on heterogeneous lookup by - // default. - template <typename Compare> - struct key_compare_to_adapter { - using type = Compare; - }; - - template <> - struct key_compare_to_adapter<std::less<std::string>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<phmap::Less<std::string>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<std::greater<std::string>> { - using type = StringBtreeDefaultGreater; - }; - -#if PHMAP_HAVE_STD_STRING_VIEW - template <> - struct key_compare_to_adapter<std::less<std::string_view>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<phmap::Less<std::string_view>> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter<std::greater<std::string_view>> { - using type = StringBtreeDefaultGreater; - }; -#endif - - template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi, typename SlotPolicy> - struct common_params { - // If Compare is a common comparator for a std::string-like type, then we adapt it - // to use heterogeneous lookup and to be a key-compare-to comparator. - using key_compare = typename key_compare_to_adapter<Compare>::type; - // A type which indicates if we have a key-compare-to functor or a plain old - // key-compare functor. - using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; - - using allocator_type = Alloc; - using key_type = Key; - using size_type = std::size_t ; - using difference_type = ptrdiff_t; - - // True if this is a multiset or multimap. - using is_multi_container = std::integral_constant<bool, Multi>; - - using slot_policy = SlotPolicy; - using slot_type = typename slot_policy::slot_type; - using value_type = typename slot_policy::value_type; - using init_type = typename slot_policy::mutable_value_type; - using pointer = value_type *; - using const_pointer = const value_type *; - using reference = value_type &; - using const_reference = const value_type &; - - enum { - kTargetNodeSize = TargetNodeSize, - - // Upper bound for the available space for values. This is largest for leaf - // nodes, which have overhead of at least a pointer + 4 bytes (for storing - // 3 field_types and an enum). - kNodeValueSpace = - TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), - }; - - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. - using node_count_type = - phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > - (std::numeric_limits<uint8_t>::max)()), - uint16_t, uint8_t>; // NOLINT - - // The following methods are necessary for passing this struct as PolicyTraits - // for node_handle and/or are used within btree. - static value_type &element(slot_type *slot) { - return slot_policy::element(slot); - } - static const value_type &element(const slot_type *slot) { - return slot_policy::element(slot); - } - template <class... Args> - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - slot_policy::construct(alloc, slot, std::forward<Args>(args)...); - } - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - slot_policy::construct(alloc, slot, other); - } - static void destroy(Alloc *alloc, slot_type *slot) { - slot_policy::destroy(alloc, slot); - } - static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { - construct(alloc, new_slot, old_slot); - destroy(alloc, old_slot); - } - static void swap(Alloc *alloc, slot_type *a, slot_type *b) { - slot_policy::swap(alloc, a, b); - } - static void move(Alloc *alloc, slot_type *src, slot_type *dest) { - slot_policy::move(alloc, src, dest); - } - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - slot_policy::move(alloc, first, last, result); - } - }; - - // A parameters structure for holding the type parameters for a btree_map. - // Compare and Alloc should be nothrow copy-constructible. - template <typename Key, typename Data, typename Compare, typename Alloc, - int TargetNodeSize, bool Multi> - struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - phmap::priv::map_slot_policy<Key, Data>> { - using super_type = typename map_params::common_params; - using mapped_type = Data; - // This type allows us to move keys when it is safe to do so. It is safe - // for maps in which value_type and mutable_value_type are layout compatible. - using slot_policy = typename super_type::slot_policy; - using slot_type = typename super_type::slot_type; - using value_type = typename super_type::value_type; - using init_type = typename super_type::init_type; - - using key_compare = typename super_type::key_compare; - // Inherit from key_compare for empty base class optimization. - struct value_compare : private key_compare { - value_compare() = default; - explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} - - template <typename T, typename U> - auto operator()(const T &left, const U &right) const - -> decltype(std::declval<key_compare>()(left.first, right.first)) { - return key_compare::operator()(left.first, right.first); - } - }; - using is_map_container = std::true_type; - - static const Key &key(const value_type &x) { return x.first; } - static const Key &key(const init_type &x) { return x.first; } - static const Key &key(const slot_type *x) { return slot_policy::key(x); } - static mapped_type &value(value_type *value) { return value->second; } - }; - - // This type implements the necessary functions from the - // btree::priv::slot_type interface. - template <typename Key> - struct set_slot_policy { - using slot_type = Key; - using value_type = Key; - using mutable_value_type = Key; - - static value_type &element(slot_type *slot) { return *slot; } - static const value_type &element(const slot_type *slot) { return *slot; } - - template <typename Alloc, class... Args> - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - phmap::allocator_traits<Alloc>::construct(*alloc, slot, - std::forward<Args>(args)...); - } - - template <typename Alloc> - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - phmap::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); - } - - template <typename Alloc> - static void destroy(Alloc *alloc, slot_type *slot) { - phmap::allocator_traits<Alloc>::destroy(*alloc, slot); - } - - template <typename Alloc> - static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { - using std::swap; - swap(*a, *b); - } - - template <typename Alloc> - static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { - *dest = std::move(*src); - } - - template <typename Alloc> - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } - }; - - // A parameters structure for holding the type parameters for a btree_set. - // Compare and Alloc should be nothrow copy-constructible. - template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi> - struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - set_slot_policy<Key>> { - using value_type = Key; - using slot_type = typename set_params::common_params::slot_type; - using value_compare = typename set_params::common_params::key_compare; - using is_map_container = std::false_type; - - static const Key &key(const value_type &x) { return x; } - static const Key &key(const slot_type *x) { return *x; } - }; - - // An adapter class that converts a lower-bound compare into an upper-bound - // compare. Note: there is no need to make a version of this adapter specialized - // for key-compare-to functors because the upper-bound (the first value greater - // than the input) is never an exact match. - template <typename Compare> - struct upper_bound_adapter { - explicit upper_bound_adapter(const Compare &c) : comp(c) {} - template <typename K, typename LK> - bool operator()(const K &a, const LK &b) const { - // Returns true when a is not greater than b. - return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); - } - - private: - Compare comp; - }; - - enum class MatchKind : uint8_t { kEq, kNe }; - - template <typename V, bool IsCompareTo> - struct SearchResult { - V value; - MatchKind match; - - static constexpr bool HasMatch() { return true; } - bool IsEq() const { return match == MatchKind::kEq; } - }; - - // When we don't use CompareTo, `match` is not present. - // This ensures that callers can't use it accidentally when it provides no - // useful information. - template <typename V> - struct SearchResult<V, false> { - V value; - - static constexpr bool HasMatch() { return false; } - static constexpr bool IsEq() { return false; } - }; - - // A node in the btree holding. The same node type is used for both internal - // and leaf nodes in the btree, though the nodes are allocated in such a way - // that the children array is only valid in internal nodes. - template <typename Params> - class btree_node { - using is_key_compare_to = typename Params::is_key_compare_to; - using is_multi_container = typename Params::is_multi_container; - using field_type = typename Params::node_count_type; - using allocator_type = typename Params::allocator_type; - using slot_type = typename Params::slot_type; - - public: - using params_type = Params; - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using key_compare = typename Params::key_compare; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - - // Btree decides whether to use linear node search as follows: - // - If the key is arithmetic and the comparator is std::less or - // std::greater, choose linear. - // - Otherwise, choose binary. - // TODO(ezb): Might make sense to add condition(s) based on node-size. - using use_linear_search = std::integral_constant< - bool, - std::is_arithmetic<key_type>::value && - (std::is_same<phmap::Less<key_type>, key_compare>::value || - std::is_same<std::less<key_type>, key_compare>::value || - std::is_same<std::greater<key_type>, key_compare>::value)>; - - - ~btree_node() = default; - btree_node(btree_node const &) = delete; - btree_node &operator=(btree_node const &) = delete; - - // Public for EmptyNodeType. - constexpr static size_type Alignment() { - static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), - "Alignment of all nodes must be equal."); - return (size_type)InternalLayout().Alignment(); - } - - protected: - btree_node() = default; - - private: - using layout_type = phmap::priv::Layout<btree_node *, field_type, - slot_type, btree_node *>; - constexpr static size_type SizeWithNValues(size_type n) { - return (size_type)layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)n, - /*children*/ 0) - .AllocSize(); - } - // A lower bound for the overhead of fields other than values in a leaf node. - constexpr static size_type MinimumOverhead() { - return (size_type)(SizeWithNValues(1) - sizeof(value_type)); - } - - // Compute how many values we can fit onto a leaf node taking into account - // padding. - constexpr static size_type NodeTargetValues(const int begin, const int end) { - return begin == end ? begin - : SizeWithNValues((begin + end) / 2 + 1) > - params_type::kTargetNodeSize - ? NodeTargetValues(begin, (begin + end) / 2) - : NodeTargetValues((begin + end) / 2 + 1, end); - } - - enum { - kTargetNodeSize = params_type::kTargetNodeSize, - kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), - - // We need a minimum of 3 values per internal node in order to perform - // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, - - // The node is internal (i.e. is not a leaf node) if and only if `max_count` - // has this value. - kInternalNodeMaxCount = 0, - }; - - // Leaves can have less than kNodeValues values. - constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)max_values, - /*children*/ 0); - } - constexpr static layout_type InternalLayout() { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ kNodeValues, - /*children*/ kNodeValues + 1); - } - constexpr static size_type LeafSize(const int max_values = kNodeValues) { - return (size_type)LeafLayout(max_values).AllocSize(); - } - constexpr static size_type InternalSize() { - return (size_type)InternalLayout().AllocSize(); - } - - // N is the index of the type in the Layout definition. - // ElementType<N> is the Nth type in the Layout definition. - template <size_type N> - inline typename layout_type::template ElementType<N> *GetField() { - // We assert that we don't read from values that aren't there. - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this)); - } - - template <size_type N> - inline const typename layout_type::template ElementType<N> *GetField() const { - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer<N>( - reinterpret_cast<const char *>(this)); - } - - void set_parent(btree_node *p) { *GetField<0>() = p; } - field_type &mutable_count() { return GetField<1>()[2]; } - slot_type *slot(size_type i) { return &GetField<2>()[i]; } - const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } - void set_position(field_type v) { GetField<1>()[0] = v; } - void set_start(field_type v) { GetField<1>()[1] = v; } - void set_count(field_type v) { GetField<1>()[2] = v; } - void set_max_count(field_type v) { GetField<1>()[3] = v; } - - public: - // Whether this is a leaf node or not. This value doesn't change after the - // node is created. - bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } - - // Getter for the position of this node in its parent. - field_type position() const { return GetField<1>()[0]; } - - // Getter for the offset of the first value in the `values` array. - field_type start() const { return GetField<1>()[1]; } - - // Getters for the number of values stored in this node. - field_type count() const { return GetField<1>()[2]; } - field_type max_count() const { - // Internal nodes have max_count==kInternalNodeMaxCount. - // Leaf nodes have max_count in [1, kNodeValues]. - const field_type max_cnt = GetField<1>()[3]; - return max_cnt == field_type{kInternalNodeMaxCount} - ? field_type{kNodeValues} - : max_cnt; - } - - // Getter for the parent of this node. - btree_node *parent() const { return *GetField<0>(); } - // Getter for whether the node is the root of the tree. The parent of the - // root of the tree is the leftmost node in the tree which is guaranteed to - // be a leaf. - bool is_root() const { return parent()->leaf(); } - void make_root() { - assert(parent()->is_root()); - set_parent(parent()->parent()); - } - - // Getters for the key/value at position i in the node. - const key_type &key(size_type i) const { return params_type::key(slot(i)); } - reference value(size_type i) { return params_type::element(slot(i)); } - const_reference value(size_type i) const { return params_type::element(slot(i)); } - - // Getters/setter for the child at position i in the node. - btree_node *child(size_type i) const { return GetField<3>()[i]; } - btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } - void clear_child(size_type i) { - phmap::priv::SanitizerPoisonObject(&mutable_child(i)); - } - void set_child(size_type i, btree_node *c) { - phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); - mutable_child(i) = c; - c->set_position((field_type)i); - } - void init_child(int i, btree_node *c) { - set_child(i, c); - c->set_parent(this); - } - - // Returns the position of the first value whose key is not less than k. - template <typename K> - SearchResult<int, is_key_compare_to::value> lower_bound( - const K &k, const key_compare &comp) const { - return use_linear_search::value ? linear_search(k, comp) - : binary_search(k, comp); - } - // Returns the position of the first value whose key is greater than k. - template <typename K> - int upper_bound(const K &k, const key_compare &comp) const { - auto upper_compare = upper_bound_adapter<key_compare>(comp); - return use_linear_search::value ? linear_search(k, upper_compare).value - : binary_search(k, upper_compare).value; - } - - template <typename K, typename Compare> - SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> - linear_search(const K &k, const Compare &comp) const { - return linear_search_impl(k, 0, count(), comp, - btree_is_key_compare_to<Compare, key_type>()); - } - - template <typename K, typename Compare> - SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> - binary_search(const K &k, const Compare &comp) const { - return binary_search_impl(k, 0, count(), comp, - btree_is_key_compare_to<Compare, key_type>()); - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using plain compare. - template <typename K, typename Compare> - SearchResult<int, false> linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s < e) { - if (!comp(key(s), k)) { - break; - } - ++s; - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using compare-to. - template <typename K, typename Compare> - SearchResult<int, true> linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::true_type /* IsCompareTo */) const { - while (s < e) { - const phmap::weak_ordering c = comp(key(s), k); - if (c == 0) { - return {s, MatchKind::kEq}; - } else if (c > 0) { - break; - } - ++s; - } - return {s, MatchKind::kNe}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using plain compare. - template <typename K, typename Compare> - SearchResult<int, false> binary_search_impl( - const K &k, int s, int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s != e) { - const int mid = (s + e) >> 1; - if (comp(key(mid), k)) { - s = mid + 1; - } else { - e = mid; - } - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using compare-to. - template <typename K, typename CompareTo> - SearchResult<int, true> binary_search_impl( - const K &k, int s, int e, const CompareTo &comp, - std::true_type /* IsCompareTo */) const { - if (is_multi_container::value) { - MatchKind exact_match = MatchKind::kNe; - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else { - e = mid; - if (c == 0) { - // Need to return the first value whose key is not less than k, - // which requires continuing the binary search if this is a - // multi-container. - exact_match = MatchKind::kEq; - } - } - } - return {s, exact_match}; - } else { // Not a multi-container. - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else if (c > 0) { - e = mid; - } else { - return {mid, MatchKind::kEq}; - } - } - return {s, MatchKind::kNe}; - } - } - - // Emplaces a value at position i, shifting all existing values and - // children at positions >= i to the right by 1. - template <typename... Args> - void emplace_value(size_type i, allocator_type *alloc, Args &&... args); - - // Removes the value at position i, shifting all existing values and children - // at positions > i to the left by 1. - void remove_value(int i, allocator_type *alloc); - - // Removes the values at positions [i, i + to_erase), shifting all values - // after that range to the left by to_erase. Does not change children at all. - void remove_values_ignore_children(int i, size_type to_erase, - allocator_type *alloc); - - // Rebalances a node with its right sibling. - void rebalance_right_to_left(int to_move, btree_node *right, - allocator_type *alloc); - void rebalance_left_to_right(int to_move, btree_node *right, - allocator_type *alloc); - - // Splits a node, moving a portion of the node's values to its right sibling. - void split(int insert_position, btree_node *dest, allocator_type *alloc); - - // Merges a node with its right sibling, moving all of the values and the - // delimiting key in the parent node onto itself. - void merge(btree_node *sibling, allocator_type *alloc); - - // Swap the contents of "this" and "src". - void swap(btree_node *src, allocator_type *alloc); - - // Node allocation/deletion routines. - static btree_node *init_leaf(btree_node *n, btree_node *parent, - int max_cnt) { - n->set_parent(parent); - n->set_position(0); - n->set_start(0); - n->set_count(0); - n->set_max_count((field_type)max_cnt); - phmap::priv::SanitizerPoisonMemoryRegion( - n->slot(0), max_cnt * sizeof(slot_type)); - return n; - } - static btree_node *init_internal(btree_node *n, btree_node *parent) { - init_leaf(n, parent, kNodeValues); - // Set `max_count` to a sentinel value to indicate that this node is - // internal. - n->set_max_count(kInternalNodeMaxCount); - phmap::priv::SanitizerPoisonMemoryRegion( - &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); - return n; - } - void destroy(allocator_type *alloc) { - for (int i = 0; i < count(); ++i) { - value_destroy(i, alloc); - } - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return use_linear_search::value; - } - - private: - template <typename... Args> - void value_init(const size_type i, allocator_type *alloc, Args &&... args) { - phmap::priv::SanitizerUnpoisonObject(slot(i)); - params_type::construct(alloc, slot(i), std::forward<Args>(args)...); - } - void value_destroy(const size_type i, allocator_type *alloc) { - params_type::destroy(alloc, slot(i)); - phmap::priv::SanitizerPoisonObject(slot(i)); - } - - // Move n values starting at value i in this node into the values starting at - // value j in node x. - void uninitialized_move_n(const size_type n, const size_type i, - const size_type j, btree_node *x, - allocator_type *alloc) { - phmap::priv::SanitizerUnpoisonMemoryRegion( - x->slot(j), n * sizeof(slot_type)); - for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); - src != end; ++src, ++dest) { - params_type::construct(alloc, dest, src); - } - } - - // Destroys a range of n values, starting at index i. - void value_destroy_n(const size_type i, const size_type n, - allocator_type *alloc) { - for (int j = 0; j < n; ++j) { - value_destroy(i + j, alloc); - } - } - - template <typename P> - friend class btree; - template <typename N, typename R, typename P> - friend struct btree_iterator; - friend class BtreeNodePeer; - }; - - template <typename Node, typename Reference, typename Pointer> - struct btree_iterator { - private: - using key_type = typename Node::key_type; - using size_type = typename Node::size_type; - using params_type = typename Node::params_type; - - using node_type = Node; - using normal_node = typename std::remove_const<Node>::type; - using const_node = const Node; - using normal_pointer = typename params_type::pointer; - using normal_reference = typename params_type::reference; - using const_pointer = typename params_type::const_pointer; - using const_reference = typename params_type::const_reference; - using slot_type = typename params_type::slot_type; - - using iterator = - btree_iterator<normal_node, normal_reference, normal_pointer>; - using const_iterator = - btree_iterator<const_node, const_reference, const_pointer>; - - public: - // These aliases are public for std::iterator_traits. - using difference_type = typename Node::difference_type; - using value_type = typename params_type::value_type; - using pointer = Pointer; - using reference = Reference; - using iterator_category = std::bidirectional_iterator_tag; - - btree_iterator() : node(nullptr), position(-1) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} - - // NOTE: this SFINAE allows for implicit conversions from iterator to - // const_iterator, but it specifically avoids defining copy constructors so - // that btree_iterator can be trivially copyable. This is for performance and - // binary size reasons. - template <typename N, typename R, typename P, - phmap::enable_if_t< - std::is_same<btree_iterator<N, R, P>, iterator>::value && - std::is_same<btree_iterator, const_iterator>::value, - int> = 0> - btree_iterator(const btree_iterator<N, R, P> &x) // NOLINT - : node(x.node), position(x.position) {} - - private: - // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids defining a copy constructor. - // NOTE: the const_cast is safe because this constructor is only called by - // non-const methods and the container owns the nodes. - template <typename N, typename R, typename P, - phmap::enable_if_t< - std::is_same<btree_iterator<N, R, P>, const_iterator>::value && - std::is_same<btree_iterator, iterator>::value, - int> = 0> - explicit btree_iterator(const btree_iterator<N, R, P> &x) - : node(const_cast<node_type *>(x.node)), position(x.position) {} - - // Increment/decrement the iterator. - void increment() { - if (node->leaf() && ++position < node->count()) { - return; - } - increment_slow(); - } - void increment_slow(); - - void decrement() { - if (node->leaf() && --position >= 0) { - return; - } - decrement_slow(); - } - void decrement_slow(); - - public: - bool operator==(const const_iterator &x) const { - return node == x.node && position == x.position; - } - bool operator!=(const const_iterator &x) const { - return node != x.node || position != x.position; - } - - // Accessors for the key/value the iterator is pointing at. - reference operator*() const { - return node->value(position); - } - pointer operator->() const { - return &node->value(position); - } - - btree_iterator& operator++() { - increment(); - return *this; - } - btree_iterator& operator--() { - decrement(); - return *this; - } - btree_iterator operator++(int) { - btree_iterator tmp = *this; - ++*this; - return tmp; - } - btree_iterator operator--(int) { - btree_iterator tmp = *this; - --*this; - return tmp; - } - - private: - template <typename Params> - friend class btree; - template <typename Tree> - friend class btree_container; - template <typename Tree> - friend class btree_set_container; - template <typename Tree> - friend class btree_map_container; - template <typename Tree> - friend class btree_multiset_container; - template <typename N, typename R, typename P> - friend struct btree_iterator; - template <typename TreeType, typename CheckerType> - friend class base_checker; - - const key_type &key() const { return node->key(position); } - slot_type *slot() { return node->slot(position); } - - // The node in the tree the iterator is pointing at. - Node *node; - // The position within the node of the tree the iterator is pointing at. - // TODO(ezb): make this a field_type - int position; - }; - - template <typename Params> - class btree { - using node_type = btree_node<Params>; - using is_key_compare_to = typename Params::is_key_compare_to; - - // We use a static empty node for the root/leftmost/rightmost of empty btrees - // in order to avoid branching in begin()/end(). - struct alignas(node_type::Alignment()) EmptyNodeType : node_type { - using field_type = typename node_type::field_type; - node_type *parent; - field_type position = 0; - field_type start = 0; - field_type count = 0; - // max_count must be != kInternalNodeMaxCount (so that this node is regarded - // as a leaf node). max_count() is never called when the tree is empty. - field_type max_count = node_type::kInternalNodeMaxCount + 1; - -#ifdef _MSC_VER - // MSVC has constexpr code generations bugs here. - EmptyNodeType() : parent(this) {} -#else - constexpr EmptyNodeType(node_type *p) : parent(p) {} -#endif - }; - - static node_type *EmptyNode() { -#ifdef _MSC_VER - static EmptyNodeType empty_node; - // This assert fails on some other construction methods. - assert(empty_node.parent == &empty_node); - return &empty_node; -#else - static constexpr EmptyNodeType empty_node( - const_cast<EmptyNodeType *>(&empty_node)); - return const_cast<EmptyNodeType *>(&empty_node); -#endif - } - - enum { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, - }; - - struct node_stats { - using size_type = typename Params::size_type; - - node_stats(size_type l, size_type i) - : leaf_nodes(l), - internal_nodes(i) { - } - - node_stats& operator+=(const node_stats &x) { - leaf_nodes += x.leaf_nodes; - internal_nodes += x.internal_nodes; - return *this; - } - - size_type leaf_nodes; - size_type internal_nodes; - }; - - public: - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - using key_compare = typename Params::key_compare; - using value_compare = typename Params::value_compare; - using allocator_type = typename Params::allocator_type; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using iterator = btree_iterator<node_type, reference, pointer>; - using const_iterator = typename iterator::const_iterator; - using reverse_iterator = std::reverse_iterator<iterator>; - using const_reverse_iterator = std::reverse_iterator<const_iterator>; - using node_handle_type = node_handle<Params, Params, allocator_type>; - - // Internal types made public for use by btree_container types. - using params_type = Params; - using slot_type = typename Params::slot_type; - - private: - // For use in copy_or_move_values_in_order. - const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } - value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } - - // Copies or moves (depending on the template parameter) the values in - // x into this btree in their order in x. This btree must be empty before this - // method is called. This method is used in copy construction, copy - // assignment, and move assignment. - template <typename Btree> - void copy_or_move_values_in_order(Btree *x); - - // Validates that various assumptions/requirements are true at compile time. - constexpr static bool static_assert_validation(); - - public: - btree(const key_compare &comp, const allocator_type &alloc); - - btree(const btree &x); - btree(btree &&x) noexcept - : root_(std::move(x.root_)), - rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), - size_(phmap::exchange(x.size_, 0)) { - x.mutable_root() = EmptyNode(); - } - - ~btree() { - // Put static_asserts in destructor to avoid triggering them before the type - // is complete. - static_assert(static_assert_validation(), "This call must be elided."); - clear(); - } - - // Assign the contents of x to *this. - btree &operator=(const btree &x); - btree &operator=(btree &&x) noexcept; - - iterator begin() { - return iterator(leftmost(), 0); - } - const_iterator begin() const { - return const_iterator(leftmost(), 0); - } - iterator end() { return iterator(rightmost_, rightmost_->count()); } - const_iterator end() const { - return const_iterator(rightmost_, rightmost_->count()); - } - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - reverse_iterator rend() { - return reverse_iterator(begin()); - } - const_reverse_iterator rend() const { - return const_reverse_iterator(begin()); - } - - // Finds the first element whose key is not less than key. - template <typename K> - iterator lower_bound(const K &key) { - return internal_end(internal_lower_bound(key)); - } - template <typename K> - const_iterator lower_bound(const K &key) const { - return internal_end(internal_lower_bound(key)); - } - - // Finds the first element whose key is greater than key. - template <typename K> - iterator upper_bound(const K &key) { - return internal_end(internal_upper_bound(key)); - } - template <typename K> - const_iterator upper_bound(const K &key) const { - return internal_end(internal_upper_bound(key)); - } - - // Finds the range of values which compare equal to key. The first member of - // the returned pair is equal to lower_bound(key). The second member pair of - // the pair is equal to upper_bound(key). - template <typename K> - std::pair<iterator, iterator> equal_range(const K &key) { - return {lower_bound(key), upper_bound(key)}; - } - template <typename K> - std::pair<const_iterator, const_iterator> equal_range(const K &key) const { - return {lower_bound(key), upper_bound(key)}; - } - - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template <typename... Args> - std::pair<iterator, bool> insert_unique(const key_type &key, Args &&... args); - - // Inserts with hint. Checks to see if the value should be placed immediately - // before `position` in the tree. If so, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_unique() were made. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template <typename... Args> - std::pair<iterator, bool> insert_hint_unique(iterator position, - const key_type &key, - Args &&... args); - - // Insert a range of values into the btree. - template <typename InputIterator> - void insert_iterator_unique(InputIterator b, InputIterator e); - - // Inserts a value into the btree. - template <typename ValueType> - iterator insert_multi(const key_type &key, ValueType &&v); - - // Inserts a value into the btree. - template <typename ValueType> - iterator insert_multi(ValueType &&v) { - return insert_multi(params_type::key(v), std::forward<ValueType>(v)); - } - - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_multi(v) were made. - template <typename ValueType> - iterator insert_hint_multi(iterator position, ValueType &&v); - - // Insert a range of values into the btree. - template <typename InputIterator> - void insert_iterator_multi(InputIterator b, InputIterator e); - - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - // Requirement: does not read the value at `*iter`. - iterator erase(iterator iter); - - // Erases range. Returns the number of keys erased and an iterator pointing - // to the element after the last erased element. - std::pair<size_type, iterator> erase(iterator begin, iterator end); - - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - template <typename K> - size_type erase_unique(const K &key); - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - template <typename K> - size_type erase_multi(const K &key); - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. - template <typename K> - iterator find(const K &key) { - return internal_end(internal_find(key)); - } - template <typename K> - const_iterator find(const K &key) const { - return internal_end(internal_find(key)); - } - - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_unique(const K &key) const { - const iterator beg = internal_find(key); - if (beg.node == nullptr) { - // The key doesn't exist in the tree. - return 0; - } - return 1; - } - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_multi(const K &key) const { - const auto range = equal_range(key); - return std::distance(range.first, range.second); - } - - // Clear the btree, deleting all of the values it contains. - void clear(); - - // Swap the contents of *this and x. - void swap(btree &x); - - const key_compare &key_comp() const noexcept { - return root_.template get<0>(); - } - template <typename K, typename LK> - bool compare_keys(const K &x, const LK &y) const { - return compare_internal::compare_result_as_less_than(key_comp()(x, y)); - } - - value_compare value_comp() const { return value_compare(key_comp()); } - - // Verifies the structure of the btree. - void verify() const; - - // Size routines. - size_type size() const { return size_; } - size_type max_size() const { return (std::numeric_limits<size_type>::max)(); } - bool empty() const { return size_ == 0; } - - // The height of the btree. An empty tree will have height 0. - size_type height() const { - size_type h = 0; - if (!empty()) { - // Count the length of the chain from the leftmost node up to the - // root. We actually count from the root back around to the level below - // the root, but the calculation is the same because of the circularity - // of that traversal. - const node_type *n = root(); - do { - ++h; - n = n->parent(); - } while (n != root()); - } - return h; - } - - // The number of internal, leaf and total nodes used by the btree. - size_type leaf_nodes() const { - return internal_stats(root()).leaf_nodes; - } - size_type internal_nodes() const { - return internal_stats(root()).internal_nodes; - } - size_type nodes() const { - node_stats stats = internal_stats(root()); - return stats.leaf_nodes + stats.internal_nodes; - } - - // The total number of bytes used by the btree. - size_type bytes_used() const { - node_stats stats = internal_stats(root()); - if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { - return sizeof(*this) + - node_type::LeafSize(root()->max_count()); - } else { - return sizeof(*this) + - stats.leaf_nodes * node_type::LeafSize() + - stats.internal_nodes * node_type::InternalSize(); - } - } - - // The average number of bytes used per value stored in the btree. - static double average_bytes_per_value() { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return node_type::LeafSize() / (kNodeValues * 0.75); - } - - // The fullness of the btree. Computed as the number of elements in the btree - // divided by the maximum number of elements a tree with the current number - // of nodes could hold. A value of 1 indicates perfect space - // utilization. Smaller values indicate space wastage. - // Returns 0 for empty trees. - double fullness() const { - if (empty()) return 0.0; - return static_cast<double>(size()) / (nodes() * kNodeValues); - } - // The overhead of the btree structure in bytes per node. Computed as the - // total number of bytes used by the btree minus the number of bytes used for - // storing elements divided by the number of elements. - // Returns 0 for empty trees. - double overhead() const { - if (empty()) return 0.0; - return (bytes_used() - size() * sizeof(value_type)) / - static_cast<double>(size()); - } - - // The allocator used by the btree. - allocator_type get_allocator() const { - return allocator(); - } - - private: - // Internal accessor routines. - node_type *root() { return root_.template get<2>(); } - const node_type *root() const { return root_.template get<2>(); } - node_type *&mutable_root() noexcept { return root_.template get<2>(); } - key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } - - // The leftmost node is stored as the parent of the root node. - node_type *leftmost() { return root()->parent(); } - const node_type *leftmost() const { return root()->parent(); } - - // Allocator routines. - allocator_type *mutable_allocator() noexcept { - return &root_.template get<1>(); - } - const allocator_type &allocator() const noexcept { - return root_.template get<1>(); - } - - // Allocates a correctly aligned node of at least size bytes using the - // allocator. - node_type *allocate(const size_type sz) { - return reinterpret_cast<node_type *>( - phmap::priv::Allocate<node_type::Alignment()>( - mutable_allocator(), (size_t)sz)); - } - - // Node creation/deletion routines. - node_type* new_internal_node(node_type *parent) { - node_type *p = allocate(node_type::InternalSize()); - return node_type::init_internal(p, parent); - } - node_type* new_leaf_node(node_type *parent) { - node_type *p = allocate(node_type::LeafSize()); - return node_type::init_leaf(p, parent, kNodeValues); - } - node_type *new_leaf_root_node(const int max_count) { - node_type *p = allocate(node_type::LeafSize(max_count)); - return node_type::init_leaf(p, p, max_count); - } - - // Deletion helper routines. - void erase_same_node(iterator begin, iterator end); - iterator erase_from_leaf_node(iterator begin, size_type to_erase); - iterator rebalance_after_delete(iterator iter); - - // Deallocates a node of a certain size in bytes using the allocator. - void deallocate(const size_type sz, node_type *node) { - phmap::priv::Deallocate<node_type::Alignment()>( - mutable_allocator(), node, (size_t)sz); - } - - void delete_internal_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::InternalSize(), node); - } - void delete_leaf_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::LeafSize(node->max_count()), node); - } - - // Rebalances or splits the node iter points to. - void rebalance_or_split(iterator *iter); - - // Merges the values of left, right and the delimiting key on their parent - // onto left, removing the delimiting key and deleting right. - void merge_nodes(node_type *left, node_type *right); - - // Tries to merge node with its left or right sibling, and failing that, - // rebalance with its left or right sibling. Returns true if a merge - // occurred, at which point it is no longer valid to access node. Returns - // false if no merging took place. - bool try_merge_or_rebalance(iterator *iter); - - // Tries to shrink the height of the tree by 1. - void try_shrink(); - - iterator internal_end(iterator iter) { - return iter.node != nullptr ? iter : end(); - } - const_iterator internal_end(const_iterator iter) const { - return iter.node != nullptr ? iter : end(); - } - - // Emplaces a value into the btree immediately before iter. Requires that - // key(v) <= iter.key() and (--iter).key() <= key(v). - template <typename... Args> - iterator internal_emplace(iterator iter, Args &&... args); - - // Returns an iterator pointing to the first value >= the value "iter" is - // pointing at. Note that "iter" might be pointing to an invalid location as - // iter.position == iter.node->count(). This routine simply moves iter up in - // the tree to a valid location. - // Requires: iter.node is non-null. - template <typename IterType> - static IterType internal_last(IterType iter); - - // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version uses a less-than comparator and is incapable of distinguishing when - // there is an exact match. The second version is for the key-compare-to - // specialization and distinguishes exact matches. The key-compare-to - // specialization allows the caller to avoid a subsequent comparison to - // determine if an exact match was made, which is important for keys with - // expensive comparison, such as strings. - template <typename K> - SearchResult<iterator, is_key_compare_to::value> internal_locate( - const K &key) const; - - template <typename K> - SearchResult<iterator, false> internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const; - - template <typename K> - SearchResult<iterator, true> internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const; - - // Internal routine which implements lower_bound(). - template <typename K> - iterator internal_lower_bound(const K &key) const; - - // Internal routine which implements upper_bound(). - template <typename K> - iterator internal_upper_bound(const K &key) const; - - // Internal routine which implements find(). - template <typename K> - iterator internal_find(const K &key) const; - - // Deletes a node and all of its children. - void internal_clear(node_type *node); - - // Verifies the tree structure of node. - int internal_verify(const node_type *node, - const key_type *lo, const key_type *hi) const; - - node_stats internal_stats(const node_type *node) const { - // The root can be a static empty node. - if (node == nullptr || (node == root() && empty())) { - return node_stats(0, 0); - } - if (node->leaf()) { - return node_stats(1, 0); - } - node_stats res(0, 1); - for (int i = 0; i <= node->count(); ++i) { - res += internal_stats(node->child(i)); - } - return res; - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return node_type::testonly_uses_linear_node_search(); - } - - private: - // We use compressed tuple in order to save space because key_compare and - // allocator_type are usually empty. - phmap::priv::CompressedTuple<key_compare, allocator_type, - node_type *> - root_; - - // A pointer to the rightmost node. Note that the leftmost node is stored as - // the root's parent. - node_type *rightmost_; - - // Number of values. - size_type size_; - }; - - //// - // btree_node methods - template <typename P> - template <typename... Args> - inline void btree_node<P>::emplace_value(const size_type i, - allocator_type *alloc, - Args &&... args) { - assert(i <= count()); - // Shift old values to create space for new value and then construct it in - // place. - if (i < count()) { - value_init(count(), alloc, slot(count() - 1)); - for (size_type j = count() - 1; j > i; --j) - params_type::move(alloc, slot(j - 1), slot(j)); - value_destroy(i, alloc); - } - value_init(i, alloc, std::forward<Args>(args)...); - set_count((field_type)(count() + 1)); - - if (!leaf() && count() > i + 1) { - for (int j = count(); j > i + 1; --j) { - set_child(j, child(j - 1)); - } - clear_child(i + 1); - } - } - - template <typename P> - inline void btree_node<P>::remove_value(const int i, allocator_type *alloc) { - if (!leaf() && count() > i + 1) { - assert(child(i + 1)->count() == 0); - for (size_type j = i + 1; j < count(); ++j) { - set_child(j, child(j + 1)); - } - clear_child(count()); - } - - remove_values_ignore_children(i, /*to_erase=*/1, alloc); - } - - template <typename P> - inline void btree_node<P>::remove_values_ignore_children( - int i, size_type to_erase, allocator_type *alloc) { - params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); - value_destroy_n(count() - to_erase, to_erase, alloc); - set_count((field_type)(count() - to_erase)); - } - - template <typename P> - void btree_node<P>::rebalance_right_to_left(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(right->count() >= count()); - assert(to_move >= 1); - assert(to_move <= right->count()); - - // 1) Move the delimiting value in the parent to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // 2) Move the (to_move - 1) values from the right node to the left node. - right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); - - // 3) Move the new delimiting value to the parent from the right node. - params_type::move(alloc, right->slot(to_move - 1), - parent()->slot(position())); - - // 4) Shift the values in the right node to their correct position. - params_type::move(alloc, right->slot(to_move), right->slot(right->count()), - right->slot(0)); - - // 5) Destroy the now-empty to_move entries in the right node. - right->value_destroy_n(right->count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i < to_move; ++i) { - init_child(count() + i + 1, right->child(i)); - } - for (int i = 0; i <= right->count() - to_move; ++i) { - assert(i + to_move <= right->max_count()); - right->init_child(i, right->child(i + to_move)); - right->clear_child(i + to_move); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() + to_move)); - right->set_count((field_type)(right->count() - to_move)); - } - - template <typename P> - void btree_node<P>::rebalance_left_to_right(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(count() >= right->count()); - assert(to_move >= 1); - assert(to_move <= count()); - - // Values in the right node are shifted to the right to make room for the - // new to_move values. Then, the delimiting value in the parent and the - // other (to_move - 1) values in the left node are moved into the right node. - // Lastly, a new delimiting value is moved from the left node into the - // parent, and the remaining empty left node entries are destroyed. - - if (right->count() >= to_move) { - // The original location of the right->count() values are sufficient to hold - // the new to_move entries from the parent and left node. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(to_move, right->count() - to_move, - right->count(), right, alloc); - for (slot_type *src = right->slot(right->count() - to_move - 1), - *dest = right->slot(right->count() - 1), - *end = right->slot(0); - src >= end; --src, --dest) { - params_type::move(alloc, src, dest); - } - - // 2) Move the delimiting value in the parent to the right node. - params_type::move(alloc, parent()->slot(position()), - right->slot(to_move - 1)); - - // 3) Move the (to_move - 1) values from the left node to the right node. - params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), - right->slot(0)); - } else { - // The right node does not have enough initialized space to hold the new - // to_move entries, so part of them will move to uninitialized space. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); - - // 2) Move the delimiting value in the parent to the right node. - right->value_init(to_move - 1, alloc, parent()->slot(position())); - - // 3) Move the (to_move - 1) values from the left node to the right node. - const size_type uninitialized_remaining = to_move - right->count() - 1; - uninitialized_move_n(uninitialized_remaining, - count() - uninitialized_remaining, right->count(), - right, alloc); - params_type::move(alloc, slot(count() - (to_move - 1)), - slot(count() - uninitialized_remaining), right->slot(0)); - } - - // 4) Move the new delimiting value to the parent from the left node. - params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); - - // 5) Destroy the now-empty to_move entries in the left node. - value_destroy_n(count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the left to the right node. - for (int i = right->count(); i >= 0; --i) { - right->init_child(i + to_move, right->child(i)); - right->clear_child(i); - } - for (int i = 1; i <= to_move; ++i) { - right->init_child(i - 1, child(count() - to_move + i)); - clear_child(count() - to_move + i); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() - to_move)); - right->set_count((field_type)(right->count() + to_move)); - } - - template <typename P> - void btree_node<P>::split(const int insert_position, btree_node *dest, - allocator_type *alloc) { - assert(dest->count() == 0); - assert(max_count() == kNodeValues); - - // We bias the split based on the position being inserted. If we're - // inserting at the beginning of the left node then bias the split to put - // more values on the right node. If we're inserting at the end of the - // right node then bias the split to put more values on the left node. - if (insert_position == 0) { - dest->set_count((field_type)(count() - 1)); - } else if (insert_position == kNodeValues) { - dest->set_count(0); - } else { - dest->set_count((field_type)(count() / 2)); - } - set_count((field_type)(count() - dest->count())); - assert(count() >= 1); - - // Move values from the left sibling to the right sibling. - uninitialized_move_n(dest->count(), count(), 0, dest, alloc); - - // Destroy the now-empty entries in the left node. - value_destroy_n(count(), dest->count(), alloc); - - // The split key is the largest value in the left sibling. - set_count((field_type)(count() - 1)); - parent()->emplace_value(position(), alloc, slot(count())); - value_destroy(count(), alloc); - parent()->init_child(position() + 1, dest); - - if (!leaf()) { - for (int i = 0; i <= dest->count(); ++i) { - assert(child(count() + i + 1) != nullptr); - dest->init_child(i, child(count() + i + 1)); - clear_child(count() + i + 1); - } - } - } - - template <typename P> - void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); - - // Move the delimiting value to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // Move the values from the right to the left node. - src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); - - // Destroy the now-empty entries in the right node. - src->value_destroy_n(0, src->count(), alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i <= src->count(); ++i) { - init_child(count() + i + 1, src->child(i)); - src->clear_child(i); - } - } - - // Fixup the counts on the src and dest nodes. - set_count((field_type)(1 + count() + src->count())); - src->set_count(0); - - // Remove the value on the parent node. - parent()->remove_value(position(), alloc); - } - - template <typename P> - void btree_node<P>::swap(btree_node *x, allocator_type *alloc) { - using std::swap; - assert(leaf() == x->leaf()); - - // Determine which is the smaller/larger node. - btree_node *smaller = this, *larger = x; - if (smaller->count() > larger->count()) { - swap(smaller, larger); - } - - // Swap the values. - for (slot_type *a = smaller->slot(0), *b = larger->slot(0), - *end = a + smaller->count(); - a != end; ++a, ++b) { - params_type::swap(alloc, a, b); - } - - // Move values that can't be swapped. - const size_type to_move = larger->count() - smaller->count(); - larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), - smaller, alloc); - larger->value_destroy_n(smaller->count(), to_move, alloc); - - if (!leaf()) { - // Swap the child pointers. - std::swap_ranges(&smaller->mutable_child(0), - &smaller->mutable_child(smaller->count() + 1), - &larger->mutable_child(0)); - // Update swapped children's parent pointers. - int i = 0; - for (; i <= smaller->count(); ++i) { - smaller->child(i)->set_parent(smaller); - larger->child(i)->set_parent(larger); - } - // Move the child pointers that couldn't be swapped. - for (; i <= larger->count(); ++i) { - smaller->init_child(i, larger->child(i)); - larger->clear_child(i); - } - } - - // Swap the counts. - swap(mutable_count(), x->mutable_count()); - } - - //// - // btree_iterator methods - template <typename N, typename R, typename P> - void btree_iterator<N, R, P>::increment_slow() { - if (node->leaf()) { - assert(position >= node->count()); - btree_iterator save(*this); - while (position == node->count() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); - } - if (position == node->count()) { - *this = save; - } - } else { - assert(position < node->count()); - node = node->child(position + 1); - while (!node->leaf()) { - node = node->child(0); - } - position = 0; - } - } - - template <typename N, typename R, typename P> - void btree_iterator<N, R, P>::decrement_slow() { - if (node->leaf()) { - assert(position <= -1); - btree_iterator save(*this); - while (position < 0 && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); - } - if (position < 0) { - *this = save; - } - } else { - assert(position >= 0); - node = node->child(position); - while (!node->leaf()) { - node = node->child(node->count()); - } - position = node->count() - 1; - } - } - - //// - // btree methods - template <typename P> - template <typename Btree> - void btree<P>::copy_or_move_values_in_order(Btree *x) { - static_assert(std::is_same<btree, Btree>::value || - std::is_same<const btree, Btree>::value, - "Btree type must be same or const."); - assert(empty()); - - // We can avoid key comparisons because we know the order of the - // values is the same order we'll store them in. - auto iter = x->begin(); - if (iter == x->end()) return; - insert_multi(maybe_move_from_iterator(iter)); - ++iter; - for (; iter != x->end(); ++iter) { - // If the btree is not empty, we can just insert the new value at the end - // of the tree. - internal_emplace(end(), maybe_move_from_iterator(iter)); - } - } - - template <typename P> - constexpr bool btree<P>::static_assert_validation() { - static_assert(std::is_nothrow_copy_constructible<key_compare>::value, - "Key comparison must be nothrow copy constructible"); - static_assert(std::is_nothrow_copy_constructible<allocator_type>::value, - "Allocator must be nothrow copy constructible"); - static_assert(type_traits_internal::is_trivially_copyable<iterator>::value, - "iterator not trivially copyable."); - - // Note: We assert that kTargetValues, which is computed from - // Params::kTargetNodeSize, must fit the node_type::field_type. - static_assert( - kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), - "target node size too large"); - - // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. - using compare_result_type = - phmap::invoke_result<key_compare, key_type, key_type>; - static_assert( - std::is_same<compare_result_type, bool>::value || - std::is_convertible<compare_result_type, phmap::weak_ordering>::value, - "key comparison function must return phmap::{weak,strong}_ordering or " - "bool."); - - // Test the assumption made in setting kNodeValueSpace. - static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, - "node space assumption incorrect"); - - return true; - } - - template <typename P> - btree<P>::btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - - template <typename P> - btree<P>::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { - copy_or_move_values_in_order(&x); - } - - template <typename P> - template <typename... Args> - auto btree<P>::insert_unique(const key_type &key, Args &&... args) - -> std::pair<iterator, bool> { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - auto res = internal_locate(key); - iterator &iter = res.value; - - if (res.HasMatch()) { - if (res.IsEq()) { - // The key already exists in the tree, do nothing. - return {iter, false}; - } - } else { - iterator last = internal_last(iter); - if (last.node && !compare_keys(key, last.key())) { - // The key already exists in the tree, do nothing. - return {last, false}; - } - } - return {internal_emplace(iter, std::forward<Args>(args)...), true}; - } - - template <typename P> - template <typename... Args> - inline auto btree<P>::insert_hint_unique(iterator position, const key_type &key, - Args &&... args) - -> std::pair<iterator, bool> { - if (!empty()) { - if (position == end() || compare_keys(key, position.key())) { - iterator prev = position; - if (position == begin() || compare_keys((--prev).key(), key)) { - // prev.key() < key < position.key() - return {internal_emplace(position, std::forward<Args>(args)...), true}; - } - } else if (compare_keys(position.key(), key)) { - ++position; - if (position == end() || compare_keys(key, position.key())) { - // {original `position`}.key() < key < {current `position`}.key() - return {internal_emplace(position, std::forward<Args>(args)...), true}; - } - } else { - // position.key() == key - return {position, false}; - } - } - return insert_unique(key, std::forward<Args>(args)...); - } - - template <typename P> - template <typename InputIterator> - void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_unique(end(), params_type::key(*b), *b); - } - } - - template <typename P> - template <typename ValueType> - auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } - - iterator iter = internal_upper_bound(key); - if (iter.node == nullptr) { - iter = end(); - } - return internal_emplace(iter, std::forward<ValueType>(v)); - } - - template <typename P> - template <typename ValueType> - auto btree<P>::insert_hint_multi(iterator position, ValueType &&v) -> iterator { - if (!empty()) { - const key_type &key = params_type::key(v); - if (position == end() || !compare_keys(position.key(), key)) { - iterator prev = position; - if (position == begin() || !compare_keys(key, (--prev).key())) { - // prev.key() <= key <= position.key() - return internal_emplace(position, std::forward<ValueType>(v)); - } - } else { - iterator next = position; - ++next; - if (next == end() || !compare_keys(next.key(), key)) { - // position.key() < key <= next.key() - return internal_emplace(next, std::forward<ValueType>(v)); - } - } - } - return insert_multi(std::forward<ValueType>(v)); - } - - template <typename P> - template <typename InputIterator> - void btree<P>::insert_iterator_multi(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_multi(end(), *b); - } - } - - template <typename P> - auto btree<P>::operator=(const btree &x) -> btree & { - if (this != &x) { - clear(); - - *mutable_key_comp() = x.key_comp(); - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - *mutable_allocator() = x.allocator(); - } - - copy_or_move_values_in_order(&x); - } - return *this; - } - - template <typename P> - auto btree<P>::operator=(btree &&x) noexcept -> btree & { - if (this != &x) { - clear(); - - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - if (allocator() == x.allocator()) { - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - // We aren't allowed to propagate the allocator and the allocator is - // different so we can't take over its memory. We must move each element - // individually. We need both `x` and `this` to have `x`s key comparator - // while moving the values so we can't swap the key comparators. - *mutable_key_comp() = x.key_comp(); - copy_or_move_values_in_order(&x); - } - } - } - return *this; - } - - template <typename P> - auto btree<P>::erase(iterator iter) -> iterator { - bool internal_delete = false; - if (!iter.node->leaf()) { - // Deletion of a value on an internal node. First, move the largest value - // from our left child here, then delete that position (in remove_value() - // below). We can get to the largest value from our left child by - // decrementing iter. - iterator internal_iter(iter); - --iter; - assert(iter.node->leaf()); - params_type::move(mutable_allocator(), iter.node->slot(iter.position), - internal_iter.node->slot(internal_iter.position)); - internal_delete = true; - } - - // Delete the key from the leaf. - iter.node->remove_value(iter.position, mutable_allocator()); - --size_; - - // We want to return the next value after the one we just erased. If we - // erased from an internal node (internal_delete == true), then the next - // value is ++(++iter). If we erased from a leaf node (internal_delete == - // false) then the next value is ++iter. Note that ++iter may point to an - // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. - - iterator res = rebalance_after_delete(iter); - - // If we erased from an internal node, advance the iterator. - if (internal_delete) { - ++res; - } - return res; - } - - template <typename P> - auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { - // Merge/rebalance as we walk back up the tree. - iterator res(iter); - bool first_iteration = true; - for (;;) { - if (iter.node == root()) { - try_shrink(); - if (empty()) { - return end(); - } - break; - } - if (iter.node->count() >= kMinNodeValues) { - break; - } - bool merged = try_merge_or_rebalance(&iter); - // On the first iteration, we should update `res` with `iter` because `res` - // may have been invalidated. - if (first_iteration) { - res = iter; - first_iteration = false; - } - if (!merged) { - break; - } - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - } - - // Adjust our return value. If we're pointing at the end of a node, advance - // the iterator. - if (res.position == res.node->count()) { - res.position = res.node->count() - 1; - ++res; - } - - return res; - } - - template <typename P> - auto btree<P>::erase(iterator _begin, iterator _end) - -> std::pair<size_type, iterator> { - difference_type count = std::distance(_begin, _end); - assert(count >= 0); - - if (count == 0) { - return {0, _begin}; - } - - if (count == (difference_type)size_) { - clear(); - return {count, this->end()}; - } - - if (_begin.node == _end.node) { - erase_same_node(_begin, _end); - size_ -= count; - return {count, rebalance_after_delete(_begin)}; - } - - const size_type target_size = size_ - count; - while (size_ > target_size) { - if (_begin.node->leaf()) { - const size_type remaining_to_erase = size_ - target_size; - const size_type remaining_in_node = _begin.node->count() - _begin.position; - _begin = erase_from_leaf_node( - _begin, (std::min)(remaining_to_erase, remaining_in_node)); - } else { - _begin = erase(_begin); - } - } - return {count, _begin}; - } - - template <typename P> - void btree<P>::erase_same_node(iterator _begin, iterator _end) { - assert(_begin.node == _end.node); - assert(_end.position > _begin.position); - - node_type *node = _begin.node; - size_type to_erase = _end.position - _begin.position; - if (!node->leaf()) { - // Delete all children between _begin and _end. - for (size_type i = 0; i < to_erase; ++i) { - internal_clear(node->child(_begin.position + i + 1)); - } - // Rotate children after _end into new positions. - for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { - node->set_child(i - to_erase, node->child(i)); - node->clear_child(i); - } - } - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - // Do not need to update rightmost_, because - // * either _end == this->end(), and therefore node == rightmost_, and still - // exists - // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since - // it wasn't covered in [_begin, _end) - } - - template <typename P> - auto btree<P>::erase_from_leaf_node(iterator _begin, size_type to_erase) - -> iterator { - node_type *node = _begin.node; - assert(node->leaf()); - assert(node->count() > _begin.position); - assert(_begin.position + to_erase <= node->count()); - - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); - - size_ -= to_erase; - - return rebalance_after_delete(_begin); - } - - template <typename P> - template <typename K> - auto btree<P>::erase_unique(const K &key) -> size_type { - const iterator iter = internal_find(key); - if (iter.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - erase(iter); - return 1; - } - - template <typename P> - template <typename K> - auto btree<P>::erase_multi(const K &key) -> size_type { - const iterator _begin = internal_lower_bound(key); - if (_begin.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - // Delete all of the keys between _begin and upper_bound(key). - const iterator _end = internal_end(internal_upper_bound(key)); - return erase(_begin, _end).first; - } - - template <typename P> - void btree<P>::clear() { - if (!empty()) { - internal_clear(root()); - } - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - size_ = 0; - } - - template <typename P> - void btree<P>::swap(btree &x) { - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_swap::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - } else { - // It's undefined behavior if the allocators are unequal here. - assert(allocator() == x.allocator()); - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - } - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } - - template <typename P> - void btree<P>::verify() const { - assert(root() != nullptr); - assert(leftmost() != nullptr); - assert(rightmost_ != nullptr); - assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost_ == (--const_iterator(root(), root()->count())).node); - assert(leftmost()->leaf()); - assert(rightmost_->leaf()); - } - - template <typename P> - void btree<P>::rebalance_or_split(iterator *iter) { - node_type *&node = iter->node; - int &insert_position = iter->position; - assert(node->count() == node->max_count()); - assert(kNodeValues == node->max_count()); - - // First try to make room on the node by rebalancing. - node_type *parent = node->parent(); - if (node != root()) { - if (node->position() > 0) { - // Try rebalancing with our left sibling. - node_type *left = parent->child(node->position() - 1); - assert(left->max_count() == kNodeValues); - if (left->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the end of the right node then we bias rebalancing to - // fill up the left node. - int to_move = (kNodeValues - left->count()) / - (1 + (insert_position < kNodeValues)); - to_move = (std::max)(1, to_move); - - if (((insert_position - to_move) >= 0) || - ((left->count() + to_move) < kNodeValues)) { - left->rebalance_right_to_left(to_move, node, mutable_allocator()); - - assert(node->max_count() - node->count() == to_move); - insert_position = insert_position - to_move; - if (insert_position < 0) { - insert_position = insert_position + left->count() + 1; - node = left; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - if (node->position() < parent->count()) { - // Try rebalancing with our right sibling. - node_type *right = parent->child(node->position() + 1); - assert(right->max_count() == kNodeValues); - if (right->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the _beginning of the left node then we bias rebalancing - // to fill up the right node. - int to_move = - (kNodeValues - right->count()) / (1 + (insert_position > 0)); - to_move = (std::max)(1, to_move); - - if ((insert_position <= (node->count() - to_move)) || - ((right->count() + to_move) < kNodeValues)) { - node->rebalance_left_to_right(to_move, right, mutable_allocator()); - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = right; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - // Rebalancing failed, make sure there is room on the parent node for a new - // value. - assert(parent->max_count() == kNodeValues); - if (parent->count() == kNodeValues) { - iterator parent_iter(node->parent(), node->position()); - rebalance_or_split(&parent_iter); - } - } else { - // Rebalancing not possible because this is the root node. - // Create a new root node and set the current root node as the child of the - // new root. - parent = new_internal_node(parent); - parent->init_child(0, root()); - mutable_root() = parent; - // If the former root was a leaf node, then it's now the rightmost node. - assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); - } - - // Split the node. - node_type *split_node; - if (node->leaf()) { - split_node = new_leaf_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - if (rightmost_ == node) rightmost_ = split_node; - } else { - split_node = new_internal_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - } - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = split_node; - } - } - - template <typename P> - void btree<P>::merge_nodes(node_type *left, node_type *right) { - left->merge(right, mutable_allocator()); - if (right->leaf()) { - if (rightmost_ == right) rightmost_ = left; - delete_leaf_node(right); - } else { - delete_internal_node(right); - } - } - - template <typename P> - bool btree<P>::try_merge_or_rebalance(iterator *iter) { - node_type *parent = iter->node->parent(); - if (iter->node->position() > 0) { - // Try merging with our left sibling. - node_type *left = parent->child(iter->node->position() - 1); - assert(left->max_count() == kNodeValues); - if ((1 + left->count() + iter->node->count()) <= kNodeValues) { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; - return true; - } - } - if (iter->node->position() < parent->count()) { - // Try merging with our right sibling. - node_type *right = parent->child(iter->node->position() + 1); - assert(right->max_count() == kNodeValues); - if ((1 + iter->node->count() + right->count()) <= kNodeValues) { - merge_nodes(iter->node, right); - return true; - } - // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the front of the tree. - if ((right->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position > 0))) { - int to_move = (right->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); - return false; - } - } - if (iter->node->position() > 0) { - // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the back of the tree. - node_type *left = parent->child(iter->node->position() - 1); - if ((left->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position < iter->node->count()))) { - int to_move = (left->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, left->count() - 1); - left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); - iter->position += to_move; - return false; - } - } - return false; - } - - template <typename P> - void btree<P>::try_shrink() { - if (root()->count() > 0) { - return; - } - // Deleted the last item on the root node, shrink the height of the tree. - if (root()->leaf()) { - assert(size() == 0); - delete_leaf_node(root()); - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - } else { - node_type *child = root()->child(0); - child->make_root(); - delete_internal_node(root()); - mutable_root() = child; - } - } - - template <typename P> - template <typename IterType> - inline IterType btree<P>::internal_last(IterType iter) { - assert(iter.node != nullptr); - while (iter.position == iter.node->count()) { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - if (iter.node->leaf()) { - iter.node = nullptr; - break; - } - } - return iter; - } - - template <typename P> - template <typename... Args> - inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) - -> iterator { - if (!iter.node->leaf()) { - // We can't insert on an internal node. Instead, we'll insert after the - // previous value which is guaranteed to be on a leaf node. - --iter; - ++iter.position; - } - const int max_count = iter.node->max_count(); - if (iter.node->count() == max_count) { - // Make room in the leaf for the new item. - if (max_count < kNodeValues) { - // Insertion into the root where the root is smaller than the full node - // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = - new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count)); - iter.node->swap(root(), mutable_allocator()); - delete_leaf_node(root()); - mutable_root() = iter.node; - rightmost_ = iter.node; - } else { - rebalance_or_split(&iter); - } - } - iter.node->emplace_value(iter.position, mutable_allocator(), - std::forward<Args>(args)...); - ++size_; - return iter; - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate(const K &key) const - -> SearchResult<iterator, is_key_compare_to::value> { - return internal_locate_impl(key, is_key_compare_to()); - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const - -> SearchResult<iterator, false> { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - // NOTE: we don't need to walk all the way down the tree if the keys are - // equal, but determining equality would require doing an extra comparison - // on each node on the way down, and we will need to go all the way to the - // leaf node in the expected case. - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter}; - } - - template <typename P> - template <typename K> - inline auto btree<P>::internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const - -> SearchResult<iterator, true> { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - SearchResult<int, true> res = iter.node->lower_bound(key, key_comp()); - iter.position = res.value; - if (res.match == MatchKind::kEq) { - return {iter, MatchKind::kEq}; - } - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter, MatchKind::kNe}; - } - - template <typename P> - template <typename K> - auto btree<P>::internal_lower_bound(const K &key) const -> iterator { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template <typename P> - template <typename K> - auto btree<P>::internal_upper_bound(const K &key) const -> iterator { - iterator iter(const_cast<node_type *>(root()), 0); - for (;;) { - iter.position = iter.node->upper_bound(key, key_comp()); - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template <typename P> - template <typename K> - auto btree<P>::internal_find(const K &key) const -> iterator { - auto res = internal_locate(key); - if (res.HasMatch()) { - if (res.IsEq()) { - return res.value; - } - } else { - const iterator iter = internal_last(res.value); - if (iter.node != nullptr && !compare_keys(key, iter.key())) { - return iter; - } - } - return {nullptr, 0}; - } - - template <typename P> - void btree<P>::internal_clear(node_type *node) { - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - internal_clear(node->child(i)); - } - delete_internal_node(node); - } else { - delete_leaf_node(node); - } - } - - template <typename P> - int btree<P>::internal_verify( - const node_type *node, const key_type *lo, const key_type *hi) const { - assert(node->count() > 0); - assert(node->count() <= node->max_count()); - if (lo) { - assert(!compare_keys(node->key(0), *lo)); - } - if (hi) { - assert(!compare_keys(*hi, node->key(node->count() - 1))); - } - for (int i = 1; i < node->count(); ++i) { - assert(!compare_keys(node->key(i), node->key(i - 1))); - } - int count = node->count(); - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - assert(node->child(i) != nullptr); - assert(node->child(i)->parent() == node); - assert(node->child(i)->position() == i); - count += internal_verify( - node->child(i), - (i == 0) ? lo : &node->key(i - 1), - (i == node->count()) ? hi : &node->key(i)); - } - } - return count; - } - - // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. - // --------------------------------------------------------------------------------- - template <typename Tree> - class btree_container { - using params_type = typename Tree::params_type; - - protected: - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template <class K> - using key_arg = - typename KeyArg<IsTransparent<typename Tree::key_compare>::value>:: - template type<K, typename Tree::key_type>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using difference_type = typename Tree::difference_type; - using key_compare = typename Tree::key_compare; - using value_compare = typename Tree::value_compare; - using allocator_type = typename Tree::allocator_type; - using reference = typename Tree::reference; - using const_reference = typename Tree::const_reference; - using pointer = typename Tree::pointer; - using const_pointer = typename Tree::const_pointer; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using reverse_iterator = typename Tree::reverse_iterator; - using const_reverse_iterator = typename Tree::const_reverse_iterator; - using node_type = typename Tree::node_handle_type; - - // Constructors/assignments. - btree_container() : tree_(key_compare(), allocator_type()) {} - explicit btree_container(const key_compare &comp, - const allocator_type &alloc = allocator_type()) - : tree_(comp, alloc) {} - btree_container(const btree_container &x) = default; - btree_container(btree_container &&x) noexcept = default; - btree_container &operator=(const btree_container &x) = default; - btree_container &operator=(btree_container &&x) noexcept( - std::is_nothrow_move_assignable<Tree>::value) = default; - - // Iterator routines. - iterator begin() { return tree_.begin(); } - const_iterator begin() const { return tree_.begin(); } - const_iterator cbegin() const { return tree_.begin(); } - iterator end() { return tree_.end(); } - const_iterator end() const { return tree_.end(); } - const_iterator cend() const { return tree_.end(); } - reverse_iterator rbegin() { return tree_.rbegin(); } - const_reverse_iterator rbegin() const { return tree_.rbegin(); } - const_reverse_iterator crbegin() const { return tree_.rbegin(); } - reverse_iterator rend() { return tree_.rend(); } - const_reverse_iterator rend() const { return tree_.rend(); } - const_reverse_iterator crend() const { return tree_.rend(); } - - // Lookup routines. - template <typename K = key_type> - iterator find(const key_arg<K> &key) { - return tree_.find(key); - } - template <typename K = key_type> - const_iterator find(const key_arg<K> &key) const { return tree_.find(key); } - - template <typename K = key_type> - bool contains(const key_arg<K> &key) const { return find(key) != end(); } - - template <typename K = key_type> - iterator lower_bound(const key_arg<K> &key) { return tree_.lower_bound(key); } - - template <typename K = key_type> - const_iterator lower_bound(const key_arg<K> &key) const { return tree_.lower_bound(key); } - - template <typename K = key_type> - iterator upper_bound(const key_arg<K> &key) { return tree_.upper_bound(key); } - - template <typename K = key_type> - const_iterator upper_bound(const key_arg<K> &key) const { return tree_.upper_bound(key); } - - template <typename K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K> &key) { return tree_.equal_range(key); } - - template <typename K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K> &key) const { - return tree_.equal_range(key); - } - - iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } - iterator erase(iterator iter) { return tree_.erase(iter); } - iterator erase(const_iterator first, const_iterator last) { - return tree_.erase(iterator(first), iterator(last)).second; - } - - node_type extract(iterator position) { - // Use Move instead of Transfer, because the rebalancing code expects to - // have a valid object to scribble metadata bits on top of. - auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot()); - erase(position); - return node; - } - - node_type extract(const_iterator position) { - return extract(iterator(position)); - } - - public: - void clear() { tree_.clear(); } - void swap(btree_container &x) { tree_.swap(x.tree_); } - void verify() const { tree_.verify(); } - - size_type size() const { return tree_.size(); } - size_type max_size() const { return tree_.max_size(); } - bool empty() const { return tree_.empty(); } - - friend bool operator==(const btree_container &x, const btree_container &y) { - if (x.size() != y.size()) return false; - return std::equal(x.begin(), x.end(), y.begin()); - } - - friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } - - friend bool operator<(const btree_container &x, const btree_container &y) { - return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); - } - - friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } - - friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } - - friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } - - // The allocator used by the btree. - allocator_type get_allocator() const { return tree_.get_allocator(); } - - // The key comparator used by the btree. - key_compare key_comp() const { return tree_.key_comp(); } - value_compare value_comp() const { return tree_.value_comp(); } - - // Support absl::Hash. - template <typename State> - friend State AbslHashValue(State h, const btree_container &b) { - for (const auto &v : b) { - h = State::combine(std::move(h), v); - } - return State::combine(std::move(h), b.size()); - } - - protected: - Tree tree_; - }; - - // A common base class for btree_set and btree_map. - // ----------------------------------------------- - template <typename Tree> - class btree_set_container : public btree_container<Tree> { - using super_type = btree_container<Tree>; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - friend class BtreeNodePeer; - - protected: - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - using insert_return_type = InsertReturnType<iterator, node_type>; - using super_type::super_type; - btree_set_container() {} - - template <class InputIterator> - btree_set_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - btree_set_container(std::initializer_list<init_type> init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_set_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_unique(key); - } - - // Insertion routines. - std::pair<iterator, bool> insert(const value_type &x) { - return this->tree_.insert_unique(params_type::key(x), x); - } - std::pair<iterator, bool> insert(value_type &&x) { - return this->tree_.insert_unique(params_type::key(x), std::move(x)); - } - template <typename... Args> - std::pair<iterator, bool> emplace(Args &&... args) { - init_type v(std::forward<Args>(args)...); - return this->tree_.insert_unique(params_type::key(v), std::move(v)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), x) - .first; - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), - std::move(x)) - .first; - } - - template <typename... Args> - iterator emplace_hint(const_iterator position, Args &&... args) { - init_type v(std::forward<Args>(args)...); - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(v), - std::move(v)) - .first; - } - - template <typename InputIterator> - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_unique(b, e); - } - - void insert(std::initializer_list<init_type> init) { - this->tree_.insert_iterator_unique(init.begin(), init.end()); - } - - insert_return_type insert(node_type &&node) { - if (!node) return {this->end(), false, node_type()}; - std::pair<iterator, bool> res = - this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) { - CommonAccess::Destroy(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - std::pair<iterator, bool> res = this->tree_.insert_hint_unique( - iterator(hint), params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) CommonAccess::Destroy(&node); - return res.first; - } - - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { return this->tree_.erase_unique(key); } - using super_type::erase; - - template <typename K = key_type> - node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - - using super_type::extract; - - // Merge routines. - // Moves elements from `src` into `this`. If the element already exists in - // `this`, it is left unmodified in `src`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &src) { // NOLINT - for (auto src_it = src.begin(); src_it != src.end();) { - if (insert(std::move(*src_it)).second) { - src_it = src.erase(src_it); - } else { - ++src_it; - } - } - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &&src) { - merge(src); - } - }; - - // Base class for btree_map. - // ------------------------- - template <typename Tree> - class btree_map_container : public btree_set_container<Tree> { - using super_type = btree_set_container<Tree>; - using params_type = typename Tree::params_type; - - protected: - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using mapped_type = typename params_type::mapped_type; - using value_type = typename Tree::value_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - - // Inherit constructors. - using super_type::super_type; - btree_map_container() {} - - // Insertion routines. - template <typename... Args> - std::pair<iterator, bool> try_emplace(const key_type &k, Args &&... args) { - return this->tree_.insert_unique( - k, std::piecewise_construct, std::forward_as_tuple(k), - std::forward_as_tuple(std::forward<Args>(args)...)); - } - template <typename... Args> - std::pair<iterator, bool> try_emplace(key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_unique guarantees that `key` is never - // referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_.insert_unique( - key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - } - template <typename... Args> - iterator try_emplace(const_iterator hint, const key_type &k, - Args &&... args) { - return this->tree_ - .insert_hint_unique(iterator(hint), k, std::piecewise_construct, - std::forward_as_tuple(k), - std::forward_as_tuple(std::forward<Args>(args)...)) - .first; - } - template <typename... Args> - iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_hint_unique guarantees that `key` is - // never referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_ - .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, - std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward<Args>(args)...)) - .first; - } - mapped_type &operator[](const key_type &k) { - return try_emplace(k).first->second; - } - mapped_type &operator[](key_type &&k) { - return try_emplace(std::move(k)).first->second; - } - - template <typename K = key_type> - mapped_type &at(const key_arg<K> &key) { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - template <typename K = key_type> - const mapped_type &at(const key_arg<K> &key) const { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - }; - - // A common base class for btree_multiset and btree_multimap. - template <typename Tree> - class btree_multiset_container : public btree_container<Tree> { - using super_type = btree_container<Tree>; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - - template <class K> - using key_arg = typename super_type::template key_arg<K>; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - - // Inherit constructors. - using super_type::super_type; - btree_multiset_container() {} - - // Range constructor. - template <class InputIterator> - btree_multiset_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - // Initializer list constructor. - btree_multiset_container(std::initializer_list<init_type> init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_multi(key); - } - - // Insertion routines. - iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } - iterator insert(value_type &&x) { - return this->tree_.insert_multi(std::move(x)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_.insert_hint_multi(iterator(position), x); - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_.insert_hint_multi(iterator(position), std::move(x)); - } - template <typename InputIterator> - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_multi(b, e); - } - void insert(std::initializer_list<init_type> init) { - this->tree_.insert_iterator_multi(init.begin(), init.end()); - } - template <typename... Args> - iterator emplace(Args &&... args) { - return this->tree_.insert_multi(init_type(std::forward<Args>(args)...)); - } - template <typename... Args> - iterator emplace_hint(const_iterator position, Args &&... args) { - return this->tree_.insert_hint_multi( - iterator(position), init_type(std::forward<Args>(args)...)); - } - iterator insert(node_type &&node) { - if (!node) return this->end(); - iterator res = - this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - CommonAccess::Destroy(&node); - return res; - } - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - iterator res = this->tree_.insert_hint_multi( - iterator(hint), - std::move(params_type::element(CommonAccess::GetSlot(node)))); - CommonAccess::Destroy(&node); - return res; - } - - // Deletion routines. - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { - return this->tree_.erase_multi(key); - } - using super_type::erase; - - // Node extraction routines. - template <typename K = key_type> - node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - using super_type::extract; - - // Merge routines. - // Moves all elements from `src` into `this`. - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &src) { // NOLINT - insert(std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); - src.clear(); - } - - template < - typename T, - typename phmap::enable_if_t< - phmap::conjunction< - std::is_same<value_type, typename T::value_type>, - std::is_same<allocator_type, typename T::allocator_type>, - std::is_same<typename params_type::is_map_container, - typename T::params_type::is_map_container>>::value, - int> = 0> - void merge(btree_container<T> &&src) { - merge(src); - } - }; - - // A base class for btree_multimap. - template <typename Tree> - class btree_multimap_container : public btree_multiset_container<Tree> { - using super_type = btree_multiset_container<Tree>; - using params_type = typename Tree::params_type; - - public: - using mapped_type = typename params_type::mapped_type; - - // Inherit constructors. - using super_type::super_type; - btree_multimap_container() {} - }; - -} // namespace priv - - - - // ---------------------------------------------------------------------- - // btree_set - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Compare, typename Alloc> - class btree_set : public priv::btree_set_container< - priv::btree<priv::set_params< - Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> - { - using Base = typename btree_set::btree_set_container; - - public: - btree_set() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_set` containers. - // ------------------------------------------------------- - template <typename K, typename C, typename A> - void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename C, typename A, typename Pred> - void erase_if(btree_set<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multiset - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Compare, typename Alloc> - class btree_multiset : public priv::btree_multiset_container< - priv::btree<priv::set_params< - Key, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> - { - using Base = typename btree_multiset::btree_multiset_container; - - public: - btree_multiset() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multiset` containers. - // ------------------------------------------------------------ - template <typename K, typename C, typename A> - void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename C, typename A, typename Pred> - void erase_if(btree_multiset<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } - } - - - // ---------------------------------------------------------------------- - // btree_map - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Value, typename Compare, typename Alloc> - class btree_map : public priv::btree_map_container< - priv::btree<priv::map_params< - Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ false>>> - { - using Base = typename btree_map::btree_map_container; - - public: - btree_map() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_map` containers. - // ------------------------------------------------------- - template <typename K, typename V, typename C, typename A> - void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) { - return x.swap(y); - } - - // ---------------------------------------------------------------------- - template <typename K, typename V, typename C, typename A, typename Pred> - void erase_if(btree_map<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - // ---------------------------------------------------------------------- - // btree_multimap - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template <typename Key, typename Value, typename Compare, typename Alloc> - class btree_multimap : public priv::btree_multimap_container< - priv::btree<priv::map_params< - Key, Value, Compare, Alloc, /*TargetNodeSize=*/ 256, /*Multi=*/ true>>> - { - using Base = typename btree_multimap::btree_multimap_container; - - public: - btree_multimap() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multimap` containers. - // ------------------------------------------------------------ - template <typename K, typename V, typename C, typename A> - void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template <typename K, typename V, typename C, typename A, typename Pred> - void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } - - -} // namespace btree - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/benchmarks/external/parallel_hashmap/conanfile.py b/benchmarks/external/parallel_hashmap/conanfile.py deleted file mode 100644 index c046377d..00000000 --- a/benchmarks/external/parallel_hashmap/conanfile.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from conans import ConanFile, tools -import os - -class SparseppConan(ConanFile): - name = "parallel_hashmap" - version = "1.27" - description = "A header-only, very fast and memory-friendly hash map" - - # Indicates License type of the packaged library - license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" - - # Packages the license for the conanfile.py - exports = ["LICENSE"] - - # Custom attributes for Bincrafters recipe conventions - source_subfolder = "source_subfolder" - - def source(self): - source_url = "https://github.com/greg7mdp/parallel-hashmap" - tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) - extracted_dir = self.name + "-" + self.version - - #Rename to "source_folder" is a convention to simplify later steps - os.rename(extracted_dir, self.source_subfolder) - - - def package(self): - include_folder = os.path.join(self.source_subfolder, "parallel_hashmap") - self.copy(pattern="LICENSE") - self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder) - - def package_id(self): - self.info.header_only() diff --git a/benchmarks/external/parallel_hashmap/meminfo.h b/benchmarks/external/parallel_hashmap/meminfo.h deleted file mode 100644 index 872f3c69..00000000 --- a/benchmarks/external/parallel_hashmap/meminfo.h +++ /dev/null @@ -1,195 +0,0 @@ -#if !defined(spp_memory_h_guard) -#define spp_memory_h_guard - -#include <cstdint> -#include <cstring> -#include <cstdlib> - -#if defined(_WIN32) || defined( __CYGWIN__) - #define SPP_WIN -#endif - -#ifdef SPP_WIN - #include <windows.h> - #include <Psapi.h> - #undef min - #undef max -#elif defined(__linux__) - #include <sys/types.h> - #include <sys/sysinfo.h> -#elif defined(__FreeBSD__) - #include <paths.h> - #include <fcntl.h> - #include <kvm.h> - #include <unistd.h> - #include <sys/sysctl.h> - #include <sys/user.h> -#endif - -namespace spp -{ - uint64_t GetSystemMemory(); - uint64_t GetTotalMemoryUsed(); - uint64_t GetProcessMemoryUsed(); - uint64_t GetPhysicalMemory(); - - uint64_t GetSystemMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo (&memInfo); - auto totalVirtualMem = memInfo.totalram; - - totalVirtualMem += memInfo.totalswap; - totalVirtualMem *= memInfo.mem_unit; - return static_cast<uint64_t>(totalVirtualMem); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageCnt; - size_t pageCntLen = sizeof(pageCnt); - u_int pageSize; - struct kvm_swap kswap; - uint64_t totalVirtualMem; - - pageSize = static_cast<u_int>(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - totalVirtualMem = pageCnt * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - totalVirtualMem += kswap.ksw_total * pageSize; - - return totalVirtualMem; -#else - return 0; -#endif - } - - uint64_t GetTotalMemoryUsed() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - auto virtualMemUsed = memInfo.totalram - memInfo.freeram; - - virtualMemUsed += memInfo.totalswap - memInfo.freeswap; - virtualMemUsed *= memInfo.mem_unit; - - return static_cast<uint64_t>(virtualMemUsed); -#elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageSize; - u_int pageCnt, freeCnt; - size_t pageCntLen = sizeof(pageCnt); - size_t freeCntLen = sizeof(freeCnt); - struct kvm_swap kswap; - uint64_t virtualMemUsed; - - pageSize = static_cast<u_int>(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); - virtualMemUsed = (pageCnt - freeCnt) * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - virtualMemUsed += kswap.ksw_used * pageSize; - - return virtualMemUsed; -#else - return 0; -#endif - } - - uint64_t GetProcessMemoryUsed() - { -#ifdef SPP_WIN - PROCESS_MEMORY_COUNTERS_EX pmc; - GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PPROCESS_MEMORY_COUNTERS>(&pmc), sizeof(pmc)); - return static_cast<uint64_t>(pmc.PrivateUsage); -#elif defined(__linux__) - auto parseLine = - [](char* line)->int - { - auto i = strlen(line); - - while(*line < '0' || *line > '9') - { - line++; - } - - line[i-3] = '\0'; - i = atoi(line); - return i; - }; - - auto file = fopen("/proc/self/status", "r"); - auto result = -1; - char line[128]; - - while(fgets(line, 128, file) != nullptr) - { - if(strncmp(line, "VmSize:", 7) == 0) - { - result = parseLine(line); - break; - } - } - - fclose(file); - return static_cast<uint64_t>(result) * 1024; -#elif defined(__FreeBSD__) - struct kinfo_proc info; - size_t infoLen = sizeof(info); - int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); - return static_cast<uint64_t>(info.ki_rssize * getpagesize()); -#else - return 0; -#endif - } - - uint64_t GetPhysicalMemory() - { -#ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast<uint64_t>(memInfo.ullTotalPhys); -#elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - - auto totalPhysMem = memInfo.totalram; - - totalPhysMem *= memInfo.mem_unit; - return static_cast<uint64_t>(totalPhysMem); -#elif defined(__FreeBSD__) - u_long physMem; - size_t physMemLen = sizeof(physMem); - int mib[] = { CTL_HW, HW_PHYSMEM }; - - sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); - return physMem; -#else - return 0; -#endif - } - -} - -#endif // spp_memory_h_guard diff --git a/benchmarks/external/parallel_hashmap/phmap.h b/benchmarks/external/parallel_hashmap/phmap.h deleted file mode 100644 index 653ae5ea..00000000 --- a/benchmarks/external/parallel_hashmap/phmap.h +++ /dev/null @@ -1,4788 +0,0 @@ -#if !defined(phmap_h_guard_) -#define phmap_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified -#endif - -#include <algorithm> -#include <cmath> -#include <cstring> -#include <iterator> -#include <limits> -#include <memory> -#include <tuple> -#include <type_traits> -#include <utility> -#include <array> -#include <cassert> -#include <atomic> - -#include "phmap_fwd_decl.h" -#include "phmap_utils.h" -#include "phmap_base.h" - -#if PHMAP_HAVE_STD_STRING_VIEW - #include <string_view> -#endif - -namespace phmap { - -namespace priv { - -// -------------------------------------------------------------------------- -template <size_t Width> -class probe_seq -{ -public: - probe_seq(size_t hashval, size_t mask) { - assert(((mask + 1) & mask) == 0 && "not a mask"); - mask_ = mask; - offset_ = hashval & mask_; - } - size_t offset() const { return offset_; } - size_t offset(size_t i) const { return (offset_ + i) & mask_; } - - void next() { - index_ += Width; - offset_ += index_; - offset_ &= mask_; - } - // 0-based probe index. The i-th probe in the probe sequence. - size_t getindex() const { return index_; } - -private: - size_t mask_; - size_t offset_; - size_t index_ = 0; -}; - -// -------------------------------------------------------------------------- -template <class ContainerKey, class Hash, class Eq> -struct RequireUsableKey -{ - template <class PassedKey, class... Args> - std::pair< - decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), - decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), - std::declval<const PassedKey&>()))>* - operator()(const PassedKey&, const Args&...) const; -}; - -// -------------------------------------------------------------------------- -template <class E, class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable : std::false_type {}; - -template <class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable< - phmap::void_t<decltype( - Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), - std::declval<Ts>()...))>, - Policy, Hash, Eq, Ts...> : std::true_type {}; - -// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. -// -------------------------------------------------------------------------- -template <class T> -constexpr bool IsNoThrowSwappable() { - using std::swap; - return noexcept(swap(std::declval<T&>(), std::declval<T&>())); -} - -// -------------------------------------------------------------------------- -template <typename T> -int TrailingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountTrailingZerosNonZero64(static_cast<uint64_t>(x)); - else - return base_internal::CountTrailingZerosNonZero32(static_cast<uint32_t>(x)); -} - -// -------------------------------------------------------------------------- -template <typename T> -int LeadingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountLeadingZeros64(static_cast<uint64_t>(x)); - else - return base_internal::CountLeadingZeros32(static_cast<uint32_t>(x)); -} - -// -------------------------------------------------------------------------- -// An abstraction over a bitmask. It provides an easy way to iterate through the -// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), -// this is a true bitmask. On non-SSE, platforms the arithematic used to -// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as -// either 0x00 or 0x80. -// -// For example: -// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 -// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 -// -------------------------------------------------------------------------- -template <class T, int SignificantBits, int Shift = 0> -class BitMask -{ - static_assert(std::is_unsigned<T>::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - -public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } - int LowestBitSet() const { - return priv::TrailingZeros(mask_) >> Shift; - } - int HighestBitSet() const { - return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - - 1) >> - Shift; - } - - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - - int TrailingZeros() const { - return priv::TrailingZeros(mask_) >> Shift; - } - - int LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return priv::LeadingZeros(mask_ << extra_bits) >> Shift; - } - -private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } - - T mask_; -}; - -// -------------------------------------------------------------------------- -using ctrl_t = signed char; -using h2_t = uint8_t; - -// -------------------------------------------------------------------------- -// The values here are selected for maximum performance. See the static asserts -// below for details. -// -------------------------------------------------------------------------- -enum Ctrl : ctrl_t -{ - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 -}; - -static_assert( - kEmpty & kDeleted & kSentinel & 0x80, - "Special markers need to have the MSB to make checking for them efficient"); -static_assert(kEmpty < kSentinel && kDeleted < kSentinel, - "kEmpty and kDeleted must be smaller than kSentinel to make the " - "SIMD test of IsEmptyOrDeleted() efficient"); -static_assert(kSentinel == -1, - "kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(kEmpty == -128, - "kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); -static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, - "kEmpty and kDeleted must share an unset bit that is not shared " - "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " - "efficient"); -static_assert(kDeleted == -2, - "kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - -// -------------------------------------------------------------------------- -// A single block of empty control bytes for tables without any slots allocated. -// This enables removing a branch in the hot path of find(). -// -------------------------------------------------------------------------- -inline ctrl_t* EmptyGroup() { - alignas(16) static constexpr ctrl_t empty_group[] = { - kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, - kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; - return const_cast<ctrl_t*>(empty_group); -} - -// -------------------------------------------------------------------------- -inline size_t HashSeed(const ctrl_t* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return reinterpret_cast<uintptr_t>(ctrl) >> 12; -} - -#ifdef PHMAP_NON_DETERMINISTIC - -inline size_t H1(size_t hashval, const ctrl_t* ctrl) { - // use ctrl_ pointer to add entropy to ensure - // non-deterministic iteration order. - return (hashval >> 7) ^ HashSeed(ctrl); -} - -#else - -inline size_t H1(size_t hashval, const ctrl_t* ) { - return (hashval >> 7); -} - -#endif - - -inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } - -inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= 0; } -inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } -inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } - -#if PHMAP_HAVE_SSE2 - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch -#endif - -// -------------------------------------------------------------------------- -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -// -------------------------------------------------------------------------- -inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { -#if defined(__GNUC__) && !defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Woverflow" - - if (std::is_unsigned<char>::value) { - const __m128i mask = _mm_set1_epi8(static_cast<char>(0x80)); - const __m128i diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } - - #pragma GCC diagnostic pop -#endif - return _mm_cmpgt_epi8(a, b); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupSse2Impl -{ - enum { kWidth = 16 }; // the number of slots per group - - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); - } - - // Returns a bitmask representing the positions of slots that match hash. - // ---------------------------------------------------------------------- - BitMask<uint32_t, kWidth> Match(h2_t hash) const { - auto match = _mm_set1_epi8((char)hash); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); - } - - // Returns a bitmask representing the positions of empty slots. - // ------------------------------------------------------------ - BitMask<uint32_t, kWidth> MatchEmpty() const { -#if PHMAP_HAVE_SSSE3 - // This only works because kEmpty is -128. - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); -#else - return Match(static_cast<h2_t>(kEmpty)); -#endif - } - - // Returns a bitmask representing the positions of empty or deleted slots. - // ----------------------------------------------------------------------- - BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); - } - - // Returns the number of trailing empty or deleted elements in the group. - // ---------------------------------------------------------------------- - uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return TrailingZeros( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); - } - - // ---------------------------------------------------------------------- - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast<char>(-128)); - auto x126 = _mm_set1_epi8(126); -#if PHMAP_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); -#else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); -#endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } - - __m128i ctrl; -}; - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // PHMAP_HAVE_SSE2 - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -struct GroupPortableImpl -{ - enum { kWidth = 8 }; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on kEmpty, kDeleted, kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } - - uint64_t ctrl; -}; - -#if PHMAP_HAVE_SSE2 - using Group = GroupSse2Impl; -#else - using Group = GroupPortableImpl; -#endif - -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set; - -inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } - -// -------------------------------------------------------------------------- -// PRECONDITION: -// IsValidCapacity(capacity) -// ctrl[capacity] == kSentinel -// ctrl[i] != kSentinel for all i < capacity -// Applies mapping for every byte in ctrl: -// DELETED -> EMPTY -// EMPTY -> EMPTY -// FULL -> DELETED -// -------------------------------------------------------------------------- -inline void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) -{ - assert(ctrl[capacity] == kSentinel); - assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { - Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); - } - // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; -} - -// -------------------------------------------------------------------------- -// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. -// -------------------------------------------------------------------------- -inline size_t NormalizeCapacity(size_t n) -{ - return n ? ~size_t{} >> LeadingZeros(n) : 1; -} - -// -------------------------------------------------------------------------- -// We use 7/8th as maximum load factor. -// For 16-wide groups, that gives an average of two empty slots per group. -// -------------------------------------------------------------------------- -inline size_t CapacityToGrowth(size_t capacity) -{ - assert(IsValidCapacity(capacity)); - // `capacity*7/8` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (capacity == 7) - { - // x-x/8 does not work when x==7. - return 6; - } - } - return capacity - capacity / 8; -} - -// -------------------------------------------------------------------------- -// From desired "growth" to a lowerbound of the necessary capacity. -// Might not be a valid one and required NormalizeCapacity(). -// -------------------------------------------------------------------------- -inline size_t GrowthToLowerboundCapacity(size_t growth) -{ - // `growth*8/7` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (growth == 7) - { - // x+(x-1)/7 does not work when x==7. - return 8; - } - } - return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); -} - -namespace hashtable_debug_internal { - -// If it is a map, call get<0>(). -using std::get; -template <typename T, typename = typename T::mapped_type> -auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { - return get<0>(pair); -} - -// If it is not a map, return the value directly. -template <typename T> -const typename T::key_type& GetKey(const typename T::key_type& key, char) { - return key; -} - -// -------------------------------------------------------------------------- -// Containers should specialize this to provide debug information for that -// container. -// -------------------------------------------------------------------------- -template <class Container, typename Enabler = void> -struct HashtableDebugAccess -{ - // Returns the number of probes required to find `key` in `c`. The "number of - // probes" is a concept that can vary by container. Implementations should - // return 0 when `key` was found in the minimum number of operations and - // should increment the result for each non-trivial operation required to find - // `key`. - // - // The default implementation uses the bucket api from the standard and thus - // works for `std::unordered_*` containers. - // -------------------------------------------------------------------------- - static size_t GetNumProbes(const Container& c, - const typename Container::key_type& key) { - if (!c.bucket_count()) return {}; - size_t num_probes = 0; - size_t bucket = c.bucket(key); - for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { - if (it == e) return num_probes; - if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes; - } - } -}; - -} // namespace hashtable_debug_internal - -// ---------------------------------------------------------------------------- -// I N F O Z S T U B S -// ---------------------------------------------------------------------------- -struct HashtablezInfo -{ - void PrepareForSampling() {} -}; - -inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} - -static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} - -static inline void RecordEraseSlow(HashtablezInfo*) {} - -static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } -static inline void UnsampleSlow(HashtablezInfo* ) {} - -class HashtablezInfoHandle -{ -public: - inline void RecordStorageChanged(size_t , size_t ) {} - inline void RecordRehash(size_t ) {} - inline void RecordInsert(size_t , size_t ) {} - inline void RecordErase() {} - friend inline void swap(HashtablezInfoHandle& , - HashtablezInfoHandle& ) noexcept {} -}; - -static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } - -class HashtablezSampler -{ -public: - // Returns a global Sampler. - static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } - HashtablezInfo* Register() { static HashtablezInfo info; return &info; } - void Unregister(HashtablezInfo* ) {} - - using DisposeCallback = void (*)(const HashtablezInfo&); - DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } - int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& ) { return 0; } -}; - -static inline void SetHashtablezEnabled(bool ) {} -static inline void SetHashtablezSampleParameter(int32_t ) {} -static inline void SetHashtablezMaxSamples(int32_t ) {} - - -namespace memory_internal { - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template <class Alloc, class T, class Tuple, size_t... I> -void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, - phmap::index_sequence<I...>) { - phmap::allocator_traits<Alloc>::construct( - *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...); -} - -template <class T, class F> -struct WithConstructedImplF { - template <class... Args> - decltype(std::declval<F>()(std::declval<T>())) operator()( - Args&&... args) const { - return std::forward<F>(f)(T(std::forward<Args>(args)...)); - } - F&& f; -}; - -template <class T, class Tuple, size_t... Is, class F> -decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl( - Tuple&& t, phmap::index_sequence<Is...>, F&& f) { - return WithConstructedImplF<T, F>{std::forward<F>(f)}( - std::get<Is>(std::forward<Tuple>(t))...); -} - -template <class T, size_t... Is> -auto TupleRefImpl(T&& t, phmap::index_sequence<Is...>) - -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) { - return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...); -} - -// Returns a tuple of references to the elements of the input tuple. T must be a -// tuple. -// ---------------------------------------------------------------------------- -template <class T> -auto TupleRef(T&& t) -> decltype( - TupleRefImpl(std::forward<T>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<T>::type>::value>())) { - return TupleRefImpl( - std::forward<T>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<T>::type>::value>()); -} - -template <class F, class K, class V> -decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct, - std::declval<std::tuple<K>>(), std::declval<V>())) -DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) { - const auto& key = std::get<0>(p.first); - return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), - std::move(p.second)); -} - -} // namespace memory_internal - - -// ---------------------------------------------------------------------------- -// R A W _ H A S H _ S E T -// ---------------------------------------------------------------------------- -// An open-addressing -// hashtable with quadratic probing. -// -// This is a low level hashtable on top of which different interfaces can be -// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. -// -// The table interface is similar to that of std::unordered_set. Notable -// differences are that most member functions support heterogeneous keys when -// BOTH the hash and eq functions are marked as transparent. They do so by -// providing a typedef called `is_transparent`. -// -// When heterogeneous lookup is enabled, functions that take key_type act as if -// they have an overload set like: -// -// iterator find(const key_type& key); -// template <class K> -// iterator find(const K& key); -// -// size_type erase(const key_type& key); -// template <class K> -// size_type erase(const K& key); -// -// std::pair<iterator, iterator> equal_range(const key_type& key); -// template <class K> -// std::pair<iterator, iterator> equal_range(const K& key); -// -// When heterogeneous lookup is disabled, only the explicit `key_type` overloads -// exist. -// -// find() also supports passing the hash explicitly: -// -// iterator find(const key_type& key, size_t hash); -// template <class U> -// iterator find(const U& key, size_t hash); -// -// In addition the pointer to element and iterator stability guarantees are -// weaker: all iterators and pointers are invalidated after a new element is -// inserted. -// -// IMPLEMENTATION DETAILS -// -// The table stores elements inline in a slot array. In addition to the slot -// array the table maintains some control state per slot. The extra state is one -// byte per slot and stores empty or deleted marks, or alternatively 7 bits from -// the hash of an occupied slot. The table is split into logical groups of -// slots, like so: -// -// Group 1 Group 2 Group 3 -// +---------------+---------------+---------------+ -// | | | | | | | | | | | | | | | | | | | | | | | | | -// +---------------+---------------+---------------+ -// -// On lookup the hash is split into two parts: -// - H2: 7 bits (those stored in the control bytes) -// - H1: the rest of the bits -// The groups are probed using H1. For each group the slots are matched to H2 in -// parallel. Because H2 is 7 bits (128 states) and the number of slots per group -// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. -// -// On insert, once the right group is found (as in lookup), its slots are -// filled in order. -// -// On erase a slot is cleared. In case the group did not have any empty slots -// before the erase, the erased slot is marked as deleted. -// -// Groups without empty slots (but maybe with deleted slots) extend the probe -// sequence. The probing algorithm is quadratic. Given N the number of groups, -// the probing function for the i'th probe is: -// -// P(0) = H1 % N -// -// P(i) = (P(i - 1) + i) % N -// -// This probing function guarantees that after N probes, all the groups of the -// table will be probed exactly once. -// ---------------------------------------------------------------------------- -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set -{ - using PolicyTraits = hash_policy_traits<Policy>; - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - -public: - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - -private: - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using Layout = phmap::priv::Layout<ctrl_t, slot_type>; - - static Layout MakeLayout(size_t capacity) { - assert(IsValidCapacity(capacity)); - return Layout(capacity + Group::kWidth + 1, capacity); - } - - using AllocTraits = phmap::allocator_traits<allocator_type>; - using SlotAlloc = typename phmap::allocator_traits< - allocator_type>::template rebind_alloc<slot_type>; - using SlotAllocTraits = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<slot_type>; - - static_assert(std::is_lvalue_reference<reference>::value, - "Policy::element() must return a reference"); - - template <typename T> - struct SameAsElementReference - : std::is_same<typename std::remove_cv< - typename std::remove_reference<reference>::type>::type, - typename std::remove_cv< - typename std::remove_reference<T>::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - template <class T> - using RequiresInsertable = typename std::enable_if< - phmap::disjunction<std::is_convertible<T, init_type>, - SameAsElementReference<T>>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template <class T> - using RequiresNotInit = - typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; - - template <class... Ts> - using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; - -public: - static_assert(std::is_same<pointer, value_type*>::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same<const_pointer, const value_type*>::value, - "Allocators with custom pointer types are not supported"); - - class iterator - { - friend class raw_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename raw_hash_set::value_type; - using reference = - phmap::conditional_t<PolicyTraits::constant_iterators::value, - const value_type&, value_type&>; - using pointer = phmap::remove_reference_t<reference>*; - using difference_type = typename raw_hash_set::difference_type; - - iterator() {} - - // PRECONDITION: not an end() iterator. - reference operator*() const { return PolicyTraits::element(slot_); } - - // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } - - // PRECONDITION: not an end() iterator. - iterator& operator++() { - ++ctrl_; - ++slot_; - skip_empty_or_deleted(); - return *this; - } - // PRECONDITION: not an end() iterator. - iterator operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } - -#if PHMAP_BIDIRECTIONAL - // PRECONDITION: not a begin() iterator. - iterator& operator--() { - assert(ctrl_); - do { - --ctrl_; - --slot_; - } while (IsEmptyOrDeleted(*ctrl_)); - return *this; - } - - // PRECONDITION: not a begin() iterator. - iterator operator--(int) { - auto tmp = *this; - --*this; - return tmp; - } -#endif - - friend bool operator==(const iterator& a, const iterator& b) { - return a.ctrl_ == b.ctrl_; - } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() - iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} - - void skip_empty_or_deleted() { - while (IsEmptyOrDeleted(*ctrl_)) { - // ctrl is not necessarily aligned to Group::kWidth. It is also likely - // to read past the space for ctrl bytes and into slots. This is ok - // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there - // is no way to read outside the combined slot array. - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); - ctrl_ += shift; - slot_ += shift; - } - } - - ctrl_t* ctrl_ = nullptr; - // To avoid uninitialized member warnings, put slot_ in an anonymous union. - // The member is not initialized on singleton and end iterators. - union { - slot_type* slot_; - }; - }; - - class const_iterator - { - friend class raw_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename raw_hash_set::value_type; - using reference = typename raw_hash_set::const_reference; - using pointer = typename raw_hash_set::const_pointer; - using difference_type = typename raw_hash_set::difference_type; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} - - reference operator*() const { return *inner_; } - pointer operator->() const { return inner_.operator->(); } - - const_iterator& operator++() { - ++inner_; - return *this; - } - const_iterator operator++(int) { return inner_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.inner_ == b.inner_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} - - iterator inner_; - }; - - using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; - using insert_return_type = InsertReturnType<iterator, node_type>; - - raw_hash_set() noexcept( - std::is_nothrow_default_constructible<hasher>::value&& - std::is_nothrow_default_constructible<key_equal>::value&& - std::is_nothrow_default_constructible<allocator_type>::value) {} - - explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { - if (bucket_cnt) { - capacity_ = NormalizeCapacity(bucket_cnt); - reset_growth_left(); - initialize_slots(); - } - } - - raw_hash_set(size_t bucket_cnt, const hasher& hashfn, - const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit raw_hash_set(const allocator_type& alloc) - : raw_hash_set(0, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { - insert(first, last); - } - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list<value_type> as the first - // argument like std::unordered_set<value_type> does, we have two overloads - // that accept std::initializer_list<T> and std::initializer_list<init_type>. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list<std::string>, then - // // copies the strings into the set. - // std::unordered_set<std::string> s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list<const char*>, then - // // copies the strings into the set. - // phmap::flat_hash_set<std::string> s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set<int> a, b{a}; - // - // RequiresNotInit<T> is a workaround for gcc prior to 7.1. - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, - const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(const raw_hash_set& that) - : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - reserve(that.size()); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(hashval); - set_ctrl(target.offset, H2(hashval)); - emplace_at(target.offset, v); - infoz_.RecordInsert(hashval, target.probe_length); - } - size_ = that.size(); - growth_left() -= that.size(); - } - - raw_hash_set(raw_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible<hasher>::value&& - std::is_nothrow_copy_constructible<key_equal>::value&& - std::is_nothrow_copy_constructible<allocator_type>::value) - : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), - slots_(phmap::exchange(that.slots_, nullptr)), - size_(phmap::exchange(that.size_, 0)), - capacity_(phmap::exchange(that.capacity_, 0)), - infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function<Key>, moving it - // would create a nullptr functor that cannot be called. - settings_(that.settings_) { - // growth_left was copied above, reset the one from `that`. - that.growth_left() = 0; - } - - raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), - settings_(0, that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); - std::swap(infoz_, that.infoz_); - } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); - } - } - - raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; - } - - raw_hash_set& operator=(raw_hash_set&& that) noexcept( - phmap::allocator_traits<allocator_type>::is_always_equal::value&& - std::is_nothrow_move_assignable<hasher>::value&& - std::is_nothrow_move_assignable<key_equal>::value) { - // TODO(sbenza): We should only use the operations from the noexcept clause - // to make sure we actually adhere to that contract. - return move_assign( - std::move(that), - typename AllocTraits::propagate_on_container_move_assignment()); - } - - ~raw_hash_set() { destroy_slots(); } - - iterator begin() { - auto it = iterator_at(0); - it.skip_empty_or_deleted(); - return it; - } - iterator end() - { -#if PHMAP_BIDIRECTIONAL - return iterator_at(capacity_); -#else - return {ctrl_ + capacity_}; -#endif - } - - const_iterator begin() const { - return const_cast<raw_hash_set*>(this)->begin(); - } - const_iterator end() const { return const_cast<raw_hash_set*>(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } - size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (empty()) - return; - if (capacity_ > 127) { - destroy_slots(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - size_ = 0; - reset_ctrl(); - reset_growth_left(); - } - assert(empty()); - infoz_.RecordStorageChanged(0, capacity_); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map<std::string, int> m; - // m.insert(std::make_pair("abc", 42)); - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - std::pair<iterator, bool> insert(T&& value) { - return emplace(std::forward<T>(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set<int> s; - // s.insert(n); - // - // flat_hash_set<std::string> s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - std::pair<iterator, bool> insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set<std::string, int> s; - // s.insert({"abc", 42}); - std::pair<iterator, bool> insert(init_type&& value) { - return emplace(std::move(value)); - } - - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward<T>(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template <typename It> - using IsRandomAccess = std::is_same<typename std::iterator_traits<It>::iterator_category, - std::random_access_iterator_tag>; - - - template<typename T> - struct has_difference_operator - { - private: - using yes = std::true_type; - using no = std::false_type; - - template<typename U> static auto test(int) -> decltype(std::declval<U>() - std::declval<U>() == 1, yes()); - template<typename> static no test(...); - - public: - static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; - }; - - template <class InputIt, typename phmap::enable_if_t<has_difference_operator<InputIt>::value, int> = 0> - void insert(InputIt first, InputIt last) { - this->reserve(this->size() + (last - first)); - for (; first != last; ++first) - emplace(*first); - } - - template <class InputIt, typename phmap::enable_if_t<!has_difference_operator<InputIt>::value, int> = 0> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) - emplace(*first); - } - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> - void insert(std::initializer_list<T> ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list<init_type> ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - insert_return_type insert(node_type&& node, size_t hashval) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlotWithHash<false>{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward<Args>(args)...); - } - - template <class... Args, typename std::enable_if<IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type - raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); - } - - template <class... Args, typename std::enable_if<!IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlotWithHash<true>{*this, std::move(*slot), hashval}, elem); - } - - template <class... Args> - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward<Args>(args)...).first; - } - - template <class... Args> - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward<Args>(args)...).first; - } - - // Extension API: support for lazy emplace. - // - // Looks up key in the table. If found, returns the iterator to the element. - // Otherwise calls f with one argument of type raw_hash_set::constructor. f - // MUST call raw_hash_set::constructor with arguments as if a - // raw_hash_set::value_type is constructed, otherwise the behavior is - // undefined. - // - // For example: - // - // std::unordered_set<ArenaString> s; - // // Makes ArenaStr even if "abc" is in the map. - // s.insert(ArenaString(&arena, "abc")); - // - // flat_hash_set<ArenaStr> s; - // // Makes ArenaStr only if "abc" is not in the map. - // s.lazy_emplace("abc", [&](const constructor& ctor) { - // ctor(&arena, "abc"); - // }); - // - // WARNING: This API is currently experimental. If there is a way to implement - // the same thing with the rest of the API, prefer that. - class constructor - { - friend class raw_hash_set; - - public: - template <class... Args> - void operator()(Args&&... args) const { - assert(*slot_); - PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); - *slot_ = nullptr; - } - - private: - constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - - allocator_type* alloc_; - slot_type** slot_; - }; - - template <class K = key_type, class F> - iterator lazy_emplace(const key_arg<K>& key, F&& f) { - auto res = find_or_prepare_insert(key); - if (res.second) { - lazy_emplace_at(res.first, std::forward<F>(f)); - } - return iterator_at(res.first); - } - - template <class K = key_type, class F> - iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t &hashval, F&& f) { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - lazy_emplace_at(res.first, std::forward<F>(f)); - } - return iterator_at(res.first); - } - - template <class K = key_type, class F> - void lazy_emplace_at(size_t& idx, F&& f) { - slot_type* slot = slots_ + idx; - std::forward<F>(f)(constructor(&alloc_ref(), &slot)); - assert(!slot); - } - - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - template <class K = key_type> - size_type erase(const key_arg<K>& key) { - auto it = find(key); - if (it == end()) return 0; - _erase(it); - return 1; - } - - - iterator erase(const_iterator cit) { return erase(cit.inner_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if (<pred>) { - // m._erase(it++); - // } else { - // ++it; - // } - // } - void _erase(iterator it) { - assert(it != end()); - PolicyTraits::destroy(&alloc_ref(), it.slot_); - erase_meta_only(it); - } - void _erase(const_iterator cit) { _erase(cit.inner_); } - - // This overload is necessary because otherwise erase<K>(const K&) would be - // a better match if non-const iterator is passed as an argument. - iterator erase(iterator it) { - auto res = it; - ++res; - _erase(it); - return res; - } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.inner_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT - assert(this != &src); - for (auto it = src.begin(), e = src.end(); it != e; ++it) { - if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) - .second) { - src.erase_meta_only(it); - } - } - } - - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - auto node = - CommonAccess::Make<node_type>(alloc_ref(), position.inner_.slot_); - erase_meta_only(position); - return node; - } - - template < - class K = key_type, - typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> - node_type extract(const key_arg<K>& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable<allocator_type>())) { - using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); - swap(hash_ref(), that.hash_ref()); - swap(eq_ref(), that.eq_ref()); - swap(infoz_, that.infoz_); - if (AllocTraits::propagate_on_container_swap::value) { - swap(alloc_ref(), that.alloc_ref()); - } else { - // If the allocators do not compare equal it is officially undefined - // behavior. We choose to do nothing. - } - } - -#ifndef PHMAP_NON_DETERMINISTIC - template<typename OutputArchive> - bool dump(OutputArchive&) const; - - template<typename InputArchive> - bool load(InputArchive&); -#endif - - void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); - infoz_.RecordStorageChanged(0, 0); - return; - } - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity((std::max)(n, size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { - resize(m); - } - } - - void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - template <class K = key_type> - size_t count(const key_arg<K>& key) const { - return find(key) == end() ? size_t(0) : size_t(1); - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - void prefetch_hash(size_t hashval) const { - (void)hashval; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - auto seq = probe(hashval); - _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); - _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); -#elif defined(__GNUC__) - auto seq = probe(hashval); - __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); -#endif // __GNUC__ - } - - template <class K = key_type> - void prefetch(const key_arg<K>& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - template <class K = key_type> - iterator find(const key_arg<K>& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return iterator_at(offset); - else - return end(); - } - - template <class K = key_type> - pointer find_ptr(const key_arg<K>& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return &PolicyTraits::element(slots_ + offset); - else - return nullptr; - } - - template <class K = key_type> - iterator find(const key_arg<K>& key) { - return find(key, this->hash(key)); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key, size_t hashval) const { - return const_cast<raw_hash_set*>(this)->find(key, hashval); - } - template <class K = key_type> - const_iterator find(const key_arg<K>& key) const { - return find(key, this->hash(key)); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key) const { - return find(key) != end(); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template <class K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - template <class K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K>& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { return capacity_; } - float load_factor() const { - return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; - } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { - if (a.size() != b.size()) return false; - const raw_hash_set* outer = &a; - const raw_hash_set* inner = &b; - if (outer->capacity() > inner->capacity()) - std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; - return true; - } - - friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { - return !(a == b); - } - - friend void swap(raw_hash_set& a, - raw_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template <class K> - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -private: - template <class Container, typename Enabler> - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - template <class K = key_type> - bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) { - auto seq = probe(hashval); - while (true) { - Group g{ ctrl_ + seq.offset() }; - for (int i : g.Match((h2_t)H2(hashval))) { - offset = seq.offset((size_t)i); - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + offset)))) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) - return false; - seq.next(); - } - } - - struct FindElement - { - template <class K, class... Args> - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const raw_hash_set& s; - }; - - struct HashElement - { - template <class K, class... Args> - size_t operator()(const K& key, Args&&...) const { - return phmap_mix<sizeof(size_t)>()(h(key)); - } - const hasher& h; - }; - - template <class K1> - struct EqualElement - { - template <class K2, class... Args> - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable(const K& key, size_t hashval, - Args&&... args) - { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - emplace_at(res.first, std::forward<Args>(args)...); - } - return {iterator_at(res.first), res.second}; - } - - struct EmplaceDecomposable - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, s.hash(key), std::forward<Args>(args)...); - } - raw_hash_set& s; - }; - - struct EmplaceDecomposableHashval { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, hashval, std::forward<Args>(args)...); - } - raw_hash_set& s; - size_t hashval; - }; - - template <bool do_destroy> - struct InsertSlot - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - }; - - template <bool do_destroy> - struct InsertSlotWithHash - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key, hashval); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - size_t &hashval; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast<size_t>(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - - set_ctrl(index, was_never_full ? kEmpty : kDeleted); - growth_left() += was_never_full; - infoz_.RecordErase(); - } - - void initialize_slots() { - assert(capacity_); - if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && - slots_ == nullptr) { - infoz_ = Sample(); - } - - auto layout = MakeLayout(capacity_); - char* mem = static_cast<char*>( - Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize())); - ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem)); - slots_ = layout.template Pointer<1>(mem); - reset_ctrl(); - reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); - } - - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - auto layout = MakeLayout(capacity_); - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize()); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; - } - - void resize(size_t new_capacity) { - assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - capacity_ = new_capacity; - initialize_slots(); - - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); - } - } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); - auto layout = MakeLayout(old_capacity); - Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, - layout.AllocSize()); - } - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); - assert(!is_small()); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type - raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - - // Verify if the old and new i fall within the same group wrt the hashval. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const auto probe_index = [&](size_t pos) { - return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - set_ctrl(i, H2(hashval)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. - // set_ctrl poisons/unpoisons the slots so we have to call it at the - // right time. - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - set_ctrl(i, kEmpty); - } else { - assert(IsDeleted(ctrl_[new_i])); - set_ctrl(new_i, H2(hashval)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(); - } - - void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); - } else if (size() <= CapacityToGrowth(capacity()) / 2) { - // Squash DELETED without growing if there is enough capacity. - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(capacity_ * 2 + 1); - } - } - - bool has_element(const value_type& elem, size_t hashval) const { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == - elem)) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; - seq.next(); - assert(seq.getindex() < capacity_ && "full table!"); - } - return false; - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - return has_element(elem, hashval); - } - - // Probes the raw_hash_set with the probe sequence for hash and returns the - // pointer to the first empty or deleted slot. - // NOTE: this function must work with tables having both kEmpty and kDelete - // in one group. Such tables appears during drop_deletes_without_resize. - // - // This function is very useful when insertions happen and: - // - the input is already a set - // - there are enough slots - // - the element with the hash is not in the table - struct FindInfo - { - size_t offset; - size_t probe_length; - }; - FindInfo find_first_non_full(size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); - if (mask) { - return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; - } - assert(seq.getindex() < capacity_ && "full table!"); - seq.next(); - } - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset((size_t)i))))) - return {seq.offset((size_t)i), false}; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; - seq.next(); - } - return {prepare_insert(hashval), true}; - } - - template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key) { - return find_or_prepare_insert(key, this->hash(key)); - } - - size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(hashval); - if (PHMAP_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { - rehash_and_grow_if_necessary(); - target = find_first_non_full(hashval); - } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); - set_ctrl(target.offset, H2(hashval)); - infoz_.RecordInsert(hashval, target.probe_length); - return target.offset; - } - - // Constructs the value in the space pointed by the iterator. This only works - // after an unsuccessful find_or_prepare_insert() and before any other - // modifications happen in the raw_hash_set. - // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward<Args>(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. - // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). - template <class... Args> - void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, - std::forward<Args>(args)...); - - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && - "constructed value does not match the lookup key"); - } - - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } - -private: - friend struct RawHashSetTestOnlyAccess; - - probe_seq<Group::kWidth> probe(size_t hashval) const { - return probe_seq<Group::kWidth>(H1(hashval, ctrl_), capacity_); - } - - // Reset all ctrl bytes back to kEmpty, except the sentinel. - void reset_ctrl() { - std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); - ctrl_[capacity_] = kSentinel; - SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - } - - void reset_growth_left() { - growth_left() = CapacityToGrowth(capacity()) - size_; - } - - // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at - // the end too. - void set_ctrl(size_t i, ctrl_t h) { - assert(i < capacity_); - - if (IsFull(h)) { - SanitizerUnpoisonObject(slots_ + i); - } else { - SanitizerPoisonObject(slots_ + i); - } - - ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + 1 + - ((Group::kWidth - 1) & capacity_)] = h; - } - - size_t& growth_left() { return settings_.template get<0>(); } - - template <size_t N, - template <class, class, class, class> class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_set; - - template <size_t N, - template <class, class, class, class> class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_map; - - // The representation of the object has two modes: - // - small: For capacities < kWidth-1 - // - large: For the rest. - // - // Differences: - // - In small mode we are able to use the whole capacity. The extra control - // bytes give us at least one "empty" control byte to stop the iteration. - // This is important to make 1 a valid capacity. - // - // - In small mode only the first `capacity()` control bytes after the - // sentinel are valid. The rest contain dummy kEmpty values that do not - // represent a real slot. This is important to take into account on - // find_first_non_full(), where we never try ShouldInsertBackwards() for - // small tables. - bool is_small() const { return capacity_ < Group::kWidth - 1; } - - hasher& hash_ref() { return settings_.template get<1>(); } - const hasher& hash_ref() const { return settings_.template get<1>(); } - key_equal& eq_ref() { return settings_.template get<2>(); } - const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } - const allocator_type& alloc_ref() const { - return settings_.template get<3>(); - } - - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array - ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] - slot_type* slots_ = nullptr; // [capacity * slot_type] - size_t size_ = 0; // number of full slots - size_t capacity_ = 0; // total number of slots - HashtablezInfoHandle infoz_; - phmap::priv::CompressedTuple<size_t /* growth_left */, hasher, - key_equal, allocator_type> - settings_{0, hasher{}, key_equal{}, allocator_type{}}; -}; - - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map<K, IncompleteType>. - // MappedReference<> may be a non-reference type. - template <class P> - using MappedReference = decltype(P::value( - std::addressof(std::declval<typename raw_hash_map::reference>()))); - - // MappedConstReference<> may be a non-reference type. - template <class P> - using MappedConstReference = decltype(P::value( - std::addressof(std::declval<typename raw_hash_map::const_reference>()))); - - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - using Base = raw_hash_set<Policy, Hash, Eq, Alloc>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - - static_assert(!std::is_reference<key_type>::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference<mapped_type>::value, ""); - - using iterator = typename raw_hash_map::raw_hash_set::iterator; - using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; - - raw_hash_map() {} - using Base::raw_hash_set; // use raw_hash_set constructor - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map<int, int> m; - // m.insert_or_assign(n, n); - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { - return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { - return insert_or_assign_impl(std::forward<K>(k), v); - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { - return insert_or_assign_impl(k, std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { - return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { - return insert_or_assign(std::forward<K>(k), v).first; - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { - return insert_or_assign(k, std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { - return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { - return try_emplace_impl(k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace(k, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class P = Policy> - MappedReference<P> at(const key_arg<K>& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy> - MappedConstReference<P> at(const key_arg<K>& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy, K* = nullptr> - MappedReference<P> operator[](key_arg<K>&& key) { - return Policy::value(&*try_emplace(std::forward<K>(key)).first); - } - - template <class K = key_type, class P = Policy> - MappedReference<P> operator[](const key_arg<K>& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template <class K, class V> - std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); - else - Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v); - return {this->iterator_at(res.first), res.second}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(res.first), res.second}; - } -}; - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -// Returns "random" seed. -inline size_t RandomSeed() -{ -#if PHMAP_HAVE_THREAD_LOCAL - static thread_local size_t counter = 0; - size_t value = ++counter; -#else // PHMAP_HAVE_THREAD_LOCAL - static std::atomic<size_t> counter(0); - size_t value = counter.fetch_add(1, std::memory_order_relaxed); -#endif // PHMAP_HAVE_THREAD_LOCAL - return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); -} - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_set -{ - using PolicyTraits = hash_policy_traits<Policy>; - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - static_assert(N <= 12, "N = 12 means 4096 hash tables!"); - constexpr static size_t num_tables = 1 << N; - constexpr static size_t mask = num_tables - 1; - -public: - using EmbeddedSet = RefSet<Policy, Hash, Eq, Alloc>; - using EmbeddedIterator= typename EmbeddedSet::iterator; - using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; - using constructor = typename EmbeddedSet::constructor; - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits<value_type>::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - // -------------------------------------------------------------------- - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - -protected: - using Lockable = phmap::LockableImpl<Mtx_>; - - // -------------------------------------------------------------------- - struct Inner : public Lockable - { - bool operator==(const Inner& o) const - { - typename Lockable::SharedLocks l(const_cast<Inner &>(*this), const_cast<Inner &>(o)); - return set_ == o.set_; - } - - EmbeddedSet set_; - }; - -private: - // Give an early error when key_type is not hashable/eq. - // -------------------------------------------------------------------- - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using AllocTraits = phmap::allocator_traits<allocator_type>; - - static_assert(std::is_lvalue_reference<reference>::value, - "Policy::element() must return a reference"); - - template <typename T> - struct SameAsElementReference : std::is_same< - typename std::remove_cv<typename std::remove_reference<reference>::type>::type, - typename std::remove_cv<typename std::remove_reference<T>::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - // -------------------------------------------------------------------- - template <class T> - using RequiresInsertable = typename std::enable_if< - phmap::disjunction<std::is_convertible<T, init_type>, - SameAsElementReference<T>>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template <class T> - using RequiresNotInit = - typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; - - template <class... Ts> - using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; - -public: - static_assert(std::is_same<pointer, value_type*>::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same<const_pointer, const value_type*>::value, - "Allocators with custom pointer types are not supported"); - - // --------------------- i t e r a t o r ------------------------------ - class iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename parallel_hash_set::value_type; - using reference = - phmap::conditional_t<PolicyTraits::constant_iterators::value, - const value_type&, value_type&>; - using pointer = phmap::remove_reference_t<reference>*; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; - using EmbeddedIterator = typename EmbeddedSet::iterator; - - iterator() {} - - reference operator*() const { return *it_; } - pointer operator->() const { return &operator*(); } - - iterator& operator++() { - assert(inner_); // null inner means we are already at the end - ++it_; - skip_empty(); - return *this; - } - - iterator operator++(int) { - assert(inner_); // null inner means we are already at the end - auto tmp = *this; - ++*this; - return tmp; - } - - friend bool operator==(const iterator& a, const iterator& b) { - return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); - } - - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : - inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() - if (inner) - it_end_ = inner->set_.end(); - } - - void skip_empty() { - while (it_ == it_end_) { - ++inner_; - if (inner_ == inner_end_) { - inner_ = nullptr; // marks end() - break; - } - else { - it_ = inner_->set_.begin(); - it_end_ = inner_->set_.end(); - } - } - } - - Inner *inner_ = nullptr; - Inner *inner_end_ = nullptr; - EmbeddedIterator it_, it_end_; - }; - - // --------------------- c o n s t i t e r a t o r ----------------- - class const_iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename parallel_hash_set::value_type; - using reference = typename parallel_hash_set::const_reference; - using pointer = typename parallel_hash_set::const_pointer; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : iter_(std::move(i)) {} - - reference operator*() const { return *(iter_); } - pointer operator->() const { return iter_.operator->(); } - - const_iterator& operator++() { - ++iter_; - return *this; - } - const_iterator operator++(int) { return iter_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.iter_ == b.iter_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) - : iter_(const_cast<Inner**>(inner), - const_cast<Inner**>(inner_end), - const_cast<EmbeddedIterator*>(it)) {} - - iterator iter_; - }; - - using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; - using insert_return_type = InsertReturnType<iterator, node_type>; - - // ------------------------- c o n s t r u c t o r s ------------------ - - parallel_hash_set() noexcept( - std::is_nothrow_default_constructible<hasher>::value&& - std::is_nothrow_default_constructible<key_equal>::value&& - std::is_nothrow_default_constructible<allocator_type>::value) {} - - explicit parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) { - for (auto& inner : sets_) - inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); - } - - parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param, - const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - - explicit parallel_hash_set(const allocator_type& alloc) - : parallel_hash_set(0, hasher(), key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { - insert(first, last); - } - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class InputIter> - parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list<value_type> as the first - // argument like std::unordered_set<value_type> does, we have two overloads - // that accept std::initializer_list<T> and std::initializer_list<init_type>. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list<std::string>, then copies - // // the strings into the set. - // std::unordered_set<std::string> s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list<const char*>, then - // // copies the strings into the set. - // phmap::flat_hash_set<std::string> s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set<int> a, b{a}; - // - // RequiresNotInit<T> is a workaround for gcc prior to 7.1. - // -------------------------------------------------------------------- - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - parallel_hash_set(std::initializer_list<T> init, const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list<init_type> init, - const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(const parallel_hash_set& that) - : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) - : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = { that.sets_[i].set_, a }; - } - - parallel_hash_set(parallel_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible<hasher>::value&& - std::is_nothrow_copy_constructible<key_equal>::value&& - std::is_nothrow_copy_constructible<allocator_type>::value) - : parallel_hash_set(std::move(that), that.alloc_ref()) { - } - - parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) - { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = { std::move(that.sets_[i]).set_, a }; - } - - parallel_hash_set& operator=(const parallel_hash_set& that) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = that.sets_[i].set_; - return *this; - } - - parallel_hash_set& operator=(parallel_hash_set&& that) noexcept( - phmap::allocator_traits<allocator_type>::is_always_equal::value && - std::is_nothrow_move_assignable<hasher>::value && - std::is_nothrow_move_assignable<key_equal>::value) { - for (size_t i=0; i<num_tables; ++i) - sets_[i].set_ = std::move(that.sets_[i].set_); - return *this; - } - - ~parallel_hash_set() {} - - iterator begin() { - auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin()); - it.skip_empty(); - return it; - } - - iterator end() { return iterator(); } - const_iterator begin() const { return const_cast<parallel_hash_set *>(this)->begin(); } - const_iterator end() const { return const_cast<parallel_hash_set *>(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - - size_t size() const { - size_t sz = 0; - for (const auto& inner : sets_) - sz += inner.set_.size(); - return sz; - } - - size_t capacity() const { - size_t c = 0; - for (const auto& inner : sets_) - c += inner.set_.capacity(); - return c; - } - - size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - } - - // extension - clears only soecified submap - // ---------------------------------------- - void clear(std::size_t submap_index) { - Inner& inner = sets_[submap_index]; - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map<std::string, int> m; - // m.insert(std::make_pair("abc", 42)); - // -------------------------------------------------------------------- - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - std::pair<iterator, bool> insert(T&& value) { - return emplace(std::forward<T>(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set<int> s; - // s.insert(n); - // - // flat_hash_set<std::string> s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - std::pair<iterator, bool> insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set<std::pair<std::string, int>> s; - // s.insert({"abc", 42}); - // -------------------------------------------------------------------- - std::pair<iterator, bool> insert(init_type&& value) { - return emplace(std::move(value)); - } - - template <class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<T>::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward<T>(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template <class InputIt> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) insert(*first); - } - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> - void insert(std::initializer_list<T> ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list<init_type> ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) - return {end(), false, node_type()}; - auto& key = node.key(); - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - - typename Lockable::UniqueLock m(inner); - auto res = set.insert(std::move(node), hashval); - return { make_iterator(&inner, res.position), - res.inserted, - res.inserted ? node_type() : std::move(res.node) }; - } - - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } - - struct ReturnKey_ - { - template <class Key, class... Args> - Key operator()(Key&& k, const Args&...) const { - return std::forward<Key>(k); - } - }; - - // -------------------------------------------------------------------- - // phmap expension: emplace_with_hash - // ---------------------------------- - // same as emplace, but hashval is provided - // -------------------------------------------------------------------- - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); - } - - struct EmplaceDecomposableHashval - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable_with_hash(key, hashval, std::forward<Args>(args)...); - } - parallel_hash_set& s; - size_t hashval; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, - std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash<true> f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template <class... Args> - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class F> - iterator lazy_emplace_with_hash(size_t hashval, const key_arg<K>& key, F&& f) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); - } - - // -------------------------------------------------------------------- - // end of phmap expension - // -------------------------------------------------------------------- - - template <class K, class... Args> - std::pair<iterator, bool> emplace_decomposable(const K& key, Args&&... args) - { - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward<Args>(args)...)); - } - - struct EmplaceDecomposable - { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, std::forward<Args>(args)...); - } - parallel_hash_set& s; - }; - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - typename std::aligned_storage<sizeof(slot_type), alignof(slot_type)>::type raw; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - size_t hashval = this->hash(PolicyTraits::key(slot)); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash<true> f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } - - template <class... Args> - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward<Args>(args)...).first; - } - - iterator make_iterator(Inner* inner, const EmbeddedIterator it) - { - if (it == inner->set_.end()) - return iterator(); - return iterator(inner, &sets_[0] + num_tables, it); - } - - std::pair<iterator, bool> make_rv(Inner* inner, - const std::pair<EmbeddedIterator, bool>& res) - { - return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; - } - - template <class K = key_type, class F> - iterator lazy_emplace(const key_arg<K>& key, F&& f) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward<F>(f))); - } - - template <class K = key_type, class FExists, class FEmplace> - bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(key, m); - Inner* inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.lazy_emplace_at(std::get<1>(res), std::forward<FEmplace>(fEmplace)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward<FExists>(fExists)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - // -------------------------------------------------------------------- - template <class K = key_type> - size_type erase(const key_arg<K>& key) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UpgradeLock m(inner); - auto it = set.find(key, hashval); - if (it == set.end()) - return 0; - - typename Lockable::UpgradeToUnique unique(m); - set._erase(it); - return 1; - } - - // -------------------------------------------------------------------- - iterator erase(const_iterator cit) { return erase(cit.iter_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if (<pred>) { - // m._erase(it++); - // } else { - // ++it; - // } - // } - // -------------------------------------------------------------------- - void _erase(iterator it) { - assert(it.inner_ != nullptr); - it.inner_->set_._erase(it.it_); - } - void _erase(const_iterator cit) { _erase(cit.iter_); } - - // This overload is necessary because otherwise erase<K>(const K&) would be - // a better match if non-const iterator is passed as an argument. - // -------------------------------------------------------------------- - iterator erase(iterator it) { _erase(it++); return it; } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.iter_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - // -------------------------------------------------------------------- - template <typename E = Eq> - void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>& src) { // NOLINT - assert(this != &src); - if (this != &src) - { - for (size_t i=0; i<num_tables; ++i) - { - typename Lockable::UniqueLocks l(sets_[i], src.sets_[i]); - sets_[i].set_.merge(src.sets_[i].set_); - } - } - } - - template <typename E = Eq> - void merge(parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, E, Alloc>&& src) { - merge(src); - } - - node_type extract(const_iterator position) { - return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); - } - - template < - class K = key_type, - typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> - node_type extract(const key_arg<K>& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(parallel_hash_set& that) noexcept( - IsNoThrowSwappable<EmbeddedSet>() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable<allocator_type>())) { - using std::swap; - for (size_t i=0; i<num_tables; ++i) - { - typename Lockable::UniqueLocks l(sets_[i], that.sets_[i]); - swap(sets_[i].set_, that.sets_[i].set_); - } - } - - void rehash(size_t n) { - size_t nn = n / num_tables; - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.rehash(nn); - } - } - - void reserve(size_t n) - { - size_t target = GrowthToLowerboundCapacity(n); - size_t normalized = 16 * NormalizeCapacity(n / num_tables); - rehash(normalized > target ? normalized : target); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - // -------------------------------------------------------------------- - template <class K = key_type> - size_t count(const key_arg<K>& key) const { - return find(key) == end() ? 0 : 1; - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - // -------------------------------------------------------------------- - void prefetch_hash(size_t hashval) const { - const Inner& inner = sets_[subidx(hashval)]; - const auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - set.prefetch_hash(hashval); - } - - template <class K = key_type> - void prefetch(const key_arg<K>& key) const { - prefetch_hash(this->hash(key)); - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - // -------------------------------------------------------------------- - template <class K = key_type> - iterator find(const key_arg<K>& key, size_t hashval) { - typename Lockable::SharedLock m; - return find(key, hashval, m); - } - - template <class K = key_type> - iterator find(const key_arg<K>& key) { - return find(key, this->hash(key)); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key, size_t hashval) const { - return const_cast<parallel_hash_set*>(this)->find(key, hashval); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key) const { - return find(key, this->hash(key)); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key) const { - return find(key) != end(); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key, size_t hashval) const { - return find(key, hashval) != end(); - } - - template <class K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - template <class K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K>& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { - size_t sz = 0; - for (const auto& inner : sets_) - { - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - sz += inner.set_.bucket_count(); - } - return sz; - } - - float load_factor() const { - size_t _capacity = bucket_count(); - return _capacity ? static_cast<float>(static_cast<double>(size()) / _capacity) : 0; - } - - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { - return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); - } - - friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { - return !(a == b); - } - - friend void swap(parallel_hash_set& a, - parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - template <class K> - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } - -#ifndef PHMAP_NON_DETERMINISTIC - template<typename OutputArchive> - bool dump(OutputArchive& ar) const; - - template<typename InputArchive> - bool load(InputArchive& ar); -#endif - -private: - template <class Container, typename Enabler> - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - struct FindElement - { - template <class K, class... Args> - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const parallel_hash_set& s; - }; - - struct HashElement - { - template <class K, class... Args> - size_t operator()(const K& key, Args&&...) const { - return phmap_mix<sizeof(size_t)>()(h(key)); - } - const hasher& h; - }; - - template <class K1> - struct EqualElement - { - template <class K2, class... Args> - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - // -------------------------------------------------------------------- - void erase_meta_only(const_iterator cit) { - auto &it = cit.iter_; - assert(it.set_ != nullptr); - it.set_.erase_meta_only(const_iterator(it.it_)); - } - - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.drop_deletes_without_resize(); - } - } - - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast<Inner&>(inner)); - return set.has_element(elem, hashval); - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - // -------------------------------------------------------------------- - parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { - parallel_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - - parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { - parallel_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - -protected: - template <class K = key_type, class L = typename Lockable::SharedLock> - pointer find_ptr(const key_arg<K>& key, size_t hashval, L& mutexlock) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return set.find_ptr(key, hashval); - } - - template <class K = key_type, class L = typename Lockable::SharedLock> - iterator find(const key_arg<K>& key, size_t hashval, L& mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return make_iterator(&inner, set.find(key, hashval)); - } - - template <class K> - std::tuple<Inner*, size_t, bool> - find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(typename Lockable::UniqueLock(inner)); - auto p = set.find_or_prepare_insert(key, hashval); // std::pair<size_t, bool> - return std::make_tuple(&inner, p.first, p.second); - } - - template <class K> - std::tuple<Inner*, size_t, bool> - find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { - return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock); - } - - iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) { - return {inner, &sets_[0] + num_tables, it}; - } - const_iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) const { - return {inner, &sets_[0] + num_tables, it}; - } - - static size_t subidx(size_t hashval) { - return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; - } - - static size_t subcnt() { - return num_tables; - } - -private: - friend struct RawHashSetTestOnlyAccess; - - size_t growth_left() { - size_t sz = 0; - for (const auto& set : sets_) - sz += set.growth_left(); - return sz; - } - - hasher& hash_ref() { return sets_[0].set_.hash_ref(); } - const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } - key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } - const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } - allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } - const allocator_type& alloc_ref() const { - return sets_[0].set_.alloc_ref(); - } - -protected: // protected in case users want to derive fromm this - std::array<Inner, num_tables> sets_; -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc> -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map<K, IncompleteType>. - // MappedReference<> may be a non-reference type. - template <class P> - using MappedReference = decltype(P::value( - std::addressof(std::declval<typename parallel_hash_map::reference>()))); - - // MappedConstReference<> may be a non-reference type. - template <class P> - using MappedConstReference = decltype(P::value( - std::addressof(std::declval<typename parallel_hash_map::const_reference>()))); - - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - using Base = typename parallel_hash_map::parallel_hash_set; - using Lockable = phmap::LockableImpl<Mtx_>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - - static_assert(!std::is_reference<key_type>::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference<mapped_type>::value, ""); - - using iterator = typename parallel_hash_map::parallel_hash_set::iterator; - using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; - - parallel_hash_map() {} - -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using parallel_hash_map::parallel_hash_set::parallel_hash_set; -#endif - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map<int, int> m; - // m.insert_or_assign(n, n); - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { - return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { - return insert_or_assign_impl(std::forward<K>(k), v); - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { - return insert_or_assign_impl(k, std::forward<V>(v)); - } - - template <class K = key_type, class V = mapped_type> - std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { - return insert_or_assign_impl(k, v); - } - - template <class K = key_type, class V = mapped_type, K* = nullptr, - V* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { - return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type, K* = nullptr> - iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { - return insert_or_assign(std::forward<K>(k), v).first; - } - - template <class K = key_type, class V = mapped_type, V* = nullptr> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { - return insert_or_assign(k, std::forward<V>(v)).first; - } - - template <class K = key_type, class V = mapped_type> - iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { - return insert_or_assign(k, v).first; - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { - return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { - return try_emplace_impl(k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace(k, std::forward<Args>(args)...).first; - } - - template <class K = key_type, class P = Policy> - MappedReference<P> at(const key_arg<K>& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - template <class K = key_type, class P = Policy> - MappedConstReference<P> at(const key_arg<K>& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } - - // ----------- phmap extensions -------------------------- - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0, - K* = nullptr> - std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, key_arg<K>&& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, - typename std::enable_if< - !std::is_convertible<K, const_iterator>::value, int>::type = 0> - std::pair<iterator, bool> try_emplace_with_hash(size_t hashval, const key_arg<K>& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, k, std::forward<Args>(args)...); - } - - template <class K = key_type, class... Args, K* = nullptr> - iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg<K>&& k, Args&&... args) { - return try_emplace_with_hash(hashval, std::forward<K>(k), std::forward<Args>(args)...).first; - } - - template <class K = key_type, class... Args> - iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg<K>& k, Args&&... args) { - return try_emplace_with_hash(hashval, k, std::forward<Args>(args)...).first; - } - - // if map contains key, lambda is called with the mapped value (under read lock protection), - // and if_contains returns true. This is a const API and lambda should not modify the value - // ----------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool if_contains(const key_arg<K>& key, F&& f) const { - return const_cast<parallel_hash_map*>(this)->template - modify_if_impl<K, F, typename Lockable::SharedLock>(key, std::forward<F>(f)); - } - - // if map contains key, lambda is called with the mapped value without read lock protection, - // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value - // This should be used only if we know that no other thread may be mutating the map at the time. - // ----------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool if_contains_unsafe(const key_arg<K>& key, F&& f) const { - return const_cast<parallel_hash_map*>(this)->template - modify_if_impl<K, F, LockableBaseImpl<phmap::NullMutex>::DoNothing>(key, std::forward<F>(f)); - } - - // if map contains key, lambda is called with the mapped value (under write lock protection), - // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value - // ---------------------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool modify_if(const key_arg<K>& key, F&& f) { - return modify_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); - } - - - // if map contains key, lambda is called with the mapped value (under write lock protection). - // If the lambda returns true, the key is subsequently erased from the map (the write lock - // is only released after erase). - // returns true if key was erased, false otherwise. - // ---------------------------------------------------------------------------------------------------- - template <class K = key_type, class F> - bool erase_if(const key_arg<K>& key, F&& f) { - return erase_if_impl<K, F, typename Lockable::UniqueLock>(key, std::forward<F>(f)); - } - - // if map does not contains key, it is inserted and the mapped value is value-constructed - // with the provided arguments (if any), as with try_emplace. - // if map already contains key, then the lambda is called with the mapped value (under - // write lock protection) and can update the mapped value. - // returns true if key was not already present, false otherwise. - // --------------------------------------------------------------------------------------- - template <class K = key_type, class F, class... Args> - bool try_emplace_l(K&& k, F&& f, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward<F>(f)(Policy::value(&*it)); - } - return std::get<2>(res); - } - - // ----------- end of phmap extensions -------------------------- - - template <class K = key_type, class P = Policy, K* = nullptr> - MappedReference<P> operator[](key_arg<K>&& key) { - return Policy::value(&*try_emplace(std::forward<K>(key)).first); - } - - template <class K = key_type, class P = Policy> - MappedReference<P> operator[](const key_arg<K>& key) { - return Policy::value(&*try_emplace(key).first); - } - -private: - template <class K = key_type, class F, class L> - bool modify_if_impl(const key_arg<K>& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable<F, mapped_type&>::value); -#endif - L m; - auto ptr = this->template find_ptr<K, L>(key, this->hash(key), m); - if (ptr == nullptr) - return false; - std::forward<F>(f)(Policy::value(ptr)); - return true; - } - - template <class K = key_type, class F, class L> - bool erase_if_impl(const key_arg<K>& key, F&& f) { -#if __cplusplus >= 201703L - static_assert(std::is_invocable<F, mapped_type&>::value); -#endif - L m; - auto it = this->template find<K, L>(key, this->hash(key), m); - if (it == this->end()) return false; - if (std::forward<F>(f)(Policy::value(&*it))) - { - this->erase(it); - return true; - } - return false; - } - - - template <class K, class V> - std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::forward<K>(k), std::forward<V>(v)); - else - Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward<V>(v); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - template <class K = key_type, class... Args> - std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(k)), - std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } - - -}; - - -// Constructs T into uninitialized storage pointed by `ptr` using the args -// specified in the tuple. -// ---------------------------------------------------------------------------- -template <class Alloc, class T, class Tuple> -void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { - memory_internal::ConstructFromTupleImpl( - alloc, ptr, std::forward<Tuple>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<Tuple>::type>::value>()); -} - -// Constructs T using the args specified in the tuple and calls F with the -// constructed value. -// ---------------------------------------------------------------------------- -template <class T, class Tuple, class F> -decltype(std::declval<F>()(std::declval<T>())) WithConstructed( - Tuple&& t, F&& f) { - return memory_internal::WithConstructedImpl<T>( - std::forward<Tuple>(t), - phmap::make_index_sequence< - std::tuple_size<typename std::decay<Tuple>::type>::value>(), - std::forward<F>(f)); -} - -// ---------------------------------------------------------------------------- -// Given arguments of an std::pair's consructor, PairArgs() returns a pair of -// tuples with references to the passed arguments. The tuples contain -// constructor arguments for the first and the second elements of the pair. -// -// The following two snippets are equivalent. -// -// 1. std::pair<F, S> p(args...); -// -// 2. auto a = PairArgs(args...); -// std::pair<F, S> p(std::piecewise_construct, -// std::move(p.first), std::move(p.second)); -// ---------------------------------------------------------------------------- -inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; } - -template <class F, class S> -std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) { - return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)), - std::forward_as_tuple(std::forward<S>(s))}; -} - -template <class F, class S> -std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs( - const std::pair<F, S>& p) { - return PairArgs(p.first, p.second); -} - -template <class F, class S> -std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) { - return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second)); -} - -template <class F, class S> -auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) - -> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), - memory_internal::TupleRef(std::forward<S>(s)))) { - return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), - memory_internal::TupleRef(std::forward<S>(s))); -} - -// A helper function for implementing apply() in map policies. -// ---------------------------------------------------------------------------- -template <class F, class... Args> -auto DecomposePair(F&& f, Args&&... args) - -> decltype(memory_internal::DecomposePairImpl( - std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) { - return memory_internal::DecomposePairImpl( - std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); -} - -// A helper function for implementing apply() in set policies. -// ---------------------------------------------------------------------------- -template <class F, class Arg> -decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>())) -DecomposeValue(F&& f, Arg&& arg) { - const auto& key = arg; - return std::forward<F>(f)(key, std::forward<Arg>(arg)); -} - - -// -------------------------------------------------------------------------- -// Policy: a policy defines how to perform different operations on -// the slots of the hashtable (see hash_policy_traits.h for the full interface -// of policy). -// -// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The -// functor should accept a key and return size_t as hash. For best performance -// it is important that the hash function provides high entropy across all bits -// of the hash. -// -// Eq: a (possibly polymorphic) functor that compares two keys for equality. It -// should accept two (of possibly different type) keys and return a bool: true -// if they are equal, false if they are not. If two keys compare equal, then -// their hash values as defined by Hash MUST be equal. -// -// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which -// the storage of the hashtable will be allocated and the elements will be -// constructed and destroyed. -// -------------------------------------------------------------------------- -template <class T> -struct FlatHashSetPolicy -{ - using slot_type = T; - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - phmap::allocator_traits<Allocator>::construct(*alloc, slot, - std::forward<Args>(args)...); - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - phmap::allocator_traits<Allocator>::destroy(*alloc, slot); - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - construct(alloc, new_slot, std::move(*old_slot)); - destroy(alloc, old_slot); - } - - static T& element(slot_type* slot) { return *slot; } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposeValue( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward<F>(f), std::forward<Args>(args)...); - } - - static size_t space_used(const T*) { return 0; } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class K, class V> -struct FlatHashMapPolicy -{ - using slot_policy = priv::map_slot_policy<K, V>; - using slot_type = typename slot_policy::slot_type; - using key_type = K; - using mapped_type = V; - using init_type = std::pair</*non const*/ key_type, mapped_type>; - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - slot_policy::construct(alloc, slot, std::forward<Args>(args)...); - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - slot_policy::transfer(alloc, new_slot, old_slot); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposePair( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward<F>(f), - std::forward<Args>(args)...); - } - - static size_t space_used(const slot_type*) { return 0; } - - static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } - - static V& value(std::pair<const K, V>* kv) { return kv->second; } - static const V& value(const std::pair<const K, V>* kv) { return kv->second; } -}; - -template <class Reference, class Policy> -struct node_hash_policy { - static_assert(std::is_lvalue_reference<Reference>::value, ""); - - using slot_type = typename std::remove_cv< - typename std::remove_reference<Reference>::type>::type*; - - template <class Alloc, class... Args> - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - *slot = Policy::new_element(alloc, std::forward<Args>(args)...); - } - - template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::delete_element(alloc, *slot); - } - - template <class Alloc> - static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { - *new_slot = *old_slot; - } - - static size_t space_used(const slot_type* slot) { - if (slot == nullptr) return Policy::element_space_used(nullptr); - return Policy::element_space_used(*slot); - } - - static Reference element(slot_type* slot) { return **slot; } - - template <class T, class P = Policy> - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - - template <class... Ts, class P = Policy> - static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward<Ts>(ts)...)) { - return P::apply(std::forward<Ts>(ts)...); - } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class T> -struct NodeHashSetPolicy - : phmap::priv::node_hash_policy<T&, NodeHashSetPolicy<T>> -{ - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template <class Allocator, class... Args> - static T* new_element(Allocator* alloc, Args&&... args) { - using ValueAlloc = - typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; - ValueAlloc value_alloc(*alloc); - T* res = phmap::allocator_traits<ValueAlloc>::allocate(value_alloc, 1); - phmap::allocator_traits<ValueAlloc>::construct(value_alloc, res, - std::forward<Args>(args)...); - return res; - } - - template <class Allocator> - static void delete_element(Allocator* alloc, T* elem) { - using ValueAlloc = - typename phmap::allocator_traits<Allocator>::template rebind_alloc<T>; - ValueAlloc value_alloc(*alloc); - phmap::allocator_traits<ValueAlloc>::destroy(value_alloc, elem); - phmap::allocator_traits<ValueAlloc>::deallocate(value_alloc, elem, 1); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposeValue( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward<F>(f), std::forward<Args>(args)...); - } - - static size_t element_space_used(const T*) { return sizeof(T); } -}; - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <class Key, class Value> -class NodeHashMapPolicy - : public phmap::priv::node_hash_policy< - std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>> -{ - using value_type = std::pair<const Key, Value>; - -public: - using key_type = Key; - using mapped_type = Value; - using init_type = std::pair</*non const*/ key_type, mapped_type>; - - template <class Allocator, class... Args> - static value_type* new_element(Allocator* alloc, Args&&... args) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc<value_type>; - PairAlloc pair_alloc(*alloc); - value_type* res = - phmap::allocator_traits<PairAlloc>::allocate(pair_alloc, 1); - phmap::allocator_traits<PairAlloc>::construct(pair_alloc, res, - std::forward<Args>(args)...); - return res; - } - - template <class Allocator> - static void delete_element(Allocator* alloc, value_type* pair) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc<value_type>; - PairAlloc pair_alloc(*alloc); - phmap::allocator_traits<PairAlloc>::destroy(pair_alloc, pair); - phmap::allocator_traits<PairAlloc>::deallocate(pair_alloc, pair, 1); - } - - template <class F, class... Args> - static decltype(phmap::priv::DecomposePair( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward<F>(f), - std::forward<Args>(args)...); - } - - static size_t element_space_used(const value_type*) { - return sizeof(value_type); - } - - static Value& value(value_type* elem) { return elem->second; } - static const Value& value(const value_type* elem) { return elem->second; } -}; - - -// -------------------------------------------------------------------------- -// hash_default -// -------------------------------------------------------------------------- - -#if PHMAP_HAVE_STD_STRING_VIEW - -// support char16_t wchar_t .... -template<class CharT> -struct StringHashT -{ - using is_transparent = void; - - size_t operator()(std::basic_string_view<CharT> v) const { - std::string_view bv{reinterpret_cast<const char*>(v.data()), v.size() * sizeof(CharT)}; - return std::hash<std::string_view>()(bv); - } -}; - -// Supports heterogeneous lookup for basic_string<T>-like elements. -template<class CharT> -struct StringHashEqT -{ - using Hash = StringHashT<CharT>; - - struct Eq { - using is_transparent = void; - - bool operator()(std::basic_string_view<CharT> lhs, std::basic_string_view<CharT> rhs) const { - return lhs == rhs; - } - }; -}; - -template <> -struct HashEq<std::string> : StringHashEqT<char> {}; - -template <> -struct HashEq<std::string_view> : StringHashEqT<char> {}; - -// char16_t -template <> -struct HashEq<std::u16string> : StringHashEqT<char16_t> {}; - -template <> -struct HashEq<std::u16string_view> : StringHashEqT<char16_t> {}; - -// wchar_t -template <> -struct HashEq<std::wstring> : StringHashEqT<wchar_t> {}; - -template <> -struct HashEq<std::wstring_view> : StringHashEqT<wchar_t> {}; - -#endif - -// Supports heterogeneous lookup for pointers and smart pointers. -// ------------------------------------------------------------- -template <class T> -struct HashEq<T*> -{ - struct Hash { - using is_transparent = void; - template <class U> - size_t operator()(const U& ptr) const { - return phmap::Hash<const T*>{}(HashEq::ToPtr(ptr)); - } - }; - - struct Eq { - using is_transparent = void; - template <class A, class B> - bool operator()(const A& a, const B& b) const { - return HashEq::ToPtr(a) == HashEq::ToPtr(b); - } - }; - -private: - static const T* ToPtr(const T* ptr) { return ptr; } - - template <class U, class D> - static const T* ToPtr(const std::unique_ptr<U, D>& ptr) { - return ptr.get(); - } - - template <class U> - static const T* ToPtr(const std::shared_ptr<U>& ptr) { - return ptr.get(); - } -}; - -template <class T, class D> -struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {}; - -template <class T> -struct HashEq<std::shared_ptr<T>> : HashEq<T*> {}; - -namespace hashtable_debug_internal { - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -template <typename Set> -struct HashtableDebugAccess<Set, phmap::void_t<typename Set::raw_hash_set>> -{ - using Traits = typename Set::PolicyTraits; - using Slot = typename Traits::slot_type; - - static size_t GetNumProbes(const Set& set, - const typename Set::key_type& key) { - size_t num_probes = 0; - size_t hashval = set.hash(key); - auto seq = set.probe(hashval); - while (true) { - priv::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(priv::H2(hashval))) { - if (Traits::apply( - typename Set::template EqualElement<typename Set::key_type>{ - key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset((size_t)i)))) - return num_probes; - ++num_probes; - } - if (g.MatchEmpty()) return num_probes; - seq.next(); - ++num_probes; - } - } - - static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); - size_t m = layout.AllocSize(); - - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * c.size(); - } else { - for (size_t i = 0; i != capacity; ++i) { - if (priv::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); - } - } - } - return m; - } - - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); - size_t m = layout.AllocSize(); - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; - } -}; - -} // namespace hashtable_debug_internal -} // namespace priv - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::flat_hash_set<T>` is an unordered associative container which has -// been optimized for both speed and memory footprint in most common use cases. -// Its interface is similar to that of `std::unordered_set<T>` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the set is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class flat_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::FlatHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename flat_hash_set::raw_hash_set; - -public: - flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::flat_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::flat_hash_map<K, V>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map<K, V>` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Invalidates any references and pointers to elements within the table after -// `rehash()`. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `_erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class K, class V, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class flat_hash_map : public phmap::priv::raw_hash_map< - phmap::priv::FlatHashMapPolicy<K, V>, - Hash, Eq, Alloc> { - using Base = typename flat_hash_map::raw_hash_map; - -public: - flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_set -// ----------------------------------------------------------------------------- -// An `phmap::node_hash_set<T>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_set<T>` with the -// following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash set. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class node_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename node_hash_set::raw_hash_set; - -public: - node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_set; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::node_hash_map -// ----------------------------------------------------------------------------- -// -// An `phmap::node_hash_map<K, V>` is an unordered associative container which -// has been optimized for both speed and memory footprint in most common use -// cases. Its interface is similar to that of `std::unordered_map<K, V>` with -// the following notable differences: -// -// * Supports heterogeneous lookup, through `find()`, `operator[]()` and -// `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. -// * Contains a `capacity()` member function indicating the number of element -// slots (open, deleted, and empty) within the hash map. -// * Returns `void` from the `erase(iterator)` overload. -// ----------------------------------------------------------------------------- -template <class Key, class Value, class Hash, class Eq, class Alloc> // default values in phmap_fwd_decl.h -class node_hash_map - : public phmap::priv::raw_hash_map< - phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, - Alloc> -{ - using Base = typename node_hash_map::raw_hash_map; - -public: - node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::raw_hash_map; -#else - using Base::Base; -#endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_set -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> // default values in phmap_fwd_decl.h -class parallel_flat_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashSetPolicy<T>, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_set::parallel_hash_set; - -public: - parallel_flat_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h -// ----------------------------------------------------------------------------- -template <class K, class V, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashMapPolicy<K, V>, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_map::parallel_hash_map; - -public: - parallel_flat_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_set -// ----------------------------------------------------------------------------- -template <class T, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_node_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashSetPolicy<T>, Hash, Eq, Alloc> -{ - using Base = typename parallel_node_hash_set::parallel_hash_set; - -public: - parallel_node_hash_set() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_set; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -// ----------------------------------------------------------------------------- -// phmap::parallel_node_hash_map -// ----------------------------------------------------------------------------- -template <class Key, class Value, class Hash, class Eq, class Alloc, size_t N, class Mtx_> -class parallel_node_hash_map - : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashMapPolicy<Key, Value>, Hash, Eq, - Alloc> -{ - using Base = typename parallel_node_hash_map::parallel_hash_map; - -public: - parallel_node_hash_map() {} -#ifdef __INTEL_COMPILER - using Base::parallel_hash_map; -#else - using Base::Base; -#endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } -}; - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // phmap_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_base.h b/benchmarks/external/parallel_hashmap/phmap_base.h deleted file mode 100644 index d0c6f3ce..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_base.h +++ /dev/null @@ -1,5171 +0,0 @@ -#if !defined(phmap_base_h_guard_) -#define phmap_base_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#include <algorithm> -#include <cassert> -#include <cstddef> -#include <initializer_list> -#include <iterator> -#include <string> -#include <type_traits> -#include <utility> -#include <functional> -#include <tuple> -#include <utility> -#include <memory> -#include <mutex> // for std::lock - -#include "phmap_config.h" - -#ifdef PHMAP_HAVE_SHARED_MUTEX - #include <shared_mutex> // after "phmap_config.h" -#endif - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4582) // constructor is not implicitly called - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member -#endif // _MSC_VER - -namespace phmap { - -template <class T> using Allocator = typename std::allocator<T>; - -template<class T1, class T2> using Pair = typename std::pair<T1, T2>; - -template <class T> -struct EqualTo -{ - inline bool operator()(const T& a, const T& b) const - { - return std::equal_to<T>()(a, b); - } -}; - -template <class T> -struct Less -{ - inline bool operator()(const T& a, const T& b) const - { - return std::less<T>()(a, b); - } -}; - -namespace type_traits_internal { - -template <typename... Ts> -struct VoidTImpl { - using type = void; -}; - -// This trick to retrieve a default alignment is necessary for our -// implementation of aligned_storage_t to be consistent with any implementation -// of std::aligned_storage. -// --------------------------------------------------------------------------- -template <size_t Len, typename T = std::aligned_storage<Len>> -struct default_alignment_of_aligned_storage; - -template <size_t Len, size_t Align> -struct default_alignment_of_aligned_storage<Len, - std::aligned_storage<Len, Align>> { - static constexpr size_t value = Align; -}; - -// NOTE: The `is_detected` family of templates here differ from the library -// fundamentals specification in that for library fundamentals, `Op<Args...>` is -// evaluated as soon as the type `is_detected<Op, Args...>` undergoes -// substitution, regardless of whether or not the `::value` is accessed. That -// is inconsistent with all other standard traits and prevents lazy evaluation -// in larger contexts (such as if the `is_detected` check is a trailing argument -// of a `conjunction`. This implementation opts to instead be lazy in the same -// way that the standard traits are (this "defect" of the detection idiom -// specifications has been reported). -// --------------------------------------------------------------------------- - -template <class Enabler, template <class...> class Op, class... Args> -struct is_detected_impl { - using type = std::false_type; -}; - -template <template <class...> class Op, class... Args> -struct is_detected_impl<typename VoidTImpl<Op<Args...>>::type, Op, Args...> { - using type = std::true_type; -}; - -template <template <class...> class Op, class... Args> -struct is_detected : is_detected_impl<void, Op, Args...>::type {}; - -template <class Enabler, class To, template <class...> class Op, class... Args> -struct is_detected_convertible_impl { - using type = std::false_type; -}; - -template <class To, template <class...> class Op, class... Args> -struct is_detected_convertible_impl< - typename std::enable_if<std::is_convertible<Op<Args...>, To>::value>::type, - To, Op, Args...> { - using type = std::true_type; -}; - -template <class To, template <class...> class Op, class... Args> -struct is_detected_convertible - : is_detected_convertible_impl<void, To, Op, Args...>::type {}; - -template <typename T> -using IsCopyAssignableImpl = - decltype(std::declval<T&>() = std::declval<const T&>()); - -template <typename T> -using IsMoveAssignableImpl = decltype(std::declval<T&>() = std::declval<T&&>()); - -} // namespace type_traits_internal - -template <typename T> -struct is_copy_assignable : type_traits_internal::is_detected< - type_traits_internal::IsCopyAssignableImpl, T> { -}; - -template <typename T> -struct is_move_assignable : type_traits_internal::is_detected< - type_traits_internal::IsMoveAssignableImpl, T> { -}; - -// --------------------------------------------------------------------------- -// void_t() -// -// Ignores the type of any its arguments and returns `void`. In general, this -// metafunction allows you to create a general case that maps to `void` while -// allowing specializations that map to specific types. -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::void_t` metafunction. -// -// NOTE: `phmap::void_t` does not use the standard-specified implementation so -// that it can remain compatible with gcc < 5.1. This can introduce slightly -// different behavior, such as when ordering partial specializations. -// --------------------------------------------------------------------------- -template <typename... Ts> -using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; - -// --------------------------------------------------------------------------- -// conjunction -// -// Performs a compile-time logical AND operation on the passed types (which -// must have `::value` members convertible to `bool`. Short-circuits if it -// encounters any `false` members (and does not compare the `::value` members -// of any remaining arguments). -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::conjunction` metafunction. -// --------------------------------------------------------------------------- -template <typename... Ts> -struct conjunction; - -template <typename T, typename... Ts> -struct conjunction<T, Ts...> - : std::conditional<T::value, conjunction<Ts...>, T>::type {}; - -template <typename T> -struct conjunction<T> : T {}; - -template <> -struct conjunction<> : std::true_type {}; - -// --------------------------------------------------------------------------- -// disjunction -// -// Performs a compile-time logical OR operation on the passed types (which -// must have `::value` members convertible to `bool`. Short-circuits if it -// encounters any `true` members (and does not compare the `::value` members -// of any remaining arguments). -// -// This metafunction is designed to be a drop-in replacement for the C++17 -// `std::disjunction` metafunction. -// --------------------------------------------------------------------------- -template <typename... Ts> -struct disjunction; - -template <typename T, typename... Ts> -struct disjunction<T, Ts...> : - std::conditional<T::value, T, disjunction<Ts...>>::type {}; - -template <typename T> -struct disjunction<T> : T {}; - -template <> -struct disjunction<> : std::false_type {}; - -template <typename T> -struct negation : std::integral_constant<bool, !T::value> {}; - -template <typename T> -struct is_trivially_destructible - : std::integral_constant<bool, __has_trivial_destructor(T) && - std::is_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_default_constructible - : std::integral_constant<bool, __has_trivial_constructor(T) && - std::is_default_constructible<T>::value && - is_trivially_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_copy_constructible - : std::integral_constant<bool, __has_trivial_copy(T) && - std::is_copy_constructible<T>::value && - is_trivially_destructible<T>::value> {}; - -template <typename T> -struct is_trivially_copy_assignable - : std::integral_constant< - bool, __has_trivial_assign(typename std::remove_reference<T>::type) && - phmap::is_copy_assignable<T>::value> {}; - -// ----------------------------------------------------------------------------- -// C++14 "_t" trait aliases -// ----------------------------------------------------------------------------- - -template <typename T> -using remove_cv_t = typename std::remove_cv<T>::type; - -template <typename T> -using remove_const_t = typename std::remove_const<T>::type; - -template <typename T> -using remove_volatile_t = typename std::remove_volatile<T>::type; - -template <typename T> -using add_cv_t = typename std::add_cv<T>::type; - -template <typename T> -using add_const_t = typename std::add_const<T>::type; - -template <typename T> -using add_volatile_t = typename std::add_volatile<T>::type; - -template <typename T> -using remove_reference_t = typename std::remove_reference<T>::type; - -template <typename T> -using add_lvalue_reference_t = typename std::add_lvalue_reference<T>::type; - -template <typename T> -using add_rvalue_reference_t = typename std::add_rvalue_reference<T>::type; - -template <typename T> -using remove_pointer_t = typename std::remove_pointer<T>::type; - -template <typename T> -using add_pointer_t = typename std::add_pointer<T>::type; - -template <typename T> -using make_signed_t = typename std::make_signed<T>::type; - -template <typename T> -using make_unsigned_t = typename std::make_unsigned<T>::type; - -template <typename T> -using remove_extent_t = typename std::remove_extent<T>::type; - -template <typename T> -using remove_all_extents_t = typename std::remove_all_extents<T>::type; - -template <size_t Len, size_t Align = type_traits_internal:: - default_alignment_of_aligned_storage<Len>::value> -using aligned_storage_t = typename std::aligned_storage<Len, Align>::type; - -template <typename T> -using decay_t = typename std::decay<T>::type; - -template <bool B, typename T = void> -using enable_if_t = typename std::enable_if<B, T>::type; - -template <bool B, typename T, typename F> -using conditional_t = typename std::conditional<B, T, F>::type; - - -template <typename... T> -using common_type_t = typename std::common_type<T...>::type; - -template <typename T> -using underlying_type_t = typename std::underlying_type<T>::type; - -template< class F, class... ArgTypes> -#if PHMAP_HAVE_CC17 - using invoke_result = typename std::invoke_result<F, ArgTypes...>; -#else - using invoke_result = typename std::result_of<F(ArgTypes...)>::type; -#endif - -namespace type_traits_internal { - -// ---------------------------------------------------------------------- -// In MSVC we can't probe std::hash or stdext::hash because it triggers a -// static_assert instead of failing substitution. Libc++ prior to 4.0 -// also used a static_assert. -// ---------------------------------------------------------------------- -#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \ - _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 -#else - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 -#endif - -#if !PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template <typename Key, typename = size_t> - struct IsHashable : std::true_type {}; -#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template <typename Key, typename = void> - struct IsHashable : std::false_type {}; - - template <typename Key> - struct IsHashable<Key, - phmap::enable_if_t<std::is_convertible< - decltype(std::declval<std::hash<Key>&>()(std::declval<Key const&>())), - std::size_t>::value>> : std::true_type {}; -#endif - -struct AssertHashEnabledHelper -{ -private: - static void Sink(...) {} - struct NAT {}; - - template <class Key> - static auto GetReturnType(int) - -> decltype(std::declval<std::hash<Key>>()(std::declval<Key const&>())); - template <class Key> - static NAT GetReturnType(...); - - template <class Key> - static std::nullptr_t DoIt() { - static_assert(IsHashable<Key>::value, - "std::hash<Key> does not provide a call operator"); - static_assert( - std::is_default_constructible<std::hash<Key>>::value, - "std::hash<Key> must be default constructible when it is enabled"); - static_assert( - std::is_copy_constructible<std::hash<Key>>::value, - "std::hash<Key> must be copy constructible when it is enabled"); - static_assert(phmap::is_copy_assignable<std::hash<Key>>::value, - "std::hash<Key> must be copy assignable when it is enabled"); - // is_destructible is unchecked as it's implied by each of the - // is_constructible checks. - using ReturnType = decltype(GetReturnType<Key>(0)); - static_assert(std::is_same<ReturnType, NAT>::value || - std::is_same<ReturnType, size_t>::value, - "std::hash<Key> must return size_t"); - return nullptr; - } - - template <class... Ts> - friend void AssertHashEnabled(); -}; - -template <class... Ts> -inline void AssertHashEnabled -() -{ - using Helper = AssertHashEnabledHelper; - Helper::Sink(Helper::DoIt<Ts>()...); -} - -} // namespace type_traits_internal - -} // namespace phmap - - -// ----------------------------------------------------------------------------- -// hash_policy_traits -// ----------------------------------------------------------------------------- -namespace phmap { -namespace priv { - -// Defines how slots are initialized/destroyed/moved. -template <class Policy, class = void> -struct hash_policy_traits -{ -private: - struct ReturnKey - { - // We return `Key` here. - // When Key=T&, we forward the lvalue reference. - // When Key=T, we return by value to avoid a dangling reference. - // eg, for string_hash_map. - template <class Key, class... Args> - Key operator()(Key&& k, const Args&...) const { - return std::forward<Key>(k); - } - }; - - template <class P = Policy, class = void> - struct ConstantIteratorsImpl : std::false_type {}; - - template <class P> - struct ConstantIteratorsImpl<P, phmap::void_t<typename P::constant_iterators>> - : P::constant_iterators {}; - -public: - // The actual object stored in the hash table. - using slot_type = typename Policy::slot_type; - - // The type of the keys stored in the hashtable. - using key_type = typename Policy::key_type; - - // The argument type for insertions into the hashtable. This is different - // from value_type for increased performance. See initializer_list constructor - // and insert() member functions for more details. - using init_type = typename Policy::init_type; - - using reference = decltype(Policy::element(std::declval<slot_type*>())); - using pointer = typename std::remove_reference<reference>::type*; - using value_type = typename std::remove_reference<reference>::type; - - // Policies can set this variable to tell raw_hash_set that all iterators - // should be constant, even `iterator`. This is useful for set-like - // containers. - // Defaults to false if not provided by the policy. - using constant_iterators = ConstantIteratorsImpl<>; - - // PRECONDITION: `slot` is UNINITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class Alloc, class... Args> - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - Policy::construct(alloc, slot, std::forward<Args>(args)...); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is UNINITIALIZED - template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); - } - - // Transfers the `old_slot` to `new_slot`. Any memory allocated by the - // allocator inside `old_slot` to `new_slot` can be transferred. - // - // OPTIONAL: defaults to: - // - // clone(new_slot, std::move(*old_slot)); - // destroy(old_slot); - // - // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED - // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is - // UNINITIALIZED - template <class Alloc> - static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, 0); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class P = Policy> - static auto element(slot_type* slot) -> decltype(P::element(slot)) { - return P::element(slot); - } - - // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. - // - // If `slot` is nullptr, returns the constant amount of memory owned by any - // full slot or -1 if slots own variable amounts of memory. - // - // PRECONDITION: `slot` is INITIALIZED or nullptr - template <class P = Policy> - static size_t space_used(const slot_type* slot) { - return P::space_used(slot); - } - - // Provides generalized access to the key for elements, both for elements in - // the table and for elements that have not yet been inserted (or even - // constructed). We would like an API that allows us to say: `key(args...)` - // but we cannot do that for all cases, so we use this more general API that - // can be used for many things, including the following: - // - // - Given an element in a table, get its key. - // - Given an element initializer, get its key. - // - Given `emplace()` arguments, get the element key. - // - // Implementations of this must adhere to a very strict technical - // specification around aliasing and consuming arguments: - // - // Let `value_type` be the result type of `element()` without ref- and - // cv-qualifiers. The first argument is a functor, the rest are constructor - // arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where - // `k` is the element key, and `xs...` are the new constructor arguments for - // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias - // `ts...`. The key won't be touched once `xs...` are used to construct an - // element; `ts...` won't be touched at all, which allows `apply()` to consume - // any rvalues among them. - // - // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not - // trigger a hard compile error unless it originates from `f`. In other words, - // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not - // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. - // - // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, - // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. - template <class F, class... Ts, class P = Policy> - static auto apply(F&& f, Ts&&... ts) - -> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) { - return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...); - } - - // Returns the "key" portion of the slot. - // Used for node handle manipulation. - template <class P = Policy> - static auto key(slot_type* slot) - -> decltype(P::apply(ReturnKey(), element(slot))) { - return P::apply(ReturnKey(), element(slot)); - } - - // Returns the "value" (as opposed to the "key") portion of the element. Used - // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. - template <class T, class P = Policy> - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - -private: - - // Use auto -> decltype as an enabler. - template <class Alloc, class P = Policy> - static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, int) - -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { - P::transfer(alloc, new_slot, old_slot); - } - - template <class Alloc> - static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, char) { - construct(alloc, new_slot, std::move(element(old_slot))); - destroy(alloc, old_slot); - } -}; - -} // namespace priv -} // namespace phmap - -// ----------------------------------------------------------------------------- -// file utility.h -// ----------------------------------------------------------------------------- - -// --------- identity.h -namespace phmap { -namespace internal { - -template <typename T> -struct identity { - typedef T type; -}; - -template <typename T> -using identity_t = typename identity<T>::type; - -} // namespace internal -} // namespace phmap - - -// --------- inline_variable.h - -#ifdef __cpp_inline_variables - -#if defined(__clang__) - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) \ - extern const ::phmap::internal::identity_t<type> name; -#else // Otherwise, just define the macro to do nothing. - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) -#endif // defined(__clang__) - -// See above comment at top of file for details. -#define PHMAP_INTERNAL_INLINE_CONSTEXPR(type, name, init) \ - PHMAP_INTERNAL_EXTERN_DECL(type, name) \ - inline constexpr ::phmap::internal::identity_t<type> name = init - -#else - -// See above comment at top of file for details. -// -// Note: -// identity_t is used here so that the const and name are in the -// appropriate place for pointer types, reference types, function pointer -// types, etc.. -#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ - template <class /*PhmapInternalDummy*/ = void> \ - struct PhmapInternalInlineVariableHolder##name { \ - static constexpr ::phmap::internal::identity_t<var_type> kInstance = init; \ - }; \ - \ - template <class PhmapInternalDummy> \ - constexpr ::phmap::internal::identity_t<var_type> \ - PhmapInternalInlineVariableHolder##name<PhmapInternalDummy>::kInstance; \ - \ - static constexpr const ::phmap::internal::identity_t<var_type>& \ - name = /* NOLINT */ \ - PhmapInternalInlineVariableHolder##name<>::kInstance; \ - static_assert(sizeof(void (*)(decltype(name))) != 0, \ - "Silence unused variable warnings.") - -#endif // __cpp_inline_variables - -// ----------- throw_delegate - -namespace phmap { -namespace base_internal { - -namespace { -template <typename T> -#ifdef PHMAP_HAVE_EXCEPTIONS -[[noreturn]] void Throw(const T& error) { - throw error; -} -#else -[[noreturn]] void Throw(const T&) { - std::abort(); -} -#endif -} // namespace - -static inline void ThrowStdLogicError(const std::string& what_arg) { - Throw(std::logic_error(what_arg)); -} -static inline void ThrowStdLogicError(const char* what_arg) { - Throw(std::logic_error(what_arg)); -} -static inline void ThrowStdInvalidArgument(const std::string& what_arg) { - Throw(std::invalid_argument(what_arg)); -} -static inline void ThrowStdInvalidArgument(const char* what_arg) { - Throw(std::invalid_argument(what_arg)); -} - -static inline void ThrowStdDomainError(const std::string& what_arg) { - Throw(std::domain_error(what_arg)); -} -static inline void ThrowStdDomainError(const char* what_arg) { - Throw(std::domain_error(what_arg)); -} - -static inline void ThrowStdLengthError(const std::string& what_arg) { - Throw(std::length_error(what_arg)); -} -static inline void ThrowStdLengthError(const char* what_arg) { - Throw(std::length_error(what_arg)); -} - -static inline void ThrowStdOutOfRange(const std::string& what_arg) { - Throw(std::out_of_range(what_arg)); -} -static inline void ThrowStdOutOfRange(const char* what_arg) { - Throw(std::out_of_range(what_arg)); -} - -static inline void ThrowStdRuntimeError(const std::string& what_arg) { - Throw(std::runtime_error(what_arg)); -} -static inline void ThrowStdRuntimeError(const char* what_arg) { - Throw(std::runtime_error(what_arg)); -} - -static inline void ThrowStdRangeError(const std::string& what_arg) { - Throw(std::range_error(what_arg)); -} -static inline void ThrowStdRangeError(const char* what_arg) { - Throw(std::range_error(what_arg)); -} - -static inline void ThrowStdOverflowError(const std::string& what_arg) { - Throw(std::overflow_error(what_arg)); -} -static inline void ThrowStdOverflowError(const char* what_arg) { - Throw(std::overflow_error(what_arg)); -} - -static inline void ThrowStdUnderflowError(const std::string& what_arg) { - Throw(std::underflow_error(what_arg)); -} -static inline void ThrowStdUnderflowError(const char* what_arg) { - Throw(std::underflow_error(what_arg)); -} - -static inline void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); } - -static inline void ThrowStdBadAlloc() { Throw(std::bad_alloc()); } - -} // namespace base_internal -} // namespace phmap - -// ----------- invoke.h - -namespace phmap { -namespace base_internal { - -template <typename Derived> -struct StrippedAccept -{ - template <typename... Args> - struct Accept : Derived::template AcceptImpl<typename std::remove_cv< - typename std::remove_reference<Args>::type>::type...> {}; -}; - -// (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T -// and t1 is an object of type T or a reference to an object of type T or a -// reference to an object of a type derived from T. -struct MemFunAndRef : StrippedAccept<MemFunAndRef> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename... Params, typename Obj, - typename... Args> - struct AcceptImpl<R (C::*)(Params...), Obj, Args...> - : std::is_base_of<C, Obj> {}; - - template <typename R, typename C, typename... Params, typename Obj, - typename... Args> - struct AcceptImpl<R (C::*)(Params...) const, Obj, Args...> - : std::is_base_of<C, Obj> {}; - - template <typename MemFun, typename Obj, typename... Args> - static decltype((std::declval<Obj>().* - std::declval<MemFun>())(std::declval<Args>()...)) - Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { - return (std::forward<Obj>(obj).* - std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); - } -}; - -// ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a -// class T and t1 is not one of the types described in the previous item. -struct MemFunAndPtr : StrippedAccept<MemFunAndPtr> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename... Params, typename Ptr, - typename... Args> - struct AcceptImpl<R (C::*)(Params...), Ptr, Args...> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename R, typename C, typename... Params, typename Ptr, - typename... Args> - struct AcceptImpl<R (C::*)(Params...) const, Ptr, Args...> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename MemFun, typename Ptr, typename... Args> - static decltype(((*std::declval<Ptr>()).* - std::declval<MemFun>())(std::declval<Args>()...)) - Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { - return ((*std::forward<Ptr>(ptr)).* - std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); - } -}; - -// t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is -// an object of type T or a reference to an object of type T or a reference -// to an object of a type derived from T. -struct DataMemAndRef : StrippedAccept<DataMemAndRef> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename Obj> - struct AcceptImpl<R C::*, Obj> : std::is_base_of<C, Obj> {}; - - template <typename DataMem, typename Ref> - static decltype(std::declval<Ref>().*std::declval<DataMem>()) Invoke( - DataMem&& data_mem, Ref&& ref) { - return std::forward<Ref>(ref).*std::forward<DataMem>(data_mem); - } -}; - -// (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 -// is not one of the types described in the previous item. -struct DataMemAndPtr : StrippedAccept<DataMemAndPtr> -{ - template <typename... Args> - struct AcceptImpl : std::false_type {}; - - template <typename R, typename C, typename Ptr> - struct AcceptImpl<R C::*, Ptr> - : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value> {}; - - template <typename DataMem, typename Ptr> - static decltype((*std::declval<Ptr>()).*std::declval<DataMem>()) Invoke( - DataMem&& data_mem, Ptr&& ptr) { - return (*std::forward<Ptr>(ptr)).*std::forward<DataMem>(data_mem); - } -}; - -// f(t1, t2, ..., tN) in all other cases. -struct Callable -{ - // Callable doesn't have Accept because it's the last clause that gets picked - // when none of the previous clauses are applicable. - template <typename F, typename... Args> - static decltype(std::declval<F>()(std::declval<Args>()...)) Invoke( - F&& f, Args&&... args) { - return std::forward<F>(f)(std::forward<Args>(args)...); - } -}; - -// Resolves to the first matching clause. -template <typename... Args> -struct Invoker -{ - typedef typename std::conditional< - MemFunAndRef::Accept<Args...>::value, MemFunAndRef, - typename std::conditional< - MemFunAndPtr::Accept<Args...>::value, MemFunAndPtr, - typename std::conditional< - DataMemAndRef::Accept<Args...>::value, DataMemAndRef, - typename std::conditional<DataMemAndPtr::Accept<Args...>::value, - DataMemAndPtr, Callable>::type>::type>:: - type>::type type; -}; - -// The result type of Invoke<F, Args...>. -template <typename F, typename... Args> -using InvokeT = decltype(Invoker<F, Args...>::type::Invoke( - std::declval<F>(), std::declval<Args>()...)); - -// Invoke(f, args...) is an implementation of INVOKE(f, args...) from section -// [func.require] of the C++ standard. -template <typename F, typename... Args> -InvokeT<F, Args...> Invoke(F&& f, Args&&... args) { - return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), - std::forward<Args>(args)...); -} -} // namespace base_internal -} // namespace phmap - - -// ----------- utility.h - -namespace phmap { - -// integer_sequence -// -// Class template representing a compile-time integer sequence. An instantiation -// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its -// type through its template arguments (which is a common need when -// working with C++11 variadic templates). `phmap::integer_sequence` is designed -// to be a drop-in replacement for C++14's `std::integer_sequence`. -// -// Example: -// -// template< class T, T... Ints > -// void user_function(integer_sequence<T, Ints...>); -// -// int main() -// { -// // user_function's `T` will be deduced to `int` and `Ints...` -// // will be deduced to `0, 1, 2, 3, 4`. -// user_function(make_integer_sequence<int, 5>()); -// } -template <typename T, T... Ints> -struct integer_sequence -{ - using value_type = T; - static constexpr size_t size() noexcept { return sizeof...(Ints); } -}; - -// index_sequence -// -// A helper template for an `integer_sequence` of `size_t`, -// `phmap::index_sequence` is designed to be a drop-in replacement for C++14's -// `std::index_sequence`. -template <size_t... Ints> -using index_sequence = integer_sequence<size_t, Ints...>; - -namespace utility_internal { - -template <typename Seq, size_t SeqSize, size_t Rem> -struct Extend; - -// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>; -}; - -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>; -}; - -// Recursion helper for 'make_integer_sequence<T, N>'. -// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'. -template <typename T, size_t N> -struct Gen { - using type = - typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type; -}; - -template <typename T> -struct Gen<T, 0> { - using type = integer_sequence<T>; -}; - -} // namespace utility_internal - -// Compile-time sequences of integers - -// make_integer_sequence -// -// This template alias is equivalent to -// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in -// replacement for C++14's `std::make_integer_sequence`. -template <typename T, T N> -using make_integer_sequence = typename utility_internal::Gen<T, N>::type; - -// make_index_sequence -// -// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, -// and is designed to be a drop-in replacement for C++14's -// `std::make_index_sequence`. -template <size_t N> -using make_index_sequence = make_integer_sequence<size_t, N>; - -// index_sequence_for -// -// Converts a typename pack into an index sequence of the same length, and -// is designed to be a drop-in replacement for C++14's -// `std::index_sequence_for()` -template <typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; - -// Tag types - -#ifdef PHMAP_HAVE_STD_OPTIONAL - -using std::in_place_t; -using std::in_place; - -#else // PHMAP_HAVE_STD_OPTIONAL - -// in_place_t -// -// Tag type used to specify in-place construction, such as with -// `phmap::optional`, designed to be a drop-in replacement for C++17's -// `std::in_place_t`. -struct in_place_t {}; - -PHMAP_INTERNAL_INLINE_CONSTEXPR(in_place_t, in_place, {}); - -#endif // PHMAP_HAVE_STD_OPTIONAL - -#if defined(PHMAP_HAVE_STD_ANY) || defined(PHMAP_HAVE_STD_VARIANT) -using std::in_place_type_t; -#else - -// in_place_type_t -// -// Tag type used for in-place construction when the type to construct needs to -// be specified, such as with `phmap::any`, designed to be a drop-in replacement -// for C++17's `std::in_place_type_t`. -template <typename T> -struct in_place_type_t {}; -#endif // PHMAP_HAVE_STD_ANY || PHMAP_HAVE_STD_VARIANT - -#ifdef PHMAP_HAVE_STD_VARIANT -using std::in_place_index_t; -#else - -// in_place_index_t -// -// Tag type used for in-place construction when the type to construct needs to -// be specified, such as with `phmap::any`, designed to be a drop-in replacement -// for C++17's `std::in_place_index_t`. -template <size_t I> -struct in_place_index_t {}; -#endif // PHMAP_HAVE_STD_VARIANT - -// Constexpr move and forward - -// move() -// -// A constexpr version of `std::move()`, designed to be a drop-in replacement -// for C++14's `std::move()`. -template <typename T> -constexpr phmap::remove_reference_t<T>&& move(T&& t) noexcept { - return static_cast<phmap::remove_reference_t<T>&&>(t); -} - -// forward() -// -// A constexpr version of `std::forward()`, designed to be a drop-in replacement -// for C++14's `std::forward()`. -template <typename T> -constexpr T&& forward( - phmap::remove_reference_t<T>& t) noexcept { // NOLINT(runtime/references) - return static_cast<T&&>(t); -} - -namespace utility_internal { -// Helper method for expanding tuple into a called method. -template <typename Functor, typename Tuple, std::size_t... Indexes> -auto apply_helper(Functor&& functor, Tuple&& t, index_sequence<Indexes...>) - -> decltype(phmap::base_internal::Invoke( - phmap::forward<Functor>(functor), - std::get<Indexes>(phmap::forward<Tuple>(t))...)) { - return phmap::base_internal::Invoke( - phmap::forward<Functor>(functor), - std::get<Indexes>(phmap::forward<Tuple>(t))...); -} - -} // namespace utility_internal - -// apply -// -// Invokes a Callable using elements of a tuple as its arguments. -// Each element of the tuple corresponds to an argument of the call (in order). -// Both the Callable argument and the tuple argument are perfect-forwarded. -// For member-function Callables, the first tuple element acts as the `this` -// pointer. `phmap::apply` is designed to be a drop-in replacement for C++17's -// `std::apply`. Unlike C++17's `std::apply`, this is not currently `constexpr`. -// -// Example: -// -// class Foo { -// public: -// void Bar(int); -// }; -// void user_function1(int, std::string); -// void user_function2(std::unique_ptr<Foo>); -// auto user_lambda = [](int, int) {}; -// -// int main() -// { -// std::tuple<int, std::string> tuple1(42, "bar"); -// // Invokes the first user function on int, std::string. -// phmap::apply(&user_function1, tuple1); -// -// std::tuple<std::unique_ptr<Foo>> tuple2(phmap::make_unique<Foo>()); -// // Invokes the user function that takes ownership of the unique -// // pointer. -// phmap::apply(&user_function2, std::move(tuple2)); -// -// auto foo = phmap::make_unique<Foo>(); -// std::tuple<Foo*, int> tuple3(foo.get(), 42); -// // Invokes the method Bar on foo with one argument, 42. -// phmap::apply(&Foo::Bar, tuple3); -// -// std::tuple<int, int> tuple4(8, 9); -// // Invokes a lambda. -// phmap::apply(user_lambda, tuple4); -// } -template <typename Functor, typename Tuple> -auto apply(Functor&& functor, Tuple&& t) - -> decltype(utility_internal::apply_helper( - phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), - phmap::make_index_sequence<std::tuple_size< - typename std::remove_reference<Tuple>::type>::value>{})) { - return utility_internal::apply_helper( - phmap::forward<Functor>(functor), phmap::forward<Tuple>(t), - phmap::make_index_sequence<std::tuple_size< - typename std::remove_reference<Tuple>::type>::value>{}); -} - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch -#endif // _MSC_VER - -// exchange -// -// Replaces the value of `obj` with `new_value` and returns the old value of -// `obj`. `phmap::exchange` is designed to be a drop-in replacement for C++14's -// `std::exchange`. -// -// Example: -// -// Foo& operator=(Foo&& other) { -// ptr1_ = phmap::exchange(other.ptr1_, nullptr); -// int1_ = phmap::exchange(other.int1_, -1); -// return *this; -// } -template <typename T, typename U = T> -T exchange(T& obj, U&& new_value) -{ - T old_value = phmap::move(obj); - obj = phmap::forward<U>(new_value); - return old_value; -} - -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - - -} // namespace phmap - -// ----------------------------------------------------------------------------- -// memory.h -// ----------------------------------------------------------------------------- - -namespace phmap { - -template <typename T> -std::unique_ptr<T> WrapUnique(T* ptr) -{ - static_assert(!std::is_array<T>::value, "array types are unsupported"); - static_assert(std::is_object<T>::value, "non-object types are unsupported"); - return std::unique_ptr<T>(ptr); -} - -namespace memory_internal { - -// Traits to select proper overload and return type for `phmap::make_unique<>`. -template <typename T> -struct MakeUniqueResult { - using scalar = std::unique_ptr<T>; -}; -template <typename T> -struct MakeUniqueResult<T[]> { - using array = std::unique_ptr<T[]>; -}; -template <typename T, size_t N> -struct MakeUniqueResult<T[N]> { - using invalid = void; -}; - -} // namespace memory_internal - -#if (__cplusplus > 201103L || defined(_MSC_VER)) && \ - !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) - using std::make_unique; -#else - - template <typename T, typename... Args> - typename memory_internal::MakeUniqueResult<T>::scalar make_unique( - Args&&... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); - } - - template <typename T> - typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) { - return std::unique_ptr<T>(new typename phmap::remove_extent_t<T>[n]()); - } - - template <typename T, typename... Args> - typename memory_internal::MakeUniqueResult<T>::invalid make_unique( - Args&&... /* args */) = delete; -#endif - -template <typename T> -auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) -{ - // ptr is a forwarding reference to support Ts with non-const operators. - return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; -} - -inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; } - -template <typename T, typename D> -std::shared_ptr<T> ShareUniquePtr(std::unique_ptr<T, D>&& ptr) { - return ptr ? std::shared_ptr<T>(std::move(ptr)) : std::shared_ptr<T>(); -} - -template <typename T> -std::weak_ptr<T> WeakenPtr(const std::shared_ptr<T>& ptr) { - return std::weak_ptr<T>(ptr); -} - -namespace memory_internal { - -// ExtractOr<E, O, D>::type evaluates to E<O> if possible. Otherwise, D. -template <template <typename> class Extract, typename Obj, typename Default, - typename> -struct ExtractOr { - using type = Default; -}; - -template <template <typename> class Extract, typename Obj, typename Default> -struct ExtractOr<Extract, Obj, Default, void_t<Extract<Obj>>> { - using type = Extract<Obj>; -}; - -template <template <typename> class Extract, typename Obj, typename Default> -using ExtractOrT = typename ExtractOr<Extract, Obj, Default, void>::type; - -// Extractors for the features of allocators. -template <typename T> -using GetPointer = typename T::pointer; - -template <typename T> -using GetConstPointer = typename T::const_pointer; - -template <typename T> -using GetVoidPointer = typename T::void_pointer; - -template <typename T> -using GetConstVoidPointer = typename T::const_void_pointer; - -template <typename T> -using GetDifferenceType = typename T::difference_type; - -template <typename T> -using GetSizeType = typename T::size_type; - -template <typename T> -using GetPropagateOnContainerCopyAssignment = - typename T::propagate_on_container_copy_assignment; - -template <typename T> -using GetPropagateOnContainerMoveAssignment = - typename T::propagate_on_container_move_assignment; - -template <typename T> -using GetPropagateOnContainerSwap = typename T::propagate_on_container_swap; - -template <typename T> -using GetIsAlwaysEqual = typename T::is_always_equal; - -template <typename T> -struct GetFirstArg; - -template <template <typename...> class Class, typename T, typename... Args> -struct GetFirstArg<Class<T, Args...>> { - using type = T; -}; - -template <typename Ptr, typename = void> -struct ElementType { - using type = typename GetFirstArg<Ptr>::type; -}; - -template <typename T> -struct ElementType<T, void_t<typename T::element_type>> { - using type = typename T::element_type; -}; - -template <typename T, typename U> -struct RebindFirstArg; - -template <template <typename...> class Class, typename T, typename... Args, - typename U> -struct RebindFirstArg<Class<T, Args...>, U> { - using type = Class<U, Args...>; -}; - -template <typename T, typename U, typename = void> -struct RebindPtr { - using type = typename RebindFirstArg<T, U>::type; -}; - -template <typename T, typename U> -struct RebindPtr<T, U, void_t<typename T::template rebind<U>>> { - using type = typename T::template rebind<U>; -}; - -template <typename T, typename U> -constexpr bool HasRebindAlloc(...) { - return false; -} - -template <typename T, typename U> -constexpr bool HasRebindAlloc(typename std::allocator_traits<T>::template rebind_alloc<U>*) { - return true; -} - -template <typename T, typename U, bool = HasRebindAlloc<T, U>(nullptr)> -struct RebindAlloc { - using type = typename RebindFirstArg<T, U>::type; -}; - -template <typename A, typename U> -struct RebindAlloc<A, U, true> { - using type = typename std::allocator_traits<A>::template rebind_alloc<U>; -}; - - -} // namespace memory_internal - -template <typename Ptr> -struct pointer_traits -{ - using pointer = Ptr; - - // element_type: - // Ptr::element_type if present. Otherwise T if Ptr is a template - // instantiation Template<T, Args...> - using element_type = typename memory_internal::ElementType<Ptr>::type; - - // difference_type: - // Ptr::difference_type if present, otherwise std::ptrdiff_t - using difference_type = - memory_internal::ExtractOrT<memory_internal::GetDifferenceType, Ptr, - std::ptrdiff_t>; - - // rebind: - // Ptr::rebind<U> if exists, otherwise Template<U, Args...> if Ptr is a - // template instantiation Template<T, Args...> - template <typename U> - using rebind = typename memory_internal::RebindPtr<Ptr, U>::type; - - // pointer_to: - // Calls Ptr::pointer_to(r) - static pointer pointer_to(element_type& r) { // NOLINT(runtime/references) - return Ptr::pointer_to(r); - } -}; - -// Specialization for T*. -template <typename T> -struct pointer_traits<T*> -{ - using pointer = T*; - using element_type = T; - using difference_type = std::ptrdiff_t; - - template <typename U> - using rebind = U*; - - // pointer_to: - // Calls std::addressof(r) - static pointer pointer_to( - element_type& r) noexcept { // NOLINT(runtime/references) - return std::addressof(r); - } -}; - -// ----------------------------------------------------------------------------- -// Class Template: allocator_traits -// ----------------------------------------------------------------------------- -// -// A C++11 compatible implementation of C++17's std::allocator_traits. -// -template <typename Alloc> -struct allocator_traits -{ - using allocator_type = Alloc; - - // value_type: - // Alloc::value_type - using value_type = typename Alloc::value_type; - - // pointer: - // Alloc::pointer if present, otherwise value_type* - using pointer = memory_internal::ExtractOrT<memory_internal::GetPointer, - Alloc, value_type*>; - - // const_pointer: - // Alloc::const_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<const value_type> - using const_pointer = - memory_internal::ExtractOrT<memory_internal::GetConstPointer, Alloc, - typename phmap::pointer_traits<pointer>:: - template rebind<const value_type>>; - - // void_pointer: - // Alloc::void_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<void> - using void_pointer = memory_internal::ExtractOrT< - memory_internal::GetVoidPointer, Alloc, - typename phmap::pointer_traits<pointer>::template rebind<void>>; - - // const_void_pointer: - // Alloc::const_void_pointer if present, otherwise - // phmap::pointer_traits<pointer>::rebind<const void> - using const_void_pointer = memory_internal::ExtractOrT< - memory_internal::GetConstVoidPointer, Alloc, - typename phmap::pointer_traits<pointer>::template rebind<const void>>; - - // difference_type: - // Alloc::difference_type if present, otherwise - // phmap::pointer_traits<pointer>::difference_type - using difference_type = memory_internal::ExtractOrT< - memory_internal::GetDifferenceType, Alloc, - typename phmap::pointer_traits<pointer>::difference_type>; - - // size_type: - // Alloc::size_type if present, otherwise - // std::make_unsigned<difference_type>::type - using size_type = memory_internal::ExtractOrT< - memory_internal::GetSizeType, Alloc, - typename std::make_unsigned<difference_type>::type>; - - // propagate_on_container_copy_assignment: - // Alloc::propagate_on_container_copy_assignment if present, otherwise - // std::false_type - using propagate_on_container_copy_assignment = memory_internal::ExtractOrT< - memory_internal::GetPropagateOnContainerCopyAssignment, Alloc, - std::false_type>; - - // propagate_on_container_move_assignment: - // Alloc::propagate_on_container_move_assignment if present, otherwise - // std::false_type - using propagate_on_container_move_assignment = memory_internal::ExtractOrT< - memory_internal::GetPropagateOnContainerMoveAssignment, Alloc, - std::false_type>; - - // propagate_on_container_swap: - // Alloc::propagate_on_container_swap if present, otherwise std::false_type - using propagate_on_container_swap = - memory_internal::ExtractOrT<memory_internal::GetPropagateOnContainerSwap, - Alloc, std::false_type>; - - // is_always_equal: - // Alloc::is_always_equal if present, otherwise std::is_empty<Alloc>::type - using is_always_equal = - memory_internal::ExtractOrT<memory_internal::GetIsAlwaysEqual, Alloc, - typename std::is_empty<Alloc>::type>; - - // rebind_alloc: - // Alloc::rebind<T>::other if present, otherwise Alloc<T, Args> if this Alloc - // is Alloc<U, Args> - template <typename T> - using rebind_alloc = typename memory_internal::RebindAlloc<Alloc, T>::type; - - // rebind_traits: - // phmap::allocator_traits<rebind_alloc<T>> - template <typename T> - using rebind_traits = phmap::allocator_traits<rebind_alloc<T>>; - - // allocate(Alloc& a, size_type n): - // Calls a.allocate(n) - static pointer allocate(Alloc& a, // NOLINT(runtime/references) - size_type n) { - return a.allocate(n); - } - - // allocate(Alloc& a, size_type n, const_void_pointer hint): - // Calls a.allocate(n, hint) if possible. - // If not possible, calls a.allocate(n) - static pointer allocate(Alloc& a, size_type n, // NOLINT(runtime/references) - const_void_pointer hint) { - return allocate_impl(0, a, n, hint); - } - - // deallocate(Alloc& a, pointer p, size_type n): - // Calls a.deallocate(p, n) - static void deallocate(Alloc& a, pointer p, // NOLINT(runtime/references) - size_type n) { - a.deallocate(p, n); - } - - // construct(Alloc& a, T* p, Args&&... args): - // Calls a.construct(p, std::forward<Args>(args)...) if possible. - // If not possible, calls - // ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...) - template <typename T, typename... Args> - static void construct(Alloc& a, T* p, // NOLINT(runtime/references) - Args&&... args) { - construct_impl(0, a, p, std::forward<Args>(args)...); - } - - // destroy(Alloc& a, T* p): - // Calls a.destroy(p) if possible. If not possible, calls p->~T(). - template <typename T> - static void destroy(Alloc& a, T* p) { // NOLINT(runtime/references) - destroy_impl(0, a, p); - } - - // max_size(const Alloc& a): - // Returns a.max_size() if possible. If not possible, returns - // std::numeric_limits<size_type>::max() / sizeof(value_type) - static size_type max_size(const Alloc& a) { return max_size_impl(0, a); } - - // select_on_container_copy_construction(const Alloc& a): - // Returns a.select_on_container_copy_construction() if possible. - // If not possible, returns a. - static Alloc select_on_container_copy_construction(const Alloc& a) { - return select_on_container_copy_construction_impl(0, a); - } - -private: - template <typename A> - static auto allocate_impl(int, A& a, // NOLINT(runtime/references) - size_type n, const_void_pointer hint) - -> decltype(a.allocate(n, hint)) { - return a.allocate(n, hint); - } - static pointer allocate_impl(char, Alloc& a, // NOLINT(runtime/references) - size_type n, const_void_pointer) { - return a.allocate(n); - } - - template <typename A, typename... Args> - static auto construct_impl(int, A& a, // NOLINT(runtime/references) - Args&&... args) - -> decltype(std::allocator_traits<A>::construct(a, std::forward<Args>(args)...)) { - std::allocator_traits<A>::construct(a, std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static void construct_impl(char, Alloc&, T* p, Args&&... args) { - ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...); - } - - template <typename A, typename T> - static auto destroy_impl(int, A& a, // NOLINT(runtime/references) - T* p) -> decltype(std::allocator_traits<A>::destroy(a, p)) { - std::allocator_traits<A>::destroy(a, p); - } - template <typename T> - static void destroy_impl(char, Alloc&, T* p) { - p->~T(); - } - - template <typename A> - static auto max_size_impl(int, const A& a) -> decltype(a.max_size()) { - return a.max_size(); - } - static size_type max_size_impl(char, const Alloc&) { - return (std::numeric_limits<size_type>::max)() / sizeof(value_type); - } - - template <typename A> - static auto select_on_container_copy_construction_impl(int, const A& a) - -> decltype(a.select_on_container_copy_construction()) { - return a.select_on_container_copy_construction(); - } - static Alloc select_on_container_copy_construction_impl(char, - const Alloc& a) { - return a; - } -}; - -namespace memory_internal { - -// This template alias transforms Alloc::is_nothrow into a metafunction with -// Alloc as a parameter so it can be used with ExtractOrT<>. -template <typename Alloc> -using GetIsNothrow = typename Alloc::is_nothrow; - -} // namespace memory_internal - -// PHMAP_ALLOCATOR_NOTHROW is a build time configuration macro for user to -// specify whether the default allocation function can throw or never throws. -// If the allocation function never throws, user should define it to a non-zero -// value (e.g. via `-DPHMAP_ALLOCATOR_NOTHROW`). -// If the allocation function can throw, user should leave it undefined or -// define it to zero. -// -// allocator_is_nothrow<Alloc> is a traits class that derives from -// Alloc::is_nothrow if present, otherwise std::false_type. It's specialized -// for Alloc = std::allocator<T> for any type T according to the state of -// PHMAP_ALLOCATOR_NOTHROW. -// -// default_allocator_is_nothrow is a class that derives from std::true_type -// when the default allocator (global operator new) never throws, and -// std::false_type when it can throw. It is a convenience shorthand for writing -// allocator_is_nothrow<std::allocator<T>> (T can be any type). -// NOTE: allocator_is_nothrow<std::allocator<T>> is guaranteed to derive from -// the same type for all T, because users should specialize neither -// allocator_is_nothrow nor std::allocator. -template <typename Alloc> -struct allocator_is_nothrow - : memory_internal::ExtractOrT<memory_internal::GetIsNothrow, Alloc, - std::false_type> {}; - -#if defined(PHMAP_ALLOCATOR_NOTHROW) && PHMAP_ALLOCATOR_NOTHROW - template <typename T> - struct allocator_is_nothrow<std::allocator<T>> : std::true_type {}; - struct default_allocator_is_nothrow : std::true_type {}; -#else - struct default_allocator_is_nothrow : std::false_type {}; -#endif - -namespace memory_internal { -template <typename Allocator, typename Iterator, typename... Args> -void ConstructRange(Allocator& alloc, Iterator first, Iterator last, - const Args&... args) -{ - for (Iterator cur = first; cur != last; ++cur) { - PHMAP_INTERNAL_TRY { - std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), - args...); - } - PHMAP_INTERNAL_CATCH_ANY { - while (cur != first) { - --cur; - std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); - } - PHMAP_INTERNAL_RETHROW; - } - } -} - -template <typename Allocator, typename Iterator, typename InputIterator> -void CopyRange(Allocator& alloc, Iterator destination, InputIterator first, - InputIterator last) -{ - for (Iterator cur = destination; first != last; - static_cast<void>(++cur), static_cast<void>(++first)) { - PHMAP_INTERNAL_TRY { - std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), - *first); - } - PHMAP_INTERNAL_CATCH_ANY { - while (cur != destination) { - --cur; - std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); - } - PHMAP_INTERNAL_RETHROW; - } - } -} -} // namespace memory_internal -} // namespace phmap - - -// ----------------------------------------------------------------------------- -// optional.h -// ----------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_OPTIONAL - -#include <optional> // IWYU pragma: export - -namespace phmap { -using std::bad_optional_access; -using std::optional; -using std::make_optional; -using std::nullopt_t; -using std::nullopt; -} // namespace phmap - -#else - -#if defined(__clang__) - #if __has_feature(cxx_inheriting_constructors) - #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 - #endif -#elif (defined(__GNUC__) && \ - (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8)) || \ - (__cpp_inheriting_constructors >= 200802) || \ - (defined(_MSC_VER) && _MSC_VER >= 1910) - - #define PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 -#endif - -namespace phmap { - -class bad_optional_access : public std::exception -{ -public: - bad_optional_access() = default; - ~bad_optional_access() override; - const char* what() const noexcept override; -}; - -template <typename T> -class optional; - -// -------------------------------- -struct nullopt_t -{ - struct init_t {}; - static init_t init; - - explicit constexpr nullopt_t(init_t& /*unused*/) {} -}; - -constexpr nullopt_t nullopt(nullopt_t::init); - -namespace optional_internal { - -// throw delegator -[[noreturn]] void throw_bad_optional_access(); - - -struct empty_struct {}; - -// This class stores the data in optional<T>. -// It is specialized based on whether T is trivially destructible. -// This is the specialization for non trivially destructible type. -template <typename T, bool unused = std::is_trivially_destructible<T>::value> -class optional_data_dtor_base -{ - struct dummy_type { - static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); - // Use an array to avoid GCC 6 placement-new warning. - empty_struct data[sizeof(T) / sizeof(empty_struct)]; - }; - -protected: - // Whether there is data or not. - bool engaged_; - // Data storage - union { - dummy_type dummy_; - T data_; - }; - - void destruct() noexcept { - if (engaged_) { - data_.~T(); - engaged_ = false; - } - } - - // dummy_ must be initialized for constexpr constructor. - constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} - - template <typename... Args> - constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(phmap::forward<Args>(args)...) {} - - ~optional_data_dtor_base() { destruct(); } -}; - -// Specialization for trivially destructible type. -template <typename T> -class optional_data_dtor_base<T, true> -{ - struct dummy_type { - static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); - // Use array to avoid GCC 6 placement-new warning. - empty_struct data[sizeof(T) / sizeof(empty_struct)]; - }; - -protected: - // Whether there is data or not. - bool engaged_; - // Data storage - union { - dummy_type dummy_; - T data_; - }; - void destruct() noexcept { engaged_ = false; } - - // dummy_ must be initialized for constexpr constructor. - constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} - - template <typename... Args> - constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(phmap::forward<Args>(args)...) {} -}; - -template <typename T> -class optional_data_base : public optional_data_dtor_base<T> -{ -protected: - using base = optional_data_dtor_base<T>; -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using base::base; -#else - optional_data_base() = default; - - template <typename... Args> - constexpr explicit optional_data_base(in_place_t t, Args&&... args) - : base(t, phmap::forward<Args>(args)...) {} -#endif - - template <typename... Args> - void construct(Args&&... args) { - // Use dummy_'s address to work around casting cv-qualified T* to void*. - ::new (static_cast<void*>(&this->dummy_)) T(std::forward<Args>(args)...); - this->engaged_ = true; - } - - template <typename U> - void assign(U&& u) { - if (this->engaged_) { - this->data_ = std::forward<U>(u); - } else { - construct(std::forward<U>(u)); - } - } -}; - -// TODO: Add another class using -// std::is_trivially_move_constructible trait when available to match -// http://cplusplus.github.io/LWG/lwg-defects.html#2900, for types that -// have trivial move but nontrivial copy. -// Also, we should be checking is_trivially_copyable here, which is not -// supported now, so we use is_trivially_* traits instead. -template <typename T, - bool unused = phmap::is_trivially_copy_constructible<T>::value&& - phmap::is_trivially_copy_assignable<typename std::remove_cv< - T>::type>::value&& std::is_trivially_destructible<T>::value> -class optional_data; - -// Trivially copyable types -template <typename T> -class optional_data<T, true> : public optional_data_base<T> -{ -protected: -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using optional_data_base<T>::optional_data_base; -#else - optional_data() = default; - - template <typename... Args> - constexpr explicit optional_data(in_place_t t, Args&&... args) - : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} -#endif -}; - -template <typename T> -class optional_data<T, false> : public optional_data_base<T> -{ -protected: -#if PHMAP_OPTIONAL_USE_INHERITING_CONSTRUCTORS - using optional_data_base<T>::optional_data_base; -#else - template <typename... Args> - constexpr explicit optional_data(in_place_t t, Args&&... args) - : optional_data_base<T>(t, phmap::forward<Args>(args)...) {} -#endif - - optional_data() = default; - - optional_data(const optional_data& rhs) : optional_data_base<T>() { - if (rhs.engaged_) { - this->construct(rhs.data_); - } - } - - optional_data(optional_data&& rhs) noexcept( - phmap::default_allocator_is_nothrow::value || - std::is_nothrow_move_constructible<T>::value) - : optional_data_base<T>() { - if (rhs.engaged_) { - this->construct(std::move(rhs.data_)); - } - } - - optional_data& operator=(const optional_data& rhs) { - if (rhs.engaged_) { - this->assign(rhs.data_); - } else { - this->destruct(); - } - return *this; - } - - optional_data& operator=(optional_data&& rhs) noexcept( - std::is_nothrow_move_assignable<T>::value&& - std::is_nothrow_move_constructible<T>::value) { - if (rhs.engaged_) { - this->assign(std::move(rhs.data_)); - } else { - this->destruct(); - } - return *this; - } -}; - -// Ordered by level of restriction, from low to high. -// Copyable implies movable. -enum class copy_traits { copyable = 0, movable = 1, non_movable = 2 }; - -// Base class for enabling/disabling copy/move constructor. -template <copy_traits> -class optional_ctor_base; - -template <> -class optional_ctor_base<copy_traits::copyable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = default; - optional_ctor_base(optional_ctor_base&&) = default; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -template <> -class optional_ctor_base<copy_traits::movable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = delete; - optional_ctor_base(optional_ctor_base&&) = default; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -template <> -class optional_ctor_base<copy_traits::non_movable> -{ -public: - constexpr optional_ctor_base() = default; - optional_ctor_base(const optional_ctor_base&) = delete; - optional_ctor_base(optional_ctor_base&&) = delete; - optional_ctor_base& operator=(const optional_ctor_base&) = default; - optional_ctor_base& operator=(optional_ctor_base&&) = default; -}; - -// Base class for enabling/disabling copy/move assignment. -template <copy_traits> -class optional_assign_base; - -template <> -class optional_assign_base<copy_traits::copyable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = default; - optional_assign_base& operator=(optional_assign_base&&) = default; -}; - -template <> -class optional_assign_base<copy_traits::movable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = delete; - optional_assign_base& operator=(optional_assign_base&&) = default; -}; - -template <> -class optional_assign_base<copy_traits::non_movable> -{ -public: - constexpr optional_assign_base() = default; - optional_assign_base(const optional_assign_base&) = default; - optional_assign_base(optional_assign_base&&) = default; - optional_assign_base& operator=(const optional_assign_base&) = delete; - optional_assign_base& operator=(optional_assign_base&&) = delete; -}; - -template <typename T> -constexpr copy_traits get_ctor_copy_traits() -{ - return std::is_copy_constructible<T>::value - ? copy_traits::copyable - : std::is_move_constructible<T>::value ? copy_traits::movable - : copy_traits::non_movable; -} - -template <typename T> -constexpr copy_traits get_assign_copy_traits() -{ - return phmap::is_copy_assignable<T>::value && - std::is_copy_constructible<T>::value - ? copy_traits::copyable - : phmap::is_move_assignable<T>::value && - std::is_move_constructible<T>::value - ? copy_traits::movable - : copy_traits::non_movable; -} - -// Whether T is constructible or convertible from optional<U>. -template <typename T, typename U> -struct is_constructible_convertible_from_optional - : std::integral_constant< - bool, std::is_constructible<T, optional<U>&>::value || - std::is_constructible<T, optional<U>&&>::value || - std::is_constructible<T, const optional<U>&>::value || - std::is_constructible<T, const optional<U>&&>::value || - std::is_convertible<optional<U>&, T>::value || - std::is_convertible<optional<U>&&, T>::value || - std::is_convertible<const optional<U>&, T>::value || - std::is_convertible<const optional<U>&&, T>::value> {}; - -// Whether T is constructible or convertible or assignable from optional<U>. -template <typename T, typename U> -struct is_constructible_convertible_assignable_from_optional - : std::integral_constant< - bool, is_constructible_convertible_from_optional<T, U>::value || - std::is_assignable<T&, optional<U>&>::value || - std::is_assignable<T&, optional<U>&&>::value || - std::is_assignable<T&, const optional<U>&>::value || - std::is_assignable<T&, const optional<U>&&>::value> {}; - -// Helper function used by [optional.relops], [optional.comp_with_t], -// for checking whether an expression is convertible to bool. -bool convertible_to_bool(bool); - -// Base class for std::hash<phmap::optional<T>>: -// If std::hash<std::remove_const_t<T>> is enabled, it provides operator() to -// compute the hash; Otherwise, it is disabled. -// Reference N4659 23.14.15 [unord.hash]. -template <typename T, typename = size_t> -struct optional_hash_base -{ - optional_hash_base() = delete; - optional_hash_base(const optional_hash_base&) = delete; - optional_hash_base(optional_hash_base&&) = delete; - optional_hash_base& operator=(const optional_hash_base&) = delete; - optional_hash_base& operator=(optional_hash_base&&) = delete; -}; - -template <typename T> -struct optional_hash_base<T, decltype(std::hash<phmap::remove_const_t<T> >()( - std::declval<phmap::remove_const_t<T> >()))> -{ - using argument_type = phmap::optional<T>; - using result_type = size_t; - size_t operator()(const phmap::optional<T>& opt) const { - phmap::type_traits_internal::AssertHashEnabled<phmap::remove_const_t<T>>(); - if (opt) { - return std::hash<phmap::remove_const_t<T> >()(*opt); - } else { - return static_cast<size_t>(0x297814aaad196e6dULL); - } - } -}; - -} // namespace optional_internal - - -// ----------------------------------------------------------------------------- -// phmap::optional class definition -// ----------------------------------------------------------------------------- - -template <typename T> -class optional : private optional_internal::optional_data<T>, - private optional_internal::optional_ctor_base< - optional_internal::get_ctor_copy_traits<T>()>, - private optional_internal::optional_assign_base< - optional_internal::get_assign_copy_traits<T>()> -{ - using data_base = optional_internal::optional_data<T>; - -public: - typedef T value_type; - - // Constructors - - // Constructs an `optional` holding an empty value, NOT a default constructed - // `T`. - constexpr optional() noexcept {} - - // Constructs an `optional` initialized with `nullopt` to hold an empty value. - constexpr optional(nullopt_t) noexcept {} // NOLINT(runtime/explicit) - - // Copy constructor, standard semantics - optional(const optional& src) = default; - - // Move constructor, standard semantics - optional(optional&& src) = default; - - // Constructs a non-empty `optional` direct-initialized value of type `T` from - // the arguments `std::forward<Args>(args)...` within the `optional`. - // (The `in_place_t` is a tag used to indicate that the contained object - // should be constructed in-place.) - template <typename InPlaceT, typename... Args, - phmap::enable_if_t<phmap::conjunction< - std::is_same<InPlaceT, in_place_t>, - std::is_constructible<T, Args&&...> >::value>* = nullptr> - constexpr explicit optional(InPlaceT, Args&&... args) - : data_base(in_place_t(), phmap::forward<Args>(args)...) {} - - // Constructs a non-empty `optional` direct-initialized value of type `T` from - // the arguments of an initializer_list and `std::forward<Args>(args)...`. - // (The `in_place_t` is a tag used to indicate that the contained object - // should be constructed in-place.) - template <typename U, typename... Args, - typename = typename std::enable_if<std::is_constructible< - T, std::initializer_list<U>&, Args&&...>::value>::type> - constexpr explicit optional(in_place_t, std::initializer_list<U> il, - Args&&... args) - : data_base(in_place_t(), il, phmap::forward<Args>(args)...) { - } - - // Value constructor (implicit) - template < - typename U = T, - typename std::enable_if< - phmap::conjunction<phmap::negation<std::is_same< - in_place_t, typename std::decay<U>::type> >, - phmap::negation<std::is_same< - optional<T>, typename std::decay<U>::type> >, - std::is_convertible<U&&, T>, - std::is_constructible<T, U&&> >::value, - bool>::type = false> - constexpr optional(U&& v) : data_base(in_place_t(), phmap::forward<U>(v)) {} - - // Value constructor (explicit) - template < - typename U = T, - typename std::enable_if< - phmap::conjunction<phmap::negation<std::is_same< - in_place_t, typename std::decay<U>::type>>, - phmap::negation<std::is_same< - optional<T>, typename std::decay<U>::type>>, - phmap::negation<std::is_convertible<U&&, T>>, - std::is_constructible<T, U&&>>::value, - bool>::type = false> - explicit constexpr optional(U&& v) - : data_base(in_place_t(), phmap::forward<U>(v)) {} - - // Converting copy constructor (implicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U> >, - std::is_constructible<T, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U> >, - std::is_convertible<const U&, T> >::value, - bool>::type = false> - optional(const optional<U>& rhs) { - if (rhs) { - this->construct(*rhs); - } - } - - // Converting copy constructor (explicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U>>, - std::is_constructible<T, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U>>, - phmap::negation<std::is_convertible<const U&, T>>>::value, - bool>::type = false> - explicit optional(const optional<U>& rhs) { - if (rhs) { - this->construct(*rhs); - } - } - - // Converting move constructor (implicit) - template <typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U> >, - std::is_constructible<T, U&&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_from_optional<T, U> >, - std::is_convertible<U&&, T> >::value, - bool>::type = false> - optional(optional<U>&& rhs) { - if (rhs) { - this->construct(std::move(*rhs)); - } - } - - // Converting move constructor (explicit) - template < - typename U, - typename std::enable_if< - phmap::conjunction< - phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, - phmap::negation< - optional_internal::is_constructible_convertible_from_optional< - T, U>>, - phmap::negation<std::is_convertible<U&&, T>>>::value, - bool>::type = false> - explicit optional(optional<U>&& rhs) { - if (rhs) { - this->construct(std::move(*rhs)); - } - } - - // Destructor. Trivial if `T` is trivially destructible. - ~optional() = default; - - // Assignment Operators - - // Assignment from `nullopt` - // - // Example: - // - // struct S { int value; }; - // optional<S> opt = phmap::nullopt; // Could also use opt = { }; - optional& operator=(nullopt_t) noexcept { - this->destruct(); - return *this; - } - - // Copy assignment operator, standard semantics - optional& operator=(const optional& src) = default; - - // Move assignment operator, standard semantics - optional& operator=(optional&& src) = default; - - // Value assignment operators - template < - typename U = T, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation< - std::is_same<optional<T>, typename std::decay<U>::type>>, - phmap::negation< - phmap::conjunction<std::is_scalar<T>, - std::is_same<T, typename std::decay<U>::type>>>, - std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type> - optional& operator=(U&& v) { - this->assign(std::forward<U>(v)); - return *this; - } - - template < - typename U, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation<std::is_same<T, U>>, - std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>, - phmap::negation< - optional_internal:: - is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> - optional& operator=(const optional<U>& rhs) { - if (rhs) { - this->assign(*rhs); - } else { - this->destruct(); - } - return *this; - } - - template <typename U, - typename = typename std::enable_if<phmap::conjunction< - phmap::negation<std::is_same<T, U>>, std::is_constructible<T, U>, - std::is_assignable<T&, U>, - phmap::negation< - optional_internal:: - is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> - optional& operator=(optional<U>&& rhs) { - if (rhs) { - this->assign(std::move(*rhs)); - } else { - this->destruct(); - } - return *this; - } - - // Modifiers - - // optional::reset() - // - // Destroys the inner `T` value of an `phmap::optional` if one is present. - PHMAP_ATTRIBUTE_REINITIALIZES void reset() noexcept { this->destruct(); } - - // optional::emplace() - // - // (Re)constructs the underlying `T` in-place with the given forwarded - // arguments. - // - // Example: - // - // optional<Foo> opt; - // opt.emplace(arg1,arg2,arg3); // Constructs Foo(arg1,arg2,arg3) - // - // If the optional is non-empty, and the `args` refer to subobjects of the - // current object, then behaviour is undefined, because the current object - // will be destructed before the new object is constructed with `args`. - template <typename... Args, - typename = typename std::enable_if< - std::is_constructible<T, Args&&...>::value>::type> - T& emplace(Args&&... args) { - this->destruct(); - this->construct(std::forward<Args>(args)...); - return reference(); - } - - // Emplace reconstruction overload for an initializer list and the given - // forwarded arguments. - // - // Example: - // - // struct Foo { - // Foo(std::initializer_list<int>); - // }; - // - // optional<Foo> opt; - // opt.emplace({1,2,3}); // Constructs Foo({1,2,3}) - template <typename U, typename... Args, - typename = typename std::enable_if<std::is_constructible< - T, std::initializer_list<U>&, Args&&...>::value>::type> - T& emplace(std::initializer_list<U> il, Args&&... args) { - this->destruct(); - this->construct(il, std::forward<Args>(args)...); - return reference(); - } - - // Swaps - - // Swap, standard semantics - void swap(optional& rhs) noexcept( - std::is_nothrow_move_constructible<T>::value&& - std::is_trivial<T>::value) { - if (*this) { - if (rhs) { - using std::swap; - swap(**this, *rhs); - } else { - rhs.construct(std::move(**this)); - this->destruct(); - } - } else { - if (rhs) { - this->construct(std::move(*rhs)); - rhs.destruct(); - } else { - // No effect (swap(disengaged, disengaged)). - } - } - } - - // Observers - - // optional::operator->() - // - // Accesses the underlying `T` value's member `m` of an `optional`. If the - // `optional` is empty, behavior is undefined. - // - // If you need myOpt->foo in constexpr, use (*myOpt).foo instead. - const T* operator->() const { - assert(this->engaged_); - return std::addressof(this->data_); - } - T* operator->() { - assert(this->engaged_); - return std::addressof(this->data_); - } - - // optional::operator*() - // - // Accesses the underlying `T` value of an `optional`. If the `optional` is - // empty, behavior is undefined. - constexpr const T& operator*() const & { return reference(); } - T& operator*() & { - assert(this->engaged_); - return reference(); - } - constexpr const T&& operator*() const && { - return phmap::move(reference()); - } - T&& operator*() && { - assert(this->engaged_); - return std::move(reference()); - } - - // optional::operator bool() - // - // Returns false if and only if the `optional` is empty. - // - // if (opt) { - // // do something with opt.value(); - // } else { - // // opt is empty. - // } - // - constexpr explicit operator bool() const noexcept { return this->engaged_; } - - // optional::has_value() - // - // Determines whether the `optional` contains a value. Returns `false` if and - // only if `*this` is empty. - constexpr bool has_value() const noexcept { return this->engaged_; } - -// Suppress bogus warning on MSVC: MSVC complains call to reference() after -// throw_bad_optional_access() is unreachable. -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4702) -#endif // _MSC_VER - // optional::value() - // - // Returns a reference to an `optional`s underlying value. The constness - // and lvalue/rvalue-ness of the `optional` is preserved to the view of - // the `T` sub-object. Throws `phmap::bad_optional_access` when the `optional` - // is empty. - constexpr const T& value() const & { - return static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference()); - } - T& value() & { - return static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference()); - } - T&& value() && { // NOLINT(build/c++11) - return std::move( - static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference())); - } - constexpr const T&& value() const && { // NOLINT(build/c++11) - return phmap::move( - static_cast<bool>(*this) - ? reference() - : (optional_internal::throw_bad_optional_access(), reference())); - } -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - - // optional::value_or() - // - // Returns either the value of `T` or a passed default `v` if the `optional` - // is empty. - template <typename U> - constexpr T value_or(U&& v) const& { - static_assert(std::is_copy_constructible<value_type>::value, - "optional<T>::value_or: T must by copy constructible"); - static_assert(std::is_convertible<U&&, value_type>::value, - "optional<T>::value_or: U must be convertible to T"); - return static_cast<bool>(*this) - ? **this - : static_cast<T>(phmap::forward<U>(v)); - } - template <typename U> - T value_or(U&& v) && { // NOLINT(build/c++11) - static_assert(std::is_move_constructible<value_type>::value, - "optional<T>::value_or: T must by move constructible"); - static_assert(std::is_convertible<U&&, value_type>::value, - "optional<T>::value_or: U must be convertible to T"); - return static_cast<bool>(*this) ? std::move(**this) - : static_cast<T>(std::forward<U>(v)); - } - -private: - // Private accessors for internal storage viewed as reference to T. - constexpr const T& reference() const { return this->data_; } - T& reference() { return this->data_; } - - // T constraint checks. You can't have an optional of nullopt_t, in_place_t - // or a reference. - static_assert( - !std::is_same<nullopt_t, typename std::remove_cv<T>::type>::value, - "optional<nullopt_t> is not allowed."); - static_assert( - !std::is_same<in_place_t, typename std::remove_cv<T>::type>::value, - "optional<in_place_t> is not allowed."); - static_assert(!std::is_reference<T>::value, - "optional<reference> is not allowed."); -}; - -// Non-member functions - -// swap() -// -// Performs a swap between two `phmap::optional` objects, using standard -// semantics. -// -// NOTE: we assume `is_swappable()` is always `true`. A compile error will -// result if this is not the case. -template <typename T, - typename std::enable_if<std::is_move_constructible<T>::value, - bool>::type = false> -void swap(optional<T>& a, optional<T>& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); -} - -// make_optional() -// -// Creates a non-empty `optional<T>` where the type of `T` is deduced. An -// `phmap::optional` can also be explicitly instantiated with -// `make_optional<T>(v)`. -// -// Note: `make_optional()` constructions may be declared `constexpr` for -// trivially copyable types `T`. Non-trivial types require copy elision -// support in C++17 for `make_optional` to support `constexpr` on such -// non-trivial types. -// -// Example: -// -// constexpr phmap::optional<int> opt = phmap::make_optional(1); -// static_assert(opt.value() == 1, ""); -template <typename T> -constexpr optional<typename std::decay<T>::type> make_optional(T&& v) { - return optional<typename std::decay<T>::type>(phmap::forward<T>(v)); -} - -template <typename T, typename... Args> -constexpr optional<T> make_optional(Args&&... args) { - return optional<T>(in_place_t(), phmap::forward<Args>(args)...); -} - -template <typename T, typename U, typename... Args> -constexpr optional<T> make_optional(std::initializer_list<U> il, - Args&&... args) { - return optional<T>(in_place_t(), il, - phmap::forward<Args>(args)...); -} - -// Relational operators [optional.relops] - -// Empty optionals are considered equal to each other and less than non-empty -// optionals. Supports relations between optional<T> and optional<U>, between -// optional<T> and U, and between optional<T> and nullopt. -// -// Note: We're careful to support T having non-bool relationals. - -// Requires: The expression, e.g. "*x == *y" shall be well-formed and its result -// shall be convertible to bool. -// The C++17 (N4606) "Returns:" statements are translated into -// code in an obvious way here, and the original text retained as function docs. -// Returns: If bool(x) != bool(y), false; otherwise if bool(x) == false, true; -// otherwise *x == *y. -template <typename T, typename U> -constexpr auto operator==(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x == *y)) { - return static_cast<bool>(x) != static_cast<bool>(y) - ? false - : static_cast<bool>(x) == false ? true - : static_cast<bool>(*x == *y); -} - -// Returns: If bool(x) != bool(y), true; otherwise, if bool(x) == false, false; -// otherwise *x != *y. -template <typename T, typename U> -constexpr auto operator!=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x != *y)) { - return static_cast<bool>(x) != static_cast<bool>(y) - ? true - : static_cast<bool>(x) == false ? false - : static_cast<bool>(*x != *y); -} -// Returns: If !y, false; otherwise, if !x, true; otherwise *x < *y. -template <typename T, typename U> -constexpr auto operator<(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x < *y)) { - return !y ? false : !x ? true : static_cast<bool>(*x < *y); -} -// Returns: If !x, false; otherwise, if !y, true; otherwise *x > *y. -template <typename T, typename U> -constexpr auto operator>(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x > *y)) { - return !x ? false : !y ? true : static_cast<bool>(*x > *y); -} -// Returns: If !x, true; otherwise, if !y, false; otherwise *x <= *y. -template <typename T, typename U> -constexpr auto operator<=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x <= *y)) { - return !x ? true : !y ? false : static_cast<bool>(*x <= *y); -} -// Returns: If !y, true; otherwise, if !x, false; otherwise *x >= *y. -template <typename T, typename U> -constexpr auto operator>=(const optional<T>& x, const optional<U>& y) - -> decltype(optional_internal::convertible_to_bool(*x >= *y)) { - return !y ? true : !x ? false : static_cast<bool>(*x >= *y); -} - -// Comparison with nullopt [optional.nullops] -// The C++17 (N4606) "Returns:" statements are used directly here. -template <typename T> -constexpr bool operator==(const optional<T>& x, nullopt_t) noexcept { - return !x; -} -template <typename T> -constexpr bool operator==(nullopt_t, const optional<T>& x) noexcept { - return !x; -} -template <typename T> -constexpr bool operator!=(const optional<T>& x, nullopt_t) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator!=(nullopt_t, const optional<T>& x) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator<(const optional<T>&, nullopt_t) noexcept { - return false; -} -template <typename T> -constexpr bool operator<(nullopt_t, const optional<T>& x) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator<=(const optional<T>& x, nullopt_t) noexcept { - return !x; -} -template <typename T> -constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept { - return true; -} -template <typename T> -constexpr bool operator>(const optional<T>& x, nullopt_t) noexcept { - return static_cast<bool>(x); -} -template <typename T> -constexpr bool operator>(nullopt_t, const optional<T>&) noexcept { - return false; -} -template <typename T> -constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept { - return true; -} -template <typename T> -constexpr bool operator>=(nullopt_t, const optional<T>& x) noexcept { - return !x; -} - -// Comparison with T [optional.comp_with_t] - -// Requires: The expression, e.g. "*x == v" shall be well-formed and its result -// shall be convertible to bool. -// The C++17 (N4606) "Equivalent to:" statements are used directly here. -template <typename T, typename U> -constexpr auto operator==(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x == v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x == v) : false; -} -template <typename T, typename U> -constexpr auto operator==(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v == *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v == *x) : false; -} -template <typename T, typename U> -constexpr auto operator!=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x != v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x != v) : true; -} -template <typename T, typename U> -constexpr auto operator!=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v != *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v != *x) : true; -} -template <typename T, typename U> -constexpr auto operator<(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x < v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x < v) : true; -} -template <typename T, typename U> -constexpr auto operator<(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v < *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v < *x) : false; -} -template <typename T, typename U> -constexpr auto operator<=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x <= v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x <= v) : true; -} -template <typename T, typename U> -constexpr auto operator<=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v <= *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v <= *x) : false; -} -template <typename T, typename U> -constexpr auto operator>(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x > v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x > v) : false; -} -template <typename T, typename U> -constexpr auto operator>(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v > *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v > *x) : true; -} -template <typename T, typename U> -constexpr auto operator>=(const optional<T>& x, const U& v) - -> decltype(optional_internal::convertible_to_bool(*x >= v)) { - return static_cast<bool>(x) ? static_cast<bool>(*x >= v) : false; -} -template <typename T, typename U> -constexpr auto operator>=(const U& v, const optional<T>& x) - -> decltype(optional_internal::convertible_to_bool(v >= *x)) { - return static_cast<bool>(x) ? static_cast<bool>(v >= *x) : true; -} - -} // namespace phmap - -namespace std { - -// std::hash specialization for phmap::optional. -template <typename T> -struct hash<phmap::optional<T> > - : phmap::optional_internal::optional_hash_base<T> {}; - -} // namespace std - -#endif - -// ----------------------------------------------------------------------------- -// common.h -// ----------------------------------------------------------------------------- -namespace phmap { -namespace priv { - -template <class, class = void> -struct IsTransparent : std::false_type {}; -template <class T> -struct IsTransparent<T, phmap::void_t<typename T::is_transparent>> - : std::true_type {}; - -template <bool is_transparent> -struct KeyArg -{ - // Transparent. Forward `K`. - template <typename K, typename key_type> - using type = K; -}; - -template <> -struct KeyArg<false> -{ - // Not transparent. Always use `key_type`. - template <typename K, typename key_type> - using type = key_type; -}; - -#ifdef _MSC_VER - #pragma warning(push) - // warning C4820: '6' bytes padding added after data member - #pragma warning(disable : 4820) -#endif - -// The node_handle concept from C++17. -// We specialize node_handle for sets and maps. node_handle_base holds the -// common API of both. -// ----------------------------------------------------------------------- -template <typename PolicyTraits, typename Alloc> -class node_handle_base -{ -protected: - using slot_type = typename PolicyTraits::slot_type; - -public: - using allocator_type = Alloc; - - constexpr node_handle_base() {} - - node_handle_base(node_handle_base&& other) noexcept { - *this = std::move(other); - } - - ~node_handle_base() { destroy(); } - - node_handle_base& operator=(node_handle_base&& other) noexcept { - destroy(); - if (!other.empty()) { - alloc_ = other.alloc_; - PolicyTraits::transfer(alloc(), slot(), other.slot()); - other.reset(); - } - return *this; - } - - bool empty() const noexcept { return !alloc_; } - explicit operator bool() const noexcept { return !empty(); } - allocator_type get_allocator() const { return *alloc_; } - -protected: - friend struct CommonAccess; - - struct transfer_tag_t {}; - node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s) - : alloc_(a) { - PolicyTraits::transfer(alloc(), slot(), s); - } - - struct move_tag_t {}; - node_handle_base(move_tag_t, const allocator_type& a, slot_type* s) - : alloc_(a) { - PolicyTraits::construct(alloc(), slot(), s); - } - - node_handle_base(const allocator_type& a, slot_type* s) : alloc_(a) { - PolicyTraits::transfer(alloc(), slot(), s); - } - - //node_handle_base(const node_handle_base&) = delete; - //node_handle_base& operator=(const node_handle_base&) = delete; - - void destroy() { - if (!empty()) { - PolicyTraits::destroy(alloc(), slot()); - reset(); - } - } - - void reset() { - assert(alloc_.has_value()); - alloc_ = phmap::nullopt; - } - - slot_type* slot() const { - assert(!empty()); - return reinterpret_cast<slot_type*>(std::addressof(slot_space_)); - } - - allocator_type* alloc() { return std::addressof(*alloc_); } - -private: - phmap::optional<allocator_type> alloc_; - mutable phmap::aligned_storage_t<sizeof(slot_type), alignof(slot_type)> slot_space_; -}; - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -// For sets. -// --------- -template <typename Policy, typename PolicyTraits, typename Alloc, - typename = void> -class node_handle : public node_handle_base<PolicyTraits, Alloc> -{ - using Base = node_handle_base<PolicyTraits, Alloc>; - -public: - using value_type = typename PolicyTraits::value_type; - - constexpr node_handle() {} - - value_type& value() const { return PolicyTraits::element(this->slot()); } - - value_type& key() const { return PolicyTraits::element(this->slot()); } - -private: - friend struct CommonAccess; - - using Base::Base; -}; - -// For maps. -// --------- -template <typename Policy, typename PolicyTraits, typename Alloc> -class node_handle<Policy, PolicyTraits, Alloc, - phmap::void_t<typename Policy::mapped_type>> - : public node_handle_base<PolicyTraits, Alloc> -{ - using Base = node_handle_base<PolicyTraits, Alloc>; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - - constexpr node_handle() {} - - auto key() const -> decltype(PolicyTraits::key(this->slot())) { - return PolicyTraits::key(this->slot()); - } - - mapped_type& mapped() const { - return PolicyTraits::value(&PolicyTraits::element(this->slot())); - } - -private: - friend struct CommonAccess; - - using Base::Base; -}; - -// Provide access to non-public node-handle functions. -struct CommonAccess -{ - template <typename Node> - static auto GetSlot(const Node& node) -> decltype(node.slot()) { - return node.slot(); - } - - template <typename Node> - static void Destroy(Node* node) { - node->destroy(); - } - - template <typename Node> - static void Reset(Node* node) { - node->reset(); - } - - template <typename T, typename... Args> - static T Make(Args&&... args) { - return T(std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static T Transfer(Args&&... args) { - return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...); - } - - template <typename T, typename... Args> - static T Move(Args&&... args) { - return T(typename T::move_tag_t{}, std::forward<Args>(args)...); - } -}; - -// Implement the insert_return_type<> concept of C++17. -template <class Iterator, class NodeType> -struct InsertReturnType -{ - Iterator position; - bool inserted; - NodeType node; -}; - -} // namespace priv -} // namespace phmap - - -#ifdef ADDRESS_SANITIZER - #include <sanitizer/asan_interface.h> -#endif - -// --------------------------------------------------------------------------- -// span.h -// --------------------------------------------------------------------------- - -namespace phmap { - -template <typename T> -class Span; - -namespace span_internal { -// A constexpr min function -constexpr size_t Min(size_t a, size_t b) noexcept { return a < b ? a : b; } - -// Wrappers for access to container data pointers. -template <typename C> -constexpr auto GetDataImpl(C& c, char) noexcept // NOLINT(runtime/references) - -> decltype(c.data()) { - return c.data(); -} - -// Before C++17, std::string::data returns a const char* in all cases. -inline char* GetDataImpl(std::string& s, // NOLINT(runtime/references) - int) noexcept { - return &s[0]; -} - -template <typename C> -constexpr auto GetData(C& c) noexcept // NOLINT(runtime/references) - -> decltype(GetDataImpl(c, 0)) { - return GetDataImpl(c, 0); -} - -// Detection idioms for size() and data(). -template <typename C> -using HasSize = - std::is_integral<phmap::decay_t<decltype(std::declval<C&>().size())>>; - -// We want to enable conversion from vector<T*> to Span<const T* const> but -// disable conversion from vector<Derived> to Span<Base>. Here we use -// the fact that U** is convertible to Q* const* if and only if Q is the same -// type or a more cv-qualified version of U. We also decay the result type of -// data() to avoid problems with classes which have a member function data() -// which returns a reference. -template <typename T, typename C> -using HasData = - std::is_convertible<phmap::decay_t<decltype(GetData(std::declval<C&>()))>*, - T* const*>; - -// Extracts value type from a Container -template <typename C> -struct ElementType { - using type = typename phmap::remove_reference_t<C>::value_type; -}; - -template <typename T, size_t N> -struct ElementType<T (&)[N]> { - using type = T; -}; - -template <typename C> -using ElementT = typename ElementType<C>::type; - -template <typename T> -using EnableIfMutable = - typename std::enable_if<!std::is_const<T>::value, int>::type; - -template <typename T> -bool EqualImpl(Span<T> a, Span<T> b) { - static_assert(std::is_const<T>::value, ""); - return std::equal(a.begin(), a.end(), b.begin(), b.end()); -} - -template <typename T> -bool LessThanImpl(Span<T> a, Span<T> b) { - static_assert(std::is_const<T>::value, ""); - return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); -} - -// The `IsConvertible` classes here are needed because of the -// `std::is_convertible` bug in libcxx when compiled with GCC. This build -// configuration is used by Android NDK toolchain. Reference link: -// https://bugs.llvm.org/show_bug.cgi?id=27538. -template <typename From, typename To> -struct IsConvertibleHelper { - static std::true_type testval(To); - static std::false_type testval(...); - - using type = decltype(testval(std::declval<From>())); -}; - -template <typename From, typename To> -struct IsConvertible : IsConvertibleHelper<From, To>::type {}; - -// TODO(zhangxy): replace `IsConvertible` with `std::is_convertible` once the -// older version of libcxx is not supported. -template <typename From, typename To> -using EnableIfConvertibleToSpanConst = - typename std::enable_if<IsConvertible<From, Span<const To>>::value>::type; -} // namespace span_internal - -//------------------------------------------------------------------------------ -// Span -//------------------------------------------------------------------------------ -// -// A `Span` is an "array view" type for holding a view of a contiguous data -// array; the `Span` object does not and cannot own such data itself. A span -// provides an easy way to provide overloads for anything operating on -// contiguous sequences without needing to manage pointers and array lengths -// manually. - -// A span is conceptually a pointer (ptr) and a length (size) into an already -// existing array of contiguous memory; the array it represents references the -// elements "ptr[0] .. ptr[size-1]". Passing a properly-constructed `Span` -// instead of raw pointers avoids many issues related to index out of bounds -// errors. -// -// Spans may also be constructed from containers holding contiguous sequences. -// Such containers must supply `data()` and `size() const` methods (e.g -// `std::vector<T>`, `phmap::InlinedVector<T, N>`). All implicit conversions to -// `phmap::Span` from such containers will create spans of type `const T`; -// spans which can mutate their values (of type `T`) must use explicit -// constructors. -// -// A `Span<T>` is somewhat analogous to an `phmap::string_view`, but for an array -// of elements of type `T`. A user of `Span` must ensure that the data being -// pointed to outlives the `Span` itself. -// -// You can construct a `Span<T>` in several ways: -// -// * Explicitly from a reference to a container type -// * Explicitly from a pointer and size -// * Implicitly from a container type (but only for spans of type `const T`) -// * Using the `MakeSpan()` or `MakeConstSpan()` factory functions. -// -// Examples: -// -// // Construct a Span explicitly from a container: -// std::vector<int> v = {1, 2, 3, 4, 5}; -// auto span = phmap::Span<const int>(v); -// -// // Construct a Span explicitly from a C-style array: -// int a[5] = {1, 2, 3, 4, 5}; -// auto span = phmap::Span<const int>(a); -// -// // Construct a Span implicitly from a container -// void MyRoutine(phmap::Span<const int> a) { -// ... -// } -// std::vector v = {1,2,3,4,5}; -// MyRoutine(v) // convert to Span<const T> -// -// Note that `Span` objects, in addition to requiring that the memory they -// point to remains alive, must also ensure that such memory does not get -// reallocated. Therefore, to avoid undefined behavior, containers with -// associated span views should not invoke operations that may reallocate memory -// (such as resizing) or invalidate iterators into the container. -// -// One common use for a `Span` is when passing arguments to a routine that can -// accept a variety of array types (e.g. a `std::vector`, `phmap::InlinedVector`, -// a C-style array, etc.). Instead of creating overloads for each case, you -// can simply specify a `Span` as the argument to such a routine. -// -// Example: -// -// void MyRoutine(phmap::Span<const int> a) { -// ... -// } -// -// std::vector v = {1,2,3,4,5}; -// MyRoutine(v); -// -// phmap::InlinedVector<int, 4> my_inline_vector; -// MyRoutine(my_inline_vector); -// -// // Explicit constructor from pointer,size -// int* my_array = new int[10]; -// MyRoutine(phmap::Span<const int>(my_array, 10)); -template <typename T> -class Span -{ -private: - // Used to determine whether a Span can be constructed from a container of - // type C. - template <typename C> - using EnableIfConvertibleFrom = - typename std::enable_if<span_internal::HasData<T, C>::value && - span_internal::HasSize<C>::value>::type; - - // Used to SFINAE-enable a function when the slice elements are const. - template <typename U> - using EnableIfConstView = - typename std::enable_if<std::is_const<T>::value, U>::type; - - // Used to SFINAE-enable a function when the slice elements are mutable. - template <typename U> - using EnableIfMutableView = - typename std::enable_if<!std::is_const<T>::value, U>::type; - -public: - using value_type = phmap::remove_cv_t<T>; - using pointer = T*; - using const_pointer = const T*; - using reference = T&; - using const_reference = const T&; - using iterator = pointer; - using const_iterator = const_pointer; - using reverse_iterator = std::reverse_iterator<iterator>; - using const_reverse_iterator = std::reverse_iterator<const_iterator>; - using size_type = size_t; - using difference_type = ptrdiff_t; - - static const size_type npos = ~(size_type(0)); - - constexpr Span() noexcept : Span(nullptr, 0) {} - constexpr Span(pointer array, size_type lgth) noexcept - : ptr_(array), len_(lgth) {} - - // Implicit conversion constructors - template <size_t N> - constexpr Span(T (&a)[N]) noexcept // NOLINT(runtime/explicit) - : Span(a, N) {} - - // Explicit reference constructor for a mutable `Span<T>` type. Can be - // replaced with MakeSpan() to infer the type parameter. - template <typename V, typename = EnableIfConvertibleFrom<V>, - typename = EnableIfMutableView<V>> - explicit Span(V& v) noexcept // NOLINT(runtime/references) - : Span(span_internal::GetData(v), v.size()) {} - - // Implicit reference constructor for a read-only `Span<const T>` type - template <typename V, typename = EnableIfConvertibleFrom<V>, - typename = EnableIfConstView<V>> - constexpr Span(const V& v) noexcept // NOLINT(runtime/explicit) - : Span(span_internal::GetData(v), v.size()) {} - - // Implicit constructor from an initializer list, making it possible to pass a - // brace-enclosed initializer list to a function expecting a `Span`. Such - // spans constructed from an initializer list must be of type `Span<const T>`. - // - // void Process(phmap::Span<const int> x); - // Process({1, 2, 3}); - // - // Note that as always the array referenced by the span must outlive the span. - // Since an initializer list constructor acts as if it is fed a temporary - // array (cf. C++ standard [dcl.init.list]/5), it's safe to use this - // constructor only when the `std::initializer_list` itself outlives the span. - // In order to meet this requirement it's sufficient to ensure that neither - // the span nor a copy of it is used outside of the expression in which it's - // created: - // - // // Assume that this function uses the array directly, not retaining any - // // copy of the span or pointer to any of its elements. - // void Process(phmap::Span<const int> ints); - // - // // Okay: the std::initializer_list<int> will reference a temporary array - // // that isn't destroyed until after the call to Process returns. - // Process({ 17, 19 }); - // - // // Not okay: the storage used by the std::initializer_list<int> is not - // // allowed to be referenced after the first line. - // phmap::Span<const int> ints = { 17, 19 }; - // Process(ints); - // - // // Not okay for the same reason as above: even when the elements of the - // // initializer list expression are not temporaries the underlying array - // // is, so the initializer list must still outlive the span. - // const int foo = 17; - // phmap::Span<const int> ints = { foo }; - // Process(ints); - // - template <typename LazyT = T, - typename = EnableIfConstView<LazyT>> - Span( - std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) - : Span(v.begin(), v.size()) {} - - // Accessors - - // Span::data() - // - // Returns a pointer to the span's underlying array of data (which is held - // outside the span). - constexpr pointer data() const noexcept { return ptr_; } - - // Span::size() - // - // Returns the size of this span. - constexpr size_type size() const noexcept { return len_; } - - // Span::length() - // - // Returns the length (size) of this span. - constexpr size_type length() const noexcept { return size(); } - - // Span::empty() - // - // Returns a boolean indicating whether or not this span is considered empty. - constexpr bool empty() const noexcept { return size() == 0; } - - // Span::operator[] - // - // Returns a reference to the i'th element of this span. - constexpr reference operator[](size_type i) const noexcept { - // MSVC 2015 accepts this as constexpr, but not ptr_[i] - return *(data() + i); - } - - // Span::at() - // - // Returns a reference to the i'th element of this span. - constexpr reference at(size_type i) const { - return PHMAP_PREDICT_TRUE(i < size()) // - ? *(data() + i) - : (base_internal::ThrowStdOutOfRange( - "Span::at failed bounds check"), - *(data() + i)); - } - - // Span::front() - // - // Returns a reference to the first element of this span. - constexpr reference front() const noexcept { - return PHMAP_ASSERT(size() > 0), *data(); - } - - // Span::back() - // - // Returns a reference to the last element of this span. - constexpr reference back() const noexcept { - return PHMAP_ASSERT(size() > 0), *(data() + size() - 1); - } - - // Span::begin() - // - // Returns an iterator to the first element of this span. - constexpr iterator begin() const noexcept { return data(); } - - // Span::cbegin() - // - // Returns a const iterator to the first element of this span. - constexpr const_iterator cbegin() const noexcept { return begin(); } - - // Span::end() - // - // Returns an iterator to the last element of this span. - constexpr iterator end() const noexcept { return data() + size(); } - - // Span::cend() - // - // Returns a const iterator to the last element of this span. - constexpr const_iterator cend() const noexcept { return end(); } - - // Span::rbegin() - // - // Returns a reverse iterator starting at the last element of this span. - constexpr reverse_iterator rbegin() const noexcept { - return reverse_iterator(end()); - } - - // Span::crbegin() - // - // Returns a reverse const iterator starting at the last element of this span. - constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); } - - // Span::rend() - // - // Returns a reverse iterator starting at the first element of this span. - constexpr reverse_iterator rend() const noexcept { - return reverse_iterator(begin()); - } - - // Span::crend() - // - // Returns a reverse iterator starting at the first element of this span. - constexpr const_reverse_iterator crend() const noexcept { return rend(); } - - // Span mutations - - // Span::remove_prefix() - // - // Removes the first `n` elements from the span. - void remove_prefix(size_type n) noexcept { - assert(size() >= n); - ptr_ += n; - len_ -= n; - } - - // Span::remove_suffix() - // - // Removes the last `n` elements from the span. - void remove_suffix(size_type n) noexcept { - assert(size() >= n); - len_ -= n; - } - - // Span::subspan() - // - // Returns a `Span` starting at element `pos` and of length `len`. Both `pos` - // and `len` are of type `size_type` and thus non-negative. Parameter `pos` - // must be <= size(). Any `len` value that points past the end of the span - // will be trimmed to at most size() - `pos`. A default `len` value of `npos` - // ensures the returned subspan continues until the end of the span. - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).subspan(1, 2); // {11, 12} - // phmap::MakeSpan(vec).subspan(2, 8); // {12, 13} - // phmap::MakeSpan(vec).subspan(1); // {11, 12, 13} - // phmap::MakeSpan(vec).subspan(4); // {} - // phmap::MakeSpan(vec).subspan(5); // throws std::out_of_range - constexpr Span subspan(size_type pos = 0, size_type len = npos) const { - return (pos <= size()) - ? Span(data() + pos, span_internal::Min(size() - pos, len)) - : (base_internal::ThrowStdOutOfRange("pos > size()"), Span()); - } - - // Span::first() - // - // Returns a `Span` containing first `len` elements. Parameter `len` is of - // type `size_type` and thus non-negative. `len` value must be <= size(). - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).first(1); // {10} - // phmap::MakeSpan(vec).first(3); // {10, 11, 12} - // phmap::MakeSpan(vec).first(5); // throws std::out_of_range - constexpr Span first(size_type len) const { - return (len <= size()) - ? Span(data(), len) - : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); - } - - // Span::last() - // - // Returns a `Span` containing last `len` elements. Parameter `len` is of - // type `size_type` and thus non-negative. `len` value must be <= size(). - // - // Examples: - // - // std::vector<int> vec = {10, 11, 12, 13}; - // phmap::MakeSpan(vec).last(1); // {13} - // phmap::MakeSpan(vec).last(3); // {11, 12, 13} - // phmap::MakeSpan(vec).last(5); // throws std::out_of_range - constexpr Span last(size_type len) const { - return (len <= size()) - ? Span(size() - len + data(), len) - : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); - } - - // Support for phmap::Hash. - template <typename H> - friend H AbslHashValue(H h, Span v) { - return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), - v.size()); - } - -private: - pointer ptr_; - size_type len_; -}; - -template <typename T> -const typename Span<T>::size_type Span<T>::npos; - -// Span relationals - -// Equality is compared element-by-element, while ordering is lexicographical. -// We provide three overloads for each operator to cover any combination on the -// left or right hand side of mutable Span<T>, read-only Span<const T>, and -// convertible-to-read-only Span<T>. -// TODO(zhangxy): Due to MSVC overload resolution bug with partial ordering -// template functions, 5 overloads per operator is needed as a workaround. We -// should update them to 3 overloads per operator using non-deduced context like -// string_view, i.e. -// - (Span<T>, Span<T>) -// - (Span<T>, non_deduced<Span<const T>>) -// - (non_deduced<Span<const T>>, Span<T>) - -// operator== -template <typename T> -bool operator==(Span<T> a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T> -bool operator==(Span<const T> a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T> -bool operator==(Span<T> a, Span<const T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator==(const U& a, Span<T> b) { - return span_internal::EqualImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator==(Span<T> a, const U& b) { - return span_internal::EqualImpl<const T>(a, b); -} - -// operator!= -template <typename T> -bool operator!=(Span<T> a, Span<T> b) { - return !(a == b); -} - -template <typename T> -bool operator!=(Span<const T> a, Span<T> b) { - return !(a == b); -} - -template <typename T> -bool operator!=(Span<T> a, Span<const T> b) { - return !(a == b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator!=(const U& a, Span<T> b) { - return !(a == b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator!=(Span<T> a, const U& b) { - return !(a == b); -} - -// operator< -template <typename T> -bool operator<(Span<T> a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T> -bool operator<(Span<const T> a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T> -bool operator<(Span<T> a, Span<const T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<(const U& a, Span<T> b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<(Span<T> a, const U& b) { - return span_internal::LessThanImpl<const T>(a, b); -} - -// operator> -template <typename T> -bool operator>(Span<T> a, Span<T> b) { - return b < a; -} - -template <typename T> -bool operator>(Span<const T> a, Span<T> b) { - return b < a; -} - -template <typename T> -bool operator>(Span<T> a, Span<const T> b) { - return b < a; -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>(const U& a, Span<T> b) { - return b < a; -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>(Span<T> a, const U& b) { - return b < a; -} - -// operator<= -template <typename T> -bool operator<=(Span<T> a, Span<T> b) { - return !(b < a); -} - -template <typename T> -bool operator<=(Span<const T> a, Span<T> b) { - return !(b < a); -} - -template <typename T> -bool operator<=(Span<T> a, Span<const T> b) { - return !(b < a); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<=(const U& a, Span<T> b) { - return !(b < a); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator<=(Span<T> a, const U& b) { - return !(b < a); -} - -// operator>= -template <typename T> -bool operator>=(Span<T> a, Span<T> b) { - return !(a < b); -} - -template <typename T> -bool operator>=(Span<const T> a, Span<T> b) { - return !(a < b); -} - -template <typename T> -bool operator>=(Span<T> a, Span<const T> b) { - return !(a < b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>=(const U& a, Span<T> b) { - return !(a < b); -} - -template <typename T, typename U, - typename = span_internal::EnableIfConvertibleToSpanConst<U, T>> -bool operator>=(Span<T> a, const U& b) { - return !(a < b); -} - -// MakeSpan() -// -// Constructs a mutable `Span<T>`, deducing `T` automatically from either a -// container or pointer+size. -// -// Because a read-only `Span<const T>` is implicitly constructed from container -// types regardless of whether the container itself is a const container, -// constructing mutable spans of type `Span<T>` from containers requires -// explicit constructors. The container-accepting version of `MakeSpan()` -// deduces the type of `T` by the constness of the pointer received from the -// container's `data()` member. Similarly, the pointer-accepting version returns -// a `Span<const T>` if `T` is `const`, and a `Span<T>` otherwise. -// -// Examples: -// -// void MyRoutine(phmap::Span<MyComplicatedType> a) { -// ... -// }; -// // my_vector is a container of non-const types -// std::vector<MyComplicatedType> my_vector; -// -// // Constructing a Span implicitly attempts to create a Span of type -// // `Span<const T>` -// MyRoutine(my_vector); // error, type mismatch -// -// // Explicitly constructing the Span is verbose -// MyRoutine(phmap::Span<MyComplicatedType>(my_vector)); -// -// // Use MakeSpan() to make an phmap::Span<T> -// MyRoutine(phmap::MakeSpan(my_vector)); -// -// // Construct a span from an array ptr+size -// phmap::Span<T> my_span() { -// return phmap::MakeSpan(&array[0], num_elements_); -// } -// -template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { - return Span<T>(ptr, size); -} - -template <int&... ExplicitArgumentBarrier, typename T> -Span<T> MakeSpan(T* begin, T* end) noexcept { - return PHMAP_ASSERT(begin <= end), Span<T>(begin, end - begin); -} - -template <int&... ExplicitArgumentBarrier, typename C> -constexpr auto MakeSpan(C& c) noexcept // NOLINT(runtime/references) - -> decltype(phmap::MakeSpan(span_internal::GetData(c), c.size())) { - return MakeSpan(span_internal::GetData(c), c.size()); -} - -template <int&... ExplicitArgumentBarrier, typename T, size_t N> -constexpr Span<T> MakeSpan(T (&array)[N]) noexcept { - return Span<T>(array, N); -} - -// MakeConstSpan() -// -// Constructs a `Span<const T>` as with `MakeSpan`, deducing `T` automatically, -// but always returning a `Span<const T>`. -// -// Examples: -// -// void ProcessInts(phmap::Span<const int> some_ints); -// -// // Call with a pointer and size. -// int array[3] = { 0, 0, 0 }; -// ProcessInts(phmap::MakeConstSpan(&array[0], 3)); -// -// // Call with a [begin, end) pair. -// ProcessInts(phmap::MakeConstSpan(&array[0], &array[3])); -// -// // Call directly with an array. -// ProcessInts(phmap::MakeConstSpan(array)); -// -// // Call with a contiguous container. -// std::vector<int> some_ints = ...; -// ProcessInts(phmap::MakeConstSpan(some_ints)); -// ProcessInts(phmap::MakeConstSpan(std::vector<int>{ 0, 0, 0 })); -// -template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept { - return Span<const T>(ptr, size); -} - -template <int&... ExplicitArgumentBarrier, typename T> -Span<const T> MakeConstSpan(T* begin, T* end) noexcept { - return PHMAP_ASSERT(begin <= end), Span<const T>(begin, end - begin); -} - -template <int&... ExplicitArgumentBarrier, typename C> -constexpr auto MakeConstSpan(const C& c) noexcept -> decltype(MakeSpan(c)) { - return MakeSpan(c); -} - -template <int&... ExplicitArgumentBarrier, typename T, size_t N> -constexpr Span<const T> MakeConstSpan(const T (&array)[N]) noexcept { - return Span<const T>(array, N); -} -} // namespace phmap - -// --------------------------------------------------------------------------- -// layout.h -// --------------------------------------------------------------------------- -#if defined(__GXX_RTTI) - #define PHMAP_INTERNAL_HAS_CXA_DEMANGLE -#endif - -#ifdef PHMAP_INTERNAL_HAS_CXA_DEMANGLE - #include <cxxabi.h> -#endif - -namespace phmap { -namespace priv { - -// A type wrapper that instructs `Layout` to use the specific alignment for the -// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API -// and behavior as `Layout<..., T, ...>` except that the first element of the -// array of `T` is aligned to `N` (the rest of the elements follow without -// padding). -// -// Requires: `N >= alignof(T)` and `N` is a power of 2. -template <class T, size_t N> -struct Aligned; - -namespace internal_layout { - -template <class T> -struct NotAligned {}; - -template <class T, size_t N> -struct NotAligned<const Aligned<T, N>> { - static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified"); -}; - -template <size_t> -using IntToSize = size_t; - -template <class> -using TypeToSize = size_t; - -template <class T> -struct Type : NotAligned<T> { - using type = T; -}; - -template <class T, size_t N> -struct Type<Aligned<T, N>> { - using type = T; -}; - -template <class T> -struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {}; - -template <class T, size_t N> -struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {}; - -// Note: workaround for https://gcc.gnu.org/PR88115 -template <class T> -struct AlignOf : NotAligned<T> { - static constexpr size_t value = alignof(T); -}; - -template <class T, size_t N> -struct AlignOf<Aligned<T, N>> { - static_assert(N % alignof(T) == 0, - "Custom alignment can't be lower than the type's alignment"); - static constexpr size_t value = N; -}; - -// Does `Ts...` contain `T`? -template <class T, class... Ts> -using Contains = phmap::disjunction<std::is_same<T, Ts>...>; - -template <class From, class To> -using CopyConst = - typename std::conditional<std::is_const<From>::value, const To, To>::type; - -// Note: We're not qualifying this with phmap:: because it doesn't compile under -// MSVC. -template <class T> -using SliceType = Span<T>; - -// This namespace contains no types. It prevents functions defined in it from -// being found by ADL. -namespace adl_barrier { - -template <class Needle, class... Ts> -constexpr size_t Find(Needle, Needle, Ts...) { - static_assert(!Contains<Needle, Ts...>(), "Duplicate element type"); - return 0; -} - -template <class Needle, class T, class... Ts> -constexpr size_t Find(Needle, T, Ts...) { - return adl_barrier::Find(Needle(), Ts()...) + 1; -} - -constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); } - -// Returns `q * m` for the smallest `q` such that `q * m >= n`. -// Requires: `m` is a power of two. It's enforced by IsLegalElementType below. -constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } - -constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; } - -constexpr size_t Max(size_t a) { return a; } - -template <class... Ts> -constexpr size_t Max(size_t a, size_t b, Ts... rest) { - return adl_barrier::Max(b < a ? a : b, rest...); -} - -} // namespace adl_barrier - -template <bool C> -using EnableIf = typename std::enable_if<C, int>::type; - -// Can `T` be a template argument of `Layout`? -// --------------------------------------------------------------------------- -template <class T> -using IsLegalElementType = std::integral_constant< - bool, !std::is_reference<T>::value && !std::is_volatile<T>::value && - !std::is_reference<typename Type<T>::type>::value && - !std::is_volatile<typename Type<T>::type>::value && - adl_barrier::IsPow2(AlignOf<T>::value)>; - -template <class Elements, class SizeSeq, class OffsetSeq> -class LayoutImpl; - -// --------------------------------------------------------------------------- -// Public base class of `Layout` and the result type of `Layout::Partial()`. -// -// `Elements...` contains all template arguments of `Layout` that created this -// instance. -// -// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments -// passed to `Layout::Partial()` or `Layout::Layout()`. -// -// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is -// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we -// can compute offsets). -// --------------------------------------------------------------------------- -template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> -class LayoutImpl<std::tuple<Elements...>, phmap::index_sequence<SizeSeq...>, - phmap::index_sequence<OffsetSeq...>> -{ -private: - static_assert(sizeof...(Elements) > 0, "At least one field is required"); - static_assert(phmap::conjunction<IsLegalElementType<Elements>...>::value, - "Invalid element type (see IsLegalElementType)"); - - enum { - NumTypes = sizeof...(Elements), - NumSizes = sizeof...(SizeSeq), - NumOffsets = sizeof...(OffsetSeq), - }; - - // These are guaranteed by `Layout`. - static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), - "Internal error"); - static_assert(NumTypes > 0, "Internal error"); - - // Returns the index of `T` in `Elements...`. Results in a compilation error - // if `Elements...` doesn't contain exactly one instance of `T`. - template <class T> - static constexpr size_t ElementIndex() { - static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(), - "Type not found"); - return adl_barrier::Find(Type<T>(), - Type<typename Type<Elements>::type>()...); - } - - template <size_t N> - using ElementAlignment = - AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>; - -public: - // Element types of all arrays packed in a tuple. - using ElementTypes = std::tuple<typename Type<Elements>::type...>; - - // Element type of the Nth array. - template <size_t N> - using ElementType = typename std::tuple_element<N, ElementTypes>::type; - - constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) - : size_{sizes...} {} - - // Alignment of the layout, equal to the strictest alignment of all elements. - // All pointers passed to the methods of layout must be aligned to this value. - static constexpr size_t Alignment() { - return adl_barrier::Max(AlignOf<Elements>::value...); - } - - // Offset in bytes of the Nth array. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<0>() == 0); // The ints starts from 0. - // assert(x.Offset<1>() == 16); // The doubles starts from 16. - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - template <size_t N, EnableIf<N == 0> = 0> - constexpr size_t Offset() const { - return 0; - } - - template <size_t N, EnableIf<N != 0> = 0> - constexpr size_t Offset() const { - static_assert(N < NumOffsets, "Index out of bounds"); - return adl_barrier::Align( - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1], - ElementAlignment<N>::value); - } - - // Offset in bytes of the array with the specified element type. There must - // be exactly one such array and its zero-based index must be at most - // `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<int>() == 0); // The ints starts from 0. - // assert(x.Offset<double>() == 16); // The doubles starts from 16. - template <class T> - constexpr size_t Offset() const { - return Offset<ElementIndex<T>()>(); - } - - // Offsets in bytes of all arrays for which the offsets are known. - constexpr std::array<size_t, NumOffsets> Offsets() const { - return {{Offset<OffsetSeq>()...}}; - } - - // The number of elements in the Nth array. This is the Nth argument of - // `Layout::Partial()` or `Layout::Layout()` (zero-based). - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<0>() == 3); - // assert(x.Size<1>() == 4); - // - // Requires: `N < NumSizes`. - template <size_t N> - constexpr size_t Size() const { - static_assert(N < NumSizes, "Index out of bounds"); - return size_[N]; - } - - // The number of elements in the array with the specified element type. - // There must be exactly one such array and its zero-based index must be - // at most `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<int>() == 3); - // assert(x.Size<double>() == 4); - template <class T> - constexpr size_t Size() const { - return Size<ElementIndex<T>()>(); - } - - // The number of elements of all arrays for which they are known. - constexpr std::array<size_t, NumSizes> Sizes() const { - return {{Size<SizeSeq>()...}}; - } - - // Pointer to the beginning of the Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<0>(p); - // double* doubles = x.Pointer<1>(p); - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - CopyConst<Char, ElementType<N>>* Pointer(Char* p) const { - using C = typename std::remove_const<Char>::type; - static_assert( - std::is_same<C, char>() || std::is_same<C, unsigned char>() || - std::is_same<C, signed char>(), - "The argument must be a pointer to [const] [signed|unsigned] char"); - constexpr size_t alignment = Alignment(); - (void)alignment; - assert(reinterpret_cast<uintptr_t>(p) % alignment == 0); - return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>()); - } - - // Pointer to the beginning of the array with the specified element type. - // There must be exactly one such array and its zero-based index must be at - // most `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<int>(p); - // double* doubles = x.Pointer<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - CopyConst<Char, T>* Pointer(Char* p) const { - return Pointer<ElementIndex<T>()>(p); - } - - // Pointers to all arrays for which pointers are known. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // int* ints; - // double* doubles; - // std::tie(ints, doubles) = x.Pointers(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<CopyConst< - Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> - Pointers(Char* p) const { - return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( - Pointer<OffsetSeq>(p)...); - } - - // The Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<0>(p); - // Span<double> doubles = x.Slice<1>(p); - // - // Requires: `N < NumSizes`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const { - return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>()); - } - - // The array with the specified element type. There must be exactly one - // such array and its zero-based index must be less than `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<int>(p); - // Span<double> doubles = x.Slice<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - SliceType<CopyConst<Char, T>> Slice(Char* p) const { - return Slice<ElementIndex<T>()>(p); - } - - // All arrays with known sizes. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // Span<int> ints; - // Span<double> doubles; - // std::tie(ints, doubles) = x.Slices(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<SliceType<CopyConst< - Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> - Slices(Char* p) const { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed - // in 6.1). - (void)p; - return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( - Slice<SizeSeq>(p)...); - } - - // The size of the allocation that fits all arrays. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes - // - // Requires: `NumSizes == sizeof...(Ts)`. - constexpr size_t AllocSize() const { - static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); - return Offset<NumTypes - 1>() + - SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1]; - } - - // If built with --config=asan, poisons padding bytes (if any) in the - // allocation. The pointer must point to a memory block at least - // `AllocSize()` bytes in length. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // Requires: `p` is aligned to `Alignment()`. - template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0> - void PoisonPadding(const Char* p) const { - Pointer<0>(p); // verify the requirements on `Char` and `p` - } - - template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0> - void PoisonPadding(const Char* p) const { - static_assert(N < NumOffsets, "Index out of bounds"); - (void)p; -#ifdef ADDRESS_SANITIZER - PoisonPadding<Char, N - 1>(p); - // The `if` is an optimization. It doesn't affect the observable behaviour. - if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { - size_t start = - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1]; - ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); - } -#endif - } - -private: - // Arguments of `Layout::Partial()` or `Layout::Layout()`. - size_t size_[NumSizes > 0 ? NumSizes : 1]; -}; - -template <size_t NumSizes, class... Ts> -using LayoutType = LayoutImpl< - std::tuple<Ts...>, phmap::make_index_sequence<NumSizes>, - phmap::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; - -} // namespace internal_layout - -// --------------------------------------------------------------------------- -// Descriptor of arrays of various types and sizes laid out in memory one after -// another. See the top of the file for documentation. -// -// Check out the public API of internal_layout::LayoutImpl above. The type is -// internal to the library but its methods are public, and they are inherited -// by `Layout`. -// --------------------------------------------------------------------------- -template <class... Ts> -class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> -{ -public: - static_assert(sizeof...(Ts) > 0, "At least one field is required"); - static_assert( - phmap::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, - "Invalid element type (see IsLegalElementType)"); - - template <size_t NumSizes> - using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; - - template <class... Sizes> - static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { - static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType<sizeof...(Sizes)>(phmap::forward<Sizes>(sizes)...); - } - - // Creates a layout with the sizes of all arrays specified. If you know - // only the sizes of the first N arrays (where N can be zero), you can use - // `Partial()` defined above. The constructor is essentially equivalent to - // calling `Partial()` and passing in all array sizes; the constructor is - // provided as a convenient abbreviation. - // - // Note: The sizes of the arrays must be specified in number of elements, - // not in bytes. - constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) - : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} -}; - -} // namespace priv -} // namespace phmap - -// --------------------------------------------------------------------------- -// compressed_tuple.h -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - // We need to mark these classes with this declspec to ensure that - // CompressedTuple happens. - #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases) -#else // _MSC_VER - #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC -#endif // _MSC_VER - -namespace phmap { -namespace priv { - -template <typename... Ts> -class CompressedTuple; - -namespace internal_compressed_tuple { - -template <typename D, size_t I> -struct Elem; -template <typename... B, size_t I> -struct Elem<CompressedTuple<B...>, I> - : std::tuple_element<I, std::tuple<B...>> {}; -template <typename D, size_t I> -using ElemT = typename Elem<D, I>::type; - -// --------------------------------------------------------------------------- -// Use the __is_final intrinsic if available. Where it's not available, classes -// declared with the 'final' specifier cannot be used as CompressedTuple -// elements. -// TODO(sbenza): Replace this with std::is_final in C++14. -// --------------------------------------------------------------------------- -template <typename T> -constexpr bool IsFinal() { -#if defined(__clang__) || defined(__GNUC__) - return __is_final(T); -#else - return false; -#endif -} - -template <typename T> -constexpr bool ShouldUseBase() { -#ifdef __INTEL_COMPILER - // avoid crash in Intel compiler - // assertion failed at: "shared/cfe/edgcpfe/lower_init.c", line 7013 - return false; -#else - return std::is_class<T>::value && std::is_empty<T>::value && !IsFinal<T>(); -#endif -} - -// The storage class provides two specializations: -// - For empty classes, it stores T as a base class. -// - For everything else, it stores T as a member. -// ------------------------------------------------ -template <typename D, size_t I, bool = ShouldUseBase<ElemT<D, I>>()> -struct Storage -{ - using T = ElemT<D, I>; - T value; - constexpr Storage() = default; - explicit constexpr Storage(T&& v) : value(phmap::forward<T>(v)) {} - constexpr const T& get() const& { return value; } - T& get() & { return value; } - constexpr const T&& get() const&& { return phmap::move(*this).value; } - T&& get() && { return std::move(*this).value; } -}; - -template <typename D, size_t I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<D, I, true> - : ElemT<D, I> -{ - using T = internal_compressed_tuple::ElemT<D, I>; - constexpr Storage() = default; - explicit constexpr Storage(T&& v) : T(phmap::forward<T>(v)) {} - constexpr const T& get() const& { return *this; } - T& get() & { return *this; } - constexpr const T&& get() const&& { return phmap::move(*this); } - T&& get() && { return std::move(*this); } -}; - -template <typename D, typename I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl; - -template <typename... Ts, size_t... I> -struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC - CompressedTupleImpl<CompressedTuple<Ts...>, phmap::index_sequence<I...>> - // We use the dummy identity function through std::integral_constant to - // convince MSVC of accepting and expanding I in that context. Without it - // you would get: - // error C3548: 'I': parameter pack cannot be used in this context - : Storage<CompressedTuple<Ts...>, - std::integral_constant<size_t, I>::value>... -{ - constexpr CompressedTupleImpl() = default; - explicit constexpr CompressedTupleImpl(Ts&&... args) - : Storage<CompressedTuple<Ts...>, I>(phmap::forward<Ts>(args))... {} -}; - -} // namespace internal_compressed_tuple - -// --------------------------------------------------------------------------- -// Helper class to perform the Empty Base Class Optimization. -// Ts can contain classes and non-classes, empty or not. For the ones that -// are empty classes, we perform the CompressedTuple. If all types in Ts are -// empty classes, then CompressedTuple<Ts...> is itself an empty class. -// -// To access the members, use member .get<N>() function. -// -// Eg: -// phmap::priv::CompressedTuple<int, T1, T2, T3> value(7, t1, t2, -// t3); -// assert(value.get<0>() == 7); -// T1& t1 = value.get<1>(); -// const T2& t2 = value.get<2>(); -// ... -// -// https://en.cppreference.com/w/cpp/language/ebo -// --------------------------------------------------------------------------- -template <typename... Ts> -class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple - : private internal_compressed_tuple::CompressedTupleImpl< - CompressedTuple<Ts...>, phmap::index_sequence_for<Ts...>> -{ -private: - template <int I> - using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>; - -public: - constexpr CompressedTuple() = default; - explicit constexpr CompressedTuple(Ts... base) - : CompressedTuple::CompressedTupleImpl(phmap::forward<Ts>(base)...) {} - - template <int I> - ElemT<I>& get() & { - return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); - } - - template <int I> - constexpr const ElemT<I>& get() const& { - return internal_compressed_tuple::Storage<CompressedTuple, I>::get(); - } - - template <int I> - ElemT<I>&& get() && { - return std::move(*this) - .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); - } - - template <int I> - constexpr const ElemT<I>&& get() const&& { - return phmap::move(*this) - .internal_compressed_tuple::template Storage<CompressedTuple, I>::get(); - } -}; - -// Explicit specialization for a zero-element tuple -// (needed to avoid ambiguous overloads for the default constructor). -// --------------------------------------------------------------------------- -template <> -class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {}; - -} // namespace priv -} // namespace phmap - - -namespace phmap { -namespace priv { - -#ifdef _MSC_VER - #pragma warning(push) - // warning warning C4324: structure was padded due to alignment specifier - #pragma warning(disable : 4324) -#endif - - -// ---------------------------------------------------------------------------- -// Allocates at least n bytes aligned to the specified alignment. -// Alignment must be a power of 2. It must be positive. -// -// Note that many allocators don't honor alignment requirements above certain -// threshold (usually either alignof(std::max_align_t) or alignof(void*)). -// Allocate() doesn't apply alignment corrections. If the underlying allocator -// returns insufficiently alignment pointer, that's what you are going to get. -// ---------------------------------------------------------------------------- -template <size_t Alignment, class Alloc> -void* Allocate(Alloc* alloc, size_t n) { - static_assert(Alignment > 0, ""); - assert(n && "n must be positive"); - struct alignas(Alignment) M {}; - using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; - using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; - A mem_alloc(*alloc); - void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); - assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 && - "allocator does not respect alignment"); - return p; -} - -// ---------------------------------------------------------------------------- -// The pointer must have been previously obtained by calling -// Allocate<Alignment>(alloc, n). -// ---------------------------------------------------------------------------- -template <size_t Alignment, class Alloc> -void Deallocate(Alloc* alloc, void* p, size_t n) { - static_assert(Alignment > 0, ""); - assert(n && "n must be positive"); - struct alignas(Alignment) M {}; - using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>; - using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>; - A mem_alloc(*alloc); - AT::deallocate(mem_alloc, static_cast<M*>(p), - (n + sizeof(M) - 1) / sizeof(M)); -} - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -// Helper functions for asan and msan. -// ---------------------------------------------------------------------------- -inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) { -#ifdef ADDRESS_SANITIZER - ASAN_POISON_MEMORY_REGION(m, s); -#endif -#ifdef MEMORY_SANITIZER - __msan_poison(m, s); -#endif - (void)m; - (void)s; -} - -inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) { -#ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(m, s); -#endif -#ifdef MEMORY_SANITIZER - __msan_unpoison(m, s); -#endif - (void)m; - (void)s; -} - -template <typename T> -inline void SanitizerPoisonObject(const T* object) { - SanitizerPoisonMemoryRegion(object, sizeof(T)); -} - -template <typename T> -inline void SanitizerUnpoisonObject(const T* object) { - SanitizerUnpoisonMemoryRegion(object, sizeof(T)); -} - -} // namespace priv -} // namespace phmap - - -// --------------------------------------------------------------------------- -// thread_annotations.h -// --------------------------------------------------------------------------- - -#if defined(__clang__) - #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#else - #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op -#endif - -#define PHMAP_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) -#define PHMAP_PT_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) - -#define PHMAP_ACQUIRED_AFTER(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) - -#define PHMAP_ACQUIRED_BEFORE(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) - -#define PHMAP_EXCLUSIVE_LOCKS_REQUIRED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) - -#define PHMAP_SHARED_LOCKS_REQUIRED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) - -#define PHMAP_LOCKS_EXCLUDED(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) - -#define PHMAP_LOCK_RETURNED(x) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -#define PHMAP_LOCKABLE \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lockable) - -#define PHMAP_SCOPED_LOCKABLE \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -#define PHMAP_EXCLUSIVE_LOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) - -#define PHMAP_SHARED_LOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) - -#define PHMAP_UNLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) - -#define PHMAP_EXCLUSIVE_TRYLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) - -#define PHMAP_SHARED_TRYLOCK_FUNCTION(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) - -#define PHMAP_ASSERT_EXCLUSIVE_LOCK(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) - -#define PHMAP_ASSERT_SHARED_LOCK(...) \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) - -#define PHMAP_NO_THREAD_SAFETY_ANALYSIS \ - PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -//------------------------------------------------------------------------------ -// Tool-Supplied Annotations -//------------------------------------------------------------------------------ - -// TS_UNCHECKED should be placed around lock expressions that are not valid -// C++ syntax, but which are present for documentation purposes. These -// annotations will be ignored by the analysis. -#define PHMAP_TS_UNCHECKED(x) "" - -// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. -// It is used by automated tools to mark and disable invalid expressions. -// The annotation should either be fixed, or changed to TS_UNCHECKED. -#define PHMAP_TS_FIXME(x) "" - -// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of -// a particular function. However, this attribute is used to mark functions -// that are incorrect and need to be fixed. It is used by automated tools to -// avoid breaking the build when the analysis is updated. -// Code owners are expected to eventually fix the routine. -#define PHMAP_NO_THREAD_SAFETY_ANALYSIS_FIXME PHMAP_NO_THREAD_SAFETY_ANALYSIS - -// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY -// annotation that needs to be fixed, because it is producing thread safety -// warning. It disables the GUARDED_BY. -#define PHMAP_GUARDED_BY_FIXME(x) - -// Disables warnings for a single read operation. This can be used to avoid -// warnings when it is known that the read is not actually involved in a race, -// but the compiler cannot confirm that. -#define PHMAP_TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) - - -namespace phmap { -namespace thread_safety_analysis { - -// Takes a reference to a guarded data member, and returns an unguarded -// reference. -template <typename T> -inline const T& ts_unchecked_read(const T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -template <typename T> -inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -} // namespace thread_safety_analysis - -namespace priv { - -namespace memory_internal { - -// ---------------------------------------------------------------------------- -// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and -// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and -// offsetof(Pair, second) respectively. Otherwise they are -1. -// -// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout -// type, which is non-portable. -// ---------------------------------------------------------------------------- -template <class Pair, class = std::true_type> -struct OffsetOf { - static constexpr size_t kFirst = (size_t)-1; - static constexpr size_t kSecond = (size_t)-1; -}; - -template <class Pair> -struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type> -{ - static constexpr size_t kFirst = offsetof(Pair, first); - static constexpr size_t kSecond = offsetof(Pair, second); -}; - -// ---------------------------------------------------------------------------- -template <class K, class V> -struct IsLayoutCompatible -{ -private: - struct Pair { - K first; - V second; - }; - - // Is P layout-compatible with Pair? - template <class P> - static constexpr bool LayoutCompatible() { - return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) && - alignof(P) == alignof(Pair) && - memory_internal::OffsetOf<P>::kFirst == - memory_internal::OffsetOf<Pair>::kFirst && - memory_internal::OffsetOf<P>::kSecond == - memory_internal::OffsetOf<Pair>::kSecond; - } - -public: - // Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are, - // then it is safe to store them in a union and read from either. - static constexpr bool value = std::is_standard_layout<K>() && - std::is_standard_layout<Pair>() && - memory_internal::OffsetOf<Pair>::kFirst == 0 && - LayoutCompatible<std::pair<K, V>>() && - LayoutCompatible<std::pair<const K, V>>(); -}; - -} // namespace memory_internal - -// ---------------------------------------------------------------------------- -// The internal storage type for key-value containers like flat_hash_map. -// -// It is convenient for the value_type of a flat_hash_map<K, V> to be -// pair<const K, V>; the "const K" prevents accidental modification of the key -// when dealing with the reference returned from find() and similar methods. -// However, this creates other problems; we want to be able to emplace(K, V) -// efficiently with move operations, and similarly be able to move a -// pair<K, V> in insert(). -// -// The solution is this union, which aliases the const and non-const versions -// of the pair. This also allows flat_hash_map<const K, V> to work, even though -// that has the same efficiency issues with move in emplace() and insert() - -// but people do it anyway. -// -// If kMutableKeys is false, only the value member can be accessed. -// -// If kMutableKeys is true, key can be accessed through all slots while value -// and mutable_value must be accessed only via INITIALIZED slots. Slots are -// created and destroyed via mutable_value so that the key can be moved later. -// -// Accessing one of the union fields while the other is active is safe as -// long as they are layout-compatible, which is guaranteed by the definition of -// kMutableKeys. For C++11, the relevant section of the standard is -// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19) -// ---------------------------------------------------------------------------- -template <class K, class V> -union map_slot_type -{ - map_slot_type() {} - ~map_slot_type() = delete; - map_slot_type(const map_slot_type&) = delete; - map_slot_type& operator=(const map_slot_type&) = delete; - - using value_type = std::pair<const K, V>; - using mutable_value_type = std::pair<K, V>; - - value_type value; - mutable_value_type mutable_value; - K key; -}; - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -template <class K, class V> -struct map_slot_policy -{ - using slot_type = map_slot_type<K, V>; - using value_type = std::pair<const K, V>; - using mutable_value_type = std::pair<K, V>; - -private: - static void emplace(slot_type* slot) { - // The construction of union doesn't do anything at runtime but it allows us - // to access its members without violating aliasing rules. - new (slot) slot_type; - } - // If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one - // or the other via slot_type. We are also free to access the key via - // slot_type::key in this case. - using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>; - -public: - static value_type& element(slot_type* slot) { return slot->value; } - static const value_type& element(const slot_type* slot) { - return slot->value; - } - - static const K& key(const slot_type* slot) { - return kMutableKeys::value ? slot->key : slot->value.first; - } - - template <class Allocator, class... Args> - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - emplace(slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value, - std::forward<Args>(args)...); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, - std::forward<Args>(args)...); - } - } - - // Construct this slot by moving from another slot. - template <class Allocator> - static void construct(Allocator* alloc, slot_type* slot, slot_type* other) { - emplace(slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct( - *alloc, &slot->mutable_value, std::move(other->mutable_value)); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &slot->value, - std::move(other->value)); - } - } - - template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value); - } else { - phmap::allocator_traits<Allocator>::destroy(*alloc, &slot->value); - } - } - - template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - emplace(new_slot); - if (kMutableKeys::value) { - phmap::allocator_traits<Allocator>::construct( - *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); - } else { - phmap::allocator_traits<Allocator>::construct(*alloc, &new_slot->value, - std::move(old_slot->value)); - } - destroy(alloc, old_slot); - } - - template <class Allocator> - static void swap(Allocator* alloc, slot_type* a, slot_type* b) { - if (kMutableKeys::value) { - using std::swap; - swap(a->mutable_value, b->mutable_value); - } else { - value_type tmp = std::move(a->value); - phmap::allocator_traits<Allocator>::destroy(*alloc, &a->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &a->value, - std::move(b->value)); - phmap::allocator_traits<Allocator>::destroy(*alloc, &b->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &b->value, - std::move(tmp)); - } - } - - template <class Allocator> - static void move(Allocator* alloc, slot_type* src, slot_type* dest) { - if (kMutableKeys::value) { - dest->mutable_value = std::move(src->mutable_value); - } else { - phmap::allocator_traits<Allocator>::destroy(*alloc, &dest->value); - phmap::allocator_traits<Allocator>::construct(*alloc, &dest->value, - std::move(src->value)); - } - } - - template <class Allocator> - static void move(Allocator* alloc, slot_type* first, slot_type* last, - slot_type* result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } -}; - -} // namespace priv -} // phmap - - -namespace phmap { - -#ifdef BOOST_THREAD_LOCK_OPTIONS_HPP - using defer_lock_t = boost::defer_lock_t; - using try_to_lock_t = boost::try_to_lock_t; - using adopt_lock_t = boost::adopt_lock_t; -#else - struct adopt_lock_t { explicit adopt_lock_t() = default; }; - struct defer_lock_t { explicit defer_lock_t() = default; }; - struct try_to_lock_t { explicit try_to_lock_t() = default; }; -#endif - -// ----------------------------------------------------------------------------- -// NullMutex -// ----------------------------------------------------------------------------- -// A class that implements the Mutex interface, but does nothing. This is to be -// used as a default template parameters for classes who provide optional -// internal locking (like phmap::parallel_flat_hash_map). -// ----------------------------------------------------------------------------- -class NullMutex { -public: - NullMutex() {} - ~NullMutex() {} - void lock() {} - void unlock() {} - bool try_lock() { return true; } - void lock_shared() {} - void unlock_shared() {} - bool try_lock_shared() { return true; } -}; - -// ------------------------ lockable object used internally ------------------------- -template <class MutexType> -class LockableBaseImpl -{ -public: - // ---------------------------------------------------- - struct DoNothing - { - using mutex_type = MutexType; - DoNothing() noexcept {} - explicit DoNothing(mutex_type& ) noexcept {} - explicit DoNothing(mutex_type& , mutex_type&) noexcept {} - DoNothing(mutex_type&, phmap::adopt_lock_t) noexcept {} - DoNothing(mutex_type&, phmap::defer_lock_t) noexcept {} - DoNothing(mutex_type&, phmap::try_to_lock_t) {} - template<class T> explicit DoNothing(T&&) {} - DoNothing& operator=(const DoNothing&) { return *this; } - DoNothing& operator=(DoNothing&&) { return *this; } - void swap(DoNothing &) {} - bool owns_lock() const noexcept { return true; } - }; - - // ---------------------------------------------------- - class WriteLock - { - public: - using mutex_type = MutexType; - - WriteLock() : m_(nullptr), locked_(false) {} - - explicit WriteLock(mutex_type &m) : m_(&m) { - m_->lock(); - locked_ = true; - } - - WriteLock(mutex_type& m, adopt_lock_t) noexcept : - m_(&m), locked_(true) - {} - - WriteLock(mutex_type& m, defer_lock_t) noexcept : - m_(&m), locked_(false) - {} - - WriteLock(mutex_type& m, try_to_lock_t) : - m_(&m), locked_(false) { - m_->try_lock(); - } - - WriteLock(WriteLock &&o) : - m_(std::move(o.m_)), locked_(std::move(o.locked_)) { - o.locked_ = false; - o.m_ = nullptr; - } - - WriteLock& operator=(WriteLock&& other) { - WriteLock temp(std::move(other)); - swap(temp); - return *this; - } - - ~WriteLock() { - if (locked_) - m_->unlock(); - } - - void lock() { - if (!locked_) { - m_->lock(); - locked_ = true; - } - } - - void unlock() { - if (locked_) { - m_->unlock(); - locked_ = false; - } - } - - bool try_lock() { - if (locked_) - return true; - locked_ = m_->try_lock(); - return locked_; - } - - bool owns_lock() const noexcept { return locked_; } - - void swap(WriteLock &o) noexcept { - std::swap(m_, o.m_); - std::swap(locked_, o.locked_); - } - - mutex_type *mutex() const noexcept { return m_; } - - private: - mutex_type *m_; - bool locked_; - }; - - // ---------------------------------------------------- - class ReadLock - { - public: - using mutex_type = MutexType; - - ReadLock() : m_(nullptr), locked_(false) {} - - explicit ReadLock(mutex_type &m) : m_(&m) { - m_->lock_shared(); - locked_ = true; - } - - ReadLock(mutex_type& m, adopt_lock_t) noexcept : - m_(&m), locked_(true) - {} - - ReadLock(mutex_type& m, defer_lock_t) noexcept : - m_(&m), locked_(false) - {} - - ReadLock(mutex_type& m, try_to_lock_t) : - m_(&m), locked_(false) { - m_->try_lock_shared(); - } - - ReadLock(ReadLock &&o) : - m_(std::move(o.m_)), locked_(std::move(o.locked_)) { - o.locked_ = false; - o.m_ = nullptr; - } - - ReadLock& operator=(ReadLock&& other) { - ReadLock temp(std::move(other)); - swap(temp); - return *this; - } - - ~ReadLock() { - if (locked_) - m_->unlock_shared(); - } - - void lock() { - if (!locked_) { - m_->lock_shared(); - locked_ = true; - } - } - - void unlock() { - if (locked_) { - m_->unlock_shared(); - locked_ = false; - } - } - - bool try_lock() { - if (locked_) - return true; - locked_ = m_->try_lock_shared(); - return locked_; - } - - bool owns_lock() const noexcept { return locked_; } - - void swap(ReadLock &o) noexcept { - std::swap(m_, o.m_); - std::swap(locked_, o.locked_); - } - - mutex_type *mutex() const noexcept { return m_; } - - private: - mutex_type *m_; - bool locked_; - }; - - // ---------------------------------------------------- - class WriteLocks - { - public: - using mutex_type = MutexType; - - explicit WriteLocks(mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { - std::lock(m1, m2); - } - - WriteLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { // adopt means we already own the mutexes - } - - ~WriteLocks() - { - _m1.unlock(); - _m2.unlock(); - } - - WriteLocks(WriteLocks const&) = delete; - WriteLocks& operator=(WriteLocks const&) = delete; - private: - mutex_type& _m1; - mutex_type& _m2; - }; - - // ---------------------------------------------------- - class ReadLocks - { - public: - using mutex_type = MutexType; - - explicit ReadLocks(mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { - _m1.lock_shared(); - _m2.lock_shared(); - } - - ReadLocks(adopt_lock_t, mutex_type& m1, mutex_type& m2) : - _m1(m1), _m2(m2) - { // adopt means we already own the mutexes - } - - ~ReadLocks() - { - _m1.unlock_shared(); - _m2.unlock_shared(); - } - - ReadLocks(ReadLocks const&) = delete; - ReadLocks& operator=(ReadLocks const&) = delete; - private: - mutex_type& _m1; - mutex_type& _m2; - }; -}; - -// ------------------------ holds a mutex ------------------------------------ -// Default implementation for Lockable, should work fine for std::mutex -// ----------------------------------- -// use as: -// using Lockable = phmap::LockableImpl<mutex_type>; -// Lockable m; -// -// Lockable::UpgradeLock read_lock(m); // take a upgradable lock -// -// { -// Lockable::UpgradeToUnique unique_lock(read_lock); -// // now locked for write -// } -// -// --------------------------------------------------------------------------- -// Generic mutex support (always write locks) -// -------------------------------------------------------------------------- -template <class Mtx_> -class LockableImpl : public Mtx_ -{ -public: - using mutex_type = Mtx_; - using Base = LockableBaseImpl<Mtx_>; - using SharedLock = typename Base::WriteLock; - using UpgradeLock = typename Base::WriteLock; - using UniqueLock = typename Base::WriteLock; - using SharedLocks = typename Base::WriteLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership -}; - -// --------------------------------------------------------------------------- -// Null mutex (no-op) - when we don't want internal synchronization -// --------------------------------------------------------------------------- -template <> -class LockableImpl<phmap::NullMutex>: public phmap::NullMutex -{ -public: - using mutex_type = phmap::NullMutex; - using Base = LockableBaseImpl<phmap::NullMutex>; - using SharedLock = typename Base::DoNothing; - using UpgradeLock = typename Base::DoNothing; - using UniqueLock = typename Base::DoNothing; - using UpgradeToUnique = typename Base::DoNothing; - using SharedLocks = typename Base::DoNothing; - using UniqueLocks = typename Base::DoNothing; -}; - -// -------------------------------------------------------------------------- -// Abseil Mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef ABSL_SYNCHRONIZATION_MUTEX_H_ - - struct AbslMutex : protected absl::Mutex - { - void lock() { this->Lock(); } - void unlock() { this->Unlock(); } - void try_lock() { this->TryLock(); } - void lock_shared() { this->ReaderLock(); } - void unlock_shared() { this->ReaderUnlock(); } - void try_lock_shared() { this->ReaderTryLock(); } - }; - - template <> - class LockableImpl<absl::Mutex> : public AbslMutex - { - public: - using mutex_type = phmap::AbslMutex; - using Base = LockableBaseImpl<phmap::AbslMutex>; - using SharedLock = typename Base::ReadLock; - using UpgradeLock = typename Base::WriteLock; - using UniqueLock = typename Base::WriteLock; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; - -#endif - -// -------------------------------------------------------------------------- -// Boost shared_mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef BOOST_THREAD_SHARED_MUTEX_HPP - -#if 1 - // --------------------------------------------------------------------------- - template <> - class LockableImpl<boost::shared_mutex> : public boost::shared_mutex - { - public: - using mutex_type = boost::shared_mutex; - using Base = LockableBaseImpl<boost::shared_mutex>; - using SharedLock = boost::shared_lock<mutex_type>; - using UpgradeLock = boost::unique_lock<mutex_type>; // assume can't upgrade - using UniqueLock = boost::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; -#else - // --------------------------------------------------------------------------- - template <> - class LockableImpl<boost::upgrade_mutex> : public boost::upgrade_mutex - { - public: - using mutex_type = boost::upgrade_mutex; - using SharedLock = boost::shared_lock<mutex_type>; - using UpgradeLock = boost::upgrade_lock<mutex_type>; - using UniqueLock = boost::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = boost::upgrade_to_unique_lock<mutex_type>; - }; -#endif - -#endif // BOOST_THREAD_SHARED_MUTEX_HPP - -// -------------------------------------------------------------------------- -// std::shared_mutex support (read and write lock support) -// -------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_SHARED_MUTEX - - // --------------------------------------------------------------------------- - template <> - class LockableImpl<std::shared_mutex> : public std::shared_mutex - { - public: - using mutex_type = std::shared_mutex; - using Base = LockableBaseImpl<std::shared_mutex>; - using SharedLock = std::shared_lock<mutex_type>; - using UpgradeLock = std::unique_lock<mutex_type>; // assume can't upgrade - using UniqueLock = std::unique_lock<mutex_type>; - using SharedLocks = typename Base::ReadLocks; - using UniqueLocks = typename Base::WriteLocks; - using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership - }; -#endif // PHMAP_HAVE_SHARED_MUTEX - - -} // phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - - -#endif // phmap_base_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_bits.h b/benchmarks/external/parallel_hashmap/phmap_bits.h deleted file mode 100644 index 6b765fff..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_bits.h +++ /dev/null @@ -1,663 +0,0 @@ -#if !defined(phmap_bits_h_guard_) -#define phmap_bits_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -// The following guarantees declaration of the byte swap functions -#ifdef _MSC_VER - #include <stdlib.h> // NOLINT(build/include) -#elif defined(__APPLE__) - // Mac OS X / Darwin features - #include <libkern/OSByteOrder.h> -#elif defined(__FreeBSD__) - #include <sys/endian.h> -#elif defined(__GLIBC__) - #include <byteswap.h> // IWYU pragma: export -#endif - -#include <string.h> -#include <cstdint> -#include "phmap_config.h" - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed -#endif - -// ----------------------------------------------------------------------------- -// unaligned APIs -// ----------------------------------------------------------------------------- -// Portable handling of unaligned loads, stores, and copies. -// On some platforms, like ARM, the copy functions can be more efficient -// then a load and a store. -// ----------------------------------------------------------------------------- - -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) -#include <stdint.h> - -extern "C" { - uint16_t __sanitizer_unaligned_load16(const void *p); - uint32_t __sanitizer_unaligned_load32(const void *p); - uint64_t __sanitizer_unaligned_load64(const void *p); - void __sanitizer_unaligned_store16(void *p, uint16_t v); - void __sanitizer_unaligned_store32(void *p, uint32_t v); - void __sanitizer_unaligned_store64(void *p, uint64_t v); -} // extern "C" - -namespace phmap { -namespace bits { - -inline uint16_t UnalignedLoad16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32_t UnalignedLoad32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64_t UnalignedLoad64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void UnalignedStore16(void *p, uint16_t v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void UnalignedStore32(void *p, uint32_t v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void UnalignedStore64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); -} - -} // namespace bits -} // namespace phmap - -#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) - -#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) - -#else - -namespace phmap { -namespace bits { - -inline uint16_t UnalignedLoad16(const void *p) { - uint16_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint32_t UnalignedLoad32(const void *p) { - uint32_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline uint64_t UnalignedLoad64(const void *p) { - uint64_t t; - memcpy(&t, p, sizeof t); - return t; -} - -inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); } - -inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } - -inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } - -} // namespace bits -} // namespace phmap - -#define PHMAP_INTERNAL_UNALIGNED_LOAD16(_p) (phmap::bits::UnalignedLoad16(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD32(_p) (phmap::bits::UnalignedLoad32(_p)) -#define PHMAP_INTERNAL_UNALIGNED_LOAD64(_p) (phmap::bits::UnalignedLoad64(_p)) - -#define PHMAP_INTERNAL_UNALIGNED_STORE16(_p, _val) (phmap::bits::UnalignedStore16(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE32(_p, _val) (phmap::bits::UnalignedStore32(_p, _val)) -#define PHMAP_INTERNAL_UNALIGNED_STORE64(_p, _val) (phmap::bits::UnalignedStore64(_p, _val)) - -#endif - -// ----------------------------------------------------------------------------- -// File: optimization.h -// ----------------------------------------------------------------------------- - -#if defined(__pnacl__) - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } -#elif defined(__clang__) - // Clang will not tail call given inline volatile assembly. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") -#elif defined(__GNUC__) - // GCC will not tail call given inline volatile assembly. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") -#elif defined(_MSC_VER) - #include <intrin.h> - // The __nop() intrinsic blocks the optimisation. - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() __nop() -#else - #define PHMAP_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } -#endif - -#if defined(__GNUC__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wpedantic" -#endif - -#ifdef PHMAP_HAVE_INTRINSIC_INT128 - __extension__ typedef unsigned __int128 phmap_uint128; - inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) - { - auto result = static_cast<phmap_uint128>(a) * static_cast<phmap_uint128>(b); - *high = static_cast<uint64_t>(result >> 64); - return static_cast<uint64_t>(result); - } - #define PHMAP_HAS_UMUL128 1 -#elif (defined(_MSC_VER)) - #if defined(_M_X64) - #pragma intrinsic(_umul128) - inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) - { - return _umul128(a, b, high); - } - #define PHMAP_HAS_UMUL128 1 - #endif -#endif - -#if defined(__GNUC__) - #pragma GCC diagnostic pop -#endif - -#if defined(__GNUC__) - // Cache line alignment - #if defined(__i386__) || defined(__x86_64__) - #define PHMAP_CACHELINE_SIZE 64 - #elif defined(__powerpc64__) - #define PHMAP_CACHELINE_SIZE 128 - #elif defined(__aarch64__) - // We would need to read special register ctr_el0 to find out L1 dcache size. - // This value is a good estimate based on a real aarch64 machine. - #define PHMAP_CACHELINE_SIZE 64 - #elif defined(__arm__) - // Cache line sizes for ARM: These values are not strictly correct since - // cache line sizes depend on implementations, not architectures. There - // are even implementations with cache line sizes configurable at boot - // time. - #if defined(__ARM_ARCH_5T__) - #define PHMAP_CACHELINE_SIZE 32 - #elif defined(__ARM_ARCH_7A__) - #define PHMAP_CACHELINE_SIZE 64 - #endif - #endif - - #ifndef PHMAP_CACHELINE_SIZE - // A reasonable default guess. Note that overestimates tend to waste more - // space, while underestimates tend to waste more time. - #define PHMAP_CACHELINE_SIZE 64 - #endif - - #define PHMAP_CACHELINE_ALIGNED __attribute__((aligned(PHMAP_CACHELINE_SIZE))) -#elif defined(_MSC_VER) - #define PHMAP_CACHELINE_SIZE 64 - #define PHMAP_CACHELINE_ALIGNED __declspec(align(PHMAP_CACHELINE_SIZE)) -#else - #define PHMAP_CACHELINE_SIZE 64 - #define PHMAP_CACHELINE_ALIGNED -#endif - - -#if PHMAP_HAVE_BUILTIN(__builtin_expect) || \ - (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_PREDICT_FALSE(x) (__builtin_expect(x, 0)) - #define PHMAP_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) -#else - #define PHMAP_PREDICT_FALSE(x) (x) - #define PHMAP_PREDICT_TRUE(x) (x) -#endif - -// ----------------------------------------------------------------------------- -// File: bits.h -// ----------------------------------------------------------------------------- - -#if defined(_MSC_VER) - // We can achieve something similar to attribute((always_inline)) with MSVC by - // using the __forceinline keyword, however this is not perfect. MSVC is - // much less aggressive about inlining, and even with the __forceinline keyword. - #define PHMAP_BASE_INTERNAL_FORCEINLINE __forceinline -#else - // Use default attribute inline. - #define PHMAP_BASE_INTERNAL_FORCEINLINE inline PHMAP_ATTRIBUTE_ALWAYS_INLINE -#endif - - -namespace phmap { -namespace base_internal { - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { - int zeroes = 60; - if (n >> 32) zeroes -= 32, n >>= 32; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { -#if defined(_MSC_VER) && defined(_M_X64) - // MSVC does not have __buitin_clzll. Use _BitScanReverse64. - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse64(&result, n)) { - return (int)(63 - result); - } - return 64; -#elif defined(_MSC_VER) && !defined(__clang__) - // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse - unsigned long result = 0; // NOLINT(runtime/int) - if ((n >> 32) && _BitScanReverse(&result, (unsigned long)(n >> 32))) { - return 31 - result; - } - if (_BitScanReverse(&result, (unsigned long)n)) { - return 63 - result; - } - return 64; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clzll, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_clzll does not take 64-bit arg"); - - // Handle 0 as a special case because __builtin_clzll(0) is undefined. - if (n == 0) { - return 64; - } - return __builtin_clzll(n); -#else - return CountLeadingZeros64Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { - int zeroes = 28; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse(&result, n)) { - return (int)(31 - result); - } - return 32; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clz, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(int) == sizeof(n), - "__builtin_clz does not take 32-bit arg"); - - // Handle 0 as a special case because __builtin_clz(0) is undefined. - if (n == 0) { - return 32; - } - return __builtin_clz(n); -#else - return CountLeadingZeros32Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) { - int c = 63; - n &= ~n + 1; - if (n & 0x00000000FFFFFFFF) c -= 32; - if (n & 0x0000FFFF0000FFFF) c -= 16; - if (n & 0x00FF00FF00FF00FF) c -= 8; - if (n & 0x0F0F0F0F0F0F0F0F) c -= 4; - if (n & 0x3333333333333333) c -= 2; - if (n & 0x5555555555555555) c -= 1; - return c; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) { -#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward64(&result, n); - return (int)result; -#elif defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (static_cast<uint32_t>(n) == 0) { - _BitScanForward(&result, (unsigned long)(n >> 32)); - return result + 32; - } - _BitScanForward(&result, (unsigned long)n); - return result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_ctzll does not take 64-bit arg"); - return __builtin_ctzll(n); -#else - return CountTrailingZerosNonZero64Slow(n); -#endif -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) { - int c = 31; - n &= ~n + 1; - if (n & 0x0000FFFF) c -= 16; - if (n & 0x00FF00FF) c -= 8; - if (n & 0x0F0F0F0F) c -= 4; - if (n & 0x33333333) c -= 2; - if (n & 0x55555555) c -= 1; - return c; -} - -PHMAP_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward(&result, n); - return (int)result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(int) == sizeof(n), - "__builtin_ctz does not take 32-bit arg"); - return __builtin_ctz(n); -#else - return CountTrailingZerosNonZero32Slow(n); -#endif -} - -#undef PHMAP_BASE_INTERNAL_FORCEINLINE - -} // namespace base_internal -} // namespace phmap - -// ----------------------------------------------------------------------------- -// File: endian.h -// ----------------------------------------------------------------------------- - -namespace phmap { - -// Use compiler byte-swapping intrinsics if they are available. 32-bit -// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. -// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. -// For simplicity, we enable them all only for GCC 4.8.0 or later. -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) - - inline uint64_t gbswap_64(uint64_t host_int) { - return __builtin_bswap64(host_int); - } - inline uint32_t gbswap_32(uint32_t host_int) { - return __builtin_bswap32(host_int); - } - inline uint16_t gbswap_16(uint16_t host_int) { - return __builtin_bswap16(host_int); - } - -#elif defined(_MSC_VER) - - inline uint64_t gbswap_64(uint64_t host_int) { - return _byteswap_uint64(host_int); - } - inline uint32_t gbswap_32(uint32_t host_int) { - return _byteswap_ulong(host_int); - } - inline uint16_t gbswap_16(uint16_t host_int) { - return _byteswap_ushort(host_int); - } - -#elif defined(__APPLE__) - - inline uint64_t gbswap_64(uint64_t host_int) { return OSSwapInt16(host_int); } - inline uint32_t gbswap_32(uint32_t host_int) { return OSSwapInt32(host_int); } - inline uint16_t gbswap_16(uint16_t host_int) { return OSSwapInt64(host_int); } - -#else - - inline uint64_t gbswap_64(uint64_t host_int) { -#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) - // Adapted from /usr/include/byteswap.h. Not available on Mac. - if (__builtin_constant_p(host_int)) { - return __bswap_constant_64(host_int); - } else { - uint64_t result; - __asm__("bswap %0" : "=r"(result) : "0"(host_int)); - return result; - } -#elif defined(__GLIBC__) - return bswap_64(host_int); -#else - return (((host_int & uint64_t{0xFF}) << 56) | - ((host_int & uint64_t{0xFF00}) << 40) | - ((host_int & uint64_t{0xFF0000}) << 24) | - ((host_int & uint64_t{0xFF000000}) << 8) | - ((host_int & uint64_t{0xFF00000000}) >> 8) | - ((host_int & uint64_t{0xFF0000000000}) >> 24) | - ((host_int & uint64_t{0xFF000000000000}) >> 40) | - ((host_int & uint64_t{0xFF00000000000000}) >> 56)); -#endif // bswap_64 - } - - inline uint32_t gbswap_32(uint32_t host_int) { -#if defined(__GLIBC__) - return bswap_32(host_int); -#else - return (((host_int & uint32_t{0xFF}) << 24) | - ((host_int & uint32_t{0xFF00}) << 8) | - ((host_int & uint32_t{0xFF0000}) >> 8) | - ((host_int & uint32_t{0xFF000000}) >> 24)); -#endif - } - - inline uint16_t gbswap_16(uint16_t host_int) { -#if defined(__GLIBC__) - return bswap_16(host_int); -#else - return (((host_int & uint16_t{0xFF}) << 8) | - ((host_int & uint16_t{0xFF00}) >> 8)); -#endif - } - -#endif // intrinics available - -#ifdef PHMAP_IS_LITTLE_ENDIAN - - // Definitions for ntohl etc. that don't require us to include - // netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather - // than just #defining them because in debug mode, gcc doesn't - // correctly handle the (rather involved) definitions of bswap_32. - // gcc guarantees that inline functions are as fast as macros, so - // this isn't a performance hit. - inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } - inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } - inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } - -#elif defined PHMAP_IS_BIG_ENDIAN - - // These definitions are simpler on big-endian machines - // These are functions instead of macros to avoid self-assignment warnings - // on calls such as "i = ghtnol(i);". This also provides type checking. - inline uint16_t ghtons(uint16_t x) { return x; } - inline uint32_t ghtonl(uint32_t x) { return x; } - inline uint64_t ghtonll(uint64_t x) { return x; } - -#else - #error \ - "Unsupported byte order: Either PHMAP_IS_BIG_ENDIAN or " \ - "PHMAP_IS_LITTLE_ENDIAN must be defined" -#endif // byte order - -inline uint16_t gntohs(uint16_t x) { return ghtons(x); } -inline uint32_t gntohl(uint32_t x) { return ghtonl(x); } -inline uint64_t gntohll(uint64_t x) { return ghtonll(x); } - -// Utilities to convert numbers between the current hosts's native byte -// order and little-endian byte order -// -// Load/Store methods are alignment safe -namespace little_endian { -// Conversion functions. -#ifdef PHMAP_IS_LITTLE_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return x; } - inline uint16_t ToHost16(uint16_t x) { return x; } - - inline uint32_t FromHost32(uint32_t x) { return x; } - inline uint32_t ToHost32(uint32_t x) { return x; } - - inline uint64_t FromHost64(uint64_t x) { return x; } - inline uint64_t ToHost64(uint64_t x) { return x; } - - inline constexpr bool IsLittleEndian() { return true; } - -#elif defined PHMAP_IS_BIG_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } - inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } - - inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } - inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } - - inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } - inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } - - inline constexpr bool IsLittleEndian() { return false; } - -#endif /* ENDIAN */ - -// Functions to do unaligned loads and stores in little-endian order. -inline uint16_t Load16(const void *p) { - return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); -} - -inline void Store16(void *p, uint16_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); -} - -inline uint32_t Load32(const void *p) { - return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); -} - -inline void Store32(void *p, uint32_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); -} - -inline uint64_t Load64(const void *p) { - return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); -} - -inline void Store64(void *p, uint64_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); -} - -} // namespace little_endian - -// Utilities to convert numbers between the current hosts's native byte -// order and big-endian byte order (same as network byte order) -// -// Load/Store methods are alignment safe -namespace big_endian { -#ifdef PHMAP_IS_LITTLE_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } - inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } - - inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } - inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } - - inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } - inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } - - inline constexpr bool IsLittleEndian() { return true; } - -#elif defined PHMAP_IS_BIG_ENDIAN - - inline uint16_t FromHost16(uint16_t x) { return x; } - inline uint16_t ToHost16(uint16_t x) { return x; } - - inline uint32_t FromHost32(uint32_t x) { return x; } - inline uint32_t ToHost32(uint32_t x) { return x; } - - inline uint64_t FromHost64(uint64_t x) { return x; } - inline uint64_t ToHost64(uint64_t x) { return x; } - - inline constexpr bool IsLittleEndian() { return false; } - -#endif /* ENDIAN */ - -// Functions to do unaligned loads and stores in big-endian order. -inline uint16_t Load16(const void *p) { - return ToHost16(PHMAP_INTERNAL_UNALIGNED_LOAD16(p)); -} - -inline void Store16(void *p, uint16_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); -} - -inline uint32_t Load32(const void *p) { - return ToHost32(PHMAP_INTERNAL_UNALIGNED_LOAD32(p)); -} - -inline void Store32(void *p, uint32_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); -} - -inline uint64_t Load64(const void *p) { - return ToHost64(PHMAP_INTERNAL_UNALIGNED_LOAD64(p)); -} - -inline void Store64(void *p, uint64_t v) { - PHMAP_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); -} - -} // namespace big_endian - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_bits_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_config.h b/benchmarks/external/parallel_hashmap/phmap_config.h deleted file mode 100644 index fa515025..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_config.h +++ /dev/null @@ -1,771 +0,0 @@ -#if !defined(phmap_config_h_guard_) -#define phmap_config_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) -// with modifications. -// -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#define PHMAP_VERSION_MAJOR 1 -#define PHMAP_VERSION_MINOR 0 -#define PHMAP_VERSION_PATCH 0 - -// Included for the __GLIBC__ macro (or similar macros on other systems). -#include <limits.h> - -#ifdef __cplusplus - // Included for __GLIBCXX__, _LIBCPP_VERSION - #include <cstddef> -#endif // __cplusplus - -#if defined(__APPLE__) - // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, - // __IPHONE_8_0. - #include <Availability.h> - #include <TargetConditionals.h> -#endif - -#define PHMAP_XSTR(x) PHMAP_STR(x) -#define PHMAP_STR(x) #x -#define PHMAP_VAR_NAME_VALUE(var) #var "=" PHMAP_STR(var) - -// ----------------------------------------------------------------------------- -// Some sanity checks -// ----------------------------------------------------------------------------- -//#if defined(__CYGWIN__) -// #error "Cygwin is not supported." -//#endif - -#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023918 && !defined(__clang__) - #error "phmap requires Visual Studio 2015 Update 2 or higher." -#endif - -// We support gcc 4.7 and later. -#if defined(__GNUC__) && !defined(__clang__) - #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) - #error "phmap requires gcc 4.7 or higher." - #endif -#endif - -// We support Apple Xcode clang 4.2.1 (version 421.11.65) and later. -// This corresponds to Apple Xcode version 4.5. -#if defined(__apple_build_version__) && __apple_build_version__ < 4211165 - #error "phmap requires __apple_build_version__ of 4211165 or higher." -#endif - -// Enforce C++11 as the minimum. -#if defined(__cplusplus) && !defined(_MSC_VER) - #if __cplusplus < 201103L - #error "C++ versions less than C++11 are not supported." - #endif -#endif - -// We have chosen glibc 2.12 as the minimum -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) - #if !__GLIBC_PREREQ(2, 12) - #error "Minimum required version of glibc is 2.12." - #endif -#endif - -#if defined(_STLPORT_VERSION) - #error "STLPort is not supported." -#endif - -#if CHAR_BIT != 8 - #error "phmap assumes CHAR_BIT == 8." -#endif - -// phmap currently assumes that an int is 4 bytes. -#if INT_MAX < 2147483647 - #error "phmap assumes that int is at least 4 bytes. " -#endif - - - -// ----------------------------------------------------------------------------- -// Compiler Feature Checks -// ----------------------------------------------------------------------------- - -#ifdef __has_builtin - #define PHMAP_HAVE_BUILTIN(x) __has_builtin(x) -#else - #define PHMAP_HAVE_BUILTIN(x) 0 -#endif - -#if (defined(_MSVC_LANG) && _MSVC_LANG >= 201703) || __cplusplus >= 201703 - #define PHMAP_HAVE_CC17 1 -#else - #define PHMAP_HAVE_CC17 0 -#endif - -#define PHMAP_BRANCHLESS 1 - -// ---------------------------------------------------------------- -// Checks whether `std::is_trivially_destructible<T>` is supported. -// ---------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE - #error PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set -#elif defined(_LIBCPP_VERSION) || \ - (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ - defined(_MSC_VER) - #define PHMAP_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 -#endif - -// -------------------------------------------------------------- -// Checks whether `std::is_trivially_default_constructible<T>` is -// supported. -// -------------------------------------------------------------- -#if defined(PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) - #error PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set -#elif defined(PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) - #error PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set -#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ - (!defined(__clang__) && defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ - (defined(_MSC_VER) && !defined(__NVCC__)) - #define PHMAP_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 - #define PHMAP_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 -#endif - -// ------------------------------------------------------------------- -// Checks whether C++11's `thread_local` storage duration specifier is -// supported. -// ------------------------------------------------------------------- -#ifdef PHMAP_HAVE_THREAD_LOCAL - #error PHMAP_HAVE_THREAD_LOCAL cannot be directly set -#elif defined(__APPLE__) - #if __has_feature(cxx_thread_local) && \ - !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0) - #define PHMAP_HAVE_THREAD_LOCAL 1 - #endif -#else // !defined(__APPLE__) - #define PHMAP_HAVE_THREAD_LOCAL 1 -#endif - -#if defined(__ANDROID__) && defined(__clang__) - - #if __has_include(<android/ndk-version.h>) - #include <android/ndk-version.h> - #endif // __has_include(<android/ndk-version.h>) - - #if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ - defined(__NDK_MINOR__) && \ - ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) - #undef PHMAP_HAVE_TLS - #undef PHMAP_HAVE_THREAD_LOCAL - #endif -#endif - -// ------------------------------------------------------------ -// Checks whether the __int128 compiler extension for a 128-bit -// integral type is supported. -// ------------------------------------------------------------ -#ifdef PHMAP_HAVE_INTRINSIC_INT128 - #error PHMAP_HAVE_INTRINSIC_INT128 cannot be directly set -#elif defined(__SIZEOF_INT128__) - #if (defined(__clang__) && !defined(_WIN32) && !defined(__aarch64__)) || \ - (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ - (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)) - #define PHMAP_HAVE_INTRINSIC_INT128 1 - #elif defined(__CUDACC__) - #if __CUDACC_VER__ >= 70000 - #define PHMAP_HAVE_INTRINSIC_INT128 1 - #endif // __CUDACC_VER__ >= 70000 - #endif // defined(__CUDACC__) -#endif - -// ------------------------------------------------------------------ -// Checks whether the compiler both supports and enables exceptions. -// ------------------------------------------------------------------ -#ifdef PHMAP_HAVE_EXCEPTIONS - #error PHMAP_HAVE_EXCEPTIONS cannot be directly set. -#elif defined(__clang__) - #if defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) - #define PHMAP_HAVE_EXCEPTIONS 1 - #endif // defined(__EXCEPTIONS) && __has_feature(cxx_exceptions) -#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ - !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ - !(defined(_MSC_VER) && !defined(_CPPUNWIND)) - #define PHMAP_HAVE_EXCEPTIONS 1 -#endif - - -// ----------------------------------------------------------------------- -// Checks whether the platform has an mmap(2) implementation as defined in -// POSIX.1-2001. -// ----------------------------------------------------------------------- -#ifdef PHMAP_HAVE_MMAP - #error PHMAP_HAVE_MMAP cannot be directly set -#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \ - defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \ - defined(__ASYLO__) - #define PHMAP_HAVE_MMAP 1 -#endif - -// ----------------------------------------------------------------------- -// Checks the endianness of the platform. -// ----------------------------------------------------------------------- -#if defined(PHMAP_IS_BIG_ENDIAN) - #error "PHMAP_IS_BIG_ENDIAN cannot be directly set." -#endif - -#if defined(PHMAP_IS_LITTLE_ENDIAN) - #error "PHMAP_IS_LITTLE_ENDIAN cannot be directly set." -#endif - -#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - #define PHMAP_IS_LITTLE_ENDIAN 1 -#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - #define PHMAP_IS_BIG_ENDIAN 1 -#elif defined(_WIN32) - #define PHMAP_IS_LITTLE_ENDIAN 1 -#else - #error "phmap endian detection needs to be set up for your compiler" -#endif - -#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ - defined(__MAC_OS_X_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400 - #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 1 -#else - #define PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE 0 -#endif - -// --------------------------------------------------------------------------- -// Checks whether C++17 std::any is available by checking whether <any> exists. -// --------------------------------------------------------------------------- -#ifdef PHMAP_HAVE_STD_ANY - #error "PHMAP_HAVE_STD_ANY cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<any>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_ANY 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_OPTIONAL - #error "PHMAP_HAVE_STD_OPTIONAL cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<optional>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_OPTIONAL 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_VARIANT - #error "PHMAP_HAVE_STD_VARIANT cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<variant>) && __cplusplus >= 201703L && \ - !PHMAP_INTERNAL_MACOS_CXX17_TYPES_UNAVAILABLE - #define PHMAP_HAVE_STD_VARIANT 1 - #endif -#endif - -#ifdef PHMAP_HAVE_STD_STRING_VIEW - #error "PHMAP_HAVE_STD_STRING_VIEW cannot be directly set." -#endif - -#ifdef __has_include - #if __has_include(<string_view>) && __cplusplus >= 201703L && \ - (!defined(_MSC_VER) || _MSC_VER >= 1920) // vs2019 - #define PHMAP_HAVE_STD_STRING_VIEW 1 - #endif -#endif - -// #pragma message(PHMAP_VAR_NAME_VALUE(_MSVC_LANG)) - -#if defined(_MSC_VER) && _MSC_VER >= 1910 && PHMAP_HAVE_CC17 - // #define PHMAP_HAVE_STD_ANY 1 - #define PHMAP_HAVE_STD_OPTIONAL 1 - #define PHMAP_HAVE_STD_VARIANT 1 - #if !defined(PHMAP_HAVE_STD_STRING_VIEW) && _MSC_VER >= 1920 - #define PHMAP_HAVE_STD_STRING_VIEW 1 - #endif -#endif - -#if PHMAP_HAVE_CC17 - #define PHMAP_HAVE_SHARED_MUTEX 1 -#endif - -#ifndef PHMAP_HAVE_STD_STRING_VIEW - #define PHMAP_HAVE_STD_STRING_VIEW 0 -#endif - -// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION -// SEH exception from emplace for variant<SomeStruct> when constructing the -// struct can throw. This defeats some of variant_test and -// variant_exception_safety_test. -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG) - #define PHMAP_INTERNAL_MSVC_2017_DBG_MODE -#endif - -// --------------------------------------------------------------------------- -// Checks whether wchar_t is treated as a native type -// (MSVC: /Zc:wchar_t- treats wchar_t as unsigned short) -// --------------------------------------------------------------------------- -#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) -#define PHMAP_HAS_NATIVE_WCHAR_T -#endif - -// ----------------------------------------------------------------------------- -// Sanitizer Attributes -// ----------------------------------------------------------------------------- -// -// Sanitizer-related attributes are not "defined" in this file (and indeed -// are not defined as such in any file). To utilize the following -// sanitizer-related attributes within your builds, define the following macros -// within your build using a `-D` flag, along with the given value for -// `-fsanitize`: -// -// * `ADDRESS_SANITIZER` + `-fsanitize=address` (Clang, GCC 4.8) -// * `MEMORY_SANITIZER` + `-fsanitize=memory` (Clang-only) -// * `THREAD_SANITIZER + `-fsanitize=thread` (Clang, GCC 4.8+) -// * `UNDEFINED_BEHAVIOR_SANITIZER` + `-fsanitize=undefined` (Clang, GCC 4.9+) -// * `CONTROL_FLOW_INTEGRITY` + -fsanitize=cfi (Clang-only) -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// A function-like feature checking macro that is a wrapper around -// `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a -// nonzero constant integer if the attribute is supported or 0 if not. -// -// It evaluates to zero if `__has_attribute` is not defined by the compiler. -// ----------------------------------------------------------------------------- -#ifdef __has_attribute - #define PHMAP_HAVE_ATTRIBUTE(x) __has_attribute(x) -#else - #define PHMAP_HAVE_ATTRIBUTE(x) 0 -#endif - -// ----------------------------------------------------------------------------- -// A function-like feature checking macro that accepts C++11 style attributes. -// It's a wrapper around `__has_cpp_attribute`, defined by ISO C++ SD-6 -// (https://en.cppreference.com/w/cpp/experimental/feature_test). If we don't -// find `__has_cpp_attribute`, will evaluate to 0. -// ----------------------------------------------------------------------------- -#if defined(__cplusplus) && defined(__has_cpp_attribute) - #define PHMAP_HAVE_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) -#else - #define PHMAP_HAVE_CPP_ATTRIBUTE(x) 0 -#endif - -// ----------------------------------------------------------------------------- -// Function Attributes -// ----------------------------------------------------------------------------- -#if PHMAP_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(__printf__, string_index, first_to_check))) - #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(__scanf__, string_index, first_to_check))) -#else - #define PHMAP_PRINTF_ATTRIBUTE(string_index, first_to_check) - #define PHMAP_SCANF_ATTRIBUTE(string_index, first_to_check) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(always_inline) || \ - (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) - #define PHMAP_HAVE_ATTRIBUTE_ALWAYS_INLINE 1 -#else - #define PHMAP_ATTRIBUTE_ALWAYS_INLINE -#endif - -#if !defined(__INTEL_COMPILER) && (PHMAP_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__))) - #define PHMAP_ATTRIBUTE_NOINLINE __attribute__((noinline)) - #define PHMAP_HAVE_ATTRIBUTE_NOINLINE 1 -#else - #define PHMAP_ATTRIBUTE_NOINLINE -#endif - -#if PHMAP_HAVE_ATTRIBUTE(disable_tail_calls) - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL __attribute__((disable_tail_calls)) -#elif defined(__GNUC__) && !defined(__clang__) - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL \ - __attribute__((optimize("no-optimize-sibling-calls"))) -#else - #define PHMAP_ATTRIBUTE_NO_TAIL_CALL - #define PHMAP_HAVE_ATTRIBUTE_NO_TAIL_CALL 0 -#endif - -#if (PHMAP_HAVE_ATTRIBUTE(weak) || \ - (defined(__GNUC__) && !defined(__clang__))) && \ - !(defined(__llvm__) && defined(_WIN32)) - #undef PHMAP_ATTRIBUTE_WEAK - #define PHMAP_ATTRIBUTE_WEAK __attribute__((weak)) - #define PHMAP_HAVE_ATTRIBUTE_WEAK 1 -#else - #define PHMAP_ATTRIBUTE_WEAK - #define PHMAP_HAVE_ATTRIBUTE_WEAK 0 -#endif - -#if PHMAP_HAVE_ATTRIBUTE(nonnull) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_NONNULL(arg_index) __attribute__((nonnull(arg_index))) -#else - #define PHMAP_ATTRIBUTE_NONNULL(...) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_NORETURN __attribute__((noreturn)) -#elif defined(_MSC_VER) - #define PHMAP_ATTRIBUTE_NORETURN __declspec(noreturn) -#else - #define PHMAP_ATTRIBUTE_NORETURN -#endif - -#if defined(__GNUC__) && defined(ADDRESS_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_ADDRESS -#endif - -#if defined(__GNUC__) && defined(MEMORY_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_MEMORY -#endif - -#if defined(__GNUC__) && defined(THREAD_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_THREAD -#endif - -#if defined(__GNUC__) && \ - (defined(UNDEFINED_BEHAVIOR_SANITIZER) || defined(ADDRESS_SANITIZER)) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED \ - __attribute__((no_sanitize("undefined"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_UNDEFINED -#endif - -#if defined(__GNUC__) && defined(CONTROL_FLOW_INTEGRITY) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI __attribute__((no_sanitize("cfi"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_CFI -#endif - -#if defined(__GNUC__) && defined(SAFESTACK_SANITIZER) - #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK \ - __attribute__((no_sanitize("safe-stack"))) -#else - #define PHMAP_ATTRIBUTE_NO_SANITIZE_SAFESTACK -#endif - -#if PHMAP_HAVE_ATTRIBUTE(returns_nonnull) || \ - (defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ - !defined(__clang__)) - #define PHMAP_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) -#else - #define PHMAP_ATTRIBUTE_RETURNS_NONNULL -#endif - -#ifdef PHMAP_HAVE_ATTRIBUTE_SECTION - #error PHMAP_HAVE_ATTRIBUTE_SECTION cannot be directly set -#elif (PHMAP_HAVE_ATTRIBUTE(section) || \ - (defined(__GNUC__) && !defined(__clang__))) && \ - !defined(__APPLE__) && PHMAP_HAVE_ATTRIBUTE_WEAK - #define PHMAP_HAVE_ATTRIBUTE_SECTION 1 - #ifndef PHMAP_ATTRIBUTE_SECTION - #define PHMAP_ATTRIBUTE_SECTION(name) \ - __attribute__((section(#name))) __attribute__((noinline)) - #endif - #ifndef PHMAP_ATTRIBUTE_SECTION_VARIABLE - #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) - #endif - #ifndef PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS - #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ - extern char __start_##name[] PHMAP_ATTRIBUTE_WEAK; \ - extern char __stop_##name[] PHMAP_ATTRIBUTE_WEAK - #endif - #ifndef PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS - #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) - #endif - #define PHMAP_ATTRIBUTE_SECTION_START(name) \ - (reinterpret_cast<void *>(__start_##name)) - #define PHMAP_ATTRIBUTE_SECTION_STOP(name) \ - (reinterpret_cast<void *>(__stop_##name)) -#else // !PHMAP_HAVE_ATTRIBUTE_SECTION - #define PHMAP_HAVE_ATTRIBUTE_SECTION 0 - #define PHMAP_ATTRIBUTE_SECTION(name) - #define PHMAP_ATTRIBUTE_SECTION_VARIABLE(name) - #define PHMAP_INIT_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DEFINE_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_DECLARE_ATTRIBUTE_SECTION_VARS(name) - #define PHMAP_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0)) - #define PHMAP_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0)) -#endif // PHMAP_ATTRIBUTE_SECTION - -#if PHMAP_HAVE_ATTRIBUTE(force_align_arg_pointer) || \ - (defined(__GNUC__) && !defined(__clang__)) - #if defined(__i386__) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC \ - __attribute__((force_align_arg_pointer)) - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) - #elif defined(__x86_64__) - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (1) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #else // !__i386__ && !__x86_64 - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #endif // __i386__ -#else - #define PHMAP_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC - #define PHMAP_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) -#endif - -#if PHMAP_HAVE_ATTRIBUTE(nodiscard) - #define PHMAP_MUST_USE_RESULT [[nodiscard]] -#elif defined(__clang__) && PHMAP_HAVE_ATTRIBUTE(warn_unused_result) - #define PHMAP_MUST_USE_RESULT __attribute__((warn_unused_result)) -#else - #define PHMAP_MUST_USE_RESULT -#endif - -#if PHMAP_HAVE_ATTRIBUTE(hot) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_HOT __attribute__((hot)) -#else - #define PHMAP_ATTRIBUTE_HOT -#endif - -#if PHMAP_HAVE_ATTRIBUTE(cold) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_COLD __attribute__((cold)) -#else - #define PHMAP_ATTRIBUTE_COLD -#endif - -#if defined(__clang__) - #if PHMAP_HAVE_CPP_ATTRIBUTE(clang::reinitializes) - #define PHMAP_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] - #else - #define PHMAP_ATTRIBUTE_REINITIALIZES - #endif -#else - #define PHMAP_ATTRIBUTE_REINITIALIZES -#endif - -#if PHMAP_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) - #undef PHMAP_ATTRIBUTE_UNUSED - #define PHMAP_ATTRIBUTE_UNUSED __attribute__((__unused__)) -#else - #define PHMAP_ATTRIBUTE_UNUSED -#endif - -#if PHMAP_HAVE_ATTRIBUTE(tls_model) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_INITIAL_EXEC __attribute__((tls_model("initial-exec"))) -#else - #define PHMAP_ATTRIBUTE_INITIAL_EXEC -#endif - -#if PHMAP_HAVE_ATTRIBUTE(packed) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_PACKED __attribute__((__packed__)) -#else - #define PHMAP_ATTRIBUTE_PACKED -#endif - -#if PHMAP_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__)) - #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes))) -#else - #define PHMAP_ATTRIBUTE_FUNC_ALIGN(bytes) -#endif - -// ---------------------------------------------------------------------- -// Figure out SSE support -// ---------------------------------------------------------------------- -#ifndef PHMAP_HAVE_SSE2 - #if defined(__SSE2__) || \ - (defined(_MSC_VER) && \ - (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) - #define PHMAP_HAVE_SSE2 1 - #else - #define PHMAP_HAVE_SSE2 0 - #endif -#endif - -#ifndef PHMAP_HAVE_SSSE3 - #if defined(__SSSE3__) || defined(__AVX2__) - #define PHMAP_HAVE_SSSE3 1 - #else - #define PHMAP_HAVE_SSSE3 0 - #endif -#endif - -#if PHMAP_HAVE_SSSE3 && !PHMAP_HAVE_SSE2 - #error "Bad configuration!" -#endif - -#if PHMAP_HAVE_SSE2 - #include <emmintrin.h> -#endif - -#if PHMAP_HAVE_SSSE3 - #include <tmmintrin.h> -#endif - - -// ---------------------------------------------------------------------- -// constexpr if -// ---------------------------------------------------------------------- -#if PHMAP_HAVE_CC17 - #define PHMAP_IF_CONSTEXPR(expr) if constexpr ((expr)) -#else - #define PHMAP_IF_CONSTEXPR(expr) if ((expr)) -#endif - -// ---------------------------------------------------------------------- -// base/macros.h -// ---------------------------------------------------------------------- - -// PHMAP_ARRAYSIZE() -// -// Returns the number of elements in an array as a compile-time constant, which -// can be used in defining new arrays. If you use this macro on a pointer by -// mistake, you will get a compile-time error. -#define PHMAP_ARRAYSIZE(array) \ - (sizeof(::phmap::macros_internal::ArraySizeHelper(array))) - -namespace phmap { -namespace macros_internal { - // Note: this internal template function declaration is used by PHMAP_ARRAYSIZE. - // The function doesn't need a definition, as we only use its type. - template <typename T, size_t N> - auto ArraySizeHelper(const T (&array)[N]) -> char (&)[N]; -} // namespace macros_internal -} // namespace phmap - -// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported. -#if defined(__clang__) && defined(__has_warning) - #if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") - #define PHMAP_FALLTHROUGH_INTENDED [[clang::fallthrough]] - #endif -#elif defined(__GNUC__) && __GNUC__ >= 7 - #define PHMAP_FALLTHROUGH_INTENDED [[gnu::fallthrough]] -#endif - -#ifndef PHMAP_FALLTHROUGH_INTENDED - #define PHMAP_FALLTHROUGH_INTENDED \ - do { } while (0) -#endif - -// PHMAP_DEPRECATED() -// -// Marks a deprecated class, struct, enum, function, method and variable -// declarations. The macro argument is used as a custom diagnostic message (e.g. -// suggestion of a better alternative). -// -// Example: -// -// class PHMAP_DEPRECATED("Use Bar instead") Foo {...}; -// PHMAP_DEPRECATED("Use Baz instead") void Bar() {...} -// -// Every usage of a deprecated entity will trigger a warning when compiled with -// clang's `-Wdeprecated-declarations` option. This option is turned off by -// default, but the warnings will be reported by clang-tidy. -#if defined(__clang__) && __cplusplus >= 201103L - #define PHMAP_DEPRECATED(message) __attribute__((deprecated(message))) -#endif - -#ifndef PHMAP_DEPRECATED - #define PHMAP_DEPRECATED(message) -#endif - -// PHMAP_BAD_CALL_IF() -// -// Used on a function overload to trap bad calls: any call that matches the -// overload will cause a compile-time error. This macro uses a clang-specific -// "enable_if" attribute, as described at -// http://clang.llvm.org/docs/AttributeReference.html#enable-if -// -// Overloads which use this macro should be bracketed by -// `#ifdef PHMAP_BAD_CALL_IF`. -// -// Example: -// -// int isdigit(int c); -// #ifdef PHMAP_BAD_CALL_IF -// int isdigit(int c) -// PHMAP_BAD_CALL_IF(c <= -1 || c > 255, -// "'c' must have the value of an unsigned char or EOF"); -// #endif // PHMAP_BAD_CALL_IF - -#if defined(__clang__) - #if __has_attribute(enable_if) - #define PHMAP_BAD_CALL_IF(expr, msg) \ - __attribute__((enable_if(expr, "Bad call trap"), unavailable(msg))) - #endif -#endif - -// PHMAP_ASSERT() -// -// In C++11, `assert` can't be used portably within constexpr functions. -// PHMAP_ASSERT functions as a runtime assert but works in C++11 constexpr -// functions. Example: -// -// constexpr double Divide(double a, double b) { -// return PHMAP_ASSERT(b != 0), a / b; -// } -// -// This macro is inspired by -// https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ -#if defined(NDEBUG) - #define PHMAP_ASSERT(expr) (false ? (void)(expr) : (void)0) -#else - #define PHMAP_ASSERT(expr) \ - (PHMAP_PREDICT_TRUE((expr)) ? (void)0 \ - : [] { assert(false && #expr); }()) // NOLINT -#endif - -#ifdef PHMAP_HAVE_EXCEPTIONS - #define PHMAP_INTERNAL_TRY try - #define PHMAP_INTERNAL_CATCH_ANY catch (...) - #define PHMAP_INTERNAL_RETHROW do { throw; } while (false) -#else // PHMAP_HAVE_EXCEPTIONS - #define PHMAP_INTERNAL_TRY if (true) - #define PHMAP_INTERNAL_CATCH_ANY else if (false) - #define PHMAP_INTERNAL_RETHROW do {} while (false) -#endif // PHMAP_HAVE_EXCEPTIONS - - -#endif // phmap_config_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_dump.h b/benchmarks/external/parallel_hashmap/phmap_dump.h deleted file mode 100644 index 0f2018ef..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_dump.h +++ /dev/null @@ -1,227 +0,0 @@ -#if !defined(phmap_dump_h_guard_) -#define phmap_dump_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// providing dump/load/mmap_load -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#include <iostream> -#include <fstream> -#include <sstream> -#include "phmap.h" -namespace phmap -{ - -namespace type_traits_internal { - -#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 - template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {}; -#else - template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {}; -#endif - -template <class T1, class T2> -struct IsTriviallyCopyable<std::pair<T1, T2>> { - static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value; -}; -} - -namespace priv { - -// ------------------------------------------------------------------------ -// dump/load for raw_hash_set -// ------------------------------------------------------------------------ -template <class Policy, class Hash, class Eq, class Alloc> -template<typename OutputArchive> -bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - if (!ar.dump(size_)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast<char*>(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast<char*>(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; -} - -template <class Policy, class Hash, class Eq, class Alloc> -template<typename InputArchive> -bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content - if (!ar.load(&size_)) { - std::cerr << "Failed to load size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_)) { - std::cerr << "Failed to load capacity_" << std::endl; - return false; - } - - // allocate memory for ctrl_ and slots_ - initialize_slots(); - if (!ar.load(reinterpret_cast<char*>(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - std::cerr << "Failed to load ctrl" << std::endl; - return false; - } - if (!ar.load(reinterpret_cast<char*>(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to load slot" << std::endl; - return false; - } - return true; -} - -// ------------------------------------------------------------------------ -// dump/load for parallel_hash_set -// ------------------------------------------------------------------------ -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template<typename OutputArchive> -bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - if (! ar.dump(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); - if (!inner.set_.dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; -} - -template <size_t N, - template <class, class, class, class> class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template<typename InputArchive> -bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value, - "value_type should be trivially copyable"); - - size_t submap_count = 0; - if (!ar.load(&submap_count)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } - - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } - - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast<Inner&>(inner)); - if (!inner.set_.load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; -} -} // namespace priv - - - -// ------------------------------------------------------------------------ -// BinaryArchive -// File is closed when archive object is destroyed -// ------------------------------------------------------------------------ - -// ------------------------------------------------------------------------ -// ------------------------------------------------------------------------ -class BinaryOutputArchive { -public: - BinaryOutputArchive(const char *file_path) { - ofs_.open(file_path, std::ios_base::binary); - } - - bool dump(const char *p, size_t sz) { - ofs_.write(p, sz); - return true; - } - - template<typename V> - typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type - dump(const V& v) { - ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V)); - return true; - } - -private: - std::ofstream ofs_; -}; - - -class BinaryInputArchive { -public: - BinaryInputArchive(const char * file_path) { - ifs_.open(file_path, std::ios_base::binary); - } - - bool load(char* p, size_t sz) { - ifs_.read(p, sz); - return true; - } - - template<typename V> - typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type - load(V* v) { - ifs_.read(reinterpret_cast<char *>(v), sizeof(V)); - return true; - } - -private: - std::ifstream ifs_; -}; - -} // namespace phmap - -#endif // phmap_dump_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h b/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h deleted file mode 100644 index a7719c49..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_fwd_decl.h +++ /dev/null @@ -1,154 +0,0 @@ -#if !defined(phmap_fwd_decl_h_guard_) -#define phmap_fwd_decl_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion -#endif - -#include <memory> -#include <utility> - -#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_) - namespace absl { template <class T> struct Hash; }; -#endif - -namespace phmap { - -#if defined(PHMAP_USE_ABSL_HASH) - template <class T> using Hash = ::absl::Hash<T>; -#else - template <class T> struct Hash; -#endif - - template <class T> struct EqualTo; - template <class T> struct Less; - template <class T> using Allocator = typename std::allocator<T>; - template<class T1, class T2> using Pair = typename std::pair<T1, T2>; - - class NullMutex; - - namespace priv { - - // The hash of an object of type T is computed by using phmap::Hash. - template <class T, class E = void> - struct HashEq - { - using Hash = phmap::Hash<T>; - using Eq = phmap::EqualTo<T>; - }; - - template <class T> - using hash_default_hash = typename priv::HashEq<T>::Hash; - - template <class T> - using hash_default_eq = typename priv::HashEq<T>::Eq; - - // type alias for std::allocator so we can forward declare without including other headers - template <class T> - using Allocator = typename phmap::Allocator<T>; - - // type alias for std::pair so we can forward declare without including other headers - template<class T1, class T2> - using Pair = typename phmap::Pair<T1, T2>; - - } // namespace priv - - // ------------- forward declarations for hash containers ---------------------------------- - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator - class flat_hash_set; - - template <class K, class V, - class Hash = phmap::priv::hash_default_hash<K>, - class Eq = phmap::priv::hash_default_eq<K>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const K, V>>> // alias for std::allocator - class flat_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>> // alias for std::allocator - class node_hash_set; - - template <class Key, class Value, - class Hash = phmap::priv::hash_default_hash<Key>, - class Eq = phmap::priv::hash_default_eq<Key>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const Key, Value>>> // alias for std::allocator - class node_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_flat_hash_set; - - template <class K, class V, - class Hash = phmap::priv::hash_default_hash<K>, - class Eq = phmap::priv::hash_default_eq<K>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const K, V>>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_flat_hash_map; - - template <class T, - class Hash = phmap::priv::hash_default_hash<T>, - class Eq = phmap::priv::hash_default_eq<T>, - class Alloc = phmap::priv::Allocator<T>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_node_hash_set; - - template <class Key, class Value, - class Hash = phmap::priv::hash_default_hash<Key>, - class Eq = phmap::priv::hash_default_eq<Key>, - class Alloc = phmap::priv::Allocator< - phmap::priv::Pair<const Key, Value>>, // alias for std::allocator - size_t N = 4, // 2**N submaps - class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks - class parallel_node_hash_map; - - // ------------- forward declarations for btree containers ---------------------------------- - template <typename Key, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<Key>> - class btree_set; - - template <typename Key, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<Key>> - class btree_multiset; - - template <typename Key, typename Value, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> - class btree_map; - - template <typename Key, typename Value, typename Compare = phmap::Less<Key>, - typename Alloc = phmap::Allocator<phmap::priv::Pair<const Key, Value>>> - class btree_multimap; - -} // namespace phmap - - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_fwd_decl_h_guard_ diff --git a/benchmarks/external/parallel_hashmap/phmap_utils.h b/benchmarks/external/parallel_hashmap/phmap_utils.h deleted file mode 100644 index 1d0c4728..00000000 --- a/benchmarks/external/parallel_hashmap/phmap_utils.h +++ /dev/null @@ -1,378 +0,0 @@ -#if !defined(phmap_utils_h_guard_) -#define phmap_utils_h_guard_ - -// --------------------------------------------------------------------------- -// Copyright (c) 2019, Gregory Popovitch - [email protected] -// -// minimal header providing phmap::HashState -// -// use as: phmap::HashState().combine(0, _first_name, _last_name, _age); -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// --------------------------------------------------------------------------- - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion -#endif - -#include <cstdint> -#include <functional> -#include <tuple> -#include "phmap_bits.h" - -// --------------------------------------------------------------- -// Absl forward declaration requires global scope. -// --------------------------------------------------------------- -#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) && !defined(ABSL_HASH_HASH_H_) - namespace absl { template <class T> struct Hash; }; -#endif - -namespace phmap -{ - -// --------------------------------------------------------------- -// --------------------------------------------------------------- -template<int n> -struct phmap_mix -{ - inline size_t operator()(size_t) const; -}; - -template<> -struct phmap_mix<4> -{ - inline size_t operator()(size_t a) const - { - static constexpr uint64_t kmul = 0xcc9e2d51UL; - // static constexpr uint64_t kmul = 0x3B9ACB93UL; // [greg] my own random prime - uint64_t l = a * kmul; - return static_cast<size_t>(l ^ (l >> 32)); - } -}; - -#if defined(PHMAP_HAS_UMUL128) - template<> - struct phmap_mix<8> - { - // Very fast mixing (similar to Abseil) - inline size_t operator()(size_t a) const - { - static constexpr uint64_t k = 0xde5fb9d2630458e9ULL; - // static constexpr uint64_t k = 0x7C9D0BF0567102A5ULL; // [greg] my own random prime - uint64_t h; - uint64_t l = umul128(a, k, &h); - return static_cast<size_t>(h + l); - } - }; -#else - template<> - struct phmap_mix<8> - { - inline size_t operator()(size_t a) const - { - a = (~a) + (a << 21); // a = (a << 21) - a - 1; - a = a ^ (a >> 24); - a = (a + (a << 3)) + (a << 8); // a * 265 - a = a ^ (a >> 14); - a = (a + (a << 2)) + (a << 4); // a * 21 - a = a ^ (a >> 28); - a = a + (a << 31); - return static_cast<size_t>(a); - } - }; -#endif - -// -------------------------------------------- -template<int n> -struct fold_if_needed -{ - inline size_t operator()(uint64_t) const; -}; - -template<> -struct fold_if_needed<4> -{ - inline size_t operator()(uint64_t a) const - { - return static_cast<size_t>(a ^ (a >> 32)); - } -}; - -template<> -struct fold_if_needed<8> -{ - inline size_t operator()(uint64_t a) const - { - return static_cast<size_t>(a); - } -}; - -// --------------------------------------------------------------- -// see if class T has a hash_value() friend method -// --------------------------------------------------------------- -template<typename T> -struct has_hash_value -{ -private: - typedef std::true_type yes; - typedef std::false_type no; - - template<typename U> static auto test(int) -> decltype(hash_value(std::declval<const U&>()) == 1, yes()); - - template<typename> static no test(...); - -public: - static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value; -}; - -#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) - template <class T> using Hash = ::absl::Hash<T>; -#elif !defined(PHMAP_USE_ABSL_HASH) -// --------------------------------------------------------------- -// phmap::Hash -// --------------------------------------------------------------- -template <class T> -struct Hash -{ - template <class U, typename std::enable_if<has_hash_value<U>::value, int>::type = 0> - size_t _hash(const T& val) const - { - return hash_value(val); - } - - template <class U, typename std::enable_if<!has_hash_value<U>::value, int>::type = 0> - size_t _hash(const T& val) const - { - return std::hash<T>()(val); - } - - inline size_t operator()(const T& val) const - { - return _hash<T>(val); - } -}; - -template <class T> -struct Hash<T *> -{ - inline size_t operator()(const T *val) const noexcept - { - return static_cast<size_t>(reinterpret_cast<const uintptr_t>(val)); - } -}; - -template<class ArgumentType, class ResultType> -struct phmap_unary_function -{ - typedef ArgumentType argument_type; - typedef ResultType result_type; -}; - -template <> -struct Hash<bool> : public phmap_unary_function<bool, size_t> -{ - inline size_t operator()(bool val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<char> : public phmap_unary_function<char, size_t> -{ - inline size_t operator()(char val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<signed char> : public phmap_unary_function<signed char, size_t> -{ - inline size_t operator()(signed char val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<unsigned char> : public phmap_unary_function<unsigned char, size_t> -{ - inline size_t operator()(unsigned char val) const noexcept - { return static_cast<size_t>(val); } -}; - -#ifdef PHMAP_HAS_NATIVE_WCHAR_T -template <> -struct Hash<wchar_t> : public phmap_unary_function<wchar_t, size_t> -{ - inline size_t operator()(wchar_t val) const noexcept - { return static_cast<size_t>(val); } -}; -#endif - -template <> -struct Hash<int16_t> : public phmap_unary_function<int16_t, size_t> -{ - inline size_t operator()(int16_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<uint16_t> : public phmap_unary_function<uint16_t, size_t> -{ - inline size_t operator()(uint16_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<int32_t> : public phmap_unary_function<int32_t, size_t> -{ - inline size_t operator()(int32_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<uint32_t> : public phmap_unary_function<uint32_t, size_t> -{ - inline size_t operator()(uint32_t val) const noexcept - { return static_cast<size_t>(val); } -}; - -template <> -struct Hash<int64_t> : public phmap_unary_function<int64_t, size_t> -{ - inline size_t operator()(int64_t val) const noexcept - { return fold_if_needed<sizeof(size_t)>()(static_cast<uint64_t>(val)); } -}; - -template <> -struct Hash<uint64_t> : public phmap_unary_function<uint64_t, size_t> -{ - inline size_t operator()(uint64_t val) const noexcept - { return fold_if_needed<sizeof(size_t)>()(val); } -}; - -template <> -struct Hash<float> : public phmap_unary_function<float, size_t> -{ - inline size_t operator()(float val) const noexcept - { - // -0.0 and 0.0 should return same hash - uint32_t *as_int = reinterpret_cast<uint32_t *>(&val); - return (val == 0) ? static_cast<size_t>(0) : - static_cast<size_t>(*as_int); - } -}; - -template <> -struct Hash<double> : public phmap_unary_function<double, size_t> -{ - inline size_t operator()(double val) const noexcept - { - // -0.0 and 0.0 should return same hash - uint64_t *as_int = reinterpret_cast<uint64_t *>(&val); - return (val == 0) ? static_cast<size_t>(0) : - fold_if_needed<sizeof(size_t)>()(*as_int); - } -}; - -#endif - -template <class H, int sz> struct Combiner -{ - H operator()(H seed, size_t value); -}; - -template <class H> struct Combiner<H, 4> -{ - H operator()(H seed, size_t value) - { - return seed ^ (value + 0x9e3779b9 + (seed << 6) + (seed >> 2)); - } -}; - -template <class H> struct Combiner<H, 8> -{ - H operator()(H seed, size_t value) - { - return seed ^ (value + size_t(0xc6a4a7935bd1e995) + (seed << 6) + (seed >> 2)); - } -}; - -// define HashState to combine member hashes... see example below -// ----------------------------------------------------------------------------- -template <typename H> -class HashStateBase { -public: - template <typename T, typename... Ts> - static H combine(H state, const T& value, const Ts&... values); - - static H combine(H state) { return state; } -}; - -template <typename H> -template <typename T, typename... Ts> -H HashStateBase<H>::combine(H seed, const T& v, const Ts&... vs) -{ - return HashStateBase<H>::combine(Combiner<H, sizeof(H)>()( - seed, phmap::Hash<T>()(v)), - vs...); -} - -using HashState = HashStateBase<size_t>; - -// ----------------------------------------------------------------------------- - -#if !defined(PHMAP_USE_ABSL_HASH) - -// define Hash for std::pair -// ------------------------- -template<class T1, class T2> -struct Hash<std::pair<T1, T2>> { - size_t operator()(std::pair<T1, T2> const& p) const noexcept { - return phmap::HashState().combine(phmap::Hash<T1>()(p.first), p.second); - } -}; - -// define Hash for std::tuple -// -------------------------- -template<class... T> -struct Hash<std::tuple<T...>> { - size_t operator()(std::tuple<T...> const& t) const noexcept { - return _hash_helper(t); - } - -private: - template<size_t I = 0, class ...P> - typename std::enable_if<I == sizeof...(P), size_t>::type - _hash_helper(const std::tuple<P...> &) const noexcept { return 0; } - - template<size_t I = 0, class ...P> - typename std::enable_if<I < sizeof...(P), size_t>::type - _hash_helper(const std::tuple<P...> &t) const noexcept { - const auto &el = std::get<I>(t); - using el_type = typename std::remove_cv<typename std::remove_reference<decltype(el)>::type>::type; - return Combiner<size_t, sizeof(size_t)>()( - phmap::Hash<el_type>()(el), _hash_helper<I + 1>(t)); - } -}; - - -#endif - - -} // namespace phmap - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif // phmap_utils_h_guard_ diff --git a/benchmarks/external/skarupke/bytell_hash_map.hpp b/benchmarks/external/skarupke/bytell_hash_map.hpp deleted file mode 100644 index 9c9a2467..00000000 --- a/benchmarks/external/skarupke/bytell_hash_map.hpp +++ /dev/null @@ -1,1260 +0,0 @@ -// Copyright Malte Skarupke 2017. -// Distributed under the Boost Software License, Version 1.0. -// (See http://www.boost.org/LICENSE_1_0.txt) - -#pragma once - -#include <cstdint> -#include <cstddef> -#include <cmath> -#include <algorithm> -#include <iterator> -#include <utility> -#include <type_traits> -#include "flat_hash_map.hpp" -#include <vector> -#include <array> - -namespace ska -{ - -namespace detailv8 -{ -using ska::detailv3::functor_storage; -using ska::detailv3::KeyOrValueHasher; -using ska::detailv3::KeyOrValueEquality; -using ska::detailv3::AssignIfTrue; -using ska::detailv3::HashPolicySelector; - -template<typename = void> -struct sherwood_v8_constants -{ - static constexpr int8_t magic_for_empty = int8_t(0b11111111); - static constexpr int8_t magic_for_reserved = int8_t(0b11111110); - static constexpr int8_t bits_for_direct_hit = int8_t(0b10000000); - static constexpr int8_t magic_for_direct_hit = int8_t(0b00000000); - static constexpr int8_t magic_for_list_entry = int8_t(0b10000000); - - static constexpr int8_t bits_for_distance = int8_t(0b01111111); - inline static int distance_from_metadata(int8_t metadata) - { - return metadata & bits_for_distance; - } - - static constexpr int num_jump_distances = 126; - // jump distances chosen like this: - // 1. pick the first 16 integers to promote staying in the same block - // 2. add the next 66 triangular numbers to get even jumps when - // the hash table is a power of two - // 3. add 44 more triangular numbers at a much steeper growth rate - // to get a sequence that allows large jumps so that a table - // with 10000 sequential numbers doesn't endlessly re-allocate - static constexpr size_t jump_distances[num_jump_distances] - { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - - 21, 28, 36, 45, 55, 66, 78, 91, 105, 120, 136, 153, 171, 190, 210, 231, - 253, 276, 300, 325, 351, 378, 406, 435, 465, 496, 528, 561, 595, 630, - 666, 703, 741, 780, 820, 861, 903, 946, 990, 1035, 1081, 1128, 1176, - 1225, 1275, 1326, 1378, 1431, 1485, 1540, 1596, 1653, 1711, 1770, 1830, - 1891, 1953, 2016, 2080, 2145, 2211, 2278, 2346, 2415, 2485, 2556, - - 3741, 8385, 18915, 42486, 95703, 215496, 485605, 1091503, 2456436, - 5529475, 12437578, 27986421, 62972253, 141700195, 318819126, 717314626, - 1614000520, 3631437253, 8170829695, 18384318876, 41364501751, - 93070021080, 209407709220, 471167588430, 1060127437995, 2385287281530, - 5366895564381, 12075513791265, 27169907873235, 61132301007778, - 137547673121001, 309482258302503, 696335090510256, 1566753939653640, - 3525196427195653, 7931691866727775, 17846306747368716, - 40154190394120111, 90346928493040500, 203280588949935750, - 457381324898247375, 1029107980662394500, 2315492957028380766, - 5209859150892887590, - }; -}; -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::magic_for_empty; -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::magic_for_reserved; -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::bits_for_direct_hit; -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::magic_for_direct_hit; -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::magic_for_list_entry; - -template<typename T> -constexpr int8_t sherwood_v8_constants<T>::bits_for_distance; - -template<typename T> -constexpr int sherwood_v8_constants<T>::num_jump_distances; -template<typename T> -constexpr size_t sherwood_v8_constants<T>::jump_distances[num_jump_distances]; - -template<typename T, uint8_t BlockSize> -struct sherwood_v8_block -{ - sherwood_v8_block() - { - } - ~sherwood_v8_block() - { - } - int8_t control_bytes[BlockSize]; - union - { - T data[BlockSize]; - }; - - static sherwood_v8_block * empty_block() - { - static std::array<int8_t, BlockSize> empty_bytes = [] - { - std::array<int8_t, BlockSize> result; - result.fill(sherwood_v8_constants<>::magic_for_empty); - return result; - }(); - return reinterpret_cast<sherwood_v8_block *>(&empty_bytes); - } - - int first_empty_index() const - { - for (int i = 0; i < BlockSize; ++i) - { - if (control_bytes[i] == sherwood_v8_constants<>::magic_for_empty) - return i; - } - return -1; - } - - void fill_control_bytes(int8_t value) - { - std::fill(std::begin(control_bytes), std::end(control_bytes), value); - } -}; - -template<typename T, typename FindKey, typename ArgumentHash, typename Hasher, typename ArgumentEqual, typename Equal, typename ArgumentAlloc, typename ByteAlloc, uint8_t BlockSize> -class sherwood_v8_table : private ByteAlloc, private Hasher, private Equal -{ - using AllocatorTraits = std::allocator_traits<ByteAlloc>; - using BlockType = sherwood_v8_block<T, BlockSize>; - using BlockPointer = BlockType *; - using BytePointer = typename AllocatorTraits::pointer; - struct convertible_to_iterator; - using Constants = sherwood_v8_constants<>; - -public: - - using value_type = T; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using hasher = ArgumentHash; - using key_equal = ArgumentEqual; - using allocator_type = ByteAlloc; - using reference = value_type &; - using const_reference = const value_type &; - using pointer = value_type *; - using const_pointer = const value_type *; - - sherwood_v8_table() - { - } - explicit sherwood_v8_table(size_type bucket_count, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) - : ByteAlloc(alloc), Hasher(hash), Equal(equal) - { - if (bucket_count) - rehash(bucket_count); - } - sherwood_v8_table(size_type bucket_count, const ArgumentAlloc & alloc) - : sherwood_v8_table(bucket_count, ArgumentHash(), ArgumentEqual(), alloc) - { - } - sherwood_v8_table(size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) - : sherwood_v8_table(bucket_count, hash, ArgumentEqual(), alloc) - { - } - explicit sherwood_v8_table(const ArgumentAlloc & alloc) - : ByteAlloc(alloc) - { - } - template<typename It> - sherwood_v8_table(It first, It last, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) - : sherwood_v8_table(bucket_count, hash, equal, alloc) - { - insert(first, last); - } - template<typename It> - sherwood_v8_table(It first, It last, size_type bucket_count, const ArgumentAlloc & alloc) - : sherwood_v8_table(first, last, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) - { - } - template<typename It> - sherwood_v8_table(It first, It last, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) - : sherwood_v8_table(first, last, bucket_count, hash, ArgumentEqual(), alloc) - { - } - sherwood_v8_table(std::initializer_list<T> il, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) - : sherwood_v8_table(bucket_count, hash, equal, alloc) - { - if (bucket_count == 0) - rehash(il.size()); - insert(il.begin(), il.end()); - } - sherwood_v8_table(std::initializer_list<T> il, size_type bucket_count, const ArgumentAlloc & alloc) - : sherwood_v8_table(il, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) - { - } - sherwood_v8_table(std::initializer_list<T> il, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) - : sherwood_v8_table(il, bucket_count, hash, ArgumentEqual(), alloc) - { - } - sherwood_v8_table(const sherwood_v8_table & other) - : sherwood_v8_table(other, AllocatorTraits::select_on_container_copy_construction(other.get_allocator())) - { - } - sherwood_v8_table(const sherwood_v8_table & other, const ArgumentAlloc & alloc) - : ByteAlloc(alloc), Hasher(other), Equal(other), _max_load_factor(other._max_load_factor) - { - rehash_for_other_container(other); - try - { - insert(other.begin(), other.end()); - } - catch(...) - { - clear(); - deallocate_data(entries, num_slots_minus_one); - throw; - } - } - sherwood_v8_table(sherwood_v8_table && other) noexcept - : ByteAlloc(std::move(other)), Hasher(std::move(other)), Equal(std::move(other)) - , _max_load_factor(other._max_load_factor) - { - swap_pointers(other); - } - sherwood_v8_table(sherwood_v8_table && other, const ArgumentAlloc & alloc) noexcept - : ByteAlloc(alloc), Hasher(std::move(other)), Equal(std::move(other)) - , _max_load_factor(other._max_load_factor) - { - swap_pointers(other); - } - sherwood_v8_table & operator=(const sherwood_v8_table & other) - { - if (this == std::addressof(other)) - return *this; - - clear(); - if (AllocatorTraits::propagate_on_container_copy_assignment::value) - { - if (static_cast<ByteAlloc &>(*this) != static_cast<const ByteAlloc &>(other)) - { - reset_to_empty_state(); - } - AssignIfTrue<ByteAlloc, AllocatorTraits::propagate_on_container_copy_assignment::value>()(*this, other); - } - _max_load_factor = other._max_load_factor; - static_cast<Hasher &>(*this) = other; - static_cast<Equal &>(*this) = other; - rehash_for_other_container(other); - insert(other.begin(), other.end()); - return *this; - } - sherwood_v8_table & operator=(sherwood_v8_table && other) noexcept - { - if (this == std::addressof(other)) - return *this; - else if (AllocatorTraits::propagate_on_container_move_assignment::value) - { - clear(); - reset_to_empty_state(); - AssignIfTrue<ByteAlloc, AllocatorTraits::propagate_on_container_move_assignment::value>()(*this, std::move(other)); - swap_pointers(other); - } - else if (static_cast<ByteAlloc &>(*this) == static_cast<ByteAlloc &>(other)) - { - swap_pointers(other); - } - else - { - clear(); - _max_load_factor = other._max_load_factor; - rehash_for_other_container(other); - for (T & elem : other) - emplace(std::move(elem)); - other.clear(); - } - static_cast<Hasher &>(*this) = std::move(other); - static_cast<Equal &>(*this) = std::move(other); - return *this; - } - ~sherwood_v8_table() - { - clear(); - deallocate_data(entries, num_slots_minus_one); - } - - const allocator_type & get_allocator() const - { - return static_cast<const allocator_type &>(*this); - } - const ArgumentEqual & key_eq() const - { - return static_cast<const ArgumentEqual &>(*this); - } - const ArgumentHash & hash_function() const - { - return static_cast<const ArgumentHash &>(*this); - } - - template<typename ValueType> - struct templated_iterator - { - private: - friend class sherwood_v8_table; - BlockPointer current = BlockPointer(); - size_t index = 0; - - public: - templated_iterator() - { - } - templated_iterator(BlockPointer entries, size_t index) - : current(entries) - , index(index) - { - } - - using iterator_category = std::forward_iterator_tag; - using value_type = ValueType; - using difference_type = ptrdiff_t; - using pointer = ValueType *; - using reference = ValueType &; - - friend bool operator==(const templated_iterator & lhs, const templated_iterator & rhs) - { - return lhs.index == rhs.index; - } - friend bool operator!=(const templated_iterator & lhs, const templated_iterator & rhs) - { - return !(lhs == rhs); - } - - templated_iterator & operator++() - { - do - { - if (index % BlockSize == 0) - --current; - if (index-- == 0) - break; - } - while(current->control_bytes[index % BlockSize] == Constants::magic_for_empty); - return *this; - } - templated_iterator operator++(int) - { - templated_iterator copy(*this); - ++*this; - return copy; - } - - ValueType & operator*() const - { - return current->data[index % BlockSize]; - } - ValueType * operator->() const - { - return current->data + index % BlockSize; - } - - operator templated_iterator<const value_type>() const - { - return { current, index }; - } - }; - using iterator = templated_iterator<value_type>; - using const_iterator = templated_iterator<const value_type>; - - iterator begin() - { - size_t num_slots = num_slots_minus_one ? num_slots_minus_one + 1 : 0; - return ++iterator{ entries + num_slots / BlockSize, num_slots }; - } - const_iterator begin() const - { - size_t num_slots = num_slots_minus_one ? num_slots_minus_one + 1 : 0; - return ++iterator{ entries + num_slots / BlockSize, num_slots }; - } - const_iterator cbegin() const - { - return begin(); - } - iterator end() - { - return { entries - 1, std::numeric_limits<size_t>::max() }; - } - const_iterator end() const - { - return { entries - 1, std::numeric_limits<size_t>::max() }; - } - const_iterator cend() const - { - return end(); - } - - inline iterator find(const FindKey & key) - { - size_t index = hash_object(key); - size_t num_slots_minus_one = this->num_slots_minus_one; - BlockPointer entries = this->entries; - index = hash_policy.index_for_hash(index, num_slots_minus_one); - bool first = true; - for (;;) - { - size_t block_index = index / BlockSize; - int index_in_block = index % BlockSize; - BlockPointer block = entries + block_index; - int8_t metadata = block->control_bytes[index_in_block]; - if (first) - { - if ((metadata & Constants::bits_for_direct_hit) != Constants::magic_for_direct_hit) - return end(); - first = false; - } - if (compares_equal(key, block->data[index_in_block])) - return { block, index }; - int8_t to_next_index = metadata & Constants::bits_for_distance; - if (to_next_index == 0) - return end(); - index += Constants::jump_distances[to_next_index]; - index = hash_policy.keep_in_range(index, num_slots_minus_one); - } - } - inline const_iterator find(const FindKey & key) const - { - return const_cast<sherwood_v8_table *>(this)->find(key); - } - size_t count(const FindKey & key) const - { - return find(key) == end() ? 0 : 1; - } - std::pair<iterator, iterator> equal_range(const FindKey & key) - { - iterator found = find(key); - if (found == end()) - return { found, found }; - else - return { found, std::next(found) }; - } - std::pair<const_iterator, const_iterator> equal_range(const FindKey & key) const - { - const_iterator found = find(key); - if (found == end()) - return { found, found }; - else - return { found, std::next(found) }; - } - - - template<typename Key, typename... Args> - inline std::pair<iterator, bool> emplace(Key && key, Args &&... args) - { - size_t index = hash_object(key); - size_t num_slots_minus_one = this->num_slots_minus_one; - BlockPointer entries = this->entries; - index = hash_policy.index_for_hash(index, num_slots_minus_one); - bool first = true; - for (;;) - { - size_t block_index = index / BlockSize; - int index_in_block = index % BlockSize; - BlockPointer block = entries + block_index; - int8_t metadata = block->control_bytes[index_in_block]; - if (first) - { - if ((metadata & Constants::bits_for_direct_hit) != Constants::magic_for_direct_hit) - return emplace_direct_hit({ index, block }, std::forward<Key>(key), std::forward<Args>(args)...); - first = false; - } - if (compares_equal(key, block->data[index_in_block])) - return { { block, index }, false }; - int8_t to_next_index = metadata & Constants::bits_for_distance; - if (to_next_index == 0) - return emplace_new_key({ index, block }, std::forward<Key>(key), std::forward<Args>(args)...); - index += Constants::jump_distances[to_next_index]; - index = hash_policy.keep_in_range(index, num_slots_minus_one); - } - } - - std::pair<iterator, bool> insert(const value_type & value) - { - return emplace(value); - } - std::pair<iterator, bool> insert(value_type && value) - { - return emplace(std::move(value)); - } - template<typename... Args> - iterator emplace_hint(const_iterator, Args &&... args) - { - return emplace(std::forward<Args>(args)...).first; - } - iterator insert(const_iterator, const value_type & value) - { - return emplace(value).first; - } - iterator insert(const_iterator, value_type && value) - { - return emplace(std::move(value)).first; - } - - template<typename It> - void insert(It begin, It end) - { - for (; begin != end; ++begin) - { - emplace(*begin); - } - } - void insert(std::initializer_list<value_type> il) - { - insert(il.begin(), il.end()); - } - - void rehash(size_t num_items) - { - num_items = std::max(num_items, static_cast<size_t>(std::ceil(num_elements / static_cast<double>(_max_load_factor)))); - if (num_items == 0) - { - reset_to_empty_state(); - return; - } - auto new_prime_index = hash_policy.next_size_over(num_items); - if (num_items == num_slots_minus_one + 1) - return; - size_t num_blocks = num_items / BlockSize; - if (num_items % BlockSize) - ++num_blocks; - size_t memory_requirement = calculate_memory_requirement(num_blocks); - unsigned char * new_memory = &*AllocatorTraits::allocate(*this, memory_requirement); - - BlockPointer new_buckets = reinterpret_cast<BlockPointer>(new_memory); - - BlockPointer special_end_item = new_buckets + num_blocks; - for (BlockPointer it = new_buckets; it <= special_end_item; ++it) - it->fill_control_bytes(Constants::magic_for_empty); - using std::swap; - swap(entries, new_buckets); - swap(num_slots_minus_one, num_items); - --num_slots_minus_one; - hash_policy.commit(new_prime_index); - num_elements = 0; - if (num_items) - ++num_items; - size_t old_num_blocks = num_items / BlockSize; - if (num_items % BlockSize) - ++old_num_blocks; - for (BlockPointer it = new_buckets, end = new_buckets + old_num_blocks; it != end; ++it) - { - for (int i = 0; i < BlockSize; ++i) - { - int8_t metadata = it->control_bytes[i]; - if (metadata != Constants::magic_for_empty && metadata != Constants::magic_for_reserved) - { - emplace(std::move(it->data[i])); - AllocatorTraits::destroy(*this, it->data + i); - } - } - } - deallocate_data(new_buckets, num_items - 1); - } - - void reserve(size_t num_elements) - { - size_t required_buckets = num_buckets_for_reserve(num_elements); - if (required_buckets > bucket_count()) - rehash(required_buckets); - } - - // the return value is a type that can be converted to an iterator - // the reason for doing this is that it's not free to find the - // iterator pointing at the next element. if you care about the - // next iterator, turn the return value into an iterator - convertible_to_iterator erase(const_iterator to_erase) - { - LinkedListIt current = { to_erase.index, to_erase.current }; - if (current.has_next()) - { - LinkedListIt previous = current; - LinkedListIt next = current.next(*this); - while (next.has_next()) - { - previous = next; - next = next.next(*this); - } - AllocatorTraits::destroy(*this, std::addressof(*current)); - AllocatorTraits::construct(*this, std::addressof(*current), std::move(*next)); - AllocatorTraits::destroy(*this, std::addressof(*next)); - next.set_metadata(Constants::magic_for_empty); - previous.clear_next(); - } - else - { - if (!current.is_direct_hit()) - find_parent_block(current).clear_next(); - AllocatorTraits::destroy(*this, std::addressof(*current)); - current.set_metadata(Constants::magic_for_empty); - } - --num_elements; - return { to_erase.current, to_erase.index }; - } - - iterator erase(const_iterator begin_it, const_iterator end_it) - { - if (begin_it == end_it) - return { begin_it.current, begin_it.index }; - if (std::next(begin_it) == end_it) - return erase(begin_it); - if (begin_it == begin() && end_it == end()) - { - clear(); - return { end_it.current, end_it.index }; - } - std::vector<std::pair<int, LinkedListIt>> depth_in_chain; - for (const_iterator it = begin_it; it != end_it; ++it) - { - LinkedListIt list_it(it.index, it.current); - if (list_it.is_direct_hit()) - depth_in_chain.emplace_back(0, list_it); - else - { - LinkedListIt root = find_direct_hit(list_it); - int distance = 1; - for (;;) - { - LinkedListIt next = root.next(*this); - if (next == list_it) - break; - ++distance; - root = next; - } - depth_in_chain.emplace_back(distance, list_it); - } - } - std::sort(depth_in_chain.begin(), depth_in_chain.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); - for (auto it = depth_in_chain.rbegin(), end = depth_in_chain.rend(); it != end; ++it) - { - erase(it->second.it()); - } - - if (begin_it.current->control_bytes[begin_it.index % BlockSize] == Constants::magic_for_empty) - return ++iterator{ begin_it.current, begin_it.index }; - else - return { begin_it.current, begin_it.index }; - } - - size_t erase(const FindKey & key) - { - auto found = find(key); - if (found == end()) - return 0; - else - { - erase(found); - return 1; - } - } - - void clear() - { - if (!num_slots_minus_one) - return; - size_t num_slots = num_slots_minus_one + 1; - size_t num_blocks = num_slots / BlockSize; - if (num_slots % BlockSize) - ++num_blocks; - for (BlockPointer it = entries, end = it + num_blocks; it != end; ++it) - { - for (int i = 0; i < BlockSize; ++i) - { - if (it->control_bytes[i] != Constants::magic_for_empty) - { - AllocatorTraits::destroy(*this, std::addressof(it->data[i])); - it->control_bytes[i] = Constants::magic_for_empty; - } - } - } - num_elements = 0; - } - - void shrink_to_fit() - { - rehash_for_other_container(*this); - } - - void swap(sherwood_v8_table & other) - { - using std::swap; - swap_pointers(other); - swap(static_cast<ArgumentHash &>(*this), static_cast<ArgumentHash &>(other)); - swap(static_cast<ArgumentEqual &>(*this), static_cast<ArgumentEqual &>(other)); - if (AllocatorTraits::propagate_on_container_swap::value) - swap(static_cast<ByteAlloc &>(*this), static_cast<ByteAlloc &>(other)); - } - - size_t size() const - { - return num_elements; - } - size_t max_size() const - { - return (AllocatorTraits::max_size(*this)) / sizeof(T); - } - size_t bucket_count() const - { - return num_slots_minus_one ? num_slots_minus_one + 1 : 0; - } - size_type max_bucket_count() const - { - return (AllocatorTraits::max_size(*this)) / sizeof(T); - } - size_t bucket(const FindKey & key) const - { - return hash_policy.index_for_hash(hash_object(key), num_slots_minus_one); - } - float load_factor() const - { - return static_cast<double>(num_elements) / (num_slots_minus_one + 1); - } - void max_load_factor(float value) - { - _max_load_factor = value; - } - float max_load_factor() const - { - return _max_load_factor; - } - - bool empty() const - { - return num_elements == 0; - } - -private: - BlockPointer entries = BlockType::empty_block(); - size_t num_slots_minus_one = 0; - typename HashPolicySelector<ArgumentHash>::type hash_policy; - float _max_load_factor = 0.9375f; - size_t num_elements = 0; - - size_t num_buckets_for_reserve(size_t num_elements) const - { - return static_cast<size_t>(std::ceil(num_elements / static_cast<double>(_max_load_factor))); - } - void rehash_for_other_container(const sherwood_v8_table & other) - { - rehash(std::min(num_buckets_for_reserve(other.size()), other.bucket_count())); - } - bool is_full() const - { - if (!num_slots_minus_one) - return true; - else - return num_elements + 1 > (num_slots_minus_one + 1) * static_cast<double>(_max_load_factor); - } - - void swap_pointers(sherwood_v8_table & other) - { - using std::swap; - swap(hash_policy, other.hash_policy); - swap(entries, other.entries); - swap(num_slots_minus_one, other.num_slots_minus_one); - swap(num_elements, other.num_elements); - swap(_max_load_factor, other._max_load_factor); - } - - struct LinkedListIt - { - size_t index = 0; - BlockPointer block = nullptr; - - LinkedListIt() - { - } - LinkedListIt(size_t index, BlockPointer block) - : index(index), block(block) - { - } - - iterator it() const - { - return { block, index }; - } - int index_in_block() const - { - return index % BlockSize; - } - bool is_direct_hit() const - { - return (metadata() & Constants::bits_for_direct_hit) == Constants::magic_for_direct_hit; - } - bool is_empty() const - { - return metadata() == Constants::magic_for_empty; - } - bool has_next() const - { - return jump_index() != 0; - } - int8_t jump_index() const - { - return Constants::distance_from_metadata(metadata()); - } - int8_t metadata() const - { - return block->control_bytes[index_in_block()]; - } - void set_metadata(int8_t metadata) - { - block->control_bytes[index_in_block()] = metadata; - } - - LinkedListIt next(sherwood_v8_table & table) const - { - int8_t distance = jump_index(); - size_t next_index = table.hash_policy.keep_in_range(index + Constants::jump_distances[distance], table.num_slots_minus_one); - return { next_index, table.entries + next_index / BlockSize }; - } - void set_next(int8_t jump_index) - { - int8_t & metadata = block->control_bytes[index_in_block()]; - metadata = (metadata & ~Constants::bits_for_distance) | jump_index; - } - void clear_next() - { - set_next(0); - } - - value_type & operator*() const - { - return block->data[index_in_block()]; - } - bool operator!() const - { - return !block; - } - explicit operator bool() const - { - return block != nullptr; - } - bool operator==(const LinkedListIt & other) const - { - return index == other.index; - } - bool operator!=(const LinkedListIt & other) const - { - return !(*this == other); - } - }; - - template<typename... Args> - SKA_NOINLINE(std::pair<iterator, bool>) emplace_direct_hit(LinkedListIt block, Args &&... args) - { - using std::swap; - if (is_full()) - { - grow(); - return emplace(std::forward<Args>(args)...); - } - if (block.metadata() == Constants::magic_for_empty) - { - AllocatorTraits::construct(*this, std::addressof(*block), std::forward<Args>(args)...); - block.set_metadata(Constants::magic_for_direct_hit); - ++num_elements; - return { block.it(), true }; - } - else - { - LinkedListIt parent_block = find_parent_block(block); - std::pair<int8_t, LinkedListIt> free_block = find_free_index(parent_block); - if (!free_block.first) - { - grow(); - return emplace(std::forward<Args>(args)...); - } - value_type new_value(std::forward<Args>(args)...); - for (LinkedListIt it = block;;) - { - AllocatorTraits::construct(*this, std::addressof(*free_block.second), std::move(*it)); - AllocatorTraits::destroy(*this, std::addressof(*it)); - parent_block.set_next(free_block.first); - free_block.second.set_metadata(Constants::magic_for_list_entry); - if (!it.has_next()) - { - it.set_metadata(Constants::magic_for_empty); - break; - } - LinkedListIt next = it.next(*this); - it.set_metadata(Constants::magic_for_empty); - block.set_metadata(Constants::magic_for_reserved); - it = next; - parent_block = free_block.second; - free_block = find_free_index(free_block.second); - if (!free_block.first) - { - grow(); - return emplace(std::move(new_value)); - } - } - AllocatorTraits::construct(*this, std::addressof(*block), std::move(new_value)); - block.set_metadata(Constants::magic_for_direct_hit); - ++num_elements; - return { block.it(), true }; - } - } - - template<typename... Args> - SKA_NOINLINE(std::pair<iterator, bool>) emplace_new_key(LinkedListIt parent, Args &&... args) - { - if (is_full()) - { - grow(); - return emplace(std::forward<Args>(args)...); - } - std::pair<int8_t, LinkedListIt> free_block = find_free_index(parent); - if (!free_block.first) - { - grow(); - return emplace(std::forward<Args>(args)...); - } - AllocatorTraits::construct(*this, std::addressof(*free_block.second), std::forward<Args>(args)...); - free_block.second.set_metadata(Constants::magic_for_list_entry); - parent.set_next(free_block.first); - ++num_elements; - return { free_block.second.it(), true }; - } - - LinkedListIt find_direct_hit(LinkedListIt child) const - { - size_t to_move_hash = hash_object(*child); - size_t to_move_index = hash_policy.index_for_hash(to_move_hash, num_slots_minus_one); - return { to_move_index, entries + to_move_index / BlockSize }; - } - LinkedListIt find_parent_block(LinkedListIt child) - { - LinkedListIt parent_block = find_direct_hit(child); - for (;;) - { - LinkedListIt next = parent_block.next(*this); - if (next == child) - return parent_block; - parent_block = next; - } - } - - std::pair<int8_t, LinkedListIt> find_free_index(LinkedListIt parent) const - { - for (int8_t jump_index = 1; jump_index < Constants::num_jump_distances; ++jump_index) - { - size_t index = hash_policy.keep_in_range(parent.index + Constants::jump_distances[jump_index], num_slots_minus_one); - BlockPointer block = entries + index / BlockSize; - if (block->control_bytes[index % BlockSize] == Constants::magic_for_empty) - return { jump_index, { index, block } }; - } - return { 0, {} }; - } - - void grow() - { - rehash(std::max(size_t(10), 2 * bucket_count())); - } - - size_t calculate_memory_requirement(size_t num_blocks) - { - size_t memory_required = sizeof(BlockType) * num_blocks; - memory_required += BlockSize; // for metadata of past-the-end pointer - return memory_required; - } - - void deallocate_data(BlockPointer begin, size_t num_slots_minus_one) - { - if (begin == BlockType::empty_block()) - return; - - ++num_slots_minus_one; - size_t num_blocks = num_slots_minus_one / BlockSize; - if (num_slots_minus_one % BlockSize) - ++num_blocks; - size_t memory = calculate_memory_requirement(num_blocks); - unsigned char * as_byte_pointer = reinterpret_cast<unsigned char *>(begin); - AllocatorTraits::deallocate(*this, typename AllocatorTraits::pointer(as_byte_pointer), memory); - } - - void reset_to_empty_state() - { - deallocate_data(entries, num_slots_minus_one); - entries = BlockType::empty_block(); - num_slots_minus_one = 0; - hash_policy.reset(); - } - - template<typename U> - size_t hash_object(const U & key) - { - return static_cast<Hasher &>(*this)(key); - } - template<typename U> - size_t hash_object(const U & key) const - { - return static_cast<const Hasher &>(*this)(key); - } - template<typename L, typename R> - bool compares_equal(const L & lhs, const R & rhs) - { - return static_cast<Equal &>(*this)(lhs, rhs); - } - - struct convertible_to_iterator - { - BlockPointer it; - size_t index; - - operator iterator() - { - if (it->control_bytes[index % BlockSize] == Constants::magic_for_empty) - return ++iterator{it, index}; - else - return { it, index }; - } - operator const_iterator() - { - if (it->control_bytes[index % BlockSize] == Constants::magic_for_empty) - return ++iterator{it, index}; - else - return { it, index }; - } - }; -}; -template<typename T, typename Enable = void> -struct AlignmentOr8Bytes -{ - static constexpr size_t value = 8; -}; -template<typename T> -struct AlignmentOr8Bytes<T, typename std::enable_if<alignof(T) >= 1>::type> -{ - static constexpr size_t value = alignof(T); -}; -template<typename... Args> -struct CalculateBytellBlockSize; -template<typename First, typename... More> -struct CalculateBytellBlockSize<First, More...> -{ - static constexpr size_t this_value = AlignmentOr8Bytes<First>::value; - static constexpr size_t base_value = CalculateBytellBlockSize<More...>::value; - static constexpr size_t value = this_value > base_value ? this_value : base_value; -}; -template<> -struct CalculateBytellBlockSize<> -{ - static constexpr size_t value = 8; -}; -} - -template<typename K, typename V, typename H = std::hash<K>, typename E = std::equal_to<K>, typename A = std::allocator<std::pair<K, V> > > -class bytell_hash_map - : public detailv8::sherwood_v8_table - < - std::pair<K, V>, - K, - H, - detailv8::KeyOrValueHasher<K, std::pair<K, V>, H>, - E, - detailv8::KeyOrValueEquality<K, std::pair<K, V>, E>, - A, - typename std::allocator_traits<A>::template rebind_alloc<unsigned char>, - detailv8::CalculateBytellBlockSize<K, V>::value - > -{ - using Table = detailv8::sherwood_v8_table - < - std::pair<K, V>, - K, - H, - detailv8::KeyOrValueHasher<K, std::pair<K, V>, H>, - E, - detailv8::KeyOrValueEquality<K, std::pair<K, V>, E>, - A, - typename std::allocator_traits<A>::template rebind_alloc<unsigned char>, - detailv8::CalculateBytellBlockSize<K, V>::value - >; -public: - - using key_type = K; - using mapped_type = V; - - using Table::Table; - bytell_hash_map() - { - } - - inline V & operator[](const K & key) - { - return emplace(key, convertible_to_value()).first->second; - } - inline V & operator[](K && key) - { - return emplace(std::move(key), convertible_to_value()).first->second; - } - V & at(const K & key) - { - auto found = this->find(key); - if (found == this->end()) - throw std::out_of_range("Argument passed to at() was not in the map."); - return found->second; - } - const V & at(const K & key) const - { - auto found = this->find(key); - if (found == this->end()) - throw std::out_of_range("Argument passed to at() was not in the map."); - return found->second; - } - - using Table::emplace; - std::pair<typename Table::iterator, bool> emplace() - { - return emplace(key_type(), convertible_to_value()); - } - template<typename M> - std::pair<typename Table::iterator, bool> insert_or_assign(const key_type & key, M && m) - { - auto emplace_result = emplace(key, std::forward<M>(m)); - if (!emplace_result.second) - emplace_result.first->second = std::forward<M>(m); - return emplace_result; - } - template<typename M> - std::pair<typename Table::iterator, bool> insert_or_assign(key_type && key, M && m) - { - auto emplace_result = emplace(std::move(key), std::forward<M>(m)); - if (!emplace_result.second) - emplace_result.first->second = std::forward<M>(m); - return emplace_result; - } - template<typename M> - typename Table::iterator insert_or_assign(typename Table::const_iterator, const key_type & key, M && m) - { - return insert_or_assign(key, std::forward<M>(m)).first; - } - template<typename M> - typename Table::iterator insert_or_assign(typename Table::const_iterator, key_type && key, M && m) - { - return insert_or_assign(std::move(key), std::forward<M>(m)).first; - } - - friend bool operator==(const bytell_hash_map & lhs, const bytell_hash_map & rhs) - { - if (lhs.size() != rhs.size()) - return false; - for (const typename Table::value_type & value : lhs) - { - auto found = rhs.find(value.first); - if (found == rhs.end()) - return false; - else if (value.second != found->second) - return false; - } - return true; - } - friend bool operator!=(const bytell_hash_map & lhs, const bytell_hash_map & rhs) - { - return !(lhs == rhs); - } - -private: - struct convertible_to_value - { - operator V() const - { - return V(); - } - }; -}; - -template<typename T, typename H = std::hash<T>, typename E = std::equal_to<T>, typename A = std::allocator<T> > -class bytell_hash_set - : public detailv8::sherwood_v8_table - < - T, - T, - H, - detailv8::functor_storage<size_t, H>, - E, - detailv8::functor_storage<bool, E>, - A, - typename std::allocator_traits<A>::template rebind_alloc<unsigned char>, - detailv8::CalculateBytellBlockSize<T>::value - > -{ - using Table = detailv8::sherwood_v8_table - < - T, - T, - H, - detailv8::functor_storage<size_t, H>, - E, - detailv8::functor_storage<bool, E>, - A, - typename std::allocator_traits<A>::template rebind_alloc<unsigned char>, - detailv8::CalculateBytellBlockSize<T>::value - >; -public: - - using key_type = T; - - using Table::Table; - bytell_hash_set() - { - } - - template<typename... Args> - std::pair<typename Table::iterator, bool> emplace(Args &&... args) - { - return Table::emplace(T(std::forward<Args>(args)...)); - } - std::pair<typename Table::iterator, bool> emplace(const key_type & arg) - { - return Table::emplace(arg); - } - std::pair<typename Table::iterator, bool> emplace(key_type & arg) - { - return Table::emplace(arg); - } - std::pair<typename Table::iterator, bool> emplace(const key_type && arg) - { - return Table::emplace(std::move(arg)); - } - std::pair<typename Table::iterator, bool> emplace(key_type && arg) - { - return Table::emplace(std::move(arg)); - } - - friend bool operator==(const bytell_hash_set & lhs, const bytell_hash_set & rhs) - { - if (lhs.size() != rhs.size()) - return false; - for (const T & value : lhs) - { - if (rhs.find(value) == rhs.end()) - return false; - } - return true; - } - friend bool operator!=(const bytell_hash_set & lhs, const bytell_hash_set & rhs) - { - return !(lhs == rhs); - } -}; - -} // end namespace ska diff --git a/benchmarks/external/tsl/hopscotch_growth_policy.h b/benchmarks/external/tsl/hopscotch_growth_policy.h deleted file mode 100644 index 0e463868..00000000 --- a/benchmarks/external/tsl/hopscotch_growth_policy.h +++ /dev/null @@ -1,404 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2018 Thibaut Goetghebuer-Planchon <[email protected]> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef TSL_HOPSCOTCH_GROWTH_POLICY_H -#define TSL_HOPSCOTCH_GROWTH_POLICY_H - -#include <algorithm> -#include <array> -#include <climits> -#include <cmath> -#include <cstddef> -#include <cstdint> -#include <iterator> -#include <limits> -#include <ratio> -#include <stdexcept> - -/** - * Only activate tsl_hh_assert if TSL_DEBUG is defined. - * This way we avoid the performance hit when NDEBUG is not defined with assert - * as tsl_hh_assert is used a lot (people usually compile with "-O3" and not - * "-O3 -DNDEBUG"). - */ -#ifdef TSL_DEBUG -#define tsl_hh_assert(expr) assert(expr) -#else -#define tsl_hh_assert(expr) (static_cast<void>(0)) -#endif - -/** - * If exceptions are enabled, throw the exception passed in parameter, otherwise - * call std::terminate. - */ -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || \ - (defined(_MSC_VER) && defined(_CPPUNWIND))) && \ - !defined(TSL_NO_EXCEPTIONS) -#define TSL_HH_THROW_OR_TERMINATE(ex, msg) throw ex(msg) -#else -#define TSL_HH_NO_EXCEPTIONS -#ifdef NDEBUG -#define TSL_HH_THROW_OR_TERMINATE(ex, msg) std::terminate() -#else -#include <iostream> -#define TSL_HH_THROW_OR_TERMINATE(ex, msg) \ - do { \ - std::cerr << msg << std::endl; \ - std::terminate(); \ - } while (0) -#endif -#endif - -namespace tsl { -namespace hh { - -/** - * Grow the hash table by a factor of GrowthFactor keeping the bucket count to a - * power of two. It allows the table to use a mask operation instead of a modulo - * operation to map a hash to a bucket. - * - * GrowthFactor must be a power of two >= 2. - */ -template <std::size_t GrowthFactor> -class power_of_two_growth_policy { - public: - /** - * Called on the hash table creation and on rehash. The number of buckets for - * the table is passed in parameter. This number is a minimum, the policy may - * update this value with a higher value if needed (but not lower). - * - * If 0 is given, min_bucket_count_in_out must still be 0 after the policy - * creation and bucket_for_hash must always return 0 in this case. - */ - explicit power_of_two_growth_policy(std::size_t& min_bucket_count_in_out) { - if (min_bucket_count_in_out > max_bucket_count()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - if (min_bucket_count_in_out > 0) { - min_bucket_count_in_out = - round_up_to_power_of_two(min_bucket_count_in_out); - m_mask = min_bucket_count_in_out - 1; - } else { - m_mask = 0; - } - } - - /** - * Return the bucket [0, bucket_count()) to which the hash belongs. - * If bucket_count() is 0, it must always return 0. - */ - std::size_t bucket_for_hash(std::size_t hash) const noexcept { - return hash & m_mask; - } - - /** - * Return the bucket count to use when the bucket array grows on rehash. - */ - std::size_t next_bucket_count() const { - if ((m_mask + 1) > max_bucket_count() / GrowthFactor) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - return (m_mask + 1) * GrowthFactor; - } - - /** - * Return the maximum number of buckets supported by the policy. - */ - std::size_t max_bucket_count() const { - // Largest power of two. - return (std::numeric_limits<std::size_t>::max() / 2) + 1; - } - - /** - * Reset the growth policy as if it was created with a bucket count of 0. - * After a clear, the policy must always return 0 when bucket_for_hash is - * called. - */ - void clear() noexcept { m_mask = 0; } - - private: - static std::size_t round_up_to_power_of_two(std::size_t value) { - if (is_power_of_two(value)) { - return value; - } - - if (value == 0) { - return 1; - } - - --value; - for (std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) { - value |= value >> i; - } - - return value + 1; - } - - static constexpr bool is_power_of_two(std::size_t value) { - return value != 0 && (value & (value - 1)) == 0; - } - - private: - static_assert(is_power_of_two(GrowthFactor) && GrowthFactor >= 2, - "GrowthFactor must be a power of two >= 2."); - - std::size_t m_mask; -}; - -/** - * Grow the hash table by GrowthFactor::num / GrowthFactor::den and use a modulo - * to map a hash to a bucket. Slower but it can be useful if you want a slower - * growth. - */ -template <class GrowthFactor = std::ratio<3, 2>> -class mod_growth_policy { - public: - explicit mod_growth_policy(std::size_t& min_bucket_count_in_out) { - if (min_bucket_count_in_out > max_bucket_count()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - if (min_bucket_count_in_out > 0) { - m_mod = min_bucket_count_in_out; - } else { - m_mod = 1; - } - } - - std::size_t bucket_for_hash(std::size_t hash) const noexcept { - return hash % m_mod; - } - - std::size_t next_bucket_count() const { - if (m_mod == max_bucket_count()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - const double next_bucket_count = - std::ceil(double(m_mod) * REHASH_SIZE_MULTIPLICATION_FACTOR); - if (!std::isnormal(next_bucket_count)) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - if (next_bucket_count > double(max_bucket_count())) { - return max_bucket_count(); - } else { - return std::size_t(next_bucket_count); - } - } - - std::size_t max_bucket_count() const { return MAX_BUCKET_COUNT; } - - void clear() noexcept { m_mod = 1; } - - private: - static constexpr double REHASH_SIZE_MULTIPLICATION_FACTOR = - 1.0 * GrowthFactor::num / GrowthFactor::den; - static const std::size_t MAX_BUCKET_COUNT = - std::size_t(double(std::numeric_limits<std::size_t>::max() / - REHASH_SIZE_MULTIPLICATION_FACTOR)); - - static_assert(REHASH_SIZE_MULTIPLICATION_FACTOR >= 1.1, - "Growth factor should be >= 1.1."); - - std::size_t m_mod; -}; - -namespace detail { - -#if SIZE_MAX >= ULLONG_MAX -#define TSL_HH_NB_PRIMES 51 -#elif SIZE_MAX >= ULONG_MAX -#define TSL_HH_NB_PRIMES 40 -#else -#define TSL_HH_NB_PRIMES 23 -#endif - -static constexpr const std::array<std::size_t, TSL_HH_NB_PRIMES> PRIMES = {{ - 1u, - 5u, - 17u, - 29u, - 37u, - 53u, - 67u, - 79u, - 97u, - 131u, - 193u, - 257u, - 389u, - 521u, - 769u, - 1031u, - 1543u, - 2053u, - 3079u, - 6151u, - 12289u, - 24593u, - 49157u, -#if SIZE_MAX >= ULONG_MAX - 98317ul, - 196613ul, - 393241ul, - 786433ul, - 1572869ul, - 3145739ul, - 6291469ul, - 12582917ul, - 25165843ul, - 50331653ul, - 100663319ul, - 201326611ul, - 402653189ul, - 805306457ul, - 1610612741ul, - 3221225473ul, - 4294967291ul, -#endif -#if SIZE_MAX >= ULLONG_MAX - 6442450939ull, - 12884901893ull, - 25769803751ull, - 51539607551ull, - 103079215111ull, - 206158430209ull, - 412316860441ull, - 824633720831ull, - 1649267441651ull, - 3298534883309ull, - 6597069766657ull, -#endif -}}; - -template <unsigned int IPrime> -static constexpr std::size_t mod(std::size_t hash) { - return hash % PRIMES[IPrime]; -} - -// MOD_PRIME[iprime](hash) returns hash % PRIMES[iprime]. This table allows for -// faster modulo as the compiler can optimize the modulo code better with a -// constant known at the compilation. -static constexpr const std::array<std::size_t (*)(std::size_t), - TSL_HH_NB_PRIMES> - MOD_PRIME = {{ - &mod<0>, &mod<1>, &mod<2>, &mod<3>, &mod<4>, &mod<5>, - &mod<6>, &mod<7>, &mod<8>, &mod<9>, &mod<10>, &mod<11>, - &mod<12>, &mod<13>, &mod<14>, &mod<15>, &mod<16>, &mod<17>, - &mod<18>, &mod<19>, &mod<20>, &mod<21>, &mod<22>, -#if SIZE_MAX >= ULONG_MAX - &mod<23>, &mod<24>, &mod<25>, &mod<26>, &mod<27>, &mod<28>, - &mod<29>, &mod<30>, &mod<31>, &mod<32>, &mod<33>, &mod<34>, - &mod<35>, &mod<36>, &mod<37>, &mod<38>, &mod<39>, -#endif -#if SIZE_MAX >= ULLONG_MAX - &mod<40>, &mod<41>, &mod<42>, &mod<43>, &mod<44>, &mod<45>, - &mod<46>, &mod<47>, &mod<48>, &mod<49>, &mod<50>, -#endif - }}; - -} // namespace detail - -/** - * Grow the hash table by using prime numbers as bucket count. Slower than - * tsl::hh::power_of_two_growth_policy in general but will probably distribute - * the values around better in the buckets with a poor hash function. - * - * To allow the compiler to optimize the modulo operation, a lookup table is - * used with constant primes numbers. - * - * With a switch the code would look like: - * \code - * switch(iprime) { // iprime is the current prime of the hash table - * case 0: hash % 5ul; - * break; - * case 1: hash % 17ul; - * break; - * case 2: hash % 29ul; - * break; - * ... - * } - * \endcode - * - * Due to the constant variable in the modulo the compiler is able to optimize - * the operation by a series of multiplications, substractions and shifts. - * - * The 'hash % 5' could become something like 'hash - (hash * 0xCCCCCCCD) >> 34) - * * 5' in a 64 bits environment. - */ -class prime_growth_policy { - public: - explicit prime_growth_policy(std::size_t& min_bucket_count_in_out) { - auto it_prime = std::lower_bound( - detail::PRIMES.begin(), detail::PRIMES.end(), min_bucket_count_in_out); - if (it_prime == detail::PRIMES.end()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - m_iprime = static_cast<unsigned int>( - std::distance(detail::PRIMES.begin(), it_prime)); - if (min_bucket_count_in_out > 0) { - min_bucket_count_in_out = *it_prime; - } else { - min_bucket_count_in_out = 0; - } - } - - std::size_t bucket_for_hash(std::size_t hash) const noexcept { - return detail::MOD_PRIME[m_iprime](hash); - } - - std::size_t next_bucket_count() const { - if (m_iprime + 1 >= detail::PRIMES.size()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The hash table exceeds its maximum size."); - } - - return detail::PRIMES[m_iprime + 1]; - } - - std::size_t max_bucket_count() const { return detail::PRIMES.back(); } - - void clear() noexcept { m_iprime = 0; } - - private: - unsigned int m_iprime; - - static_assert(std::numeric_limits<decltype(m_iprime)>::max() >= - detail::PRIMES.size(), - "The type of m_iprime is not big enough."); -}; - -} // namespace hh -} // namespace tsl - -#endif diff --git a/benchmarks/external/tsl/hopscotch_hash.h b/benchmarks/external/tsl/hopscotch_hash.h deleted file mode 100644 index 6611ff93..00000000 --- a/benchmarks/external/tsl/hopscotch_hash.h +++ /dev/null @@ -1,1894 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2017 Thibaut Goetghebuer-Planchon <[email protected]> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef TSL_HOPSCOTCH_HASH_H -#define TSL_HOPSCOTCH_HASH_H - -#include <algorithm> -#include <cassert> -#include <cmath> -#include <cstddef> -#include <cstdint> -#include <exception> -#include <functional> -#include <initializer_list> -#include <iterator> -#include <limits> -#include <memory> -#include <new> -#include <stdexcept> -#include <tuple> -#include <type_traits> -#include <utility> -#include <vector> - -#include "hopscotch_growth_policy.h" - -#if (defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ < 9)) -#define TSL_HH_NO_RANGE_ERASE_WITH_CONST_ITERATOR -#endif - -namespace tsl { -namespace detail_hopscotch_hash { - -template <typename T> -struct make_void { - using type = void; -}; - -template <typename T, typename = void> -struct has_is_transparent : std::false_type {}; - -template <typename T> -struct has_is_transparent<T, - typename make_void<typename T::is_transparent>::type> - : std::true_type {}; - -template <typename T, typename = void> -struct has_key_compare : std::false_type {}; - -template <typename T> -struct has_key_compare<T, typename make_void<typename T::key_compare>::type> - : std::true_type {}; - -template <typename U> -struct is_power_of_two_policy : std::false_type {}; - -template <std::size_t GrowthFactor> -struct is_power_of_two_policy<tsl::hh::power_of_two_growth_policy<GrowthFactor>> - : std::true_type {}; - -template <typename T, typename U> -static T numeric_cast(U value, - const char* error_message = "numeric_cast() failed.") { - T ret = static_cast<T>(value); - if (static_cast<U>(ret) != value) { - TSL_HH_THROW_OR_TERMINATE(std::runtime_error, error_message); - } - - const bool is_same_signedness = - (std::is_unsigned<T>::value && std::is_unsigned<U>::value) || - (std::is_signed<T>::value && std::is_signed<U>::value); - if (!is_same_signedness && (ret < T{}) != (value < U{})) { - TSL_HH_THROW_OR_TERMINATE(std::runtime_error, error_message); - } - - return ret; -} - -/* - * smallest_type_for_min_bits::type returns the smallest type that can fit - * MinBits. - */ -static const std::size_t SMALLEST_TYPE_MAX_BITS_SUPPORTED = 64; -template <unsigned int MinBits, typename Enable = void> -class smallest_type_for_min_bits {}; - -template <unsigned int MinBits> -class smallest_type_for_min_bits< - MinBits, typename std::enable_if<(MinBits > 0) && (MinBits <= 8)>::type> { - public: - using type = std::uint_least8_t; -}; - -template <unsigned int MinBits> -class smallest_type_for_min_bits< - MinBits, typename std::enable_if<(MinBits > 8) && (MinBits <= 16)>::type> { - public: - using type = std::uint_least16_t; -}; - -template <unsigned int MinBits> -class smallest_type_for_min_bits< - MinBits, typename std::enable_if<(MinBits > 16) && (MinBits <= 32)>::type> { - public: - using type = std::uint_least32_t; -}; - -template <unsigned int MinBits> -class smallest_type_for_min_bits< - MinBits, typename std::enable_if<(MinBits > 32) && (MinBits <= 64)>::type> { - public: - using type = std::uint_least64_t; -}; - -/* - * Each bucket may store up to three elements: - * - An aligned storage to store a value_type object with placement-new. - * - An (optional) hash of the value in the bucket. - * - An unsigned integer of type neighborhood_bitmap used to tell us which - * buckets in the neighborhood of the current bucket contain a value with a hash - * belonging to the current bucket. - * - * For a bucket 'bct', a bit 'i' (counting from 0 and from the least significant - * bit to the most significant) set to 1 means that the bucket 'bct + i' - * contains a value with a hash belonging to bucket 'bct'. The bits used for - * that, start from the third least significant bit. The two least significant - * bits are reserved: - * - The least significant bit is set to 1 if there is a value in the bucket - * storage. - * - The second least significant bit is set to 1 if there is an overflow. More - * than NeighborhoodSize values give the same hash, all overflow values are - * stored in the m_overflow_elements list of the map. - * - * Details regarding hopscotch hashing an its implementation can be found here: - * https://tessil.github.io/2016/08/29/hopscotch-hashing.html - */ -static const std::size_t NB_RESERVED_BITS_IN_NEIGHBORHOOD = 2; - -using truncated_hash_type = std::uint_least32_t; - -/** - * Helper class that stores a truncated hash if StoreHash is true and nothing - * otherwise. - */ -template <bool StoreHash> -class hopscotch_bucket_hash { - public: - bool bucket_hash_equal(std::size_t /*hash*/) const noexcept { return true; } - - truncated_hash_type truncated_bucket_hash() const noexcept { return 0; } - - protected: - void copy_hash(const hopscotch_bucket_hash&) noexcept {} - - void set_hash(truncated_hash_type /*hash*/) noexcept {} -}; - -template <> -class hopscotch_bucket_hash<true> { - public: - bool bucket_hash_equal(std::size_t hash) const noexcept { - return m_hash == truncated_hash_type(hash); - } - - truncated_hash_type truncated_bucket_hash() const noexcept { return m_hash; } - - protected: - void copy_hash(const hopscotch_bucket_hash& bucket) noexcept { - m_hash = bucket.m_hash; - } - - void set_hash(truncated_hash_type hash) noexcept { m_hash = hash; } - - private: - truncated_hash_type m_hash; -}; - -template <typename ValueType, unsigned int NeighborhoodSize, bool StoreHash> -class hopscotch_bucket : public hopscotch_bucket_hash<StoreHash> { - private: - static const std::size_t MIN_NEIGHBORHOOD_SIZE = 4; - static const std::size_t MAX_NEIGHBORHOOD_SIZE = - SMALLEST_TYPE_MAX_BITS_SUPPORTED - NB_RESERVED_BITS_IN_NEIGHBORHOOD; - - static_assert(NeighborhoodSize >= 4, "NeighborhoodSize should be >= 4."); - // We can't put a variable in the message, ensure coherence - static_assert(MIN_NEIGHBORHOOD_SIZE == 4, ""); - - static_assert(NeighborhoodSize <= 62, "NeighborhoodSize should be <= 62."); - // We can't put a variable in the message, ensure coherence - static_assert(MAX_NEIGHBORHOOD_SIZE == 62, ""); - - static_assert(!StoreHash || NeighborhoodSize <= 30, - "NeighborhoodSize should be <= 30 if StoreHash is true."); - // We can't put a variable in the message, ensure coherence - static_assert(MAX_NEIGHBORHOOD_SIZE - 32 == 30, ""); - - using bucket_hash = hopscotch_bucket_hash<StoreHash>; - - public: - using value_type = ValueType; - using neighborhood_bitmap = typename smallest_type_for_min_bits< - NeighborhoodSize + NB_RESERVED_BITS_IN_NEIGHBORHOOD>::type; - - hopscotch_bucket() noexcept : bucket_hash(), m_neighborhood_infos(0) { - tsl_hh_assert(empty()); - } - - hopscotch_bucket(const hopscotch_bucket& bucket) noexcept( - std::is_nothrow_copy_constructible<value_type>::value) - : bucket_hash(bucket), m_neighborhood_infos(0) { - if (!bucket.empty()) { - ::new (static_cast<void*>(std::addressof(m_value))) - value_type(bucket.value()); - } - - m_neighborhood_infos = bucket.m_neighborhood_infos; - } - - hopscotch_bucket(hopscotch_bucket&& bucket) noexcept( - std::is_nothrow_move_constructible<value_type>::value) - : bucket_hash(std::move(bucket)), m_neighborhood_infos(0) { - if (!bucket.empty()) { - ::new (static_cast<void*>(std::addressof(m_value))) - value_type(std::move(bucket.value())); - } - - m_neighborhood_infos = bucket.m_neighborhood_infos; - } - - hopscotch_bucket& operator=(const hopscotch_bucket& bucket) noexcept( - std::is_nothrow_copy_constructible<value_type>::value) { - if (this != &bucket) { - remove_value(); - - bucket_hash::operator=(bucket); - if (!bucket.empty()) { - ::new (static_cast<void*>(std::addressof(m_value))) - value_type(bucket.value()); - } - - m_neighborhood_infos = bucket.m_neighborhood_infos; - } - - return *this; - } - - hopscotch_bucket& operator=(hopscotch_bucket&&) = delete; - - ~hopscotch_bucket() noexcept { - if (!empty()) { - destroy_value(); - } - } - - neighborhood_bitmap neighborhood_infos() const noexcept { - return neighborhood_bitmap(m_neighborhood_infos >> - NB_RESERVED_BITS_IN_NEIGHBORHOOD); - } - - void set_overflow(bool has_overflow) noexcept { - if (has_overflow) { - m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos | 2); - } else { - m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos & ~2); - } - } - - bool has_overflow() const noexcept { return (m_neighborhood_infos & 2) != 0; } - - bool empty() const noexcept { return (m_neighborhood_infos & 1) == 0; } - - void toggle_neighbor_presence(std::size_t ineighbor) noexcept { - tsl_hh_assert(ineighbor <= NeighborhoodSize); - m_neighborhood_infos = neighborhood_bitmap( - m_neighborhood_infos ^ - (1ull << (ineighbor + NB_RESERVED_BITS_IN_NEIGHBORHOOD))); - } - - bool check_neighbor_presence(std::size_t ineighbor) const noexcept { - tsl_hh_assert(ineighbor <= NeighborhoodSize); - if (((m_neighborhood_infos >> - (ineighbor + NB_RESERVED_BITS_IN_NEIGHBORHOOD)) & - 1) == 1) { - return true; - } - - return false; - } - - value_type& value() noexcept { - tsl_hh_assert(!empty()); -#if defined(__cplusplus) && __cplusplus >= 201703L - return *std::launder( - reinterpret_cast<value_type*>(std::addressof(m_value))); -#else - return *reinterpret_cast<value_type*>(std::addressof(m_value)); -#endif - } - - const value_type& value() const noexcept { - tsl_hh_assert(!empty()); -#if defined(__cplusplus) && __cplusplus >= 201703L - return *std::launder( - reinterpret_cast<const value_type*>(std::addressof(m_value))); -#else - return *reinterpret_cast<const value_type*>(std::addressof(m_value)); -#endif - } - - template <typename... Args> - void set_value_of_empty_bucket(truncated_hash_type hash, - Args&&... value_type_args) { - tsl_hh_assert(empty()); - - ::new (static_cast<void*>(std::addressof(m_value))) - value_type(std::forward<Args>(value_type_args)...); - set_empty(false); - this->set_hash(hash); - } - - void swap_value_into_empty_bucket(hopscotch_bucket& empty_bucket) { - tsl_hh_assert(empty_bucket.empty()); - if (!empty()) { - ::new (static_cast<void*>(std::addressof(empty_bucket.m_value))) - value_type(std::move(value())); - empty_bucket.copy_hash(*this); - empty_bucket.set_empty(false); - - destroy_value(); - set_empty(true); - } - } - - void remove_value() noexcept { - if (!empty()) { - destroy_value(); - set_empty(true); - } - } - - void clear() noexcept { - if (!empty()) { - destroy_value(); - } - - m_neighborhood_infos = 0; - tsl_hh_assert(empty()); - } - - static truncated_hash_type truncate_hash(std::size_t hash) noexcept { - return truncated_hash_type(hash); - } - - private: - void set_empty(bool is_empty) noexcept { - if (is_empty) { - m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos & ~1); - } else { - m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos | 1); - } - } - - void destroy_value() noexcept { - tsl_hh_assert(!empty()); - value().~value_type(); - } - - private: - using storage = typename std::aligned_storage<sizeof(value_type), - alignof(value_type)>::type; - - neighborhood_bitmap m_neighborhood_infos; - storage m_value; -}; - -/** - * Internal common class used by (b)hopscotch_map and (b)hopscotch_set. - * - * ValueType is what will be stored by hopscotch_hash (usually std::pair<Key, T> - * for a map and Key for a set). - * - * KeySelect should be a FunctionObject which takes a ValueType in parameter and - * returns a reference to the key. - * - * ValueSelect should be a FunctionObject which takes a ValueType in parameter - * and returns a reference to the value. ValueSelect should be void if there is - * no value (in a set for example). - * - * OverflowContainer will be used as containers for overflown elements. Usually - * it should be a list<ValueType> or a set<Key>/map<Key, T>. - */ -template <class ValueType, class KeySelect, class ValueSelect, class Hash, - class KeyEqual, class Allocator, unsigned int NeighborhoodSize, - bool StoreHash, class GrowthPolicy, class OverflowContainer> -class hopscotch_hash : private Hash, private KeyEqual, private GrowthPolicy { - private: - template <typename U> - using has_mapped_type = - typename std::integral_constant<bool, !std::is_same<U, void>::value>; - - static_assert( - noexcept(std::declval<GrowthPolicy>().bucket_for_hash(std::size_t(0))), - "GrowthPolicy::bucket_for_hash must be noexcept."); - static_assert(noexcept(std::declval<GrowthPolicy>().clear()), - "GrowthPolicy::clear must be noexcept."); - - public: - template <bool IsConst> - class hopscotch_iterator; - - using key_type = typename KeySelect::key_type; - using value_type = ValueType; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using hasher = Hash; - using key_equal = KeyEqual; - using allocator_type = Allocator; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; - using iterator = hopscotch_iterator<false>; - using const_iterator = hopscotch_iterator<true>; - - private: - using hopscotch_bucket = - tsl::detail_hopscotch_hash::hopscotch_bucket<ValueType, NeighborhoodSize, - StoreHash>; - using neighborhood_bitmap = typename hopscotch_bucket::neighborhood_bitmap; - - using buckets_allocator = typename std::allocator_traits< - allocator_type>::template rebind_alloc<hopscotch_bucket>; - using buckets_container_type = - std::vector<hopscotch_bucket, buckets_allocator>; - - using overflow_container_type = OverflowContainer; - - static_assert(std::is_same<typename overflow_container_type::value_type, - ValueType>::value, - "OverflowContainer should have ValueType as type."); - - static_assert(std::is_same<typename overflow_container_type::allocator_type, - Allocator>::value, - "Invalid allocator, not the same type as the value_type."); - - using iterator_buckets = typename buckets_container_type::iterator; - using const_iterator_buckets = - typename buckets_container_type::const_iterator; - - using iterator_overflow = typename overflow_container_type::iterator; - using const_iterator_overflow = - typename overflow_container_type::const_iterator; - - public: - /** - * The `operator*()` and `operator->()` methods return a const reference and - * const pointer respectively to the stored value type. - * - * In case of a map, to get a modifiable reference to the value associated to - * a key (the `.second` in the stored pair), you have to call `value()`. - */ - template <bool IsConst> - class hopscotch_iterator { - friend class hopscotch_hash; - - private: - using iterator_bucket = typename std::conditional< - IsConst, typename hopscotch_hash::const_iterator_buckets, - typename hopscotch_hash::iterator_buckets>::type; - using iterator_overflow = typename std::conditional< - IsConst, typename hopscotch_hash::const_iterator_overflow, - typename hopscotch_hash::iterator_overflow>::type; - - hopscotch_iterator(iterator_bucket buckets_iterator, - iterator_bucket buckets_end_iterator, - iterator_overflow overflow_iterator) noexcept - : m_buckets_iterator(buckets_iterator), - m_buckets_end_iterator(buckets_end_iterator), - m_overflow_iterator(overflow_iterator) {} - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = const typename hopscotch_hash::value_type; - using difference_type = std::ptrdiff_t; - using reference = value_type&; - using pointer = value_type*; - - hopscotch_iterator() noexcept {} - - // Copy constructor from iterator to const_iterator. - template <bool TIsConst = IsConst, - typename std::enable_if<TIsConst>::type* = nullptr> - hopscotch_iterator(const hopscotch_iterator<!TIsConst>& other) noexcept - : m_buckets_iterator(other.m_buckets_iterator), - m_buckets_end_iterator(other.m_buckets_end_iterator), - m_overflow_iterator(other.m_overflow_iterator) {} - - hopscotch_iterator(const hopscotch_iterator& other) = default; - hopscotch_iterator(hopscotch_iterator&& other) = default; - hopscotch_iterator& operator=(const hopscotch_iterator& other) = default; - hopscotch_iterator& operator=(hopscotch_iterator&& other) = default; - - const typename hopscotch_hash::key_type& key() const { - if (m_buckets_iterator != m_buckets_end_iterator) { - return KeySelect()(m_buckets_iterator->value()); - } - - return KeySelect()(*m_overflow_iterator); - } - - template < - class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - typename std::conditional<IsConst, const typename U::value_type&, - typename U::value_type&>::type - value() const { - if (m_buckets_iterator != m_buckets_end_iterator) { - return U()(m_buckets_iterator->value()); - } - - return U()(*m_overflow_iterator); - } - - reference operator*() const { - if (m_buckets_iterator != m_buckets_end_iterator) { - return m_buckets_iterator->value(); - } - - return *m_overflow_iterator; - } - - pointer operator->() const { - if (m_buckets_iterator != m_buckets_end_iterator) { - return std::addressof(m_buckets_iterator->value()); - } - - return std::addressof(*m_overflow_iterator); - } - - hopscotch_iterator& operator++() { - if (m_buckets_iterator == m_buckets_end_iterator) { - ++m_overflow_iterator; - return *this; - } - - do { - ++m_buckets_iterator; - } while (m_buckets_iterator != m_buckets_end_iterator && - m_buckets_iterator->empty()); - - return *this; - } - - hopscotch_iterator operator++(int) { - hopscotch_iterator tmp(*this); - ++*this; - - return tmp; - } - - friend bool operator==(const hopscotch_iterator& lhs, - const hopscotch_iterator& rhs) { - return lhs.m_buckets_iterator == rhs.m_buckets_iterator && - lhs.m_overflow_iterator == rhs.m_overflow_iterator; - } - - friend bool operator!=(const hopscotch_iterator& lhs, - const hopscotch_iterator& rhs) { - return !(lhs == rhs); - } - - private: - iterator_bucket m_buckets_iterator; - iterator_bucket m_buckets_end_iterator; - iterator_overflow m_overflow_iterator; - }; - - public: - template < - class OC = OverflowContainer, - typename std::enable_if<!has_key_compare<OC>::value>::type* = nullptr> - hopscotch_hash(size_type bucket_count, const Hash& hash, - const KeyEqual& equal, const Allocator& alloc, - float max_load_factor) - : Hash(hash), - KeyEqual(equal), - GrowthPolicy(bucket_count), - m_buckets_data(alloc), - m_overflow_elements(alloc), - m_buckets(static_empty_bucket_ptr()), - m_nb_elements(0) { - if (bucket_count > max_bucket_count()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The map exceeds its maximum size."); - } - - if (bucket_count > 0) { - static_assert(NeighborhoodSize - 1 > 0, ""); - - // Can't directly construct with the appropriate size in the initializer - // as m_buckets_data(bucket_count, alloc) is not supported by GCC 4.8 - m_buckets_data.resize(bucket_count + NeighborhoodSize - 1); - m_buckets = m_buckets_data.data(); - } - - this->max_load_factor(max_load_factor); - - // Check in the constructor instead of outside of a function to avoid - // compilation issues when value_type is not complete. - static_assert(std::is_nothrow_move_constructible<value_type>::value || - std::is_copy_constructible<value_type>::value, - "value_type must be either copy constructible or nothrow " - "move constructible."); - } - - template < - class OC = OverflowContainer, - typename std::enable_if<has_key_compare<OC>::value>::type* = nullptr> - hopscotch_hash(size_type bucket_count, const Hash& hash, - const KeyEqual& equal, const Allocator& alloc, - float max_load_factor, const typename OC::key_compare& comp) - : Hash(hash), - KeyEqual(equal), - GrowthPolicy(bucket_count), - m_buckets_data(alloc), - m_overflow_elements(comp, alloc), - m_buckets(static_empty_bucket_ptr()), - m_nb_elements(0) { - if (bucket_count > max_bucket_count()) { - TSL_HH_THROW_OR_TERMINATE(std::length_error, - "The map exceeds its maximum size."); - } - - if (bucket_count > 0) { - static_assert(NeighborhoodSize - 1 > 0, ""); - - // Can't directly construct with the appropriate size in the initializer - // as m_buckets_data(bucket_count, alloc) is not supported by GCC 4.8 - m_buckets_data.resize(bucket_count + NeighborhoodSize - 1); - m_buckets = m_buckets_data.data(); - } - - this->max_load_factor(max_load_factor); - - // Check in the constructor instead of outside of a function to avoid - // compilation issues when value_type is not complete. - static_assert(std::is_nothrow_move_constructible<value_type>::value || - std::is_copy_constructible<value_type>::value, - "value_type must be either copy constructible or nothrow " - "move constructible."); - } - - hopscotch_hash(const hopscotch_hash& other) - : Hash(other), - KeyEqual(other), - GrowthPolicy(other), - m_buckets_data(other.m_buckets_data), - m_overflow_elements(other.m_overflow_elements), - m_buckets(m_buckets_data.empty() ? static_empty_bucket_ptr() - : m_buckets_data.data()), - m_nb_elements(other.m_nb_elements), - m_min_load_threshold_rehash(other.m_min_load_threshold_rehash), - m_max_load_threshold_rehash(other.m_max_load_threshold_rehash), - m_max_load_factor(other.m_max_load_factor) {} - - hopscotch_hash(hopscotch_hash&& other) noexcept( - std::is_nothrow_move_constructible<Hash>::value&& - std::is_nothrow_move_constructible<KeyEqual>::value&& - std::is_nothrow_move_constructible<GrowthPolicy>::value&& std:: - is_nothrow_move_constructible<buckets_container_type>::value&& - std::is_nothrow_move_constructible< - overflow_container_type>::value) - : Hash(std::move(static_cast<Hash&>(other))), - KeyEqual(std::move(static_cast<KeyEqual&>(other))), - GrowthPolicy(std::move(static_cast<GrowthPolicy&>(other))), - m_buckets_data(std::move(other.m_buckets_data)), - m_overflow_elements(std::move(other.m_overflow_elements)), - m_buckets(m_buckets_data.empty() ? static_empty_bucket_ptr() - : m_buckets_data.data()), - m_nb_elements(other.m_nb_elements), - m_min_load_threshold_rehash(other.m_min_load_threshold_rehash), - m_max_load_threshold_rehash(other.m_max_load_threshold_rehash), - m_max_load_factor(other.m_max_load_factor) { - other.GrowthPolicy::clear(); - other.m_buckets_data.clear(); - other.m_overflow_elements.clear(); - other.m_buckets = static_empty_bucket_ptr(); - other.m_nb_elements = 0; - other.m_min_load_threshold_rehash = 0; - other.m_max_load_threshold_rehash = 0; - } - - hopscotch_hash& operator=(const hopscotch_hash& other) { - if (&other != this) { - Hash::operator=(other); - KeyEqual::operator=(other); - GrowthPolicy::operator=(other); - - m_buckets_data = other.m_buckets_data; - m_overflow_elements = other.m_overflow_elements; - m_buckets = m_buckets_data.empty() ? static_empty_bucket_ptr() - : m_buckets_data.data(); - m_nb_elements = other.m_nb_elements; - - m_min_load_threshold_rehash = other.m_min_load_threshold_rehash; - m_max_load_threshold_rehash = other.m_max_load_threshold_rehash; - m_max_load_factor = other.m_max_load_factor; - } - - return *this; - } - - hopscotch_hash& operator=(hopscotch_hash&& other) { - other.swap(*this); - other.clear(); - - return *this; - } - - allocator_type get_allocator() const { - return m_buckets_data.get_allocator(); - } - - /* - * Iterators - */ - iterator begin() noexcept { - auto begin = m_buckets_data.begin(); - while (begin != m_buckets_data.end() && begin->empty()) { - ++begin; - } - - return iterator(begin, m_buckets_data.end(), m_overflow_elements.begin()); - } - - const_iterator begin() const noexcept { return cbegin(); } - - const_iterator cbegin() const noexcept { - auto begin = m_buckets_data.cbegin(); - while (begin != m_buckets_data.cend() && begin->empty()) { - ++begin; - } - - return const_iterator(begin, m_buckets_data.cend(), - m_overflow_elements.cbegin()); - } - - iterator end() noexcept { - return iterator(m_buckets_data.end(), m_buckets_data.end(), - m_overflow_elements.end()); - } - - const_iterator end() const noexcept { return cend(); } - - const_iterator cend() const noexcept { - return const_iterator(m_buckets_data.cend(), m_buckets_data.cend(), - m_overflow_elements.cend()); - } - - /* - * Capacity - */ - bool empty() const noexcept { return m_nb_elements == 0; } - - size_type size() const noexcept { return m_nb_elements; } - - size_type max_size() const noexcept { return m_buckets_data.max_size(); } - - /* - * Modifiers - */ - void clear() noexcept { - for (auto& bucket : m_buckets_data) { - bucket.clear(); - } - - m_overflow_elements.clear(); - m_nb_elements = 0; - } - - std::pair<iterator, bool> insert(const value_type& value) { - return insert_impl(value); - } - - template <class P, typename std::enable_if<std::is_constructible< - value_type, P&&>::value>::type* = nullptr> - std::pair<iterator, bool> insert(P&& value) { - return insert_impl(value_type(std::forward<P>(value))); - } - - std::pair<iterator, bool> insert(value_type&& value) { - return insert_impl(std::move(value)); - } - - iterator insert(const_iterator hint, const value_type& value) { - if (hint != cend() && - compare_keys(KeySelect()(*hint), KeySelect()(value))) { - return mutable_iterator(hint); - } - - return insert(value).first; - } - - template <class P, typename std::enable_if<std::is_constructible< - value_type, P&&>::value>::type* = nullptr> - iterator insert(const_iterator hint, P&& value) { - return emplace_hint(hint, std::forward<P>(value)); - } - - iterator insert(const_iterator hint, value_type&& value) { - if (hint != cend() && - compare_keys(KeySelect()(*hint), KeySelect()(value))) { - return mutable_iterator(hint); - } - - return insert(std::move(value)).first; - } - - template <class InputIt> - void insert(InputIt first, InputIt last) { - if (std::is_base_of< - std::forward_iterator_tag, - typename std::iterator_traits<InputIt>::iterator_category>::value) { - const auto nb_elements_insert = std::distance(first, last); - const std::size_t nb_elements_in_buckets = - m_nb_elements - m_overflow_elements.size(); - const std::size_t nb_free_buckets = - m_max_load_threshold_rehash - nb_elements_in_buckets; - tsl_hh_assert(m_nb_elements >= m_overflow_elements.size()); - tsl_hh_assert(m_max_load_threshold_rehash >= nb_elements_in_buckets); - - if (nb_elements_insert > 0 && - nb_free_buckets < std::size_t(nb_elements_insert)) { - reserve(nb_elements_in_buckets + std::size_t(nb_elements_insert)); - } - } - - for (; first != last; ++first) { - insert(*first); - } - } - - template <class M> - std::pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj) { - return insert_or_assign_impl(k, std::forward<M>(obj)); - } - - template <class M> - std::pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj) { - return insert_or_assign_impl(std::move(k), std::forward<M>(obj)); - } - - template <class M> - iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { - if (hint != cend() && compare_keys(KeySelect()(*hint), k)) { - auto it = mutable_iterator(hint); - it.value() = std::forward<M>(obj); - - return it; - } - - return insert_or_assign(k, std::forward<M>(obj)).first; - } - - template <class M> - iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { - if (hint != cend() && compare_keys(KeySelect()(*hint), k)) { - auto it = mutable_iterator(hint); - it.value() = std::forward<M>(obj); - - return it; - } - - return insert_or_assign(std::move(k), std::forward<M>(obj)).first; - } - - template <class... Args> - std::pair<iterator, bool> emplace(Args&&... args) { - return insert(value_type(std::forward<Args>(args)...)); - } - - template <class... Args> - iterator emplace_hint(const_iterator hint, Args&&... args) { - return insert(hint, value_type(std::forward<Args>(args)...)); - } - - template <class... Args> - std::pair<iterator, bool> try_emplace(const key_type& k, Args&&... args) { - return try_emplace_impl(k, std::forward<Args>(args)...); - } - - template <class... Args> - std::pair<iterator, bool> try_emplace(key_type&& k, Args&&... args) { - return try_emplace_impl(std::move(k), std::forward<Args>(args)...); - } - - template <class... Args> - iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { - if (hint != cend() && compare_keys(KeySelect()(*hint), k)) { - return mutable_iterator(hint); - } - - return try_emplace(k, std::forward<Args>(args)...).first; - } - - template <class... Args> - iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { - if (hint != cend() && compare_keys(KeySelect()(*hint), k)) { - return mutable_iterator(hint); - } - - return try_emplace(std::move(k), std::forward<Args>(args)...).first; - } - - /** - * Here to avoid `template<class K> size_type erase(const K& key)` being used - * when we use an iterator instead of a const_iterator. - */ - iterator erase(iterator pos) { return erase(const_iterator(pos)); } - - iterator erase(const_iterator pos) { - const std::size_t ibucket_for_hash = bucket_for_hash(hash_key(pos.key())); - - if (pos.m_buckets_iterator != pos.m_buckets_end_iterator) { - auto it_bucket = - m_buckets_data.begin() + - std::distance(m_buckets_data.cbegin(), pos.m_buckets_iterator); - erase_from_bucket(*it_bucket, ibucket_for_hash); - - return ++iterator(it_bucket, m_buckets_data.end(), - m_overflow_elements.begin()); - } else { - auto it_next_overflow = - erase_from_overflow(pos.m_overflow_iterator, ibucket_for_hash); - return iterator(m_buckets_data.end(), m_buckets_data.end(), - it_next_overflow); - } - } - - iterator erase(const_iterator first, const_iterator last) { - if (first == last) { - return mutable_iterator(first); - } - - auto to_delete = erase(first); - while (to_delete != last) { - to_delete = erase(to_delete); - } - - return to_delete; - } - - template <class K> - size_type erase(const K& key) { - return erase(key, hash_key(key)); - } - - template <class K> - size_type erase(const K& key, std::size_t hash) { - const std::size_t ibucket_for_hash = bucket_for_hash(hash); - - hopscotch_bucket* bucket_found = - find_in_buckets(key, hash, m_buckets + ibucket_for_hash); - if (bucket_found != nullptr) { - erase_from_bucket(*bucket_found, ibucket_for_hash); - - return 1; - } - - if (m_buckets[ibucket_for_hash].has_overflow()) { - auto it_overflow = find_in_overflow(key); - if (it_overflow != m_overflow_elements.end()) { - erase_from_overflow(it_overflow, ibucket_for_hash); - - return 1; - } - } - - return 0; - } - - void swap(hopscotch_hash& other) { - using std::swap; - - swap(static_cast<Hash&>(*this), static_cast<Hash&>(other)); - swap(static_cast<KeyEqual&>(*this), static_cast<KeyEqual&>(other)); - swap(static_cast<GrowthPolicy&>(*this), static_cast<GrowthPolicy&>(other)); - swap(m_buckets_data, other.m_buckets_data); - swap(m_overflow_elements, other.m_overflow_elements); - swap(m_buckets, other.m_buckets); - swap(m_nb_elements, other.m_nb_elements); - swap(m_min_load_threshold_rehash, other.m_min_load_threshold_rehash); - swap(m_max_load_threshold_rehash, other.m_max_load_threshold_rehash); - swap(m_max_load_factor, other.m_max_load_factor); - } - - /* - * Lookup - */ - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - typename U::value_type& at(const K& key) { - return at(key, hash_key(key)); - } - - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - typename U::value_type& at(const K& key, std::size_t hash) { - return const_cast<typename U::value_type&>( - static_cast<const hopscotch_hash*>(this)->at(key, hash)); - } - - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - const typename U::value_type& at(const K& key) const { - return at(key, hash_key(key)); - } - - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - const typename U::value_type& at(const K& key, std::size_t hash) const { - using T = typename U::value_type; - - const T* value = - find_value_impl(key, hash, m_buckets + bucket_for_hash(hash)); - if (value == nullptr) { - TSL_HH_THROW_OR_TERMINATE(std::out_of_range, "Couldn't find key."); - } else { - return *value; - } - } - - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - typename U::value_type& operator[](K&& key) { - using T = typename U::value_type; - - const std::size_t hash = hash_key(key); - const std::size_t ibucket_for_hash = bucket_for_hash(hash); - - T* value = find_value_impl(key, hash, m_buckets + ibucket_for_hash); - if (value != nullptr) { - return *value; - } else { - return insert_value(ibucket_for_hash, hash, std::piecewise_construct, - std::forward_as_tuple(std::forward<K>(key)), - std::forward_as_tuple()) - .first.value(); - } - } - - template <class K> - size_type count(const K& key) const { - return count(key, hash_key(key)); - } - - template <class K> - size_type count(const K& key, std::size_t hash) const { - return count_impl(key, hash, m_buckets + bucket_for_hash(hash)); - } - - template <class K> - iterator find(const K& key) { - return find(key, hash_key(key)); - } - - template <class K> - iterator find(const K& key, std::size_t hash) { - return find_impl(key, hash, m_buckets + bucket_for_hash(hash)); - } - - template <class K> - const_iterator find(const K& key) const { - return find(key, hash_key(key)); - } - - template <class K> - const_iterator find(const K& key, std::size_t hash) const { - return find_impl(key, hash, m_buckets + bucket_for_hash(hash)); - } - - template <class K> - bool contains(const K& key) const { - return contains(key, hash_key(key)); - } - - template <class K> - bool contains(const K& key, std::size_t hash) const { - return count(key, hash) != 0; - } - - template <class K> - std::pair<iterator, iterator> equal_range(const K& key) { - return equal_range(key, hash_key(key)); - } - - template <class K> - std::pair<iterator, iterator> equal_range(const K& key, std::size_t hash) { - iterator it = find(key, hash); - return std::make_pair(it, (it == end()) ? it : std::next(it)); - } - - template <class K> - std::pair<const_iterator, const_iterator> equal_range(const K& key) const { - return equal_range(key, hash_key(key)); - } - - template <class K> - std::pair<const_iterator, const_iterator> equal_range( - const K& key, std::size_t hash) const { - const_iterator it = find(key, hash); - return std::make_pair(it, (it == cend()) ? it : std::next(it)); - } - - /* - * Bucket interface - */ - size_type bucket_count() const { - /* - * So that the last bucket can have NeighborhoodSize neighbors, the size of - * the bucket array is a little bigger than the real number of buckets when - * not empty. We could use some of the buckets at the beginning, but it is - * faster this way as we avoid extra checks. - */ - if (m_buckets_data.empty()) { - return 0; - } - - return m_buckets_data.size() - NeighborhoodSize + 1; - } - - size_type max_bucket_count() const { - const std::size_t max_bucket_count = - std::min(GrowthPolicy::max_bucket_count(), m_buckets_data.max_size()); - return max_bucket_count - NeighborhoodSize + 1; - } - - /* - * Hash policy - */ - float load_factor() const { - if (bucket_count() == 0) { - return 0; - } - - return float(m_nb_elements) / float(bucket_count()); - } - - float max_load_factor() const { return m_max_load_factor; } - - void max_load_factor(float ml) { - m_max_load_factor = std::max(0.1f, std::min(ml, 0.95f)); - m_min_load_threshold_rehash = - size_type(float(bucket_count()) * MIN_LOAD_FACTOR_FOR_REHASH); - m_max_load_threshold_rehash = - size_type(float(bucket_count()) * m_max_load_factor); - } - - void rehash(size_type count_) { - count_ = std::max(count_, - size_type(std::ceil(float(size()) / max_load_factor()))); - rehash_impl(count_); - } - - void reserve(size_type count_) { - rehash(size_type(std::ceil(float(count_) / max_load_factor()))); - } - - /* - * Observers - */ - hasher hash_function() const { return static_cast<const Hash&>(*this); } - - key_equal key_eq() const { return static_cast<const KeyEqual&>(*this); } - - /* - * Other - */ - iterator mutable_iterator(const_iterator pos) { - if (pos.m_buckets_iterator != pos.m_buckets_end_iterator) { - // Get a non-const iterator - auto it = m_buckets_data.begin() + - std::distance(m_buckets_data.cbegin(), pos.m_buckets_iterator); - return iterator(it, m_buckets_data.end(), m_overflow_elements.begin()); - } else { - // Get a non-const iterator - auto it = mutable_overflow_iterator(pos.m_overflow_iterator); - return iterator(m_buckets_data.end(), m_buckets_data.end(), it); - } - } - - size_type overflow_size() const noexcept { - return m_overflow_elements.size(); - } - - template <class U = OverflowContainer, - typename std::enable_if<has_key_compare<U>::value>::type* = nullptr> - typename U::key_compare key_comp() const { - return m_overflow_elements.key_comp(); - } - - private: - template <class K> - std::size_t hash_key(const K& key) const { - return Hash::operator()(key); - } - - template <class K1, class K2> - bool compare_keys(const K1& key1, const K2& key2) const { - return KeyEqual::operator()(key1, key2); - } - - std::size_t bucket_for_hash(std::size_t hash) const { - const std::size_t bucket = GrowthPolicy::bucket_for_hash(hash); - tsl_hh_assert(bucket < m_buckets_data.size() || - (bucket == 0 && m_buckets_data.empty())); - - return bucket; - } - - template <typename U = value_type, - typename std::enable_if< - std::is_nothrow_move_constructible<U>::value>::type* = nullptr> - void rehash_impl(size_type count_) { - hopscotch_hash new_map = new_hopscotch_hash(count_); - - if (!m_overflow_elements.empty()) { - new_map.m_overflow_elements.swap(m_overflow_elements); - new_map.m_nb_elements += new_map.m_overflow_elements.size(); - - for (const value_type& value : new_map.m_overflow_elements) { - const std::size_t ibucket_for_hash = - new_map.bucket_for_hash(new_map.hash_key(KeySelect()(value))); - new_map.m_buckets[ibucket_for_hash].set_overflow(true); - } - } - -#ifndef TSL_HH_NO_EXCEPTIONS - try { -#endif - const bool use_stored_hash = - USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); - for (auto it_bucket = m_buckets_data.begin(); - it_bucket != m_buckets_data.end(); ++it_bucket) { - if (it_bucket->empty()) { - continue; - } - - const std::size_t hash = - use_stored_hash ? it_bucket->truncated_bucket_hash() - : new_map.hash_key(KeySelect()(it_bucket->value())); - const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); - - new_map.insert_value(ibucket_for_hash, hash, - std::move(it_bucket->value())); - - erase_from_bucket(*it_bucket, bucket_for_hash(hash)); - } -#ifndef TSL_HH_NO_EXCEPTIONS - } - /* - * The call to insert_value may throw an exception if an element is added to - * the overflow list and the memory allocation fails. Rollback the elements - * in this case. - */ - catch (...) { - m_overflow_elements.swap(new_map.m_overflow_elements); - - const bool use_stored_hash = - USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); - for (auto it_bucket = new_map.m_buckets_data.begin(); - it_bucket != new_map.m_buckets_data.end(); ++it_bucket) { - if (it_bucket->empty()) { - continue; - } - - const std::size_t hash = - use_stored_hash ? it_bucket->truncated_bucket_hash() - : hash_key(KeySelect()(it_bucket->value())); - const std::size_t ibucket_for_hash = bucket_for_hash(hash); - - // The elements we insert were not in the overflow list before the - // switch. They will not be go in the overflow list if we rollback the - // switch. - insert_value(ibucket_for_hash, hash, std::move(it_bucket->value())); - } - - throw; - } -#endif - - new_map.swap(*this); - } - - template <typename U = value_type, - typename std::enable_if< - std::is_copy_constructible<U>::value && - !std::is_nothrow_move_constructible<U>::value>::type* = nullptr> - void rehash_impl(size_type count_) { - hopscotch_hash new_map = new_hopscotch_hash(count_); - - const bool use_stored_hash = - USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); - for (const hopscotch_bucket& bucket : m_buckets_data) { - if (bucket.empty()) { - continue; - } - - const std::size_t hash = - use_stored_hash ? bucket.truncated_bucket_hash() - : new_map.hash_key(KeySelect()(bucket.value())); - const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); - - new_map.insert_value(ibucket_for_hash, hash, bucket.value()); - } - - for (const value_type& value : m_overflow_elements) { - const std::size_t hash = new_map.hash_key(KeySelect()(value)); - const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); - - new_map.insert_value(ibucket_for_hash, hash, value); - } - - new_map.swap(*this); - } - -#ifdef TSL_HH_NO_RANGE_ERASE_WITH_CONST_ITERATOR - iterator_overflow mutable_overflow_iterator(const_iterator_overflow it) { - return std::next(m_overflow_elements.begin(), - std::distance(m_overflow_elements.cbegin(), it)); - } -#else - iterator_overflow mutable_overflow_iterator(const_iterator_overflow it) { - return m_overflow_elements.erase(it, it); - } -#endif - - // iterator is in overflow list - iterator_overflow erase_from_overflow(const_iterator_overflow pos, - std::size_t ibucket_for_hash) { -#ifdef TSL_HH_NO_RANGE_ERASE_WITH_CONST_ITERATOR - auto it_next = m_overflow_elements.erase(mutable_overflow_iterator(pos)); -#else - auto it_next = m_overflow_elements.erase(pos); -#endif - m_nb_elements--; - - // Check if we can remove the overflow flag - tsl_hh_assert(m_buckets[ibucket_for_hash].has_overflow()); - for (const value_type& value : m_overflow_elements) { - const std::size_t bucket_for_value = - bucket_for_hash(hash_key(KeySelect()(value))); - if (bucket_for_value == ibucket_for_hash) { - return it_next; - } - } - - m_buckets[ibucket_for_hash].set_overflow(false); - return it_next; - } - - /** - * bucket_for_value is the bucket in which the value is. - * ibucket_for_hash is the bucket where the value belongs. - */ - void erase_from_bucket(hopscotch_bucket& bucket_for_value, - std::size_t ibucket_for_hash) noexcept { - const std::size_t ibucket_for_value = - std::distance(m_buckets_data.data(), &bucket_for_value); - tsl_hh_assert(ibucket_for_value >= ibucket_for_hash); - - bucket_for_value.remove_value(); - m_buckets[ibucket_for_hash].toggle_neighbor_presence(ibucket_for_value - - ibucket_for_hash); - m_nb_elements--; - } - - template <class K, class M> - std::pair<iterator, bool> insert_or_assign_impl(K&& key, M&& obj) { - auto it = try_emplace_impl(std::forward<K>(key), std::forward<M>(obj)); - if (!it.second) { - it.first.value() = std::forward<M>(obj); - } - - return it; - } - - template <typename P, class... Args> - std::pair<iterator, bool> try_emplace_impl(P&& key, Args&&... args_value) { - const std::size_t hash = hash_key(key); - const std::size_t ibucket_for_hash = bucket_for_hash(hash); - - // Check if already presents - auto it_find = find_impl(key, hash, m_buckets + ibucket_for_hash); - if (it_find != end()) { - return std::make_pair(it_find, false); - } - - return insert_value( - ibucket_for_hash, hash, std::piecewise_construct, - std::forward_as_tuple(std::forward<P>(key)), - std::forward_as_tuple(std::forward<Args>(args_value)...)); - } - - template <typename P> - std::pair<iterator, bool> insert_impl(P&& value) { - const std::size_t hash = hash_key(KeySelect()(value)); - const std::size_t ibucket_for_hash = bucket_for_hash(hash); - - // Check if already presents - auto it_find = - find_impl(KeySelect()(value), hash, m_buckets + ibucket_for_hash); - if (it_find != end()) { - return std::make_pair(it_find, false); - } - - return insert_value(ibucket_for_hash, hash, std::forward<P>(value)); - } - - template <typename... Args> - std::pair<iterator, bool> insert_value(std::size_t ibucket_for_hash, - std::size_t hash, - Args&&... value_type_args) { - if ((m_nb_elements - m_overflow_elements.size()) >= - m_max_load_threshold_rehash) { - rehash(GrowthPolicy::next_bucket_count()); - ibucket_for_hash = bucket_for_hash(hash); - } - - std::size_t ibucket_empty = find_empty_bucket(ibucket_for_hash); - if (ibucket_empty < m_buckets_data.size()) { - do { - tsl_hh_assert(ibucket_empty >= ibucket_for_hash); - - // Empty bucket is in range of NeighborhoodSize, use it - if (ibucket_empty - ibucket_for_hash < NeighborhoodSize) { - auto it = insert_in_bucket(ibucket_empty, ibucket_for_hash, hash, - std::forward<Args>(value_type_args)...); - return std::make_pair( - iterator(it, m_buckets_data.end(), m_overflow_elements.begin()), - true); - } - } - // else, try to swap values to get a closer empty bucket - while (swap_empty_bucket_closer(ibucket_empty)); - } - - // Load factor is too low or a rehash will not change the neighborhood, put - // the value in overflow list - if (size() < m_min_load_threshold_rehash || - !will_neighborhood_change_on_rehash(ibucket_for_hash)) { - auto it = insert_in_overflow(ibucket_for_hash, - std::forward<Args>(value_type_args)...); - return std::make_pair( - iterator(m_buckets_data.end(), m_buckets_data.end(), it), true); - } - - rehash(GrowthPolicy::next_bucket_count()); - ibucket_for_hash = bucket_for_hash(hash); - - return insert_value(ibucket_for_hash, hash, - std::forward<Args>(value_type_args)...); - } - - /* - * Return true if a rehash will change the position of a key-value in the - * neighborhood of ibucket_neighborhood_check. In this case a rehash is needed - * instead of puting the value in overflow list. - */ - bool will_neighborhood_change_on_rehash( - size_t ibucket_neighborhood_check) const { - std::size_t expand_bucket_count = GrowthPolicy::next_bucket_count(); - GrowthPolicy expand_growth_policy(expand_bucket_count); - - const bool use_stored_hash = USE_STORED_HASH_ON_REHASH(expand_bucket_count); - for (size_t ibucket = ibucket_neighborhood_check; - ibucket < m_buckets_data.size() && - (ibucket - ibucket_neighborhood_check) < NeighborhoodSize; - ++ibucket) { - tsl_hh_assert(!m_buckets[ibucket].empty()); - - const size_t hash = - use_stored_hash ? m_buckets[ibucket].truncated_bucket_hash() - : hash_key(KeySelect()(m_buckets[ibucket].value())); - if (bucket_for_hash(hash) != expand_growth_policy.bucket_for_hash(hash)) { - return true; - } - } - - return false; - } - - /* - * Return the index of an empty bucket in m_buckets_data. - * If none, the returned index equals m_buckets_data.size() - */ - std::size_t find_empty_bucket(std::size_t ibucket_start) const { - const std::size_t limit = std::min( - ibucket_start + MAX_PROBES_FOR_EMPTY_BUCKET, m_buckets_data.size()); - for (; ibucket_start < limit; ibucket_start++) { - if (m_buckets[ibucket_start].empty()) { - return ibucket_start; - } - } - - return m_buckets_data.size(); - } - - /* - * Insert value in ibucket_empty where value originally belongs to - * ibucket_for_hash - * - * Return bucket iterator to ibucket_empty - */ - template <typename... Args> - iterator_buckets insert_in_bucket(std::size_t ibucket_empty, - std::size_t ibucket_for_hash, - std::size_t hash, - Args&&... value_type_args) { - tsl_hh_assert(ibucket_empty >= ibucket_for_hash); - tsl_hh_assert(m_buckets[ibucket_empty].empty()); - m_buckets[ibucket_empty].set_value_of_empty_bucket( - hopscotch_bucket::truncate_hash(hash), - std::forward<Args>(value_type_args)...); - - tsl_hh_assert(!m_buckets[ibucket_for_hash].empty()); - m_buckets[ibucket_for_hash].toggle_neighbor_presence(ibucket_empty - - ibucket_for_hash); - m_nb_elements++; - - return m_buckets_data.begin() + ibucket_empty; - } - - template < - class... Args, class U = OverflowContainer, - typename std::enable_if<!has_key_compare<U>::value>::type* = nullptr> - iterator_overflow insert_in_overflow(std::size_t ibucket_for_hash, - Args&&... value_type_args) { - auto it = m_overflow_elements.emplace( - m_overflow_elements.end(), std::forward<Args>(value_type_args)...); - - m_buckets[ibucket_for_hash].set_overflow(true); - m_nb_elements++; - - return it; - } - - template <class... Args, class U = OverflowContainer, - typename std::enable_if<has_key_compare<U>::value>::type* = nullptr> - iterator_overflow insert_in_overflow(std::size_t ibucket_for_hash, - Args&&... value_type_args) { - auto it = - m_overflow_elements.emplace(std::forward<Args>(value_type_args)...) - .first; - - m_buckets[ibucket_for_hash].set_overflow(true); - m_nb_elements++; - - return it; - } - - /* - * Try to swap the bucket ibucket_empty_in_out with a bucket preceding it - * while keeping the neighborhood conditions correct. - * - * If a swap was possible, the position of ibucket_empty_in_out will be closer - * to 0 and true will re returned. - */ - bool swap_empty_bucket_closer(std::size_t& ibucket_empty_in_out) { - tsl_hh_assert(ibucket_empty_in_out >= NeighborhoodSize); - const std::size_t neighborhood_start = - ibucket_empty_in_out - NeighborhoodSize + 1; - - for (std::size_t to_check = neighborhood_start; - to_check < ibucket_empty_in_out; to_check++) { - neighborhood_bitmap neighborhood_infos = - m_buckets[to_check].neighborhood_infos(); - std::size_t to_swap = to_check; - - while (neighborhood_infos != 0 && to_swap < ibucket_empty_in_out) { - if ((neighborhood_infos & 1) == 1) { - tsl_hh_assert(m_buckets[ibucket_empty_in_out].empty()); - tsl_hh_assert(!m_buckets[to_swap].empty()); - - m_buckets[to_swap].swap_value_into_empty_bucket( - m_buckets[ibucket_empty_in_out]); - - tsl_hh_assert(!m_buckets[to_check].check_neighbor_presence( - ibucket_empty_in_out - to_check)); - tsl_hh_assert( - m_buckets[to_check].check_neighbor_presence(to_swap - to_check)); - - m_buckets[to_check].toggle_neighbor_presence(ibucket_empty_in_out - - to_check); - m_buckets[to_check].toggle_neighbor_presence(to_swap - to_check); - - ibucket_empty_in_out = to_swap; - - return true; - } - - to_swap++; - neighborhood_infos = neighborhood_bitmap(neighborhood_infos >> 1); - } - } - - return false; - } - - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - typename U::value_type* find_value_impl(const K& key, std::size_t hash, - hopscotch_bucket* bucket_for_hash) { - return const_cast<typename U::value_type*>( - static_cast<const hopscotch_hash*>(this)->find_value_impl( - key, hash, bucket_for_hash)); - } - - /* - * Avoid the creation of an iterator to just get the value for operator[] and - * at() in maps. Faster this way. - * - * Return null if no value for the key (TODO use std::optional when - * available). - */ - template <class K, class U = ValueSelect, - typename std::enable_if<has_mapped_type<U>::value>::type* = nullptr> - const typename U::value_type* find_value_impl( - const K& key, std::size_t hash, - const hopscotch_bucket* bucket_for_hash) const { - const hopscotch_bucket* bucket_found = - find_in_buckets(key, hash, bucket_for_hash); - if (bucket_found != nullptr) { - return std::addressof(ValueSelect()(bucket_found->value())); - } - - if (bucket_for_hash->has_overflow()) { - auto it_overflow = find_in_overflow(key); - if (it_overflow != m_overflow_elements.end()) { - return std::addressof(ValueSelect()(*it_overflow)); - } - } - - return nullptr; - } - - template <class K> - size_type count_impl(const K& key, std::size_t hash, - const hopscotch_bucket* bucket_for_hash) const { - if (find_in_buckets(key, hash, bucket_for_hash) != nullptr) { - return 1; - } else if (bucket_for_hash->has_overflow() && - find_in_overflow(key) != m_overflow_elements.cend()) { - return 1; - } else { - return 0; - } - } - - template <class K> - iterator find_impl(const K& key, std::size_t hash, - hopscotch_bucket* bucket_for_hash) { - hopscotch_bucket* bucket_found = - find_in_buckets(key, hash, bucket_for_hash); - if (bucket_found != nullptr) { - return iterator(m_buckets_data.begin() + - std::distance(m_buckets_data.data(), bucket_found), - m_buckets_data.end(), m_overflow_elements.begin()); - } - - if (!bucket_for_hash->has_overflow()) { - return end(); - } - - return iterator(m_buckets_data.end(), m_buckets_data.end(), - find_in_overflow(key)); - } - - template <class K> - const_iterator find_impl(const K& key, std::size_t hash, - const hopscotch_bucket* bucket_for_hash) const { - const hopscotch_bucket* bucket_found = - find_in_buckets(key, hash, bucket_for_hash); - if (bucket_found != nullptr) { - return const_iterator( - m_buckets_data.cbegin() + - std::distance(m_buckets_data.data(), bucket_found), - m_buckets_data.cend(), m_overflow_elements.cbegin()); - } - - if (!bucket_for_hash->has_overflow()) { - return cend(); - } - - return const_iterator(m_buckets_data.cend(), m_buckets_data.cend(), - find_in_overflow(key)); - } - - template <class K> - hopscotch_bucket* find_in_buckets(const K& key, std::size_t hash, - hopscotch_bucket* bucket_for_hash) { - const hopscotch_bucket* bucket_found = - static_cast<const hopscotch_hash*>(this)->find_in_buckets( - key, hash, bucket_for_hash); - return const_cast<hopscotch_bucket*>(bucket_found); - } - - /** - * Return a pointer to the bucket which has the value, nullptr otherwise. - */ - template <class K> - const hopscotch_bucket* find_in_buckets( - const K& key, std::size_t hash, - const hopscotch_bucket* bucket_for_hash) const { - (void)hash; // Avoid warning of unused variable when StoreHash is false; - - // TODO Try to optimize the function. - // I tried to use ffs and __builtin_ffs functions but I could not reduce - // the time the function takes with -march=native - - neighborhood_bitmap neighborhood_infos = - bucket_for_hash->neighborhood_infos(); - while (neighborhood_infos != 0) { - if ((neighborhood_infos & 1) == 1) { - // Check StoreHash before calling bucket_hash_equal. Functionally it - // doesn't change anythin. If StoreHash is false, bucket_hash_equal is a - // no-op. Avoiding the call is there to help GCC optimizes `hash` - // parameter away, it seems to not be able to do without this hint. - if ((!StoreHash || bucket_for_hash->bucket_hash_equal(hash)) && - compare_keys(KeySelect()(bucket_for_hash->value()), key)) { - return bucket_for_hash; - } - } - - ++bucket_for_hash; - neighborhood_infos = neighborhood_bitmap(neighborhood_infos >> 1); - } - - return nullptr; - } - - template < - class K, class U = OverflowContainer, - typename std::enable_if<!has_key_compare<U>::value>::type* = nullptr> - iterator_overflow find_in_overflow(const K& key) { - return std::find_if(m_overflow_elements.begin(), m_overflow_elements.end(), - [&](const value_type& value) { - return compare_keys(key, KeySelect()(value)); - }); - } - - template < - class K, class U = OverflowContainer, - typename std::enable_if<!has_key_compare<U>::value>::type* = nullptr> - const_iterator_overflow find_in_overflow(const K& key) const { - return std::find_if(m_overflow_elements.cbegin(), - m_overflow_elements.cend(), - [&](const value_type& value) { - return compare_keys(key, KeySelect()(value)); - }); - } - - template <class K, class U = OverflowContainer, - typename std::enable_if<has_key_compare<U>::value>::type* = nullptr> - iterator_overflow find_in_overflow(const K& key) { - return m_overflow_elements.find(key); - } - - template <class K, class U = OverflowContainer, - typename std::enable_if<has_key_compare<U>::value>::type* = nullptr> - const_iterator_overflow find_in_overflow(const K& key) const { - return m_overflow_elements.find(key); - } - - template < - class U = OverflowContainer, - typename std::enable_if<!has_key_compare<U>::value>::type* = nullptr> - hopscotch_hash new_hopscotch_hash(size_type bucket_count) { - return hopscotch_hash(bucket_count, static_cast<Hash&>(*this), - static_cast<KeyEqual&>(*this), get_allocator(), - m_max_load_factor); - } - - template <class U = OverflowContainer, - typename std::enable_if<has_key_compare<U>::value>::type* = nullptr> - hopscotch_hash new_hopscotch_hash(size_type bucket_count) { - return hopscotch_hash(bucket_count, static_cast<Hash&>(*this), - static_cast<KeyEqual&>(*this), get_allocator(), - m_max_load_factor, m_overflow_elements.key_comp()); - } - - public: - static const size_type DEFAULT_INIT_BUCKETS_SIZE = 0; - static constexpr float DEFAULT_MAX_LOAD_FACTOR = - (NeighborhoodSize <= 30) ? 0.8f : 0.9f; - - private: - static const std::size_t MAX_PROBES_FOR_EMPTY_BUCKET = 12 * NeighborhoodSize; - static constexpr float MIN_LOAD_FACTOR_FOR_REHASH = 0.1f; - - /** - * We can only use the hash on rehash if the size of the hash type is the same - * as the stored one or if we use a power of two modulo. In the case of the - * power of two modulo, we just mask the least significant bytes, we just have - * to check that the truncated_hash_type didn't truncated too much bytes. - */ - template <class T = size_type, - typename std::enable_if< - std::is_same<T, truncated_hash_type>::value>::type* = nullptr> - static bool USE_STORED_HASH_ON_REHASH(size_type /*bucket_count*/) { - return StoreHash; - } - - template <class T = size_type, - typename std::enable_if< - !std::is_same<T, truncated_hash_type>::value>::type* = nullptr> - static bool USE_STORED_HASH_ON_REHASH(size_type bucket_count) { - (void)bucket_count; - if (StoreHash && is_power_of_two_policy<GrowthPolicy>::value) { - tsl_hh_assert(bucket_count > 0); - return (bucket_count - 1) <= - std::numeric_limits<truncated_hash_type>::max(); - } else { - return false; - } - } - - /** - * Return an always valid pointer to an static empty hopscotch_bucket. - */ - hopscotch_bucket* static_empty_bucket_ptr() { - static hopscotch_bucket empty_bucket; - return &empty_bucket; - } - - private: - buckets_container_type m_buckets_data; - overflow_container_type m_overflow_elements; - - /** - * Points to m_buckets_data.data() if !m_buckets_data.empty() otherwise points - * to static_empty_bucket_ptr. This variable is useful to avoid the cost of - * checking if m_buckets_data is empty when trying to find an element. - * - * TODO Remove m_buckets_data and only use a pointer+size instead of a - * pointer+vector to save some space in the hopscotch_hash object. - */ - hopscotch_bucket* m_buckets; - - size_type m_nb_elements; - - /** - * Min size of the hash table before a rehash can occurs automatically (except - * if m_max_load_threshold_rehash os reached). If the neighborhood of a bucket - * is full before the min is reacher, the elements are put into - * m_overflow_elements. - */ - size_type m_min_load_threshold_rehash; - - /** - * Max size of the hash table before a rehash occurs automatically to grow the - * table. - */ - size_type m_max_load_threshold_rehash; - - float m_max_load_factor; -}; - -} // end namespace detail_hopscotch_hash - -} // end namespace tsl - -#endif diff --git a/benchmarks/external/tsl/hopscotch_map.h b/benchmarks/external/tsl/hopscotch_map.h deleted file mode 100644 index 15c9e398..00000000 --- a/benchmarks/external/tsl/hopscotch_map.h +++ /dev/null @@ -1,735 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2017 Thibaut Goetghebuer-Planchon <[email protected]> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef TSL_HOPSCOTCH_MAP_H -#define TSL_HOPSCOTCH_MAP_H - -#include <algorithm> -#include <cstddef> -#include <functional> -#include <initializer_list> -#include <list> -#include <memory> -#include <type_traits> -#include <utility> - -#include "hopscotch_hash.h" - -namespace tsl { - -/** - * Implementation of a hash map using the hopscotch hashing algorithm. - * - * The Key and the value T must be either nothrow move-constructible, - * copy-constructible or both. - * - * The size of the neighborhood (NeighborhoodSize) must be > 0 and <= 62 if - * StoreHash is false. When StoreHash is true, 32-bits of the hash will be - * stored alongside the neighborhood limiting the NeighborhoodSize to <= 30. - * There is no memory usage difference between 'NeighborhoodSize 62; StoreHash - * false' and 'NeighborhoodSize 30; StoreHash true'. - * - * Storing the hash may improve performance on insert during the rehash process - * if the hash takes time to compute. It may also improve read performance if - * the KeyEqual function takes time (or incurs a cache-miss). If used with - * simple Hash and KeyEqual it may slow things down. - * - * StoreHash can only be set if the GrowthPolicy is set to - * tsl::power_of_two_growth_policy. - * - * GrowthPolicy defines how the map grows and consequently how a hash value is - * mapped to a bucket. By default the map uses tsl::power_of_two_growth_policy. - * This policy keeps the number of buckets to a power of two and uses a mask to - * map the hash to a bucket instead of the slow modulo. You may define your own - * growth policy, check tsl::power_of_two_growth_policy for the interface. - * - * If the destructors of Key or T throw an exception, behaviour of the class is - * undefined. - * - * Iterators invalidation: - * - clear, operator=, reserve, rehash: always invalidate the iterators. - * - insert, emplace, emplace_hint, operator[]: if there is an effective - * insert, invalidate the iterators if a displacement is needed to resolve a - * collision (which mean that most of the time, insert will invalidate the - * iterators). Or if there is a rehash. - * - erase: iterator on the erased element is the only one which become - * invalid. - */ -template <class Key, class T, class Hash = std::hash<Key>, - class KeyEqual = std::equal_to<Key>, - class Allocator = std::allocator<std::pair<Key, T>>, - unsigned int NeighborhoodSize = 62, bool StoreHash = false, - class GrowthPolicy = tsl::hh::power_of_two_growth_policy<2>> -class hopscotch_map { - private: - template <typename U> - using has_is_transparent = tsl::detail_hopscotch_hash::has_is_transparent<U>; - - class KeySelect { - public: - using key_type = Key; - - const key_type& operator()(const std::pair<Key, T>& key_value) const { - return key_value.first; - } - - key_type& operator()(std::pair<Key, T>& key_value) { - return key_value.first; - } - }; - - class ValueSelect { - public: - using value_type = T; - - const value_type& operator()(const std::pair<Key, T>& key_value) const { - return key_value.second; - } - - value_type& operator()(std::pair<Key, T>& key_value) { - return key_value.second; - } - }; - - using overflow_container_type = std::list<std::pair<Key, T>, Allocator>; - using ht = detail_hopscotch_hash::hopscotch_hash< - std::pair<Key, T>, KeySelect, ValueSelect, Hash, KeyEqual, Allocator, - NeighborhoodSize, StoreHash, GrowthPolicy, overflow_container_type>; - - public: - using key_type = typename ht::key_type; - using mapped_type = T; - using value_type = typename ht::value_type; - using size_type = typename ht::size_type; - using difference_type = typename ht::difference_type; - using hasher = typename ht::hasher; - using key_equal = typename ht::key_equal; - using allocator_type = typename ht::allocator_type; - using reference = typename ht::reference; - using const_reference = typename ht::const_reference; - using pointer = typename ht::pointer; - using const_pointer = typename ht::const_pointer; - using iterator = typename ht::iterator; - using const_iterator = typename ht::const_iterator; - - /* - * Constructors - */ - hopscotch_map() : hopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE) {} - - explicit hopscotch_map(size_type bucket_count, const Hash& hash = Hash(), - const KeyEqual& equal = KeyEqual(), - const Allocator& alloc = Allocator()) - : m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) {} - - hopscotch_map(size_type bucket_count, const Allocator& alloc) - : hopscotch_map(bucket_count, Hash(), KeyEqual(), alloc) {} - - hopscotch_map(size_type bucket_count, const Hash& hash, - const Allocator& alloc) - : hopscotch_map(bucket_count, hash, KeyEqual(), alloc) {} - - explicit hopscotch_map(const Allocator& alloc) - : hopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) {} - - template <class InputIt> - hopscotch_map(InputIt first, InputIt last, - size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, - const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), - const Allocator& alloc = Allocator()) - : hopscotch_map(bucket_count, hash, equal, alloc) { - insert(first, last); - } - - template <class InputIt> - hopscotch_map(InputIt first, InputIt last, size_type bucket_count, - const Allocator& alloc) - : hopscotch_map(first, last, bucket_count, Hash(), KeyEqual(), alloc) {} - - template <class InputIt> - hopscotch_map(InputIt first, InputIt last, size_type bucket_count, - const Hash& hash, const Allocator& alloc) - : hopscotch_map(first, last, bucket_count, hash, KeyEqual(), alloc) {} - - hopscotch_map(std::initializer_list<value_type> init, - size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, - const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), - const Allocator& alloc = Allocator()) - : hopscotch_map(init.begin(), init.end(), bucket_count, hash, equal, - alloc) {} - - hopscotch_map(std::initializer_list<value_type> init, size_type bucket_count, - const Allocator& alloc) - : hopscotch_map(init.begin(), init.end(), bucket_count, Hash(), - KeyEqual(), alloc) {} - - hopscotch_map(std::initializer_list<value_type> init, size_type bucket_count, - const Hash& hash, const Allocator& alloc) - : hopscotch_map(init.begin(), init.end(), bucket_count, hash, KeyEqual(), - alloc) {} - - hopscotch_map& operator=(std::initializer_list<value_type> ilist) { - m_ht.clear(); - - m_ht.reserve(ilist.size()); - m_ht.insert(ilist.begin(), ilist.end()); - - return *this; - } - - allocator_type get_allocator() const { return m_ht.get_allocator(); } - - /* - * Iterators - */ - iterator begin() noexcept { return m_ht.begin(); } - const_iterator begin() const noexcept { return m_ht.begin(); } - const_iterator cbegin() const noexcept { return m_ht.cbegin(); } - - iterator end() noexcept { return m_ht.end(); } - const_iterator end() const noexcept { return m_ht.end(); } - const_iterator cend() const noexcept { return m_ht.cend(); } - - /* - * Capacity - */ - bool empty() const noexcept { return m_ht.empty(); } - size_type size() const noexcept { return m_ht.size(); } - size_type max_size() const noexcept { return m_ht.max_size(); } - - /* - * Modifiers - */ - void clear() noexcept { m_ht.clear(); } - - std::pair<iterator, bool> insert(const value_type& value) { - return m_ht.insert(value); - } - - template <class P, typename std::enable_if<std::is_constructible< - value_type, P&&>::value>::type* = nullptr> - std::pair<iterator, bool> insert(P&& value) { - return m_ht.insert(std::forward<P>(value)); - } - - std::pair<iterator, bool> insert(value_type&& value) { - return m_ht.insert(std::move(value)); - } - - iterator insert(const_iterator hint, const value_type& value) { - return m_ht.insert(hint, value); - } - - template <class P, typename std::enable_if<std::is_constructible< - value_type, P&&>::value>::type* = nullptr> - iterator insert(const_iterator hint, P&& value) { - return m_ht.insert(hint, std::forward<P>(value)); - } - - iterator insert(const_iterator hint, value_type&& value) { - return m_ht.insert(hint, std::move(value)); - } - - template <class InputIt> - void insert(InputIt first, InputIt last) { - m_ht.insert(first, last); - } - - void insert(std::initializer_list<value_type> ilist) { - m_ht.insert(ilist.begin(), ilist.end()); - } - - template <class M> - std::pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj) { - return m_ht.insert_or_assign(k, std::forward<M>(obj)); - } - - template <class M> - std::pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj) { - return m_ht.insert_or_assign(std::move(k), std::forward<M>(obj)); - } - - template <class M> - iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { - return m_ht.insert_or_assign(hint, k, std::forward<M>(obj)); - } - - template <class M> - iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { - return m_ht.insert_or_assign(hint, std::move(k), std::forward<M>(obj)); - } - - /** - * Due to the way elements are stored, emplace will need to move or copy the - * key-value once. The method is equivalent to - * insert(value_type(std::forward<Args>(args)...)); - * - * Mainly here for compatibility with the std::unordered_map interface. - */ - template <class... Args> - std::pair<iterator, bool> emplace(Args&&... args) { - return m_ht.emplace(std::forward<Args>(args)...); - } - - /** - * Due to the way elements are stored, emplace_hint will need to move or copy - * the key-value once. The method is equivalent to insert(hint, - * value_type(std::forward<Args>(args)...)); - * - * Mainly here for compatibility with the std::unordered_map interface. - */ - template <class... Args> - iterator emplace_hint(const_iterator hint, Args&&... args) { - return m_ht.emplace_hint(hint, std::forward<Args>(args)...); - } - - template <class... Args> - std::pair<iterator, bool> try_emplace(const key_type& k, Args&&... args) { - return m_ht.try_emplace(k, std::forward<Args>(args)...); - } - - template <class... Args> - std::pair<iterator, bool> try_emplace(key_type&& k, Args&&... args) { - return m_ht.try_emplace(std::move(k), std::forward<Args>(args)...); - } - - template <class... Args> - iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { - return m_ht.try_emplace(hint, k, std::forward<Args>(args)...); - } - - template <class... Args> - iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { - return m_ht.try_emplace(hint, std::move(k), std::forward<Args>(args)...); - } - - iterator erase(iterator pos) { return m_ht.erase(pos); } - iterator erase(const_iterator pos) { return m_ht.erase(pos); } - iterator erase(const_iterator first, const_iterator last) { - return m_ht.erase(first, last); - } - size_type erase(const key_type& key) { return m_ht.erase(key); } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup to the value if you already have the hash. - */ - size_type erase(const key_type& key, std::size_t precalculated_hash) { - return m_ht.erase(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - size_type erase(const K& key) { - return m_ht.erase(key); - } - - /** - * @copydoc erase(const K& key) - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup to the value if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - size_type erase(const K& key, std::size_t precalculated_hash) { - return m_ht.erase(key, precalculated_hash); - } - - void swap(hopscotch_map& other) { other.m_ht.swap(m_ht); } - - /* - * Lookup - */ - T& at(const Key& key) { return m_ht.at(key); } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - T& at(const Key& key, std::size_t precalculated_hash) { - return m_ht.at(key, precalculated_hash); - } - - const T& at(const Key& key) const { return m_ht.at(key); } - - /** - * @copydoc at(const Key& key, std::size_t precalculated_hash) - */ - const T& at(const Key& key, std::size_t precalculated_hash) const { - return m_ht.at(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - T& at(const K& key) { - return m_ht.at(key); - } - - /** - * @copydoc at(const K& key) - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - T& at(const K& key, std::size_t precalculated_hash) { - return m_ht.at(key, precalculated_hash); - } - - /** - * @copydoc at(const K& key) - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - const T& at(const K& key) const { - return m_ht.at(key); - } - - /** - * @copydoc at(const K& key, std::size_t precalculated_hash) - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - const T& at(const K& key, std::size_t precalculated_hash) const { - return m_ht.at(key, precalculated_hash); - } - - T& operator[](const Key& key) { return m_ht[key]; } - T& operator[](Key&& key) { return m_ht[std::move(key)]; } - - size_type count(const Key& key) const { return m_ht.count(key); } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - size_type count(const Key& key, std::size_t precalculated_hash) const { - return m_ht.count(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - size_type count(const K& key) const { - return m_ht.count(key); - } - - /** - * @copydoc count(const K& key) const - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - size_type count(const K& key, std::size_t precalculated_hash) const { - return m_ht.count(key, precalculated_hash); - } - - iterator find(const Key& key) { return m_ht.find(key); } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - iterator find(const Key& key, std::size_t precalculated_hash) { - return m_ht.find(key, precalculated_hash); - } - - const_iterator find(const Key& key) const { return m_ht.find(key); } - - /** - * @copydoc find(const Key& key, std::size_t precalculated_hash) - */ - const_iterator find(const Key& key, std::size_t precalculated_hash) const { - return m_ht.find(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - iterator find(const K& key) { - return m_ht.find(key); - } - - /** - * @copydoc find(const K& key) - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - iterator find(const K& key, std::size_t precalculated_hash) { - return m_ht.find(key, precalculated_hash); - } - - /** - * @copydoc find(const K& key) - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - const_iterator find(const K& key) const { - return m_ht.find(key); - } - - /** - * @copydoc find(const K& key) - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - const_iterator find(const K& key, std::size_t precalculated_hash) const { - return m_ht.find(key, precalculated_hash); - } - - bool contains(const Key& key) const { return m_ht.contains(key); } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - bool contains(const Key& key, std::size_t precalculated_hash) const { - return m_ht.contains(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - bool contains(const K& key) const { - return m_ht.contains(key); - } - - /** - * @copydoc contains(const K& key) const - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - bool contains(const K& key, std::size_t precalculated_hash) const { - return m_ht.contains(key, precalculated_hash); - } - - std::pair<iterator, iterator> equal_range(const Key& key) { - return m_ht.equal_range(key); - } - - /** - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - std::pair<iterator, iterator> equal_range(const Key& key, - std::size_t precalculated_hash) { - return m_ht.equal_range(key, precalculated_hash); - } - - std::pair<const_iterator, const_iterator> equal_range(const Key& key) const { - return m_ht.equal_range(key); - } - - /** - * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) - */ - std::pair<const_iterator, const_iterator> equal_range( - const Key& key, std::size_t precalculated_hash) const { - return m_ht.equal_range(key, precalculated_hash); - } - - /** - * This overload only participates in the overload resolution if the typedef - * KeyEqual::is_transparent exists. If so, K must be hashable and comparable - * to Key. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - std::pair<iterator, iterator> equal_range(const K& key) { - return m_ht.equal_range(key); - } - - /** - * @copydoc equal_range(const K& key) - * - * Use the hash value 'precalculated_hash' instead of hashing the key. The - * hash value should be the same as hash_function()(key). Useful to speed-up - * the lookup if you already have the hash. - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - std::pair<iterator, iterator> equal_range(const K& key, - std::size_t precalculated_hash) { - return m_ht.equal_range(key, precalculated_hash); - } - - /** - * @copydoc equal_range(const K& key) - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - std::pair<const_iterator, const_iterator> equal_range(const K& key) const { - return m_ht.equal_range(key); - } - - /** - * @copydoc equal_range(const K& key, std::size_t precalculated_hash) - */ - template < - class K, class KE = KeyEqual, - typename std::enable_if<has_is_transparent<KE>::value>::type* = nullptr> - std::pair<const_iterator, const_iterator> equal_range( - const K& key, std::size_t precalculated_hash) const { - return m_ht.equal_range(key, precalculated_hash); - } - - /* - * Bucket interface - */ - size_type bucket_count() const { return m_ht.bucket_count(); } - size_type max_bucket_count() const { return m_ht.max_bucket_count(); } - - /* - * Hash policy - */ - float load_factor() const { return m_ht.load_factor(); } - float max_load_factor() const { return m_ht.max_load_factor(); } - void max_load_factor(float ml) { m_ht.max_load_factor(ml); } - - void rehash(size_type count_) { m_ht.rehash(count_); } - void reserve(size_type count_) { m_ht.reserve(count_); } - - /* - * Observers - */ - hasher hash_function() const { return m_ht.hash_function(); } - key_equal key_eq() const { return m_ht.key_eq(); } - - /* - * Other - */ - - /** - * Convert a const_iterator to an iterator. - */ - iterator mutable_iterator(const_iterator pos) { - return m_ht.mutable_iterator(pos); - } - - size_type overflow_size() const noexcept { return m_ht.overflow_size(); } - - friend bool operator==(const hopscotch_map& lhs, const hopscotch_map& rhs) { - if (lhs.size() != rhs.size()) { - return false; - } - - for (const auto& element_lhs : lhs) { - const auto it_element_rhs = rhs.find(element_lhs.first); - if (it_element_rhs == rhs.cend() || - element_lhs.second != it_element_rhs->second) { - return false; - } - } - - return true; - } - - friend bool operator!=(const hopscotch_map& lhs, const hopscotch_map& rhs) { - return !operator==(lhs, rhs); - } - - friend void swap(hopscotch_map& lhs, hopscotch_map& rhs) { lhs.swap(rhs); } - - private: - ht m_ht; -}; - -/** - * Same as `tsl::hopscotch_map<Key, T, Hash, KeyEqual, Allocator, - * NeighborhoodSize, StoreHash, tsl::hh::prime_growth_policy>`. - */ -template <class Key, class T, class Hash = std::hash<Key>, - class KeyEqual = std::equal_to<Key>, - class Allocator = std::allocator<std::pair<Key, T>>, - unsigned int NeighborhoodSize = 62, bool StoreHash = false> -using hopscotch_pg_map = - hopscotch_map<Key, T, Hash, KeyEqual, Allocator, NeighborhoodSize, - StoreHash, tsl::hh::prime_growth_policy>; - -} // end namespace tsl - -#endif diff --git a/benchmarks/external/update.sh b/benchmarks/external/update.sh index 00b587cb..c9681485 100644 --- a/benchmarks/external/update.sh +++ b/benchmarks/external/update.sh @@ -1,32 +1,38 @@ -tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl/" -tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl/" -smap="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp/" -pmap="https://raw.github.com/greg7mdp/parallel-hashmap/" -martinus_r="https://raw.github.com/martinus/robin-hood-hashing/master/src/include/" -martinus_d="https://raw.github.com/martinus/unordered_dense/master/include/ankerl/" -skarupke="https://raw.github.com/skarupke/flat_hash_map/master/" - -mkdir -p ankerl skarupke sparsepp tsl - -wget $martinus_r"robin_hood.h" -O "ankerl/robin_hood.h" -wget $martinus_d"unordered_dense.h" -O "ankerl/unordered_dense.h" - -#wget $skarupke"bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp" -wget $skarupke"flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp" - -#wget $smap"spp.h" -O "sparsepp/spp.h" -#wget $smap"spp_config.h" -O "sparsepp/spp_config.h" -#wget $smap"spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h" -#wget $smap"spp_memory.h" -O "sparsepp/spp_memory.h" -#wget $smap"spp_smartptr.h" -O "sparsepp/spp_smartptr.h" -#wget $smap"spp_stdint.h" -O "sparsepp/spp_stdint.h" -#wget $smap"spp_timer.h" -O "sparsepp/spp_timer.h" -#wget $smap"spp_traits.h" -O "sparsepp/spp_traits.h" -#wget $smap"spp_utils.h" -O "sparsepp/spp_utils.h" - -#wget $tsl_h"hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h" -#wget $tsl_h"hopscotch_hash.h" -O "tsl/hopscotch_hash.h" -#wget $tsl_h"hopscotch_map.h" -O "tsl/hopscotch_map.h" -wget $tsl_r"robin_growth_policy.h" -O "tsl/robin_growth_policy.h" -wget $tsl_r"robin_hash.h" -O "tsl/robin_hash.h" -wget $tsl_r"robin_map.h" -O "tsl/robin_map.h" +tsl_h="https://raw.github.com/Tessil/hopscotch-map/master/include/tsl" +tsl_r="https://raw.github.com/Tessil/robin-map/master/include/tsl" +greg_s="https://raw.github.com/greg7mdp/sparsepp/master/sparsepp" +greg_p="https://raw.github.com/greg7mdp/parallel-hashmap" +martinus_r="https://raw.github.com/martinus/robin-hood-hashing/master/src/include" +martinus_d="https://raw.github.com/martinus/unordered_dense/master/include/ankerl" +skarupke="https://raw.github.com/skarupke/flat_hash_map/master" +ktprime="https://raw.github.com/ktprime/emhash/master" + +mkdir -p ankerl skarupke tsl emhash # sparsepp + +wget "$martinus_r/robin_hood.h" -O "ankerl/robin_hood.h" +wget "$martinus_d/unordered_dense.h" -O "ankerl/unordered_dense.h" + +wget "$skarupke/flat_hash_map.hpp" -O "skarupke/flat_hash_map.hpp" + +wget "$tsl_r/robin_growth_policy.h" -O "tsl/robin_growth_policy.h" +wget "$tsl_r/robin_hash.h" -O "tsl/robin_hash.h" +wget "$tsl_r/robin_map.h" -O "tsl/robin_map.h" + +wget "$ktprime/thirdparty/wyhash.h" -O "emhash/wyhash.h" +wget "$ktprime/hash_table7.hpp" -O "emhash/hash_table7.hpp" + +#wget "$tsl_h/hopscotch_growth_policy.h" -O "tsl/hopscotch_growth_policy.h" +#wget "$tsl_h/hopscotch_hash.h" -O "tsl/hopscotch_hash.h" +#wget "$tsl_h/hopscotch_map.h" -O "tsl/hopscotch_map.h" + +#wget "$skarupke/bytell_hash_map.hpp" -O "skarupke/bytell_hash_map.hpp" + +#wget "$greg_s/spp.h" -O "sparsepp/spp.h" +#wget "$greg_s/spp_config.h" -O "sparsepp/spp_config.h" +#wget "$greg_s/spp_dlalloc.h" -O "sparsepp/spp_dlalloc.h" +#wget "$greg_s/spp_memory.h" -O "sparsepp/spp_memory.h" +#wget "$greg_s/spp_smartptr.h" -O "sparsepp/spp_smartptr.h" +#wget "$greg_s/spp_stdint.h" -O "sparsepp/spp_stdint.h" +#wget "$greg_s/spp_timer.h" -O "sparsepp/spp_timer.h" +#wget "$greg_s/spp_traits.h" -O "sparsepp/spp_traits.h" +#wget "$greg_s/spp_utils.h" -O "sparsepp/spp_utils.h" |
