Tulip  6.0.0
Large graphs analysis and drawing
unordered_dense.h
1 ///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////
2 
3 // A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
4 // Version 4.4.0
5 // https://github.com/martinus/unordered_dense
6 //
7 // Licensed under the MIT License <http://opensource.org/licenses/MIT>.
8 // SPDX-License-Identifier: MIT
9 // Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
10 //
11 // Permission is hereby granted, free of charge, to any person obtaining a copy
12 // of this software and associated documentation files (the "Software"), to deal
13 // in the Software without restriction, including without limitation the rights
14 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15 // copies of the Software, and to permit persons to whom the Software is
16 // furnished to do so, subject to the following conditions:
17 //
18 // The above copyright notice and this permission notice shall be included in all
19 // copies or substantial portions of the Software.
20 //
21 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 // SOFTWARE.
28 
29 #ifndef ANKERL_UNORDERED_DENSE_H
30 #define ANKERL_UNORDERED_DENSE_H
31 
32 // see https://semver.org/spec/v2.0.0.html
33 #define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
34 #define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
35 #define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes
36 
37 // API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
38 
39 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
40 #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
41 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
42 #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
43 #define ANKERL_UNORDERED_DENSE_NAMESPACE \
44  ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \
45  ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH)
46 
47 #if defined(_MSVC_LANG)
48 # define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
49 #else
50 # define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
51 #endif
52 
53 #if defined(__GNUC__)
54 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
55 # define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
56 #elif defined(_MSC_VER)
57 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
58 # define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
59 #endif
60 
61 // exceptions
62 #if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
63 # define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage)
64 #else
65 # define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage)
66 #endif
67 #ifdef _MSC_VER
68 # define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline)
69 #else
70 # define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline))
71 #endif
72 
73 // defined in unordered_dense.cpp
74 #if !defined(ANKERL_UNORDERED_DENSE_EXPORT)
75 # define ANKERL_UNORDERED_DENSE_EXPORT
76 #endif
77 
78 #if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
79 # error ankerl::unordered_dense requires C++17 or higher
80 #else
81 # include <array> // for array
82 # include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C
83 # include <cstring> // for size_t, memcpy, memset
84 # include <functional> // for equal_to, hash
85 # include <initializer_list> // for initializer_list
86 # include <iterator> // for pair, distance
87 # include <limits> // for numeric_limits
88 # include <memory> // for allocator, allocator_traits, shared_ptr
89 # include <optional> // for optional
90 # include <stdexcept> // for out_of_range
91 # include <string> // for basic_string
92 # include <string_view> // for basic_string_view, hash
93 # include <tuple> // for forward_as_tuple
94 # include <type_traits> // for enable_if_t, declval, conditional_t, ena...
95 # include <utility> // for forward, exchange, pair, as_const, piece...
96 # include <vector> // for vector
97 # if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0
98 # include <cstdlib> // for abort
99 # endif
100 
101 # if defined(__has_include)
102 # if __has_include(<memory_resource>)
103 # define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage)
104 # include <memory_resource> // for polymorphic_allocator
105 # elif __has_include(<experimental/memory_resource>)
106 # define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage)
107 # include <experimental/memory_resource> // for polymorphic_allocator
108 # endif
109 # endif
110 
111 # if defined(_MSC_VER) && defined(_M_X64)
112 # include <intrin.h>
113 # pragma intrinsic(_umul128)
114 # endif
115 
116 # if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
117 # define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage)
118 # define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage)
119 # else
120 # define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
121 # define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
122 # endif
123 
124 namespace ankerl::unordered_dense {
125 inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {
126 
127 namespace detail {
128 
129 # if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS()
130 
131 // make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
132 // inlinings more difficult. Throws are also generally the slow path.
133 [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() {
134  throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
135 }
136 [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() {
137  throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
138 }
139 [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() {
140  throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
141 }
142 
143 # else
144 
145 [[noreturn]] inline void on_error_key_not_found() {
146  abort();
147 }
148 [[noreturn]] inline void on_error_bucket_overflow() {
149  abort();
150 }
151 [[noreturn]] inline void on_error_too_many_elements() {
152  abort();
153 }
154 
155 # endif
156 
157 } // namespace detail
158 
159 // hash ///////////////////////////////////////////////////////////////////////
160 
161 // This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
162 // No big-endian support (because different values on different machines don't matter),
163 // hardcodes seed and the secret, reformats the code, and clang-tidy fixes.
164 namespace detail::wyhash {
165 
166 inline void mum(uint64_t* a, uint64_t* b) {
167 # if defined(__SIZEOF_INT128__)
168  __uint128_t r = *a;
169  r *= *b;
170  *a = static_cast<uint64_t>(r);
171  *b = static_cast<uint64_t>(r >> 64U);
172 # elif defined(_MSC_VER) && defined(_M_X64)
173  *a = _umul128(*a, *b, b);
174 # else
175  uint64_t ha = *a >> 32U;
176  uint64_t hb = *b >> 32U;
177  uint64_t la = static_cast<uint32_t>(*a);
178  uint64_t lb = static_cast<uint32_t>(*b);
179  uint64_t hi{};
180  uint64_t lo{};
181  uint64_t rh = ha * hb;
182  uint64_t rm0 = ha * lb;
183  uint64_t rm1 = hb * la;
184  uint64_t rl = la * lb;
185  uint64_t t = rl + (rm0 << 32U);
186  auto c = static_cast<uint64_t>(t < rl);
187  lo = t + (rm1 << 32U);
188  c += static_cast<uint64_t>(lo < t);
189  hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
190  *a = lo;
191  *b = hi;
192 # endif
193 }
194 
195 // multiply and xor mix function, aka MUM
196 [[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
197  mum(&a, &b);
198  return a ^ b;
199 }
200 
201 // read functions. WARNING: we don't care about endianness, so results are different on big endian!
202 [[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t {
203  uint64_t v{};
204  std::memcpy(&v, p, 8U);
205  return v;
206 }
207 
208 [[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t {
209  uint32_t v{};
210  std::memcpy(&v, p, 4);
211  return v;
212 }
213 
214 // reads 1, 2, or 3 bytes
215 [[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t {
216  return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
217 }
218 
219 [[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t {
220  static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
221  UINT64_C(0xe7037ed1a0b428db),
222  UINT64_C(0x8ebc6af09c88c6e3),
223  UINT64_C(0x589965cc75374cc3)};
224 
225  auto const* p = static_cast<uint8_t const*>(key);
226  uint64_t seed = secret[0];
227  uint64_t a{};
228  uint64_t b{};
229  if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
230  if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
231  a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
232  b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
233  } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
234  a = r3(p, len);
235  b = 0;
236  } else {
237  a = 0;
238  b = 0;
239  }
240  } else {
241  size_t i = len;
242  if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
243  uint64_t see1 = seed;
244  uint64_t see2 = seed;
245  do {
246  seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
247  see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
248  see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
249  p += 48;
250  i -= 48;
251  } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
252  seed ^= see1 ^ see2;
253  }
254  while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
255  seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
256  i -= 16;
257  p += 16;
258  }
259  a = r8(p + i - 16);
260  b = r8(p + i - 8);
261  }
262 
263  return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
264 }
265 
266 [[nodiscard]] inline auto hash(uint64_t x) -> uint64_t {
267  return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
268 }
269 
270 } // namespace detail::wyhash
271 
272 ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void>
273 struct hash {
274  auto operator()(T const& obj) const
275  noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) -> uint64_t {
276  return std::hash<T>{}(obj);
277  }
278 };
279 
280 template <typename T>
281 struct hash<T, typename std::hash<T>::is_avalanching> {
282  using is_avalanching = void;
283  auto operator()(T const& obj) const
284  noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) -> uint64_t {
285  return std::hash<T>{}(obj);
286  }
287 };
288 
289 template <typename CharT>
290 struct hash<std::basic_string<CharT>> {
291  using is_avalanching = void;
292  auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t {
293  return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
294  }
295 };
296 
297 template <typename CharT>
298 struct hash<std::basic_string_view<CharT>> {
299  using is_avalanching = void;
300  auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t {
301  return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
302  }
303 };
304 
305 template <class T>
306 struct hash<T*> {
307  using is_avalanching = void;
308  auto operator()(T* ptr) const noexcept -> uint64_t {
309  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
310  return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
311  }
312 };
313 
314 template <class T>
315 struct hash<std::unique_ptr<T>> {
316  using is_avalanching = void;
317  auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t {
318  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
319  return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
320  }
321 };
322 
323 template <class T>
324 struct hash<std::shared_ptr<T>> {
325  using is_avalanching = void;
326  auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t {
327  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
328  return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
329  }
330 };
331 
332 template <typename Enum>
333 struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
334  using is_avalanching = void;
335  auto operator()(Enum e) const noexcept -> uint64_t {
336  using underlying = typename std::underlying_type_t<Enum>;
337  return detail::wyhash::hash(static_cast<underlying>(e));
338  }
339 };
340 
341 template <typename... Args>
342 struct tuple_hash_helper {
343  // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest.
344  // If it isn't an integral we need to hash it.
345  template <typename Arg>
346  [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t {
347  if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) {
348  return static_cast<uint64_t>(arg);
349  } else {
350  return hash<Arg>{}(arg);
351  }
352  }
353 
354  [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t {
355  return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69});
356  }
357 
358  // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If
359  // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized
360  // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer.
361  template <typename T, std::size_t... Idx>
362  [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t {
363  auto h = uint64_t{};
364  ((h = mix64(h, to64(std::get<Idx>(t)))), ...);
365  return h;
366  }
367 };
368 
369 template <typename... Args>
370 struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> {
371  using is_avalanching = void;
372  auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t {
373  return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{});
374  }
375 };
376 
377 template <typename A, typename B>
378 struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> {
379  using is_avalanching = void;
380  auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t {
381  return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{});
382  }
383 };
384 
385 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
386 # define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \
387  template <> \
388  struct hash<T> { \
389  using is_avalanching = void; \
390  auto operator()(T const& obj) const noexcept -> uint64_t { \
391  return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
392  } \
393  }
394 
395 # if defined(__GNUC__) && !defined(__clang__)
396 # pragma GCC diagnostic push
397 # pragma GCC diagnostic ignored "-Wuseless-cast"
398 # endif
399 // see https://en.cppreference.com/w/cpp/utility/hash
400 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
401 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
402 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
403 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
404 # if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t)
405 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
406 # endif
407 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
408 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
409 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
410 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
411 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
412 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
413 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
414 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
415 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
416 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
417 ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);
418 
419 # if defined(__GNUC__) && !defined(__clang__)
420 # pragma GCC diagnostic pop
421 # endif
422 
423 // bucket_type //////////////////////////////////////////////////////////
424 
425 namespace bucket_type {
426 
427 struct standard {
428  static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
429  static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
430 
431  uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
432  uint32_t m_value_idx; // index into the m_values vector.
433 };
434 
435 ANKERL_UNORDERED_DENSE_PACK(struct big {
436  static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
437  static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
438 
439  uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
440  size_t m_value_idx; // index into the m_values vector.
441 });
442 
443 } // namespace bucket_type
444 
445 namespace detail {
446 
447 struct nonesuch {};
448 struct default_container_t {};
449 
450 template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
451 struct detector {
452  using value_t = std::false_type;
453  using type = Default;
454 };
455 
456 template <class Default, template <class...> class Op, class... Args>
457 struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
458  using value_t = std::true_type;
459  using type = Op<Args...>;
460 };
461 
462 template <template <class...> class Op, class... Args>
463 using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;
464 
465 template <template <class...> class Op, class... Args>
466 constexpr bool is_detected_v = is_detected<Op, Args...>::value;
467 
468 template <typename T>
469 using detect_avalanching = typename T::is_avalanching;
470 
471 template <typename T>
472 using detect_is_transparent = typename T::is_transparent;
473 
474 template <typename T>
475 using detect_iterator = typename T::iterator;
476 
477 template <typename T>
478 using detect_reserve = decltype(std::declval<T&>().reserve(size_t{}));
479 
480 // enable_if helpers
481 
482 template <typename Mapped>
483 constexpr bool is_map_v = !std::is_void_v<Mapped>;
484 
485 // clang-format off
486 template <typename Hash, typename KeyEqual>
487 constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>;
488 // clang-format on
489 
490 template <typename From, typename To1, typename To2>
491 constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;
492 
493 template <typename T>
494 constexpr bool has_reserve = is_detected_v<detect_reserve, T>;
495 
496 // base type for map has mapped_type
497 template <class T>
498 struct base_table_type_map {
499  using mapped_type = T;
500 };
501 
502 // base type for set doesn't have mapped_type
503 struct base_table_type_set {};
504 
505 } // namespace detail
506 
507 // Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector
508 // API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}.
509 // It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new
510 // block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not
511 // linear and thus there is one more indirection necessary for indexing.
512 template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096>
513 class segmented_vector {
514  template <bool IsConst>
515  class iter_t;
516 
517 public:
518  using allocator_type = Allocator;
519  using pointer = typename std::allocator_traits<allocator_type>::pointer;
520  using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
521  using difference_type = typename std::allocator_traits<allocator_type>::difference_type;
522  using value_type = T;
523  using size_type = std::size_t;
524  using reference = T&;
525  using const_reference = T const&;
526  using iterator = iter_t<false>;
527  using const_iterator = iter_t<true>;
528 
529 private:
530  using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>;
531  std::vector<pointer, vec_alloc> m_blocks{};
532  size_t m_size{};
533 
534  // Calculates the maximum number for x in (s << x) <= max_val
535  static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t {
536  auto f = size_t{0};
537  while (s << (f + 1) <= max_val) {
538  ++f;
539  }
540  return f;
541  }
542 
543  using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>;
544  static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T));
545  static constexpr auto num_elements_in_block = 1U << num_bits;
546  static constexpr auto mask = num_elements_in_block - 1U;
547 
548  /**
549  * Iterator class doubles as const_iterator and iterator
550  */
551  template <bool IsConst>
552  class iter_t {
553  using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>;
554  ptr_t m_data{};
555  size_t m_idx{};
556 
557  template <bool B>
558  friend class iter_t;
559 
560  public:
561  using difference_type = segmented_vector::difference_type;
562  using value_type = T;
563  using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>;
564  using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>;
565  using iterator_category = std::forward_iterator_tag;
566 
567  iter_t() noexcept = default;
568 
569  template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
570  // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
571  constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept
572  : m_data(other.m_data)
573  , m_idx(other.m_idx) {}
574 
575  constexpr iter_t(ptr_t data, size_t idx) noexcept
576  : m_data(data)
577  , m_idx(idx) {}
578 
579  template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
580  constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& {
581  m_data = other.m_data;
582  m_idx = other.m_idx;
583  return *this;
584  }
585 
586  constexpr auto operator++() noexcept -> iter_t& {
587  ++m_idx;
588  return *this;
589  }
590 
591  constexpr auto operator+(difference_type diff) noexcept -> iter_t {
592  return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)};
593  }
594 
595  template <bool OtherIsConst>
596  constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type {
597  return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx);
598  }
599 
600  constexpr auto operator*() const noexcept -> reference {
601  return m_data[m_idx >> num_bits][m_idx & mask];
602  }
603 
604  constexpr auto operator->() const noexcept -> pointer {
605  return &m_data[m_idx >> num_bits][m_idx & mask];
606  }
607 
608  template <bool O>
609  constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool {
610  return m_idx == o.m_idx;
611  }
612 
613  template <bool O>
614  constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool {
615  return !(*this == o);
616  }
617  };
618 
619  // slow path: need to allocate a new segment every once in a while
620  void increase_capacity() {
621  auto ba = Allocator(m_blocks.get_allocator());
622  pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block);
623  m_blocks.push_back(block);
624  }
625 
626  // Moves everything from other
627  void append_everything_from(segmented_vector&& other) {
628  reserve(size() + other.size());
629  for (auto&& o : other) {
630  emplace_back(std::move(o));
631  }
632  }
633 
634  // Copies everything from other
635  void append_everything_from(segmented_vector const& other) {
636  reserve(size() + other.size());
637  for (auto const& o : other) {
638  emplace_back(o);
639  }
640  }
641 
642  void dealloc() {
643  auto ba = Allocator(m_blocks.get_allocator());
644  for (auto ptr : m_blocks) {
645  std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block);
646  }
647  }
648 
649  [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) {
650  return (capacity + num_elements_in_block - 1U) / num_elements_in_block;
651  }
652 
653 public:
654  segmented_vector() = default;
655 
656  // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
657  segmented_vector(Allocator alloc)
658  : m_blocks(vec_alloc(alloc)) {}
659 
660  segmented_vector(segmented_vector&& other, Allocator alloc)
661  : segmented_vector(alloc) {
662  *this = std::move(other);
663  }
664 
665  segmented_vector(segmented_vector const& other, Allocator alloc)
666  : m_blocks(vec_alloc(alloc)) {
667  append_everything_from(other);
668  }
669 
670  segmented_vector(segmented_vector&& other) noexcept
671  : segmented_vector(std::move(other), get_allocator()) {}
672 
673  segmented_vector(segmented_vector const& other) {
674  append_everything_from(other);
675  }
676 
677  auto operator=(segmented_vector const& other) -> segmented_vector& {
678  if (this == &other) {
679  return *this;
680  }
681  clear();
682  append_everything_from(other);
683  return *this;
684  }
685 
686  auto operator=(segmented_vector&& other) noexcept -> segmented_vector& {
687  clear();
688  dealloc();
689  if (other.get_allocator() == get_allocator()) {
690  m_blocks = std::move(other.m_blocks);
691  m_size = std::exchange(other.m_size, {});
692  } else {
693  // make sure to construct with other's allocator!
694  m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator()));
695  append_everything_from(std::move(other));
696  }
697  return *this;
698  }
699 
700  ~segmented_vector() {
701  clear();
702  dealloc();
703  }
704 
705  [[nodiscard]] constexpr auto size() const -> size_t {
706  return m_size;
707  }
708 
709  [[nodiscard]] constexpr auto capacity() const -> size_t {
710  return m_blocks.size() * num_elements_in_block;
711  }
712 
713  // Indexing is highly performance critical
714  [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& {
715  return m_blocks[i >> num_bits][i & mask];
716  }
717 
718  [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& {
719  return m_blocks[i >> num_bits][i & mask];
720  }
721 
722  [[nodiscard]] constexpr auto begin() -> iterator {
723  return {m_blocks.data(), 0U};
724  }
725  [[nodiscard]] constexpr auto begin() const -> const_iterator {
726  return {m_blocks.data(), 0U};
727  }
728  [[nodiscard]] constexpr auto cbegin() const -> const_iterator {
729  return {m_blocks.data(), 0U};
730  }
731 
732  [[nodiscard]] constexpr auto end() -> iterator {
733  return {m_blocks.data(), m_size};
734  }
735  [[nodiscard]] constexpr auto end() const -> const_iterator {
736  return {m_blocks.data(), m_size};
737  }
738  [[nodiscard]] constexpr auto cend() const -> const_iterator {
739  return {m_blocks.data(), m_size};
740  }
741 
742  [[nodiscard]] constexpr auto back() -> reference {
743  return operator[](m_size - 1);
744  }
745  [[nodiscard]] constexpr auto back() const -> const_reference {
746  return operator[](m_size - 1);
747  }
748 
749  void pop_back() {
750  back().~T();
751  --m_size;
752  }
753 
754  [[nodiscard]] auto empty() const {
755  return 0 == m_size;
756  }
757 
758  void reserve(size_t new_capacity) {
759  m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity));
760  while (new_capacity > capacity()) {
761  increase_capacity();
762  }
763  }
764 
765  [[nodiscard]] auto get_allocator() const -> allocator_type {
766  return allocator_type{m_blocks.get_allocator()};
767  }
768 
769  template <class... Args>
770  auto emplace_back(Args&&... args) -> reference {
771  if (m_size == capacity()) {
772  increase_capacity();
773  }
774  auto* ptr = static_cast<void*>(&operator[](m_size));
775  auto& ref = *new (ptr) T(std::forward<Args>(args)...);
776  ++m_size;
777  return ref;
778  }
779 
780  void clear() {
781  if constexpr (!std::is_trivially_destructible_v<T>) {
782  for (size_t i = 0, s = size(); i < s; ++i) {
783  operator[](i).~T();
784  }
785  }
786  m_size = 0;
787  }
788 
789  void shrink_to_fit() {
790  auto ba = Allocator(m_blocks.get_allocator());
791  auto num_blocks_required = calc_num_blocks_for_capacity(m_size);
792  while (m_blocks.size() > num_blocks_required) {
793  std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block);
794  m_blocks.pop_back();
795  }
796  m_blocks.shrink_to_fit();
797  }
798 };
799 
800 namespace detail {
801 
802 // This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
803 template <class Key,
804  class T, // when void, treat it as a set.
805  class Hash,
806  class KeyEqual,
807  class AllocatorOrContainer,
808  class Bucket,
809  class BucketContainer,
810  bool IsSegmented>
811 class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
812  using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
813  using underlying_container_type = std::conditional_t<IsSegmented,
814  segmented_vector<underlying_value_type, AllocatorOrContainer>,
815  std::vector<underlying_value_type, AllocatorOrContainer>>;
816 
817 public:
818  using value_container_type = std::
819  conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>;
820 
821 private:
822  using bucket_alloc =
823  typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
824  using default_bucket_container_type =
825  std::conditional_t<IsSegmented, segmented_vector<Bucket, bucket_alloc>, std::vector<Bucket, bucket_alloc>>;
826 
827  using bucket_container_type = std::conditional_t<std::is_same_v<BucketContainer, detail::default_container_t>,
828  default_bucket_container_type,
829  BucketContainer>;
830 
831  static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
832  static constexpr float default_max_load_factor = 0.8F;
833 
834 public:
835  using key_type = Key;
836  using value_type = typename value_container_type::value_type;
837  using size_type = typename value_container_type::size_type;
838  using difference_type = typename value_container_type::difference_type;
839  using hasher = Hash;
840  using key_equal = KeyEqual;
841  using allocator_type = typename value_container_type::allocator_type;
842  using reference = typename value_container_type::reference;
843  using const_reference = typename value_container_type::const_reference;
844  using pointer = typename value_container_type::pointer;
845  using const_pointer = typename value_container_type::const_pointer;
846  using const_iterator = typename value_container_type::const_iterator;
847  using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>;
848  using bucket_type = Bucket;
849 
850 private:
851  using value_idx_type = decltype(Bucket::m_value_idx);
852  using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);
853 
854  static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
855  static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");
856 
857  value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
858  bucket_container_type m_buckets{};
859  size_t m_max_bucket_capacity = 0;
860  float m_max_load_factor = default_max_load_factor;
861  Hash m_hash{};
862  KeyEqual m_equal{};
863  uint8_t m_shifts = initial_shifts;
864 
865  [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
866  return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == bucket_count())
867  ? 0
868  : static_cast<value_idx_type>(bucket_idx + 1U);
869  }
870 
871  // Helper to access bucket through pointer types
872  [[nodiscard]] static constexpr auto at(bucket_container_type& bucket, size_t offset) -> Bucket& {
873  return bucket[offset];
874  }
875 
876  [[nodiscard]] static constexpr auto at(const bucket_container_type& bucket, size_t offset) -> const Bucket& {
877  return bucket[offset];
878  }
879 
880  // use the dist_inc and dist_dec functions so that uint16_t types work without warning
881  [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
882  return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
883  }
884 
885  [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
886  return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
887  }
888 
889  // The goal of mixed_hash is to always produce a high quality 64bit hash.
890  template <typename K>
891  [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t {
892  if constexpr (is_detected_v<detect_avalanching, Hash>) {
893  // we know that the hash is good because is_avalanching.
894  if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
895  // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
896  return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
897  } else {
898  // 64bit and is_avalanching => only use the hash itself.
899  return m_hash(key);
900  }
901  } else {
902  // not is_avalanching => apply wyhash
903  return wyhash::hash(m_hash(key));
904  }
905  }
906 
907  [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
908  return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
909  }
910 
911  [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
912  return static_cast<value_idx_type>(hash >> m_shifts);
913  }
914 
915  [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& {
916  if constexpr (is_map_v<T>) {
917  return vt.first;
918  } else {
919  return vt;
920  }
921  }
922 
923  template <typename K>
924  [[nodiscard]] auto next_while_less(K const& key) const -> Bucket {
925  auto hash = mixed_hash(key);
926  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
927  auto bucket_idx = bucket_idx_from_hash(hash);
928 
929  while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
930  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
931  bucket_idx = next(bucket_idx);
932  }
933  return {dist_and_fingerprint, bucket_idx};
934  }
935 
936  void place_and_shift_up(Bucket bucket, value_idx_type place) {
937  while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
938  bucket = std::exchange(at(m_buckets, place), bucket);
939  bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
940  place = next(place);
941  }
942  at(m_buckets, place) = bucket;
943  }
944 
945  [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
946  return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts));
947  }
948 
949  [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
950  auto shifts = initial_shifts;
951  while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
952  --shifts;
953  }
954  return shifts;
955  }
956 
957  // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
958  void copy_buckets(table const& other) {
959  // assumes m_values has already the correct data copied over.
960  if (empty()) {
961  // when empty, at least allocate an initial buckets and clear them.
962  allocate_buckets_from_shift();
963  clear_buckets();
964  } else {
965  m_shifts = other.m_shifts;
966  allocate_buckets_from_shift();
967  if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
968  for (auto i = 0UL; i < bucket_count(); ++i) {
969  at(m_buckets, i) = at(other.m_buckets, i);
970  }
971  } else {
972  std::memcpy(m_buckets.data(), other.m_buckets.data(), sizeof(Bucket) * bucket_count());
973  }
974  }
975  }
976 
977  /**
978  * True when no element can be added any more without increasing the size
979  */
980  [[nodiscard]] auto is_full() const -> bool {
981  return size() > m_max_bucket_capacity;
982  }
983 
984  void deallocate_buckets() {
985  m_buckets.clear();
986  m_buckets.shrink_to_fit();
987  m_max_bucket_capacity = 0;
988  }
989 
990  void allocate_buckets_from_shift() {
991  auto num_buckets = calc_num_buckets(m_shifts);
992  if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
993  if constexpr (has_reserve<bucket_container_type>) {
994  m_buckets.reserve(num_buckets);
995  }
996  for (size_t i = m_buckets.size(); i < num_buckets; ++i) {
997  m_buckets.emplace_back();
998  }
999  } else {
1000  m_buckets.resize(num_buckets);
1001  }
1002  if (num_buckets == max_bucket_count()) {
1003  // reached the maximum, make sure we can use each bucket
1004  m_max_bucket_capacity = max_bucket_count();
1005  } else {
1006  m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(num_buckets) * max_load_factor());
1007  }
1008  }
1009 
1010  void clear_buckets() {
1011  if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
1012  for (auto&& e : m_buckets) {
1013  std::memset(&e, 0, sizeof(e));
1014  }
1015  } else {
1016  std::memset(m_buckets.data(), 0, sizeof(Bucket) * bucket_count());
1017  }
1018  }
1019 
1020  void clear_and_fill_buckets_from_values() {
1021  clear_buckets();
1022  for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
1023  ++value_idx) {
1024  auto const& key = get_key(m_values[value_idx]);
1025  auto [dist_and_fingerprint, bucket] = next_while_less(key);
1026 
1027  // we know for certain that key has not yet been inserted, so no need to check it.
1028  place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
1029  }
1030  }
1031 
1032  void increase_size() {
1033  if (m_max_bucket_capacity == max_bucket_count()) {
1034  // remove the value again, we can't add it!
1035  m_values.pop_back();
1036  on_error_bucket_overflow();
1037  }
1038  --m_shifts;
1039  if constexpr (!IsSegmented || std::is_same_v<BucketContainer, default_container_t>) {
1040  deallocate_buckets();
1041  }
1042  allocate_buckets_from_shift();
1043  clear_and_fill_buckets_from_values();
1044  }
1045 
1046  template <typename Op>
1047  void do_erase(value_idx_type bucket_idx, Op handle_erased_value) {
1048  auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;
1049 
1050  // shift down until either empty or an element with correct spot is found
1051  auto next_bucket_idx = next(bucket_idx);
1052  while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
1053  at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
1054  at(m_buckets, next_bucket_idx).m_value_idx};
1055  bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
1056  }
1057  at(m_buckets, bucket_idx) = {};
1058  handle_erased_value(std::move(m_values[value_idx_to_remove]));
1059 
1060  // update m_values
1061  if (value_idx_to_remove != m_values.size() - 1) {
1062  // no luck, we'll have to replace the value with the last one and update the index accordingly
1063  auto& val = m_values[value_idx_to_remove];
1064  val = std::move(m_values.back());
1065 
1066  // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
1067  auto mh = mixed_hash(get_key(val));
1068  bucket_idx = bucket_idx_from_hash(mh);
1069 
1070  auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
1071  while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
1072  bucket_idx = next(bucket_idx);
1073  }
1074  at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
1075  }
1076  m_values.pop_back();
1077  }
1078 
1079  template <typename K, typename Op>
1080  auto do_erase_key(K&& key, Op handle_erased_value) -> size_t {
1081  if (empty()) {
1082  return 0;
1083  }
1084 
1085  auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);
1086 
1087  while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1088  !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
1089  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1090  bucket_idx = next(bucket_idx);
1091  }
1092 
1093  if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1094  return 0;
1095  }
1096  do_erase(bucket_idx, handle_erased_value);
1097  return 1;
1098  }
1099 
1100  template <class K, class M>
1101  auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
1102  auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
1103  if (!it_isinserted.second) {
1104  it_isinserted.first->second = std::forward<M>(mapped);
1105  }
1106  return it_isinserted;
1107  }
1108 
1109  template <typename... Args>
1110  auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint,
1111  value_idx_type bucket_idx,
1112  Args&&... args) -> std::pair<iterator, bool> {
1113 
1114  // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
1115  m_values.emplace_back(std::forward<Args>(args)...);
1116 
1117  auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
1118  if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
1119  increase_size();
1120  } else {
1121  place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1122  }
1123 
1124  // place element and shift up until we find an empty spot
1125  return {begin() + static_cast<difference_type>(value_idx), true};
1126  }
1127 
1128  template <typename K, typename... Args>
1129  auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
1130  auto hash = mixed_hash(key);
1131  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1132  auto bucket_idx = bucket_idx_from_hash(hash);
1133 
1134  while (true) {
1135  auto* bucket = &at(m_buckets, bucket_idx);
1136  if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
1137  if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1138  return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
1139  }
1140  } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
1141  return do_place_element(dist_and_fingerprint,
1142  bucket_idx,
1143  std::piecewise_construct,
1144  std::forward_as_tuple(std::forward<K>(key)),
1145  std::forward_as_tuple(std::forward<Args>(args)...));
1146  }
1147  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1148  bucket_idx = next(bucket_idx);
1149  }
1150  }
1151 
1152  template <typename K>
1153  auto do_find(K const& key) -> iterator {
1154  if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
1155  return end();
1156  }
1157 
1158  auto mh = mixed_hash(key);
1159  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
1160  auto bucket_idx = bucket_idx_from_hash(mh);
1161  auto* bucket = &at(m_buckets, bucket_idx);
1162 
1163  // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
1164  if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1165  return begin() + static_cast<difference_type>(bucket->m_value_idx);
1166  }
1167  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1168  bucket_idx = next(bucket_idx);
1169  bucket = &at(m_buckets, bucket_idx);
1170 
1171  if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1172  return begin() + static_cast<difference_type>(bucket->m_value_idx);
1173  }
1174  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1175  bucket_idx = next(bucket_idx);
1176  bucket = &at(m_buckets, bucket_idx);
1177 
1178  while (true) {
1179  if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
1180  if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1181  return begin() + static_cast<difference_type>(bucket->m_value_idx);
1182  }
1183  } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
1184  return end();
1185  }
1186  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1187  bucket_idx = next(bucket_idx);
1188  bucket = &at(m_buckets, bucket_idx);
1189  }
1190  }
1191 
1192  template <typename K>
1193  auto do_find(K const& key) const -> const_iterator {
1194  return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
1195  }
1196 
1197  template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1198  auto do_at(K const& key) -> Q& {
1199  if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) {
1200  return it->second;
1201  }
1202  on_error_key_not_found();
1203  }
1204 
1205  template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1206  auto do_at(K const& key) const -> Q const& {
1207  return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
1208  }
1209 
1210 public:
1211  explicit table(size_t bucket_count,
1212  Hash const& hash = Hash(),
1213  KeyEqual const& equal = KeyEqual(),
1214  allocator_type const& alloc_or_container = allocator_type())
1215  : m_values(alloc_or_container)
1216  , m_buckets(alloc_or_container)
1217  , m_hash(hash)
1218  , m_equal(equal) {
1219  if (0 != bucket_count) {
1220  reserve(bucket_count);
1221  } else {
1222  allocate_buckets_from_shift();
1223  clear_buckets();
1224  }
1225  }
1226 
1227  table()
1228  : table(0) {}
1229 
1230  table(size_t bucket_count, allocator_type const& alloc)
1231  : table(bucket_count, Hash(), KeyEqual(), alloc) {}
1232 
1233  table(size_t bucket_count, Hash const& hash, allocator_type const& alloc)
1234  : table(bucket_count, hash, KeyEqual(), alloc) {}
1235 
1236  explicit table(allocator_type const& alloc)
1237  : table(0, Hash(), KeyEqual(), alloc) {}
1238 
1239  template <class InputIt>
1240  table(InputIt first,
1241  InputIt last,
1242  size_type bucket_count = 0,
1243  Hash const& hash = Hash(),
1244  KeyEqual const& equal = KeyEqual(),
1245  allocator_type const& alloc = allocator_type())
1246  : table(bucket_count, hash, equal, alloc) {
1247  insert(first, last);
1248  }
1249 
1250  template <class InputIt>
1251  table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc)
1252  : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}
1253 
1254  template <class InputIt>
1255  table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
1256  : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}
1257 
1258  table(table const& other)
1259  : table(other, other.m_values.get_allocator()) {}
1260 
1261  table(table const& other, allocator_type const& alloc)
1262  : m_values(other.m_values, alloc)
1263  , m_max_load_factor(other.m_max_load_factor)
1264  , m_hash(other.m_hash)
1265  , m_equal(other.m_equal) {
1266  copy_buckets(other);
1267  }
1268 
1269  table(table&& other) noexcept
1270  : table(std::move(other), other.m_values.get_allocator()) {}
1271 
1272  table(table&& other, allocator_type const& alloc) noexcept
1273  : m_values(alloc) {
1274  *this = std::move(other);
1275  }
1276 
1277  table(std::initializer_list<value_type> ilist,
1278  size_t bucket_count = 0,
1279  Hash const& hash = Hash(),
1280  KeyEqual const& equal = KeyEqual(),
1281  allocator_type const& alloc = allocator_type())
1282  : table(bucket_count, hash, equal, alloc) {
1283  insert(ilist);
1284  }
1285 
1286  table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc)
1287  : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}
1288 
1289  table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
1290  : table(init, bucket_count, hash, KeyEqual(), alloc) {}
1291 
1292  ~table() {}
1293 
1294  auto operator=(table const& other) -> table& {
1295  if (&other != this) {
1296  deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
1297  m_values = other.m_values;
1298  m_max_load_factor = other.m_max_load_factor;
1299  m_hash = other.m_hash;
1300  m_equal = other.m_equal;
1301  m_shifts = initial_shifts;
1302  copy_buckets(other);
1303  }
1304  return *this;
1305  }
1306 
1307  auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&
1308  std::is_nothrow_move_assignable_v<Hash> &&
1309  std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& {
1310  if (&other != this) {
1311  deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
1312  m_values = std::move(other.m_values);
1313  other.m_values.clear();
1314 
1315  // we can only reuse m_buckets when both maps have the same allocator!
1316  if (get_allocator() == other.get_allocator()) {
1317  m_buckets = std::move(other.m_buckets);
1318  other.m_buckets.clear();
1319  m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
1320  m_shifts = std::exchange(other.m_shifts, initial_shifts);
1321  m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
1322  m_hash = std::exchange(other.m_hash, {});
1323  m_equal = std::exchange(other.m_equal, {});
1324  other.allocate_buckets_from_shift();
1325  other.clear_buckets();
1326  } else {
1327  // set max_load_factor *before* copying the other's buckets, so we have the same
1328  // behavior
1329  m_max_load_factor = other.m_max_load_factor;
1330 
1331  // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts
1332  copy_buckets(other);
1333  // clear's the other's buckets so other is now already usable.
1334  other.clear_buckets();
1335  m_hash = other.m_hash;
1336  m_equal = other.m_equal;
1337  }
1338  // map "other" is now already usable, it's empty.
1339  }
1340  return *this;
1341  }
1342 
1343  auto operator=(std::initializer_list<value_type> ilist) -> table& {
1344  clear();
1345  insert(ilist);
1346  return *this;
1347  }
1348 
1349  auto get_allocator() const noexcept -> allocator_type {
1350  return m_values.get_allocator();
1351  }
1352 
1353  // iterators //////////////////////////////////////////////////////////////
1354 
1355  auto begin() noexcept -> iterator {
1356  return m_values.begin();
1357  }
1358 
1359  auto begin() const noexcept -> const_iterator {
1360  return m_values.begin();
1361  }
1362 
1363  auto cbegin() const noexcept -> const_iterator {
1364  return m_values.cbegin();
1365  }
1366 
1367  auto end() noexcept -> iterator {
1368  return m_values.end();
1369  }
1370 
1371  auto cend() const noexcept -> const_iterator {
1372  return m_values.cend();
1373  }
1374 
1375  auto end() const noexcept -> const_iterator {
1376  return m_values.end();
1377  }
1378 
1379  // capacity ///////////////////////////////////////////////////////////////
1380 
1381  [[nodiscard]] auto empty() const noexcept -> bool {
1382  return m_values.empty();
1383  }
1384 
1385  [[nodiscard]] auto size() const noexcept -> size_t {
1386  return m_values.size();
1387  }
1388 
1389  [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
1390  if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) {
1391  return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
1392  } else {
1393  return size_t{1} << (sizeof(value_idx_type) * 8);
1394  }
1395  }
1396 
1397  // modifiers //////////////////////////////////////////////////////////////
1398 
1399  void clear() {
1400  m_values.clear();
1401  clear_buckets();
1402  }
1403 
1404  auto insert(value_type const& value) -> std::pair<iterator, bool> {
1405  return emplace(value);
1406  }
1407 
1408  auto insert(value_type&& value) -> std::pair<iterator, bool> {
1409  return emplace(std::move(value));
1410  }
1411 
1412  template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
1413  auto insert(P&& value) -> std::pair<iterator, bool> {
1414  return emplace(std::forward<P>(value));
1415  }
1416 
1417  auto insert(const_iterator /*hint*/, value_type const& value) -> iterator {
1418  return insert(value).first;
1419  }
1420 
1421  auto insert(const_iterator /*hint*/, value_type&& value) -> iterator {
1422  return insert(std::move(value)).first;
1423  }
1424 
1425  template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
1426  auto insert(const_iterator /*hint*/, P&& value) -> iterator {
1427  return insert(std::forward<P>(value)).first;
1428  }
1429 
1430  template <class InputIt>
1431  void insert(InputIt first, InputIt last) {
1432  while (first != last) {
1433  insert(*first);
1434  ++first;
1435  }
1436  }
1437 
1438  void insert(std::initializer_list<value_type> ilist) {
1439  insert(ilist.begin(), ilist.end());
1440  }
1441 
1442  // nonstandard API: *this is emptied.
1443  // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
1444  auto extract() && -> value_container_type {
1445  return std::move(m_values);
1446  }
1447 
1448  // nonstandard API:
1449  // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
1450  auto replace(value_container_type&& container) {
1451  if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) {
1452  on_error_too_many_elements();
1453  }
1454  auto shifts = calc_shifts_for_size(container.size());
1455  if (0 == bucket_count() || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
1456  m_shifts = shifts;
1457  deallocate_buckets();
1458  allocate_buckets_from_shift();
1459  }
1460  clear_buckets();
1461 
1462  m_values = std::move(container);
1463 
1464  // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
1465  auto value_idx = value_idx_type{};
1466 
1467  // loop until we reach the end of the container. duplicated entries will be replaced with back().
1468  while (value_idx != static_cast<value_idx_type>(m_values.size())) {
1469  auto const& key = get_key(m_values[value_idx]);
1470 
1471  auto hash = mixed_hash(key);
1472  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1473  auto bucket_idx = bucket_idx_from_hash(hash);
1474 
1475  bool key_found = false;
1476  while (true) {
1477  auto const& bucket = at(m_buckets, bucket_idx);
1478  if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
1479  break;
1480  }
1481  if (dist_and_fingerprint == bucket.m_dist_and_fingerprint &&
1482  m_equal(key, get_key(m_values[bucket.m_value_idx]))) {
1483  key_found = true;
1484  break;
1485  }
1486  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1487  bucket_idx = next(bucket_idx);
1488  }
1489 
1490  if (key_found) {
1491  if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
1492  m_values[value_idx] = std::move(m_values.back());
1493  }
1494  m_values.pop_back();
1495  } else {
1496  place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1497  ++value_idx;
1498  }
1499  }
1500  }
1501 
1502  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1503  auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> {
1504  return do_insert_or_assign(key, std::forward<M>(mapped));
1505  }
1506 
1507  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1508  auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> {
1509  return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
1510  }
1511 
1512  template <typename K,
1513  typename M,
1514  typename Q = T,
1515  typename H = Hash,
1516  typename KE = KeyEqual,
1517  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1518  auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
1519  return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
1520  }
1521 
1522  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1523  auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator {
1524  return do_insert_or_assign(key, std::forward<M>(mapped)).first;
1525  }
1526 
1527  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1528  auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator {
1529  return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
1530  }
1531 
1532  template <typename K,
1533  typename M,
1534  typename Q = T,
1535  typename H = Hash,
1536  typename KE = KeyEqual,
1537  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1538  auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator {
1539  return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
1540  }
1541 
1542  // Single arguments for unordered_set can be used without having to construct the value_type
1543  template <class K,
1544  typename Q = T,
1545  typename H = Hash,
1546  typename KE = KeyEqual,
1547  std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1548  auto emplace(K&& key) -> std::pair<iterator, bool> {
1549  auto hash = mixed_hash(key);
1550  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1551  auto bucket_idx = bucket_idx_from_hash(hash);
1552 
1553  while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1554  if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1555  m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
1556  // found it, return without ever actually creating anything
1557  return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
1558  }
1559  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1560  bucket_idx = next(bucket_idx);
1561  }
1562 
1563  // value is new, insert element first, so when exception happens we are in a valid state
1564  return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key));
1565  }
1566 
1567  template <class... Args>
1568  auto emplace(Args&&... args) -> std::pair<iterator, bool> {
1569  // we have to instantiate the value_type to be able to access the key.
1570  // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
1571  auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
1572  auto hash = mixed_hash(key);
1573  auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1574  auto bucket_idx = bucket_idx_from_hash(hash);
1575 
1576  while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1577  if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1578  m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
1579  m_values.pop_back(); // value was already there, so get rid of it
1580  return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
1581  }
1582  dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1583  bucket_idx = next(bucket_idx);
1584  }
1585 
1586  // value is new, place the bucket and shift up until we find an empty spot
1587  auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
1588  if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
1589  // increase_size just rehashes all the data we have in m_values
1590  increase_size();
1591  } else {
1592  // place element and shift up until we find an empty spot
1593  place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1594  }
1595  return {begin() + static_cast<difference_type>(value_idx), true};
1596  }
1597 
1598  template <class... Args>
1599  auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator {
1600  return emplace(std::forward<Args>(args)...).first;
1601  }
1602 
1603  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1604  auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> {
1605  return do_try_emplace(key, std::forward<Args>(args)...);
1606  }
1607 
1608  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1609  auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> {
1610  return do_try_emplace(std::move(key), std::forward<Args>(args)...);
1611  }
1612 
1613  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1614  auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator {
1615  return do_try_emplace(key, std::forward<Args>(args)...).first;
1616  }
1617 
1618  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1619  auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator {
1620  return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
1621  }
1622 
1623  template <
1624  typename K,
1625  typename... Args,
1626  typename Q = T,
1627  typename H = Hash,
1628  typename KE = KeyEqual,
1629  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
1630  bool> = true>
1631  auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
1632  return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
1633  }
1634 
1635  template <
1636  typename K,
1637  typename... Args,
1638  typename Q = T,
1639  typename H = Hash,
1640  typename KE = KeyEqual,
1641  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
1642  bool> = true>
1643  auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator {
1644  return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
1645  }
1646 
1647  auto erase(iterator it) -> iterator {
1648  auto hash = mixed_hash(get_key(*it));
1649  auto bucket_idx = bucket_idx_from_hash(hash);
1650 
1651  auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
1652  while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
1653  bucket_idx = next(bucket_idx);
1654  }
1655 
1656  do_erase(bucket_idx, [](value_type&& /*unused*/) {
1657  });
1658  return begin() + static_cast<difference_type>(value_idx_to_remove);
1659  }
1660 
1661  auto extract(iterator it) -> value_type {
1662  auto hash = mixed_hash(get_key(*it));
1663  auto bucket_idx = bucket_idx_from_hash(hash);
1664 
1665  auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
1666  while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
1667  bucket_idx = next(bucket_idx);
1668  }
1669 
1670  auto tmp = std::optional<value_type>{};
1671  do_erase(bucket_idx, [&tmp](value_type&& val) {
1672  tmp = std::move(val);
1673  });
1674  return std::move(tmp).value();
1675  }
1676 
1677  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1678  auto erase(const_iterator it) -> iterator {
1679  return erase(begin() + (it - cbegin()));
1680  }
1681 
1682  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1683  auto extract(const_iterator it) -> value_type {
1684  return extract(begin() + (it - cbegin()));
1685  }
1686 
1687  auto erase(const_iterator first, const_iterator last) -> iterator {
1688  auto const idx_first = first - cbegin();
1689  auto const idx_last = last - cbegin();
1690  auto const first_to_last = std::distance(first, last);
1691  auto const last_to_end = std::distance(last, cend());
1692 
1693  // remove elements from left to right which moves elements from the end back
1694  auto const mid = idx_first + (std::min)(first_to_last, last_to_end);
1695  auto idx = idx_first;
1696  while (idx != mid) {
1697  erase(begin() + idx);
1698  ++idx;
1699  }
1700 
1701  // all elements from the right are moved, now remove the last element until all done
1702  idx = idx_last;
1703  while (idx != mid) {
1704  --idx;
1705  erase(begin() + idx);
1706  }
1707 
1708  return begin() + idx_first;
1709  }
1710 
1711  auto erase(Key const& key) -> size_t {
1712  return do_erase_key(key, [](value_type&& /*unused*/) {
1713  });
1714  }
1715 
1716  auto extract(Key const& key) -> std::optional<value_type> {
1717  auto tmp = std::optional<value_type>{};
1718  do_erase_key(key, [&tmp](value_type&& val) {
1719  tmp = std::move(val);
1720  });
1721  return tmp;
1722  }
1723 
1724  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1725  auto erase(K&& key) -> size_t {
1726  return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) {
1727  });
1728  }
1729 
1730  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1731  auto extract(K&& key) -> std::optional<value_type> {
1732  auto tmp = std::optional<value_type>{};
1733  do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) {
1734  tmp = std::move(val);
1735  });
1736  return tmp;
1737  }
1738 
1739  void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> &&
1740  std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) {
1741  using std::swap;
1742  swap(other, *this);
1743  }
1744 
1745  // lookup /////////////////////////////////////////////////////////////////
1746 
1747  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1748  auto at(key_type const& key) -> Q& {
1749  return do_at(key);
1750  }
1751 
1752  template <typename K,
1753  typename Q = T,
1754  typename H = Hash,
1755  typename KE = KeyEqual,
1756  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1757  auto at(K const& key) -> Q& {
1758  return do_at(key);
1759  }
1760 
1761  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1762  auto at(key_type const& key) const -> Q const& {
1763  return do_at(key);
1764  }
1765 
1766  template <typename K,
1767  typename Q = T,
1768  typename H = Hash,
1769  typename KE = KeyEqual,
1770  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1771  auto at(K const& key) const -> Q const& {
1772  return do_at(key);
1773  }
1774 
1775  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1776  auto operator[](Key const& key) -> Q& {
1777  return try_emplace(key).first->second;
1778  }
1779 
1780  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1781  auto operator[](Key&& key) -> Q& {
1782  return try_emplace(std::move(key)).first->second;
1783  }
1784 
1785  template <typename K,
1786  typename Q = T,
1787  typename H = Hash,
1788  typename KE = KeyEqual,
1789  std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1790  auto operator[](K&& key) -> Q& {
1791  return try_emplace(std::forward<K>(key)).first->second;
1792  }
1793 
1794  auto count(Key const& key) const -> size_t {
1795  return find(key) == end() ? 0 : 1;
1796  }
1797 
1798  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1799  auto count(K const& key) const -> size_t {
1800  return find(key) == end() ? 0 : 1;
1801  }
1802 
1803  auto find(Key const& key) -> iterator {
1804  return do_find(key);
1805  }
1806 
1807  auto find(Key const& key) const -> const_iterator {
1808  return do_find(key);
1809  }
1810 
1811  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1812  auto find(K const& key) -> iterator {
1813  return do_find(key);
1814  }
1815 
1816  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1817  auto find(K const& key) const -> const_iterator {
1818  return do_find(key);
1819  }
1820 
1821  auto contains(Key const& key) const -> bool {
1822  return find(key) != end();
1823  }
1824 
1825  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1826  auto contains(K const& key) const -> bool {
1827  return find(key) != end();
1828  }
1829 
1830  auto equal_range(Key const& key) -> std::pair<iterator, iterator> {
1831  auto it = do_find(key);
1832  return {it, it == end() ? end() : it + 1};
1833  }
1834 
1835  auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> {
1836  auto it = do_find(key);
1837  return {it, it == end() ? end() : it + 1};
1838  }
1839 
1840  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1841  auto equal_range(K const& key) -> std::pair<iterator, iterator> {
1842  auto it = do_find(key);
1843  return {it, it == end() ? end() : it + 1};
1844  }
1845 
1846  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1847  auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> {
1848  auto it = do_find(key);
1849  return {it, it == end() ? end() : it + 1};
1850  }
1851 
1852  // bucket interface ///////////////////////////////////////////////////////
1853 
1854  auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
1855  return m_buckets.size();
1856  }
1857 
1858  static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
1859  return max_size();
1860  }
1861 
1862  // hash policy ////////////////////////////////////////////////////////////
1863 
1864  [[nodiscard]] auto load_factor() const -> float {
1865  return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
1866  }
1867 
1868  [[nodiscard]] auto max_load_factor() const -> float {
1869  return m_max_load_factor;
1870  }
1871 
1872  void max_load_factor(float ml) {
1873  m_max_load_factor = ml;
1874  if (bucket_count() != max_bucket_count()) {
1875  m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
1876  }
1877  }
1878 
1879  void rehash(size_t count) {
1880  count = (std::min)(count, max_size());
1881  auto shifts = calc_shifts_for_size((std::max)(count, size()));
1882  if (shifts != m_shifts) {
1883  m_shifts = shifts;
1884  deallocate_buckets();
1885  m_values.shrink_to_fit();
1886  allocate_buckets_from_shift();
1887  clear_and_fill_buckets_from_values();
1888  }
1889  }
1890 
1891  void reserve(size_t capa) {
1892  capa = (std::min)(capa, max_size());
1893  if constexpr (has_reserve<value_container_type>) {
1894  // std::deque doesn't have reserve(). Make sure we only call when available
1895  m_values.reserve(capa);
1896  }
1897  auto shifts = calc_shifts_for_size((std::max)(capa, size()));
1898  if (0 == bucket_count() || shifts < m_shifts) {
1899  m_shifts = shifts;
1900  deallocate_buckets();
1901  allocate_buckets_from_shift();
1902  clear_and_fill_buckets_from_values();
1903  }
1904  }
1905 
1906  // observers //////////////////////////////////////////////////////////////
1907 
1908  auto hash_function() const -> hasher {
1909  return m_hash;
1910  }
1911 
1912  auto key_eq() const -> key_equal {
1913  return m_equal;
1914  }
1915 
1916  // nonstandard API: expose the underlying values container
1917  [[nodiscard]] auto values() const noexcept -> value_container_type const& {
1918  return m_values;
1919  }
1920 
1921  // non-member functions ///////////////////////////////////////////////////
1922 
1923  friend auto operator==(table const& a, table const& b) -> bool {
1924  if (&a == &b) {
1925  return true;
1926  }
1927  if (a.size() != b.size()) {
1928  return false;
1929  }
1930  for (auto const& b_entry : b) {
1931  auto it = a.find(get_key(b_entry));
1932  if constexpr (is_map_v<T>) {
1933  // map: check that key is here, then also check that value is the same
1934  if (a.end() == it || !(b_entry.second == it->second)) {
1935  return false;
1936  }
1937  } else {
1938  // set: only check that the key is here
1939  if (a.end() == it) {
1940  return false;
1941  }
1942  }
1943  }
1944  return true;
1945  }
1946 
1947  friend auto operator!=(table const& a, table const& b) -> bool {
1948  return !(a == b);
1949  }
1950 };
1951 
1952 } // namespace detail
1953 
1954 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1955  class T,
1956  class Hash = hash<Key>,
1957  class KeyEqual = std::equal_to<Key>,
1958  class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
1959  class Bucket = bucket_type::standard,
1960  class BucketContainer = detail::default_container_t>
1961 using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;
1962 
1963 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1964  class T,
1965  class Hash = hash<Key>,
1966  class KeyEqual = std::equal_to<Key>,
1967  class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
1968  class Bucket = bucket_type::standard,
1969  class BucketContainer = detail::default_container_t>
1970 using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, true>;
1971 
1972 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1973  class Hash = hash<Key>,
1974  class KeyEqual = std::equal_to<Key>,
1975  class AllocatorOrContainer = std::allocator<Key>,
1976  class Bucket = bucket_type::standard,
1977  class BucketContainer = detail::default_container_t>
1978 using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;
1979 
1980 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1981  class Hash = hash<Key>,
1982  class KeyEqual = std::equal_to<Key>,
1983  class AllocatorOrContainer = std::allocator<Key>,
1984  class Bucket = bucket_type::standard,
1985  class BucketContainer = detail::default_container_t>
1986 using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, true>;
1987 
1988 # if defined(ANKERL_UNORDERED_DENSE_PMR)
1989 
1990 namespace pmr {
1991 
1992 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1993  class T,
1994  class Hash = hash<Key>,
1995  class KeyEqual = std::equal_to<Key>,
1996  class Bucket = bucket_type::standard>
1997 using map = detail::table<Key,
1998  T,
1999  Hash,
2000  KeyEqual,
2001  ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>,
2002  Bucket,
2003  detail::default_container_t,
2004  false>;
2005 
2006 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
2007  class T,
2008  class Hash = hash<Key>,
2009  class KeyEqual = std::equal_to<Key>,
2010  class Bucket = bucket_type::standard>
2011 using segmented_map = detail::table<Key,
2012  T,
2013  Hash,
2014  KeyEqual,
2015  ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>,
2016  Bucket,
2017  detail::default_container_t,
2018  true>;
2019 
2020 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
2021  class Hash = hash<Key>,
2022  class KeyEqual = std::equal_to<Key>,
2023  class Bucket = bucket_type::standard>
2024 using set = detail::table<Key,
2025  void,
2026  Hash,
2027  KeyEqual,
2028  ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>,
2029  Bucket,
2030  detail::default_container_t,
2031  false>;
2032 
2033 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
2034  class Hash = hash<Key>,
2035  class KeyEqual = std::equal_to<Key>,
2036  class Bucket = bucket_type::standard>
2037 using segmented_set = detail::table<Key,
2038  void,
2039  Hash,
2040  KeyEqual,
2041  ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>,
2042  Bucket,
2043  detail::default_container_t,
2044  true>;
2045 
2046 } // namespace pmr
2047 
2048 # endif
2049 
2050 // deduction guides ///////////////////////////////////////////////////////////
2051 
2052 // deduction guides for alias templates are only possible since C++20
2053 // see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction
2054 
2055 } // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
2056 } // namespace ankerl::unordered_dense
2057 
2058 // std extensions /////////////////////////////////////////////////////////////
2059 
2060 namespace std { // NOLINT(cert-dcl58-cpp)
2061 
2062 ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
2063  class T,
2064  class Hash,
2065  class KeyEqual,
2066  class AllocatorOrContainer,
2067  class Bucket,
2068  class Pred,
2069  class BucketContainer,
2070  bool IsSegmented>
2071 // NOLINTNEXTLINE(cert-dcl58-cpp)
2072 auto erase_if(
2073  ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, IsSegmented>&
2074  map,
2075  Pred pred) -> size_t {
2076  using map_t = ankerl::unordered_dense::detail::
2077  table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, IsSegmented>;
2078 
2079  // going back to front because erase() invalidates the end iterator
2080  auto const old_size = map.size();
2081  auto idx = old_size;
2082  while (idx) {
2083  --idx;
2084  auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
2085  if (pred(*it)) {
2086  map.erase(it);
2087  }
2088  }
2089 
2090  return old_size - map.size();
2091 }
2092 
2093 } // namespace std
2094 
2095 #endif
2096 #endif