diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index 10df684c476..bc756c94a38 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -688,7 +688,7 @@ jobs: gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv else if [ ! -f "~/data.osm.pbf" ]; then - wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf + wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf --quiet else echo "Using cached data.osm.pbf" fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 5327820678f..7fd33bec173 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -299,6 +299,10 @@ include_directories(SYSTEM ${PROTOZERO_INCLUDE_DIR}) set(VTZERO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/vtzero/include") include_directories(SYSTEM ${VTZERO_INCLUDE_DIR}) +set(ANKERL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/unordered_dense/include") +include_directories(SYSTEM ${ANKERL_INCLUDE_DIR}) + + set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Disable the build of Flatbuffers tests and samples.") set(FLATBUFFERS_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers") set(FLATBUFFERS_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include") diff --git a/include/util/query_heap.hpp b/include/util/query_heap.hpp index 9ded36c9e86..cb020802027 100644 --- a/include/util/query_heap.hpp +++ b/include/util/query_heap.hpp @@ -5,58 +5,19 @@ #include #include +#include +#include #include #include #include #include #include +#include #include namespace osrm::util { -template class GenerationArrayStorage -{ - using GenerationCounter = std::uint16_t; - - public: - explicit GenerationArrayStorage(std::size_t size) - : positions(size, 0), generation(1), generations(size, 0) - { - } - - Key &operator[](NodeID node) - { - generation[node] = generation; - return positions[node]; - } - - Key peek_index(const NodeID node) const - { - if (generations[node] < generation) - { - return std::numeric_limits::max(); - } - return positions[node]; - } - - void Clear() - { - generation++; - // if generation overflows we end up at 0 again and need to clear the vector - if (generation == 0) - { - generation = 1; - std::fill(generations.begin(), generations.end(), 0); - } - } - - private: - GenerationCounter generation; - std::vector generations; - std::vector positions; -}; - template class ArrayStorage { public: @@ -72,33 +33,10 @@ template class ArrayStorage std::vector positions; }; -template class MapStorage -{ - public: - explicit MapStorage(std::size_t) {} - - Key &operator[](NodeID node) { return nodes[node]; } - - void Clear() { nodes.clear(); } - - Key peek_index(const NodeID node) const - { - const auto iter = nodes.find(node); - if (nodes.end() != iter) - { - return iter->second; - } - return std::numeric_limits::max(); - } - - private: - std::map nodes; -}; - template class UnorderedMapStorage { public: - explicit UnorderedMapStorage(std::size_t) { nodes.rehash(1000); } + explicit UnorderedMapStorage(std::size_t) {} Key &operator[](const NodeID node) { return nodes[node]; } @@ -121,7 +59,7 @@ template class UnorderedMapStorage void Clear() { nodes.clear(); } private: - std::unordered_map nodes; + ankerl::unordered_dense::segmented_map nodes; }; template + $) + +target_compile_features(unordered_dense INTERFACE cxx_std_17) + +if(_unordered_dense_is_toplevel_project) + # locations are provided by GNUInstallDirs + install( + TARGETS unordered_dense + EXPORT unordered_dense_Targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + + include(CMakePackageConfigHelpers) + write_basic_package_version_file( + "unordered_denseConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) + + configure_package_config_file( + "${PROJECT_SOURCE_DIR}/cmake/unordered_denseConfig.cmake.in" + "${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) + + install( + EXPORT unordered_dense_Targets + FILE unordered_denseTargets.cmake + NAMESPACE unordered_dense:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) + + install( + FILES "${PROJECT_BINARY_DIR}/unordered_denseConfig.cmake" + "${PROJECT_BINARY_DIR}/unordered_denseConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) + + install( + DIRECTORY ${PROJECT_SOURCE_DIR}/include/ankerl + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endif() diff --git a/third_party/unordered_dense/CODE_OF_CONDUCT.md b/third_party/unordered_dense/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..1665c54f2c5 --- /dev/null +++ b/third_party/unordered_dense/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at martin.ankerl@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/third_party/unordered_dense/CONTRIBUTING.md b/third_party/unordered_dense/CONTRIBUTING.md new file mode 100644 index 00000000000..58f8c7aacaa --- /dev/null +++ b/third_party/unordered_dense/CONTRIBUTING.md @@ -0,0 +1,3 @@ +* Coding style should be consistent with the code around you. +* Use automatic formatting with clang-format. +* One feature per pull request diff --git a/third_party/unordered_dense/LICENSE b/third_party/unordered_dense/LICENSE new file mode 100644 index 00000000000..c4d1a0e48eb --- /dev/null +++ b/third_party/unordered_dense/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Martin Leitner-Ankerl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/unordered_dense/README.md b/third_party/unordered_dense/README.md new file mode 100644 index 00000000000..c46a571902e --- /dev/null +++ b/third_party/unordered_dense/README.md @@ -0,0 +1,374 @@ + + +[![Release](https://img.shields.io/github/release/martinus/unordered_dense.svg)](https://github.com/martinus/unordered_dense/releases) +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/martinus/unordered_dense/main/LICENSE) +[![meson_build_test](https://github.com/martinus/unordered_dense/actions/workflows/main.yml/badge.svg)](https://github.com/martinus/unordered_dense/actions) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6220/badge)](https://bestpractices.coreinfrastructure.org/projects/6220) +[![Sponsors](https://img.shields.io/github/sponsors/martinus?style=social)](https://github.com/sponsors/martinus) + +# 🚀 ankerl::unordered_dense::{map, set} + +A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion for C++17 and later. + +The classes `ankerl::unordered_dense::map` and `ankerl::unordered_dense::set` are (almost) drop-in replacements of `std::unordered_map` and `std::unordered_set`. While they don't have as strong iterator / reference stability guaranties, they are typically *much* faster. + +Additionally, there are `ankerl::unordered_dense::segmented_map` and `ankerl::unordered_dense::segmented_set` with lower peak memory usage. and stable iterator/references on insert. + +- [1. Overview](#1-overview) +- [2. Installation](#2-installation) + - [2.1. Installing using cmake](#21-installing-using-cmake) +- [3. Usage](#3-usage) + - [3.1. Modules](#31-modules) + - [3.2. Hash](#32-hash) + - [3.2.1. Simple Hash](#321-simple-hash) + - [3.2.2. High Quality Hash](#322-high-quality-hash) + - [3.2.3. Specialize `ankerl::unordered_dense::hash`](#323-specialize-ankerlunordered_densehash) + - [3.2.4. Heterogeneous Overloads using `is_transparent`](#324-heterogeneous-overloads-using-is_transparent) + - [3.2.5. Automatic Fallback to `std::hash`](#325-automatic-fallback-to-stdhash) + - [3.2.6. Hash the Whole Memory](#326-hash-the-whole-memory) + - [3.3. Container API](#33-container-api) + - [3.3.1. `auto extract() && -> value_container_type`](#331-auto-extract----value_container_type) + - [3.3.2. `extract()` single Elements](#332-extract-single-elements) + - [3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&`](#333-nodiscard-auto-values-const-noexcept---value_container_type-const) + - [3.3.4. `auto replace(value_container_type&& container)`](#334-auto-replacevalue_container_type-container) + - [3.4. Custom Container Types](#34-custom-container-types) + - [3.5. Custom Bucket Types](#35-custom-bucket-types) + - [3.5.1. `ankerl::unordered_dense::bucket_type::standard`](#351-ankerlunordered_densebucket_typestandard) + - [3.5.2. `ankerl::unordered_dense::bucket_type::big`](#352-ankerlunordered_densebucket_typebig) +- [4. `segmented_map` and `segmented_set`](#4-segmented_map-and-segmented_set) +- [5. Design](#5-design) + - [5.1. Inserts](#51-inserts) + - [5.2. Lookups](#52-lookups) + - [5.3. Removals](#53-removals) +- [6. Real World Usage](#6-real-world-usage) + +## 1. Overview + +The chosen design has a few advantages over `std::unordered_map`: + +* Perfect iteration speed - Data is stored in a `std::vector`, all data is contiguous! +* Very fast insertion & lookup speed, in the same ballpark as [`absl::flat_hash_map`](https://abseil.io/docs/cpp/guides/container`) +* Low memory usage +* Full support for `std::allocators`, and [polymorphic allocators](https://en.cppreference.com/w/cpp/memory/polymorphic_allocator). There are `ankerl::unordered_dense::pmr` typedefs available +* Customizeable storage type: with a template parameter you can e.g. switch from `std::vector` to `boost::interprocess::vector` or any other compatible random-access container. +* Better debugging: the underlying data can be easily seen in any debugger that can show an `std::vector`. + +There's no free lunch, so there are a few disadvantages: + +* Deletion speed is relatively slow. This needs two lookups: one for the element to delete, and one for the element that is moved onto the newly empty spot. +* no `const Key` in `std::pair` +* Iterators and references are not stable on insert or erase. + +## 2. Installation + + +The default installation location is `/usr/local`. + +### 2.1. Installing using cmake + +Clone the repository and run these commands in the cloned folder: + +```sh +mkdir build && cd build +cmake .. +cmake --build . --target install +``` + +Consider setting an install prefix if you do not want to install `unordered_dense` system wide, like so: + +```sh +mkdir build && cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${HOME}/unordered_dense_install .. +cmake --build . --target install +``` + +To make use of the installed library, add this to your project: + +```cmake +find_package(unordered_dense CONFIG REQUIRED) +target_link_libraries(your_project_name unordered_dense::unordered_dense) +``` + +## 3. Usage + +### 3.1. Modules + +`ankerl::unordered_dense` supports c++20 modules. Simply compile `src/ankerl.unordered_dense.cpp` and use the resulting module, e.g. like so: + +```sh +clang++ -std=c++20 -I include --precompile -x c++-module src/ankerl.unordered_dense.cpp +clang++ -std=c++20 -c ankerl.unordered_dense.pcm +``` + +To use the module with e.g. in `module_test.cpp`, use + +```cpp +import ankerl.unordered_dense; +``` + +and compile with e.g. + +```sh +clang++ -std=c++20 -fprebuilt-module-path=. ankerl.unordered_dense.o module_test.cpp -o main +``` + +A simple demo script can be found in `test/modules`. + +### 3.2. Hash + +`ankerl::unordered_dense::hash` is a fast and high quality hash, based on [wyhash](https://github.com/wangyi-fudan/wyhash). The `ankerl::unordered_dense` map/set differentiates between hashes of high quality (good [avalanching effect](https://en.wikipedia.org/wiki/Avalanche_effect)) and bad quality. Hashes with good quality contain a special marker: + +```cpp +using is_avalanching = void; +``` + +This is the cases for the specializations `bool`, `char`, `signed char`, `unsigned char`, `char8_t`, `char16_t`, `char32_t`, `wchar_t`, `short`, `unsigned short`, `int`, `unsigned int`, `long`, `long long`, `unsigned long`, `unsigned long long`, `T*`, `std::unique_ptr`, `std::shared_ptr`, `enum`, `std::basic_string`, and `std::basic_string_view`. + +Hashes that do not contain such a marker are assumed to be of bad quality and receive an additional mixing step inside the map/set implementation. + +#### 3.2.1. Simple Hash + +Consider a simple custom key type: + +```cpp +struct id { + uint64_t value{}; + + auto operator==(id const& other) const -> bool { + return value == other.value; + } +}; +``` + +The simplest implementation of a hash is this: + +```cpp +struct custom_hash_simple { + auto operator()(id const& x) const noexcept -> uint64_t { + return x.value; + } +}; +``` +This can be used e.g. with + +```cpp +auto ids = ankerl::unordered_dense::set(); +``` + +Since `custom_hash_simple` doesn't have a `using is_avalanching = void;` marker it is considered to be of bad quality and additional mixing of `x.value` is automatically provided inside the set. + +#### 3.2.2. High Quality Hash + +Back to the `id` example, we can easily implement a higher quality hash: + +```cpp +struct custom_hash_avalanching { + using is_avalanching = void; + + auto operator()(id const& x) const noexcept -> uint64_t { + return ankerl::unordered_dense::detail::wyhash::hash(x.value); + } +}; +``` + +We know `wyhash::hash` is of high quality, so we can add `using is_avalanching = void;` which makes the map/set directly use the returned value. + + +#### 3.2.3. Specialize `ankerl::unordered_dense::hash` + +Instead of creating a new class you can also specialize `ankerl::unordered_dense::hash`: + +```cpp +template <> +struct ankerl::unordered_dense::hash { + using is_avalanching = void; + + [[nodiscard]] auto operator()(id const& x) const noexcept -> uint64_t { + return detail::wyhash::hash(x.value); + } +}; +``` + +#### 3.2.4. Heterogeneous Overloads using `is_transparent` + +This map/set supports heterogeneous overloads as described in [P2363 Extending associative containers with the remaining heterogeneous overloads](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html) which is [targeted for C++26](https://wg21.link/p2077r2). This has overloads for `find`, `count`, `contains`, `equal_range` (see [P0919R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0919r3.html)), `erase` (see [P2077R2](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2077r2.html)), and `try_emplace`, `insert_or_assign`, `operator[]`, `at`, and `insert` & `emplace` for sets (see [P2363R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2363r3.html)). + +For heterogeneous overloads to take affect, both `hasher` and `key_equal` need to have the attribute `is_transparent` set. + +Here is an example implementation that's usable with any string types that is convertible to `std::string_view` (e.g. `char const*` and `std::string`): + +```cpp +struct string_hash { + using is_transparent = void; // enable heterogeneous overloads + using is_avalanching = void; // mark class as high quality avalanching hash + + [[nodiscard]] auto operator()(std::string_view str) const noexcept -> uint64_t { + return ankerl::unordered_dense::hash{}(str); + } +}; +``` + +To make use of this hash you'll need to specify it as a type, and also a `key_equal` with `is_transparent` like [std::equal_to<>](https://en.cppreference.com/w/cpp/utility/functional/equal_to_void): + +```cpp +auto map = ankerl::unordered_dense::map>(); +``` + +For more information see the examples in `test/unit/transparent.cpp`. + + +#### 3.2.5. Automatic Fallback to `std::hash` + +When an implementation for `std::hash` of a custom type is available, this is automatically used and assumed to be of bad quality (thus `std::hash` is used, but an additional mixing step is performed). + + +#### 3.2.6. Hash the Whole Memory + +When the type [has a unique object representation](https://en.cppreference.com/w/cpp/types/has_unique_object_representations) (no padding, trivially copyable), one can just hash the object's memory. Consider a simple class + +```cpp +struct point { + int x{}; + int y{}; + + auto operator==(point const& other) const -> bool { + return x == other.x && y == other.y; + } +}; +``` + +A fast and high quality hash can be easily provided like so: + +```cpp +struct custom_hash_unique_object_representation { + using is_avalanching = void; + + [[nodiscard]] auto operator()(point const& f) const noexcept -> uint64_t { + static_assert(std::has_unique_object_representations_v); + return ankerl::unordered_dense::detail::wyhash::hash(&f, sizeof(f)); + } +}; +``` + +### 3.3. Container API + +In addition to the standard `std::unordered_map` API (see https://en.cppreference.com/w/cpp/container/unordered_map) we have additional API that is somewhat similar to the node API, but leverages the fact that we're using a random access container internally: + +#### 3.3.1. `auto extract() && -> value_container_type` + +Extracts the internally used container. `*this` is emptied. + +#### 3.3.2. `extract()` single Elements + +Similar to `erase()` I have an API call `extract()`. It behaves exactly the same as `erase`, except that the return value is the moved element that is removed from the container: + +* `auto extract(const_iterator it) -> value_type` +* `auto extract(Key const& key) -> std::optional` +* `template auto extract(K&& key) -> std::optional` + +Note that the `extract(key)` API returns an `std::optional` that is empty when the key is not found. + +#### 3.3.3. `[[nodiscard]] auto values() const noexcept -> value_container_type const&` + +Exposes the underlying values container. + +#### 3.3.4. `auto replace(value_container_type&& container)` + +Discards the internally held container and replaces it with the one passed. Non-unique elements are +removed, and the container will be partly reordered when non-unique elements are found. + +### 3.4. Custom Container Types + +`unordered_dense` accepts a custom allocator, but you can also specify a custom container for that template argument. That way it is possible to replace the internally used `std::vector` with e.g. `std::deque` or any other container like `boost::interprocess::vector`. This supports fancy pointers (e.g. [offset_ptr](https://www.boost.org/doc/libs/1_80_0/doc/html/interprocess/offset_ptr.html)), so the container can be used with e.g. shared memory provided by `boost::interprocess`. + +### 3.5. Custom Bucket Types + +The map/set supports two different bucket types. The default should be good for pretty much everyone. + +#### 3.5.1. `ankerl::unordered_dense::bucket_type::standard` + +* Up to 2^32 = 4.29 billion elements. +* 8 bytes overhead per bucket. + +#### 3.5.2. `ankerl::unordered_dense::bucket_type::big` + +* up to 2^63 = 9223372036854775808 elements. +* 12 bytes overhead per bucket. + +## 4. `segmented_map` and `segmented_set` + +`ankerl::unordered_dense` provides a custom container implementation that has lower memory requirements than the default `std::vector`. Memory is not contiguous, but it can allocate segments without having to reallocate and move all the elements. In summary, this leads to + +* Much smoother memory usage, memory usage increases continuously. +* No high peak memory usage. +* Faster insertion because elements never need to be moved to new allocated blocks +* Slightly slower indexing compared to `std::vector` because an additional indirection is needed. + +Here is a comparison against `absl::flat_hash_map` and the `ankerl::unordered_dense::map` when inserting 10 million entries +![allocated memory](doc/allocated_memory.png) + +Abseil is fastest for this simple inserting test, taking a bit over 0.8 seconds. It's peak memory usage is about 430 MB. Note how the memory usage goes down after the last peak; when it goes down to ~290MB it has finished rehashing and could free the previously used memory block. + +`ankerl::unordered_dense::segmented_map` doesn't have these peaks, and instead has a smooth increase of memory usage. Note there are still sudden drops & increases in memory because the indexing data structure needs still needs to increase by a fixed factor. But due to holding the data in a separate container we are able to first free the old data structure, and then allocate a new, bigger indexing structure; thus we do not have peaks. + +## 5. Design + +The map/set has two data structures: +* `std::vector` which holds all data. map/set iterators are just `std::vector::iterator`! +* An indexing structure (bucket array), which is a flat array with 8-byte buckets. + +### 5.1. Inserts + +Whenever an element is added it is `emplace_back` to the vector. The key is hashed, and an entry (bucket) is added at the +corresponding location in the bucket array. The bucket has this structure: + +```cpp +struct Bucket { + uint32_t dist_and_fingerprint; + uint32_t value_idx; +}; +``` + +Each bucket stores 3 things: +* The distance of that value from the original hashed location (3 most significant bytes in `dist_and_fingerprint`) +* A fingerprint; 1 byte of the hash (lowest significant byte in `dist_and_fingerprint`) +* An index where in the vector the actual data is stored. + +This structure is especially designed for the collision resolution strategy robin-hood hashing with backward shift +deletion. + +### 5.2. Lookups + +The key is hashed and the bucket array is searched if it has an entry at that location with that fingerprint. When found, +the key in the data vector is compared, and when equal the value is returned. + +### 5.3. Removals + +Since all data is stored in a vector, removals are a bit more complicated: + +1. First, lookup the element to delete in the index array. +2. When found, replace that element in the vector with the last element in the vector. +3. Update *two* locations in the bucket array: First remove the bucket for the removed element +4. Then, update the `value_idx` of the moved element. This requires another lookup. + + +## 6. Real World Usage + +On 2023-09-10 I did a quick search on github to see if this map is used in any popular open source projects. Here are some of the projects +I found. Please send me a note if you want on that list! + +* [PruaSlicer](https://github.com/prusa3d/PrusaSlicer) - G-code generator for 3D printers (RepRap, Makerbot, Ultimaker etc.) +* [Kismet](https://github.com/kismetwireless/kismet): Wi-Fi, Bluetooth, RF, and more. Kismet is a sniffer, WIDS, and wardriving tool for Wi-Fi, Bluetooth, Zigbee, RF, and more, which runs on Linux and macOS +* [Rspamd](https://github.com/rspamd/rspamd) - Fast, free and open-source spam filtering system. +* [kallisto](https://github.com/pachterlab/kallisto) - Near-optimal RNA-Seq quantification +* [Slang](https://github.com/shader-slang/slang) - Slang is a shading language that makes it easier to build and maintain large shader codebases in a modular and extensible fashion. +* [CyberFSR2](https://github.com/PotatoOfDoom/CyberFSR2) - Drop-in DLSS replacement with FSR 2.0 for various games such as Cyberpunk 2077. +* [ossia score](https://github.com/ossia/score) - A free, open-source, cross-platform intermedia sequencer for precise and flexible scripting of interactive scenarios. +* [HiveWE](https://github.com/stijnherfst/HiveWE) - A Warcraft III World Editor (WE) that focusses on speed and ease of use. +* [opentxs](https://github.com/Open-Transactions/opentxs) - The Open-Transactions project is a collaborative effort to develop a robust, commercial-grade, fully-featured, free-software toolkit implementing the OTX protocol as well as a full-strength financial cryptography library, API, GUI, command-line interface, and prototype notary server. +* [LuisaCompute](https://github.com/LuisaGroup/LuisaCompute) - High-Performance Rendering Framework on Stream Architectures +* [Lethe](https://github.com/lethe-cfd/lethe) - Lethe (pronounced /ˈliːθiː/) is open-source computational fluid dynamics (CFD) software which uses high-order continuous Galerkin formulations to solve the incompressible Navier–Stokes equations (among others). +* [PECOS](https://github.com/amzn/pecos) - PECOS is a versatile and modular machine learning (ML) framework for fast learning and inference on problems with large output spaces, such as extreme multi-label ranking (XMR) and large-scale retrieval. +* [Operon](https://github.com/heal-research/operon) - A modern C++ framework for symbolic regression that uses genetic programming to explore a hypothesis space of possible mathematical expressions in order to find the best-fitting model for a given regression target. +* [MashMap](https://github.com/marbl/MashMap) - A fast approximate aligner for long DNA sequences +* [minigpt4.cpp](https://github.com/Maknee/minigpt4.cpp) - Port of MiniGPT4 in C++ (4bit, 5bit, 6bit, 8bit, 16bit CPU inference with GGML) diff --git a/third_party/unordered_dense/cmake/unordered_denseConfig.cmake.in b/third_party/unordered_dense/cmake/unordered_denseConfig.cmake.in new file mode 100644 index 00000000000..ff0fa6724ed --- /dev/null +++ b/third_party/unordered_dense/cmake/unordered_denseConfig.cmake.in @@ -0,0 +1,4 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +check_required_components("@PROJECT_NAME@") \ No newline at end of file diff --git a/third_party/unordered_dense/doc/allocated_memory.gnuplot b/third_party/unordered_dense/doc/allocated_memory.gnuplot new file mode 100755 index 00000000000..de16ff91825 --- /dev/null +++ b/third_party/unordered_dense/doc/allocated_memory.gnuplot @@ -0,0 +1,44 @@ +#!/usr/bin/gnuplot + +#set terminal pngcairo +#set terminal pngcairo size 730,510 enhanced font 'Verdana,10' +set terminal pngcairo size 800,600 enhanced font 'Verdana,10' + +# define axis +# remove border on top and right and set color to gray +set style line 11 lc rgb '#808080' lt 1 +set border 3 back ls 11 +set tics nomirror +# define grid +set style line 12 lc rgb '#808080' lt 0 lw 1 +set grid back ls 12 + +# line styles +set style line 1 lt 1 lc rgb '#1B9E77' # dark teal +set style line 2 lt 1 lc rgb '#D95F02' # dark orange +set style line 3 lt 1 lc rgb '#7570B3' # dark lilac +set style line 4 lt 1 lc rgb '#E7298A' # dark magenta +set style line 5 lt 1 lc rgb '#66A61E' # dark lime green +set style line 6 lt 1 lc rgb '#E6AB02' # dark banana +set style line 7 lt 1 lc rgb '#A6761D' # dark tan +set style line 8 lt 1 lc rgb '#666666' # dark gray + + +set style line 101 lc rgb '#808080' lt 1 lw 1 +set border 3 front ls 101 +set tics nomirror out scale 0.75 + +set key left top + +set output 'allocated_memory.png' + +set xlabel "Runtime [s]" +set ylabel "Allocated memory [MB]" + +set title "Inserting 10 Million uint64\\\_t -> uint64\\\_t pairs" + +# allocated_memory_segmented_vector.txt allocated_memory_std_unordered_map.txt allocated_memory_std_vector.txt +plot \ + 'allocated_memory_segmented_vector.txt' using ($1):($2/1e6) w steps ls 1 lw 2 title "ankerl::unordered\\\_dense::segmented\\\_map" , \ + 'allocated_memory_std_vector.txt' using ($1):($2/1e6) w steps ls 2 lw 2 title "ankerl::unordered\\\_dense::map" , \ + 'allocated_memory_absl_flat_hash_map.txt' using ($1):($2/1e6) w steps ls 3 lw 2 title "absl::flat\\\_hash\\\_map" diff --git a/third_party/unordered_dense/doc/allocated_memory.png b/third_party/unordered_dense/doc/allocated_memory.png new file mode 100644 index 00000000000..29d09a9e96b Binary files /dev/null and b/third_party/unordered_dense/doc/allocated_memory.png differ diff --git a/third_party/unordered_dense/example/CMakeLists.txt b/third_party/unordered_dense/example/CMakeLists.txt new file mode 100644 index 00000000000..19e2194872d --- /dev/null +++ b/third_party/unordered_dense/example/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.12) +project("UnorderedDenseExample") + +add_executable(UnorderedDenseExample main.cpp) +find_package(unordered_dense CONFIG REQUIRED) +target_link_libraries(UnorderedDenseExample unordered_dense::unordered_dense) diff --git a/third_party/unordered_dense/example/README.md b/third_party/unordered_dense/example/README.md new file mode 100644 index 00000000000..14fef9b4cfd --- /dev/null +++ b/third_party/unordered_dense/example/README.md @@ -0,0 +1,16 @@ +A simple example that demonstrats how to make use of `unordered_dense` with cmake. + +Use globally installed `unordered_dense`: +```sh +mkdir build && cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${HOME}/unordered_dense_install .. +make +``` + +Use locall installed `unordered_dense`, as in the main README.md: + +```sh +mkdir build && cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${HOME}/unordered_dense_install .. +make +``` diff --git a/third_party/unordered_dense/example/main.cpp b/third_party/unordered_dense/example/main.cpp new file mode 100644 index 00000000000..8e9c65e463c --- /dev/null +++ b/third_party/unordered_dense/example/main.cpp @@ -0,0 +1,13 @@ +#include + +#include + +auto main() -> int { + auto map = ankerl::unordered_dense::map(); + map[123] = "hello"; + map[987] = "world!"; + + for (auto const& [key, val] : map) { + std::cout << key << " => " << val << std::endl; + } +} diff --git a/third_party/unordered_dense/include/ankerl/unordered_dense.h b/third_party/unordered_dense/include/ankerl/unordered_dense.h new file mode 100644 index 00000000000..2aaacd617a6 --- /dev/null +++ b/third_party/unordered_dense/include/ankerl/unordered_dense.h @@ -0,0 +1,2032 @@ +///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// + +// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. +// Version 4.4.0 +// https://github.com/martinus/unordered_dense +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2022-2023 Martin Leitner-Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_UNORDERED_DENSE_H +#define ANKERL_UNORDERED_DENSE_H + +// see https://semver.org/spec/v2.0.0.html +#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes +#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality +#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes + +// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) +#define ANKERL_UNORDERED_DENSE_NAMESPACE \ + ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \ + ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH) + +#if defined(_MSVC_LANG) +# define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG +#else +# define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus +#endif + +#if defined(__GNUC__) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__)) +#elif defined(_MSC_VER) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop)) +#endif + +// exceptions +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage) +#else +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage) +#endif +#ifdef _MSC_VER +# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline) +#else +# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) +#endif + +// defined in unordered_dense.cpp +#if !defined(ANKERL_UNORDERED_DENSE_EXPORT) +# define ANKERL_UNORDERED_DENSE_EXPORT +#endif + +#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L +# error ankerl::unordered_dense requires C++17 or higher +#else +# include // for array +# include // for uint64_t, uint32_t, uint8_t, UINT64_C +# include // for size_t, memcpy, memset +# include // for equal_to, hash +# include // for initializer_list +# include // for pair, distance +# include // for numeric_limits +# include // for allocator, allocator_traits, shared_ptr +# include // for optional +# include // for out_of_range +# include // for basic_string +# include // for basic_string_view, hash +# include // for forward_as_tuple +# include // for enable_if_t, declval, conditional_t, ena... +# include // for forward, exchange, pair, as_const, piece... +# include // for vector +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0 +# include // for abort +# endif + +# if defined(__has_include) +# if __has_include() +# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include // for polymorphic_allocator +# elif __has_include() +# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include // for polymorphic_allocator +# endif +# endif + +# if defined(_MSC_VER) && defined(_M_X64) +# include +# pragma intrinsic(_umul128) +# endif + +# if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +# define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage) +# else +# define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +# endif + +namespace ankerl::unordered_dense { +inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE { + +namespace detail { + +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() { + throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() { + throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() { + throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); +} + +# else + +[[noreturn]] inline void on_error_key_not_found() { + abort(); +} +[[noreturn]] inline void on_error_bucket_overflow() { + abort(); +} +[[noreturn]] inline void on_error_too_many_elements() { + abort(); +} + +# endif + +} // namespace detail + +// hash /////////////////////////////////////////////////////////////////////// + +// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash +// No big-endian support (because different values on different machines don't matter), +// hardcodes seed and the secret, reformats the code, and clang-tidy fixes. +namespace detail::wyhash { + +inline void mum(uint64_t* a, uint64_t* b) { +# if defined(__SIZEOF_INT128__) + __uint128_t r = *a; + r *= *b; + *a = static_cast(r); + *b = static_cast(r >> 64U); +# elif defined(_MSC_VER) && defined(_M_X64) + *a = _umul128(*a, *b, b); +# else + uint64_t ha = *a >> 32U; + uint64_t hb = *b >> 32U; + uint64_t la = static_cast(*a); + uint64_t lb = static_cast(*b); + uint64_t hi{}; + uint64_t lo{}; + uint64_t rh = ha * hb; + uint64_t rm0 = ha * lb; + uint64_t rm1 = hb * la; + uint64_t rl = la * lb; + uint64_t t = rl + (rm0 << 32U); + auto c = static_cast(t < rl); + lo = t + (rm1 << 32U); + c += static_cast(lo < t); + hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c; + *a = lo; + *b = hi; +# endif +} + +// multiply and xor mix function, aka MUM +[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t { + mum(&a, &b); + return a ^ b; +} + +// read functions. WARNING: we don't care about endianness, so results are different on big endian! +[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t { + uint64_t v{}; + std::memcpy(&v, p, 8U); + return v; +} + +[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t { + uint32_t v{}; + std::memcpy(&v, p, 4); + return v; +} + +// reads 1, 2, or 3 bytes +[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t { + return (static_cast(p[0]) << 16U) | (static_cast(p[k >> 1U]) << 8U) | p[k - 1]; +} + +[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t { + static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f), + UINT64_C(0xe7037ed1a0b428db), + UINT64_C(0x8ebc6af09c88c6e3), + UINT64_C(0x589965cc75374cc3)}; + + auto const* p = static_cast(key); + uint64_t seed = secret[0]; + uint64_t a{}; + uint64_t b{}; + if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) { + if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) { + a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U)); + b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U)); + } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) { + a = r3(p, len); + b = 0; + } else { + a = 0; + b = 0; + } + } else { + size_t i = len; + if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) { + uint64_t see1 = seed; + uint64_t see2 = seed; + do { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); + see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); + p += 48; + i -= 48; + } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48)); + seed ^= see1 ^ see2; + } + while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + i -= 16; + p += 16; + } + a = r8(p + i - 16); + b = r8(p + i - 8); + } + + return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); +} + +[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t { + return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); +} + +} // namespace detail::wyhash + +ANKERL_UNORDERED_DENSE_EXPORT template +struct hash { + auto operator()(T const& obj) const noexcept(noexcept(std::declval>().operator()(std::declval()))) + -> uint64_t { + return std::hash{}(obj); + } +}; + +template +struct hash> { + using is_avalanching = void; + auto operator()(std::basic_string const& str) const noexcept -> uint64_t { + return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size()); + } +}; + +template +struct hash> { + using is_avalanching = void; + auto operator()(std::basic_string_view const& sv) const noexcept -> uint64_t { + return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size()); + } +}; + +template +struct hash { + using is_avalanching = void; + auto operator()(T* ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr)); + } +}; + +template +struct hash> { + using is_avalanching = void; + auto operator()(std::unique_ptr const& ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash> { + using is_avalanching = void; + auto operator()(std::shared_ptr const& ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash::value>::type> { + using is_avalanching = void; + auto operator()(Enum e) const noexcept -> uint64_t { + using underlying = typename std::underlying_type_t; + return detail::wyhash::hash(static_cast(e)); + } +}; + +template +struct tuple_hash_helper { + // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest. + // If it isn't an integral we need to hash it. + template + [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t { + if constexpr (std::is_integral_v || std::is_enum_v) { + return static_cast(arg); + } else { + return hash{}(arg); + } + } + + [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t { + return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69}); + } + + // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If + // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized + // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer. + template + [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence) noexcept -> uint64_t { + auto h = uint64_t{}; + ((h = mix64(h, to64(std::get(t)))), ...); + return h; + } +}; + +template +struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::tuple const& t) const noexcept -> uint64_t { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + +template +struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::pair const& t) const noexcept -> uint64_t { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \ + template <> \ + struct hash { \ + using is_avalanching = void; \ + auto operator()(T const& obj) const noexcept -> uint64_t { \ + return detail::wyhash::hash(static_cast(obj)); \ + } \ + } + +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wuseless-cast" +# endif +// see https://en.cppreference.com/w/cpp/utility/hash +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char); +# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t) +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t); +# endif +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long); + +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +# endif + +// bucket_type ////////////////////////////////////////////////////////// + +namespace bucket_type { + +struct standard { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + uint32_t m_value_idx; // index into the m_values vector. +}; + +ANKERL_UNORDERED_DENSE_PACK(struct big { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + size_t m_value_idx; // index into the m_values vector. +}); + +} // namespace bucket_type + +namespace detail { + +struct nonesuch {}; + +template class Op, class... Args> +struct detector { + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> { + using value_t = std::true_type; + using type = Op; +}; + +template