From 418627959684a64b011505edce2942b468c88cf9 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Mon, 20 Nov 2023 16:44:20 +0800 Subject: [PATCH 01/39] add support for fast_varint encoding --- include/ylt/struct_pack/calculate_size.hpp | 192 +++- include/ylt/struct_pack/derived_helper.hpp | 2 +- include/ylt/struct_pack/endian_wrapper.hpp | 64 +- include/ylt/struct_pack/error_code.hpp | 1 + include/ylt/struct_pack/packer.hpp | 164 +++- include/ylt/struct_pack/reflection.hpp | 28 +- include/ylt/struct_pack/type_calculate.hpp | 45 +- include/ylt/struct_pack/type_id.hpp | 61 +- include/ylt/struct_pack/unpacker.hpp | 187 +++- include/ylt/struct_pack/varint.hpp | 27 + src/struct_pack/tests/CMakeLists.txt | 1 + .../tests/test_compile_time_calculate.cpp | 33 + src/struct_pack/tests/test_fast_varint.cpp | 917 ++++++++++++++++++ 13 files changed, 1618 insertions(+), 104 deletions(-) create mode 100644 src/struct_pack/tests/test_fast_varint.cpp diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index a1c691ea6..389b57f64 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -15,6 +15,10 @@ */ #pragma once +#include +#include +#include + #include "alignment.hpp" #include "marco.h" #include "reflection.hpp" @@ -23,18 +27,23 @@ #include "type_trait.hpp" #include "util.h" #include "varint.hpp" +#include "ylt/struct_pack/type_calculate.hpp" namespace struct_pack { struct serialize_buffer_size; namespace detail { -template +template constexpr size_info STRUCT_PACK_INLINE calculate_payload_size(const Args &...items); -template +template +constexpr std::size_t STRUCT_PACK_INLINE +calculate_fast_varint_size(const Args &...items); + +template constexpr size_info inline calculate_one_size(const T &item) { - constexpr auto id = get_type_id>(); + constexpr auto id = get_type_id, parent_tag>(); static_assert(id != detail::type_id::type_end_flag); using type = remove_cvref_t; static_assert(!std::is_pointer_v); @@ -50,7 +59,12 @@ constexpr size_info inline calculate_one_size(const T &item) { return calculate_one_size(item.get()); } else if constexpr (detail::varint_t) { - ret.total = detail::calculate_varint_size(item); + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // skip it. It has been calculated in parent. + } + else { + ret.total = detail::calculate_varint_size(item); + } } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value) { @@ -147,8 +161,14 @@ constexpr size_info inline calculate_one_size(const T &item) { }); } else { + constexpr uint64_t tag = get_parent_tag(); + if constexpr (is_enable_fast_varint_coding(tag)) { + visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + ret.total += calculate_fast_varint_size(items...); + }); + } visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - ret += calculate_payload_size(items...); + ret += calculate_payload_size(items...); }); } } @@ -158,11 +178,169 @@ constexpr size_info inline calculate_one_size(const T &item) { return ret; } -template +template constexpr size_info STRUCT_PACK_INLINE calculate_payload_size(const Args &...items) { - return (calculate_one_size(items) + ...); + return (calculate_one_size(items) + ...); } + +struct fast_varint_result {}; + +template +constexpr std::size_t STRUCT_PACK_INLINE calculate_fast_varint_count() { + if constexpr (sizeof...(Args) == 0) { + return varint_t ? 1 : 0; + } + else { + return calculate_fast_varint_count() + + (varint_t ? 1 : 0); + } +} + +template +constexpr bool STRUCT_PACK_INLINE has_signed_varint() { + if constexpr (sizeof...(Args) == 0) { + if constexpr (varint_t) { + return std::is_signed_v; + } + else { + return false; + } + } + else { + if constexpr (varint_t) { + return std::is_signed_v || + has_signed_varint(); + } + else { + return has_signed_varint(); + } + } +} + +template +constexpr bool STRUCT_PACK_INLINE has_unsigned_varint() { + if constexpr (sizeof...(Args) == 0) { + if constexpr (varint_t) { + return std::is_unsigned_v; + } + else { + return false; + } + } + else { + if constexpr (varint_t) { + return std::is_unsigned_v || + has_unsigned_varint(); + } + else { + return has_unsigned_varint(); + } + } +} + +template +constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( + const Arg &item, int &non_zero_cnt32, int &non_zero_cnt64, + uint64_t &unsigned_max, int64_t &signed_max) { + if (item.get()) { + if constexpr (sizeof(Arg) == 4) { + ++non_zero_cnt32; + } + else if constexpr (sizeof(Arg) == 8) { + ++non_zero_cnt64; + } + else { + static_assert(!sizeof(Arg), "illegal branch"); + } + if constexpr (varint_t) { + if constexpr (std::is_unsigned_v< + std::remove_reference_t>) { + unsigned_max = std::max(unsigned_max, item.get()); + } + else { + signed_max = std::max( + signed_max, item.get() > 0 ? item.get() : -(item.get() + 1)); + } + } + } +} + +template +constexpr int STRUCT_PACK_INLINE +get_fast_varint_width_from_max(uint64_t unsigned_max, int64_t signed_max) { + int width_unsigned, width_signed; + if constexpr (has_unsigned_varint()) { + if SP_LIKELY (unsigned_max <= UINT8_MAX) { + width_unsigned = 0; + } + else if (unsigned_max <= UINT16_MAX) { + width_unsigned = 1; + } + else if (unsigned_max <= UINT32_MAX) { + width_unsigned = 2; + } + else { + width_unsigned = 3; + } + } + if constexpr (has_signed_varint()) { + if SP_LIKELY (signed_max <= INT8_MAX) { + width_signed = 0; + } + else if (signed_max <= INT16_MAX) { + width_signed = 1; + } + else if (signed_max <= INT32_MAX) { + width_signed = 2; + } + else { + width_signed = 3; + } + } + if constexpr (has_signed_varint()&& + has_unsigned_varint()) { + return std::max(width_unsigned, width_signed); + } + else if constexpr (has_signed_varint()) { + return width_signed; + } + else if constexpr (has_unsigned_varint()) { + return width_unsigned; + } + else { + static_assert(sizeof...(Args), "there should has a varint"); + return 0; + } +} + +template +constexpr int STRUCT_PACK_INLINE get_fast_varint_width(const Args &...items) { + uint64_t unsigned_max = 0; + int64_t signed_max = 0; + int non_zero_cnt32 = 0, non_zero_cnt64 = 0; + (get_fast_varint_width_impl(items, non_zero_cnt32, non_zero_cnt64, + unsigned_max, signed_max), + ...); + auto width = (1 << struct_pack::detail::get_fast_varint_width_from_max< + parent_tag, Args...>(unsigned_max, signed_max)); + return width * non_zero_cnt64 + (width > 4 ? 4 : width) * non_zero_cnt32; +} + +template +constexpr std::size_t STRUCT_PACK_INLINE +calculate_fast_varint_size(const Args &...items) { + constexpr auto cnt = calculate_fast_varint_count(); + constexpr auto bitset_size = ((cnt + 2) + 7) / 8; + if constexpr (cnt == 0) { + return 0; + } + else { + auto width = get_fast_varint_width(items...); + return width + bitset_size; + } +} + template STRUCT_PACK_INLINE constexpr serialize_buffer_size get_serialize_runtime_info( const Args &...args); diff --git a/include/ylt/struct_pack/derived_helper.hpp b/include/ylt/struct_pack/derived_helper.hpp index ebc1e69a4..ab7b0a5e5 100644 --- a/include/ylt/struct_pack/derived_helper.hpp +++ b/include/ylt/struct_pack/derived_helper.hpp @@ -112,7 +112,7 @@ struct deserialize_derived_class_helper { } }; -template +template constexpr size_info inline calculate_one_size(const T &item); template diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 7b23fa07c..063ce5be3 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -19,6 +19,7 @@ #include #include "reflection.hpp" +#include "ylt/struct_pack/marco.h" #include "ylt/struct_pack/util.h" namespace struct_pack::detail { @@ -131,7 +132,7 @@ inline uint64_t bswap64(uint64_t raw) { }; template -void write_wrapper(writer_t& writer, const char* data) { +void write_wrapper(writer_t& writer, const char* SP_RESTRICT data) { if constexpr (is_system_little_endian || block_size == 1) { writer.write(data, block_size); } @@ -164,6 +165,33 @@ void write_bytes_array(writer_t& writer, const char* data, std::size_t length) { else writer.write(data, length); } +template +void low_bytes_write_wrapper(writer_t& writer, const T& elem) { + static_assert(sizeof(T) >= block_size); + if constexpr (is_system_little_endian || block_size == sizeof(T)) { + const char* SP_RESTRICT data = (const char*)&elem; + writer.write(data, block_size); + } + else { + const char* SP_RESTRICT data = sizeof(T) - block_size + (const char*)&elem; + if constexpr (block_size == 2) { + auto tmp = bswap16(*(uint16_t*)data); + writer.write((char*)&tmp, block_size); + } + else if constexpr (block_size == 4) { + auto tmp = bswap32(*(uint32_t*)data); + writer.write((char*)&tmp, block_size); + } + else if constexpr (block_size == 8) { + auto tmp = bswap64(*(uint64_t*)data); + writer.write((char*)&tmp, block_size); + } + else { + static_assert(!sizeof(writer), + "illegal block size(should be 1,2,4,8,16)"); + } + } +} template bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { if constexpr (is_system_little_endian || block_size == 1) { @@ -203,4 +231,38 @@ bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, else return static_cast(reader.read(data, length)); } +template +bool low_bytes_read_wrapper(reader_t& reader, T& elem) { + static_assert(sizeof(T) >= block_size); + if constexpr (is_system_little_endian || block_size == sizeof(T)) { + char* SP_RESTRICT data = (char* )&elem; + return static_cast(reader.read(data, block_size)); + } + else { + char tmp[block_size]; + char* SP_RESTRICT data = (char* )&elem + sizeof(T) - block_size; + bool res = static_cast(reader.read(tmp, block_size)); + if SP_UNLIKELY (!res) { + return res; + } + if constexpr (block_size == 2) { + *(uint16_t*)data = bswap16(*(uint16_t*)tmp); + } + else if constexpr (block_size == 4) { + *(uint32_t*)data = bswap32(*(uint32_t*)tmp); + } + else if constexpr (block_size == 8) { + *(uint64_t*)data = bswap64(*(uint64_t*)tmp); + } + else if constexpr (block_size == 16) { + *(uint64_t*)(data + 8) = bswap64(*(uint64_t*)tmp); + *(uint64_t*)data = bswap64(*(uint64_t*)(tmp + 8)); + } + else { + static_assert(!sizeof(reader), + "illegal block size(should be 1,2,4,8,16)"); + } + return true; + } +} }; // namespace struct_pack::detail \ No newline at end of file diff --git a/include/ylt/struct_pack/error_code.hpp b/include/ylt/struct_pack/error_code.hpp index 268cf87e0..81b67df79 100644 --- a/include/ylt/struct_pack/error_code.hpp +++ b/include/ylt/struct_pack/error_code.hpp @@ -24,6 +24,7 @@ enum class errc { invalid_buffer, hash_conflict, seek_failed, + too_width_size }; namespace detail { diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 4a4e937ec..9ca5bed80 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -14,11 +14,16 @@ * limitations under the License. */ #pragma once +#include +#include #include +#include #include "calculate_size.hpp" #include "endian_wrapper.hpp" #include "reflection.hpp" +#include "ylt/struct_pack/util.h" +#include "ylt/struct_pack/varint.hpp" namespace struct_pack::detail { template < #if __cpp_concepts >= 201907L @@ -95,23 +100,23 @@ class packer { } if constexpr (hash_head % 2) { // has more metainfo auto metainfo = info.metainfo(); + auto sz = info.size(); write_wrapper(writer_, (char *)&metainfo); if constexpr (serialize_static_config::has_compatible) { - uint16_t len16; - uint32_t len32; - uint64_t len64; switch (metainfo & 0b11) { case 1: - len16 = info.size(); - write_wrapper<2>(writer_, (char *)&len16); + low_bytes_write_wrapper<2>(writer_, sz); break; case 2: - len32 = info.size(); - write_wrapper<4>(writer_, (char *)&len32); + low_bytes_write_wrapper<4>(writer_, sz); break; case 3: - len64 = info.size(); - write_wrapper<8>(writer_, (char *)&len64); + if constexpr (sizeof(std::size_t) >= 8) { + low_bytes_write_wrapper<8>(writer_, sz); + } + else { + unreachable(); + } break; default: unreachable(); @@ -127,13 +132,13 @@ class packer { } private: - template + template constexpr void STRUCT_PACK_INLINE serialize_many(const First &first_item, const Args &...items) { - serialize_one(first_item); + serialize_one(first_item); if constexpr (sizeof...(items) > 0) { - serialize_many(items...); + serialize_many(items...); } } constexpr void STRUCT_PACK_INLINE write_padding(std::size_t sz) { @@ -143,11 +148,89 @@ class packer { } } - template + template + static constexpr void STRUCT_PACK_INLINE + get_fast_varint_width_impl(std::bitset &vec, int &i, const Arg &item, + uint64_t &unsigned_max, int64_t &signed_max) { + if constexpr (varint_t) { + if (item.get() != 0) { + vec[i] = 1; + if constexpr (std::is_unsigned_v< + std::remove_reference_t>) { + unsigned_max = std::max(unsigned_max, item.get()); + } + else { + signed_max = std::max( + signed_max, item.get() > 0 ? item.get() : -(item.get() + 1)); + } + } + else { + vec[i] = 0; + } + ++i; + } + } + + template + static constexpr int STRUCT_PACK_INLINE + get_fast_varint_width(std::bitset &vec, const Args &...items) { + uint64_t unsigned_max = 0; + int64_t signed_max = 0; + int i = 0; + (get_fast_varint_width_impl(vec, i, items, unsigned_max, + signed_max), + ...); + return get_fast_varint_width_from_max(unsigned_max, + signed_max); + } + + template + constexpr void STRUCT_PACK_INLINE serialize_one_fast_varint(const Arg &item) { + if constexpr (varint_t) { + if (item.get()) + low_bytes_write_wrapper(writer_, item.get()); + } + } + + template + constexpr void STRUCT_PACK_INLINE + serialize_fast_varint(const Args &...items) { + constexpr auto cnt = calculate_fast_varint_count(); + constexpr auto bitset_size = ((cnt + 2) + 7) / 8; + if constexpr (cnt == 0) { + return; + } + else { + std::bitset vec; + auto width = get_fast_varint_width(vec, items...); + vec[cnt] = width & 0b1; + vec[cnt + 1] = width & 0b10; + write_bytes_array(writer_, (char *)&vec, bitset_size); + switch (width) { + case 0: + (serialize_one_fast_varint(items), ...); + break; + case 1: + (serialize_one_fast_varint(items), ...); + break; + case 2: + (serialize_one_fast_varint(items), ...); + break; + case 3: + (serialize_one_fast_varint(items), ...); + break; + default: + unreachable(); + } + } + } + + template constexpr void inline serialize_one(const T &item) { using type = remove_cvref_t; static_assert(!std::is_pointer_v); - constexpr auto id = get_type_id(); + constexpr auto id = get_type_id(); if constexpr (is_trivial_view_v) { return serialize_one(item.get()); } @@ -188,7 +271,12 @@ class packer { } } else if constexpr (detail::varint_t) { - detail::serialize_varint(writer_, item); + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // do nothing + } + else { + detail::serialize_varint(writer_, item); + } } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value && @@ -202,43 +290,45 @@ class packer { } } else if constexpr (map_container || container) { + auto size = item.size(); if constexpr (size_type == 1) { - uint8_t size = item.size(); - write_wrapper(writer_, (char *)&size); + low_bytes_write_wrapper(writer_, size); } #ifdef STRUCT_PACK_OPTIMIZE else if constexpr (size_type == 2) { - uint16_t size = item.size(); - write_wrapper(writer_, (char *)&size); + low_bytes_write_wrapper(writer_, size); } else if constexpr (size_type == 4) { - uint32_t size = item.size(); - write_wrapper(writer_, (char *)&size); + low_bytes_write_wrapper(writer_, size); } else if constexpr (size_type == 8) { - uint64_t size = item.size(); - write_wrapper(writer_, (char *)&size); + if constexpr (sizeof(std::size_t) >= 8) { + low_bytes_write_wrapper(writer_, size); + } + else { + static_assert(!sizeof(T), "illegal size_type"); + } } else { static_assert(!sizeof(item), "illegal size_type."); } #else else { - uint16_t size16; - uint32_t size32; - uint64_t size64; + auto size = item.size(); switch ((info.metainfo() & 0b11000) >> 3) { case 1: - size16 = item.size(); - write_wrapper<2>(writer_, (char *)&size16); + low_bytes_write_wrapper<2>(writer_, size); break; case 2: - size32 = item.size(); - write_wrapper<4>(writer_, (char *)&size32); + low_bytes_write_wrapper<4>(writer_, size); break; case 3: - size64 = item.size(); - write_wrapper<8>(writer_, (char *)&size64); + if constexpr (sizeof(std::size_t) >= 8) { + low_bytes_write_wrapper<8>(writer_, size); + } + else { + unreachable(); + } break; default: unreachable(); @@ -322,8 +412,14 @@ class packer { }); } else { + constexpr uint64_t tag = get_parent_tag(); + if constexpr (is_enable_fast_varint_coding(tag)) { + visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + serialize_fast_varint(items...); + }); + } visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - serialize_many(items...); + serialize_many(items...); }); } } diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index f130c9f14..c81d4aa90 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -38,11 +38,13 @@ namespace struct_pack { -enum sp_config { +enum sp_config : uint64_t { DEFAULT = 0, DISABLE_TYPE_INFO = 0b1, ENABLE_TYPE_INFO = 0b10, - DISABLE_ALL_META_INFO = 0b11 + DISABLE_ALL_META_INFO = 0b11, + ENCODING_WITH_VARINT = 0b100, + USE_FAST_VARINT = 0b1000 }; namespace detail { @@ -544,14 +546,32 @@ template #if __cpp_concepts >= 201907L template - concept user_defined_config = std::is_same_v())),struct_pack::sp_config>; + concept user_defined_config_by_ADL = std::is_same_v())),struct_pack::sp_config>; +#else + template + struct user_defined_config_by_ADL_impl : std::false_type {}; + + template + struct user_defined_config_by_ADL_impl())),struct_pack::sp_config>>>> + : std::true_type {}; + + template + constexpr bool user_defined_config_by_ADL = user_defined_config_by_ADL_impl::value; +#endif + +#if __cpp_concepts >= 201907L + template + concept user_defined_config = requires { + std::is_same_v; + }; #else template struct user_defined_config_impl : std::false_type {}; template struct user_defined_config_impl())),struct_pack::sp_config>>>> + std::enable_if_t>>> : std::true_type {}; template diff --git a/include/ylt/struct_pack/type_calculate.hpp b/include/ylt/struct_pack/type_calculate.hpp index 365283830..ad0337d3d 100644 --- a/include/ylt/struct_pack/type_calculate.hpp +++ b/include/ylt/struct_pack/type_calculate.hpp @@ -153,6 +153,29 @@ constexpr decltype(auto) get_type_end_flag() { } } +template +constexpr uint64_t get_parent_tag_impl() { + if constexpr (user_defined_config_by_ADL) { + return static_cast(set_sp_config((Arg *)nullptr)); + } + else if constexpr (user_defined_config) { + return static_cast(Arg::struct_pack_config); + } + else { + return 0; + } +} + +template +constexpr uint64_t get_parent_tag() { + if constexpr (sizeof...(Args) == 0) { + return 0; + } + else { + return get_parent_tag_impl(); + } +} + template constexpr decltype(auto) get_type_literal(std::index_sequence); @@ -173,7 +196,8 @@ constexpr decltype(auto) get_type_literal() { get_size_literal(); } else { - constexpr auto id = get_type_id(); + constexpr auto parent_tag = get_parent_tag(); + constexpr auto id = get_type_id(); constexpr auto begin = string_literal{{static_cast(id)}}; if constexpr (id == type_id::struct_t) { using Args = decltype(get_types()); @@ -659,7 +683,7 @@ template constexpr bool check_if_add_type_literal() { constexpr auto config = conf & 0b11; if constexpr (config == sp_config::DEFAULT) { - if constexpr (struct_pack::detail::user_defined_config) { + if constexpr (struct_pack::detail::user_defined_config_by_ADL) { constexpr auto config = set_sp_config((T *)nullptr) & 0b11; if constexpr (config == sp_config::DEFAULT) { return serialize_static_config::has_type_literal; @@ -668,6 +692,15 @@ constexpr bool check_if_add_type_literal() { return config == sp_config::ENABLE_TYPE_INFO; } } + else if constexpr (struct_pack::detail::user_defined_config) { + constexpr auto config = T::sp_config & 0b11; + if constexpr (config == sp_config::DEFAULT) { + return serialize_static_config::has_type_literal; + } + else { + return config == sp_config::ENABLE_TYPE_INFO; + } + } else { return serialize_static_config::has_type_literal; } @@ -760,12 +793,18 @@ template constexpr bool check_if_disable_hash_head_impl() { constexpr auto config = conf & 0b11; if constexpr (config != sp_config::DISABLE_ALL_META_INFO) { - if constexpr (struct_pack::detail::user_defined_config) { + if constexpr (struct_pack::detail::user_defined_config_by_ADL) { constexpr auto config = set_sp_config((T *)nullptr) & 0b11; if constexpr (config == sp_config::DISABLE_ALL_META_INFO) { return true; } } + else if constexpr (struct_pack::detail::user_defined_config_by_ADL) { + constexpr auto config = T::sp_config & 0b11; + if constexpr (config == sp_config::DISABLE_ALL_META_INFO) { + return true; + } + } return false; } return true; diff --git a/include/ylt/struct_pack/type_id.hpp b/include/ylt/struct_pack/type_id.hpp index dba4aeccb..175a33496 100644 --- a/include/ylt/struct_pack/type_id.hpp +++ b/include/ylt/struct_pack/type_id.hpp @@ -47,10 +47,14 @@ enum class type_id { float32_t, float64_t, float128_t, - v_int32_t, // variable size int - v_int64_t, // variable size int - v_uint32_t, // variable size unsigned int - v_uint64_t, // variable size unsigned int + vint32_t, // variable size int + vint64_t, // variable size int + vuint32_t, // variable size unsigned int + vuint64_t, // variable size unsigned int + fast_vint32_t, // variable size int with fast encoding + fast_vint64_t, // variable size int with fast encoding + fast_vuint32_t, // variable size unsigned int with fast encoding + fast_vuint64_t, // variable size unsigned int with fast encoding // template type string_t = 128, array_t, @@ -74,22 +78,41 @@ enum class type_id { type_end_flag = 255, }; -template +template constexpr type_id get_varint_type() { - if constexpr (std::is_same_v) { - return type_id::v_int32_t; - } - else if constexpr (std::is_same_v) { - return type_id::v_int64_t; - } - else if constexpr (std::is_same_v) { - return type_id::v_uint32_t; - } - else if constexpr (std::is_same_v) { - return type_id::v_uint64_t; + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + if constexpr (std::is_same_v) { + return type_id::fast_vint32_t; + } + else if constexpr (std::is_same_v) { + return type_id::fast_vint64_t; + } + else if constexpr (std::is_same_v) { + return type_id::fast_vuint32_t; + } + else if constexpr (std::is_same_v) { + return type_id::fast_vuint64_t; + } + else { + static_assert(!sizeof(T), "unsupported varint type!"); + } } else { - static_assert(!std::is_same_v, "unsupported varint type!"); + if constexpr (std::is_same_v) { + return type_id::vint32_t; + } + else if constexpr (std::is_same_v) { + return type_id::vint64_t; + } + else if constexpr (std::is_same_v) { + return type_id::vuint32_t; + } + else if constexpr (std::is_same_v) { + return type_id::vuint64_t; + } + else { + static_assert(!sizeof(T), "unsupported varint type!"); + } } } @@ -239,7 +262,7 @@ constexpr type_id get_floating_point_type() { } } -template +template constexpr type_id get_type_id() { static_assert(CHAR_BIT == 8); // compatible member, which should be ignored in MD5 calculated. @@ -261,7 +284,7 @@ constexpr type_id get_type_id() { return get_floating_point_type(); } else if constexpr (detail::varint_t) { - return get_varint_type(); + return get_varint_type(); } else if constexpr (std::is_same_v || std::is_same_v || std::is_abstract_v) { diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 1ecf9615a..ccc2dc285 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -111,6 +111,11 @@ class unpacker { data_len_ = reader_.tellg(); } auto &&[err_code, buffer_len] = deserialize_metainfo(); + if constexpr (sizeof(std::size_t) < 8) { + if (size_type_ > sizeof(std::size_t)) { + return errc::too_width_size; + } + } if SP_UNLIKELY (err_code != struct_pack::errc{}) { return err_code; } @@ -129,7 +134,12 @@ class unpacker { err_code = deserialize_many<4, UINT64_MAX, true>(t, args...); break; case 3: - err_code = deserialize_many<8, UINT64_MAX, true>(t, args...); + if constexpr (sizeof(std::size_t) >= 8) { + err_code = deserialize_many<8, UINT64_MAX, true>(t, args...); + } + else { + static_assert(!sizeof(T), "illegal size width"); + } break; #else case 1: @@ -447,24 +457,27 @@ class unpacker { deserialize_compatible(unsigned compatible_sz_len) { constexpr std::size_t sz[] = {0, 2, 4, 8}; auto len_sz = sz[compatible_sz_len]; - uint64_t data_len64; - uint32_t data_len32; - uint16_t data_len16; + uint64_t data_len = 0; bool result; switch (compatible_sz_len) { case 1: - if SP_LIKELY (read_wrapper<2>(reader_, (char *)&data_len16)) { - return {errc{}, data_len16}; + if SP_LIKELY (low_bytes_read_wrapper<2>(reader_, data_len)) { + return {errc{}, data_len}; } break; case 2: - if SP_LIKELY (read_wrapper<4>(reader_, (char *)&data_len32)) { - return {errc{}, data_len32}; + if SP_LIKELY (low_bytes_read_wrapper<4>(reader_, data_len)) { + return {errc{}, data_len}; } break; case 3: - if SP_LIKELY (read_wrapper<8>(reader_, (char *)&data_len64)) { - return {errc{}, data_len64}; + if constexpr (sizeof(std::size_t) >= 8) { + if SP_LIKELY (low_bytes_read_wrapper<8>(reader_, data_len)) { + return {errc{}, data_len}; + } + } + else { + return {errc::too_width_size, data_len}; } break; default: @@ -575,15 +588,22 @@ class unpacker { constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_many() { return {}; } - template + template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_many(First &&first_item, Args &&...items) { - auto code = deserialize_one(first_item); + auto code = + deserialize_one(first_item); if SP_UNLIKELY (code != struct_pack::errc{}) { return code; } - return deserialize_many(items...); + if constexpr (sizeof...(items)) { + return deserialize_many( + items...); + } + else { + return code; + } } constexpr struct_pack::errc STRUCT_PACK_INLINE @@ -596,12 +616,103 @@ class unpacker { } } - template + template + constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_one_fast_varint( + std::bitset &vec, int &i, Arg &item) { + if constexpr (varint_t) { + constexpr auto real_width = + std::min(width, sizeof(typename Arg::value_type)); + if (!vec[i++]) + return {}; + else { + bool ec; + if constexpr (std::is_unsigned_v< + std::remove_reference_t>) { + item.get() = 0; + bool ec = low_bytes_read_wrapper(reader_, item.get()); + if SP_UNLIKELY (!ec) { + return errc::no_buffer_space; + } + } + else { + typename int_t::type target; + ec = read_wrapper(reader_, (char *)&target); + if SP_UNLIKELY (!ec) { + return errc::no_buffer_space; + } + item = target; + } + return {}; + } + } + else { + return {}; + } + } + template + constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_fast_varint_helper( + std::bitset &vec, int &i, Arg &item, Args &...items) { + auto ec = deserialize_one_fast_varint(vec, i, item); + if constexpr (sizeof...(items)) { + if SP_UNLIKELY (ec != errc{}) { + return ec; + } + else { + return deserialize_fast_varint_helper(vec, i, + items...); + } + } + else { + return ec; + } + } + + template + constexpr struct_pack::errc STRUCT_PACK_INLINE + deserialize_fast_varint(Args &...items) { + constexpr auto cnt = calculate_fast_varint_count(); + constexpr auto bitset_size = ((cnt + 2) + 7) / 8; + if constexpr (cnt == 0) { + static_assert(cnt != 0, "ilegal branch"); + return {}; + } + else { + std::bitset vec; + if (auto ec = read_bytes_array(reader_, (char *)&vec, bitset_size); !ec) { + return errc::no_buffer_space; + } + std::size_t width = vec[cnt] + vec[cnt + 1] * 2; + int i = 0; + struct_pack::errc ec; + switch (width) { + case 0: + ec = deserialize_fast_varint_helper(vec, i, items...); + break; + case 1: + ec = deserialize_fast_varint_helper(vec, i, items...); + break; + case 2: + ec = deserialize_fast_varint_helper(vec, i, items...); + break; + case 3: + ec = deserialize_fast_varint_helper(vec, i, items...); + break; + default: + unreachable(); + } + return ec; + } + } + + template constexpr struct_pack::errc inline deserialize_one(T &item) { struct_pack::errc code{}; using type = remove_cvref_t; static_assert(!std::is_pointer_v); - constexpr auto id = get_type_id(); + constexpr auto id = get_type_id(); if constexpr (is_trivial_view_v) { static_assert(view_reader_t, "The Reader isn't a view_reader, can't deserialize " @@ -680,7 +791,12 @@ class unpacker { } } else if constexpr (detail::varint_t) { - code = detail::deserialize_varint(reader_, item); + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // do nothing, we have deserialized it in parent. + } + else { + code = detail::deserialize_varint(reader_, item); + } } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value && @@ -706,32 +822,26 @@ class unpacker { } } else if constexpr (container) { - uint16_t size16; - uint32_t size32; - uint64_t size64; + uint64_t size64 = 0; bool result; if constexpr (size_type == 1) { - uint8_t size8; - if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size8)) { + if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; } - size64 = size8; } #ifdef STRUCT_PACK_OPTIMIZE else if constexpr (size_type == 2) { - if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size16)) { + if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; } - size64 = size16; } else if constexpr (size_type == 4) { - if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size32)) { + if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; } - size64 = size32; } else if constexpr (size_type == 8) { - if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size64)) { + if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; } } @@ -742,19 +852,17 @@ class unpacker { else { switch (size_type_) { case 1: - if SP_UNLIKELY (!read_wrapper<2>(reader_, (char *)&size16)) { + if SP_UNLIKELY (!low_bytes_read_wrapper<2>(reader_, size64)) { return struct_pack::errc::no_buffer_space; } - size64 = size16; break; case 2: - if SP_UNLIKELY (!read_wrapper<4>(reader_, (char *)&size32)) { + if SP_UNLIKELY (!low_bytes_read_wrapper<4>(reader_, size64)) { return struct_pack::errc::no_buffer_space; } - size64 = size32; break; case 3: - if SP_UNLIKELY (!read_wrapper<8>(reader_, (char *)&size64)) { + if SP_UNLIKELY (!low_bytes_read_wrapper<8>(reader_, size64)) { return struct_pack::errc::no_buffer_space; } break; @@ -763,7 +871,7 @@ class unpacker { } } #endif - if SP_UNLIKELY (size64 == 0) { + if (size64 == 0) { return {}; } if constexpr (map_container) { @@ -972,8 +1080,17 @@ class unpacker { }); } else { + constexpr uint64_t tag = get_parent_tag(); + if constexpr (is_enable_fast_varint_coding(tag)) { + visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + code = deserialize_fast_varint(items...); + }); + if SP_UNLIKELY (code != errc::ok) { + return code; + } + } visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - code = deserialize_many(items...); + code = deserialize_many(items...); }); } } diff --git a/include/ylt/struct_pack/varint.hpp b/include/ylt/struct_pack/varint.hpp index 8a03df84c..0164beb60 100644 --- a/include/ylt/struct_pack/varint.hpp +++ b/include/ylt/struct_pack/varint.hpp @@ -25,6 +25,33 @@ namespace struct_pack { namespace detail { +constexpr inline bool is_enable_fast_varint_coding(uint64_t tag) { + return tag & struct_pack::USE_FAST_VARINT; +} + +template +struct int_t; + +template <> +struct int_t<1> { + using type = int8_t; +}; + +template <> +struct int_t<2> { + using type = int16_t; +}; + +template <> +struct int_t<4> { + using type = int32_t; +}; + +template <> +struct int_t<8> { + using type = int64_t; +}; + template class varint { public: diff --git a/src/struct_pack/tests/CMakeLists.txt b/src/struct_pack/tests/CMakeLists.txt index fa4c284c0..bf06ac3e9 100644 --- a/src/struct_pack/tests/CMakeLists.txt +++ b/src/struct_pack/tests/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(struct_pack_test test_pragma_pack.cpp test_pragma_pack_and_alignas_mix.cpp test_varint.cpp + test_fast_varint.cpp test_stream.cpp test_compatible.cpp test_non_aggregated_type.cpp diff --git a/src/struct_pack/tests/test_compile_time_calculate.cpp b/src/struct_pack/tests/test_compile_time_calculate.cpp index e949f3b24..bc9c6fe37 100644 --- a/src/struct_pack/tests/test_compile_time_calculate.cpp +++ b/src/struct_pack/tests/test_compile_time_calculate.cpp @@ -5,6 +5,7 @@ #include "doctest.h" #include "test_struct.hpp" +#include "ylt/struct_pack/reflection.hpp" #include "ylt/struct_pack/type_calculate.hpp" #if __cplusplus >= 202002L #include @@ -474,4 +475,36 @@ TEST_CASE("test has container") { test_has_container>>()); static_assert(struct_pack::detail::check_if_has_container< test_has_container>>()); +} + +struct fast_varint_example_1 { + var_int32_t a; + var_int32_t b; + var_int32_t c; + var_int32_t d; + bool operator==(const fast_varint_example_1& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } + static constexpr struct_pack::sp_config struct_pack_config = USE_FAST_VARINT; +}; + +struct fast_varint_example_2 { + var_int32_t a; + var_int32_t b; + var_int32_t c; + var_int32_t d; + bool operator==(const fast_varint_example_1& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +constexpr auto set_sp_config(fast_varint_example_2*) { + return struct_pack::sp_config::USE_FAST_VARINT; +} + +TEST_CASE("test fast varint tag") { + static_assert( + struct_pack::detail::user_defined_config); + static_assert( + struct_pack::detail::user_defined_config_by_ADL); } \ No newline at end of file diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp new file mode 100644 index 000000000..f3e754df7 --- /dev/null +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -0,0 +1,917 @@ +#include +#include + +#include "doctest.h" +#include "test_struct.hpp" +#include "ylt/struct_pack/reflection.hpp" +#include "ylt/struct_pack/varint.hpp" + +using namespace struct_pack; + +struct fast_varint_example_1 { + var_int32_t a; + var_int32_t b; + var_int32_t c; + var_int32_t d; + bool operator==(const fast_varint_example_1& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varint_example_2 { + var_int32_t a; + var_int64_t b; + var_int32_t c; + var_int64_t d; + bool operator==(const fast_varint_example_2& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varint_example_3 { + var_int64_t a; + var_int64_t b; + var_int64_t c; + var_int64_t d; + bool operator==(const fast_varint_example_3& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varint_example_4 { + var_int64_t a; + var_int64_t b; + var_int64_t c; + var_int64_t d; + var_int64_t e; + var_int64_t f; + var_int64_t g; + bool operator==(const fast_varint_example_4& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && + f == o.f && g == o.g; + } +}; + +struct fast_varint_example_5 { + var_int64_t a; + var_int64_t b; + var_int64_t c; + var_int64_t d; + var_int64_t e; + var_int64_t f; + bool operator==(const fast_varint_example_5& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && f == o.f; + } +}; + +struct fast_varuint_example_1 { + var_uint64_t a; + var_uint64_t b; + var_uint64_t c; + var_uint64_t d; + bool operator==(const fast_varuint_example_1& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varuint_example_2 { + var_uint32_t a; + var_uint32_t b; + var_uint32_t c; + var_uint32_t d; + bool operator==(const fast_varuint_example_2& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varuint_example_3 { + var_uint32_t a; + var_uint64_t b; + var_uint32_t c; + var_uint64_t d; + bool operator==(const fast_varuint_example_3& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } +}; + +struct fast_varmixedint_example_1 { + var_int32_t a; + var_uint64_t b; + var_uint32_t c; + var_int64_t d; + bool operator==(const fast_varmixedint_example_1& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT; +}; + +constexpr sp_config set_sp_config(fast_varint_example_1*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varint_example_2*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varint_example_3*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varint_example_4*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varint_example_5*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varuint_example_1*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varuint_example_2*) { + return USE_FAST_VARINT; +} + +constexpr sp_config set_sp_config(fast_varuint_example_3*) { + return USE_FAST_VARINT; +} + +TEST_CASE("fast varint test 0") { + fast_varint_example_1 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varint test 1") { + fast_varint_example_1 o{1, -1, 127, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 2") { + fast_varint_example_1 o{1, -127, 127, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 3") { + fast_varint_example_1 o{1, -127, 0, 127}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 4") { + fast_varint_example_1 o{1, 0, -127, 127}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 5") { + fast_varint_example_1 o{0, 1, -127, 127}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 6") { + fast_varint_example_1 o{1, -1, 128, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 7); + return; +} + +TEST_CASE("fast varint test 7") { + fast_varint_example_1 o{1, -128, 1, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint test 8") { + fast_varint_example_1 o{32767, -32767, 1123, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 7); + return; +} + +TEST_CASE("fast varint test 8") { + fast_varint_example_1 o{0, 0, 1123, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varint test 9") { + fast_varint_example_1 o{32768, 1, 0, 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint test 10") { + fast_varint_example_1 o{-32768, 1, 0, 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 7); + return; +} + +TEST_CASE("fast varint test 12") { + fast_varint_example_1 o{-83219132, 114514, 0, 2123321213}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint test 12") { + fast_varint_example_1 o{INT_MIN, INT_MAX, 0, INT_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint2 test 1") { + fast_varint_example_2 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varint2 test 2") { + fast_varint_example_2 o{0, 1, -1, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varint2 test 2") { + fast_varint_example_2 o{0, 127, -128, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varint2 test 3") { + fast_varint_example_2 o{0, 128, -127, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varint2 test 4") { + fast_varint_example_2 o{0, INT16_MAX, INT16_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varint2 test 5") { + fast_varint_example_2 o{0, INT16_MAX + 1, INT16_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varint2 test 6") { + fast_varint_example_2 o{0, INT16_MAX, INT16_MIN - 1, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varint2 test 7") { + fast_varint_example_2 o{0, INT32_MAX, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varint2 test 8") { + fast_varint_example_2 o{0, INT32_MAX, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varint2 test 9") { + fast_varint_example_2 o{0, INT32_MAX + 1ll, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint2 test 10") { + fast_varint_example_2 o{0, INT32_MIN - 1ll, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint2 test 11") { + fast_varint_example_2 o{0, INT64_MIN, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint2 test 12") { + fast_varint_example_2 o{0, INT64_MAX, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varint3 test 1") { + fast_varint_example_3 o{0, INT64_MAX, 0, INT64_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varint3 test 2") { + fast_varint_example_3 o{0, INT32_MAX, 0, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varint4 test 1") { + fast_varint_example_4 o{0, 0, 0, 0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 2); + return; +} + +TEST_CASE("fast varint4 test 2") { + fast_varint_example_4 o{0, 0, 0, 0, 0, 0, 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varint4 test 3") { + fast_varint_example_4 o{0, 0, 0, 0, 0, 0, INT16_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varint4 test 4") { + fast_varint_example_4 o{0, 0, 0, 0, 0, 0, INT32_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 6); + return; +} + +TEST_CASE("fast varint4 test 5") { + fast_varint_example_4 o{0, 0, 0, 0, 0, 0, INT64_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 10); + return; +} + +TEST_CASE("fast varint5 test 1") { + fast_varint_example_5 o{0, 0, 0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varint5 test 2") { + fast_varint_example_5 o{0, 0, 0, 0, 0, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 2); + return; +} + +TEST_CASE("fast varint5 test 3") { + fast_varint_example_5 o{0, 0, 0, 0, 0, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varint5 test 4") { + fast_varint_example_5 o{0, 0, 0, 0, 0, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varint5 test 5") { + fast_varint_example_5 o{0, 0, 0, 0, 0, INT64_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varuint1 test 1") { + fast_varuint_example_1 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varuint1 test 2") { + fast_varuint_example_1 o{UINT8_MAX, UINT8_MAX, 0, UINT8_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 4); + return; +} + +TEST_CASE("fast varuint1 test 3") { + fast_varuint_example_1 o{UINT8_MAX, UINT8_MAX + 1, 0, UINT8_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 7); + return; +} + +TEST_CASE("fast varuint1 test 4") { + fast_varuint_example_1 o{UINT16_MAX, UINT16_MAX, 0, UINT16_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 7); + return; +} + +TEST_CASE("fast varuint1 test 5") { + fast_varuint_example_1 o{UINT16_MAX, UINT16_MAX, 0, UINT16_MAX + 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varuint1 test 6") { + fast_varuint_example_1 o{UINT32_MAX, UINT32_MAX, 0, UINT32_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varuint1 test 7") { + fast_varuint_example_1 o{UINT32_MAX + 1ull, UINT32_MAX, 0, UINT32_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} + +TEST_CASE("fast varuint1 test 8") { + fast_varuint_example_1 o{UINT64_MAX, UINT64_MAX, 0, UINT64_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} + +TEST_CASE("fast varuint2 test 1") { + fast_varuint_example_2 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varuint2 test 2") { + fast_varuint_example_2 o{0, 0, 0, UINT8_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 2); + return; +} + +TEST_CASE("fast varuint2 test 3") { + fast_varuint_example_2 o{0, 0, 0, UINT8_MAX + 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varuint2 test 4") { + fast_varuint_example_2 o{0, 0, 0, UINT16_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 3); + return; +} + +TEST_CASE("fast varuint2 test 5") { + fast_varuint_example_2 o{0, 0, 0, UINT16_MAX + 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} +TEST_CASE("fast varuint2 test 6") { + fast_varuint_example_2 o{0, 0, 0, UINT32_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varuint3 test 1") { + fast_varuint_example_3 o{0, 0, UINT32_MAX, UINT32_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varuint3 test 2") { + fast_varuint_example_3 o{0, 0, UINT32_MAX, UINT32_MAX + 1ull}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varuint3 test 3") { + fast_varuint_example_3 o{0, 0, UINT32_MAX, UINT64_MAX}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 13); + return; +} + +TEST_CASE("fast varmixedint1 test 1") { + fast_varmixedint_example_1 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varmixedint1 test 2") { + fast_varmixedint_example_1 o{INT8_MIN, UINT8_MAX, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varmixedint1 test 3") { + fast_varmixedint_example_1 o{INT8_MIN - 1, UINT8_MAX, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varmixedint1 test 5") { + fast_varmixedint_example_1 o{INT8_MIN, UINT8_MAX + 1, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint1 test 6") { + fast_varmixedint_example_1 o{INT8_MIN, UINT8_MAX, UINT8_MAX + 1, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint1 test 7") { + fast_varmixedint_example_1 o{INT8_MIN, UINT8_MAX, UINT8_MAX, INT8_MIN - 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint1 test 8") { + fast_varmixedint_example_1 o{INT16_MIN, UINT16_MAX, UINT16_MAX, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint1 test 9") { + fast_varmixedint_example_1 o{INT16_MIN - 1, UINT16_MAX, UINT16_MAX, + INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} +TEST_CASE("fast varmixedint1 test 10") { + fast_varmixedint_example_1 o{INT16_MIN, UINT16_MAX + 1, UINT16_MAX, + INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} +TEST_CASE("fast varmixedint1 test 11") { + fast_varmixedint_example_1 o{INT16_MIN, UINT16_MAX, UINT16_MAX + 1, + INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint1 test 12") { + fast_varmixedint_example_1 o{INT16_MIN, UINT16_MAX, UINT16_MAX, + INT16_MIN - 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint1 test 13") { + fast_varmixedint_example_1 o{INT32_MIN, UINT32_MAX, UINT32_MAX, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint1 test 14") { + fast_varmixedint_example_1 o{INT32_MIN, UINT32_MAX + 1ull, UINT32_MAX, + INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} +TEST_CASE("fast varmixedint1 test 15") { + fast_varmixedint_example_1 o{INT32_MIN, UINT32_MAX, UINT32_MAX, + INT32_MIN - 1ull}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} +TEST_CASE("fast varmixedint1 test 16") { + fast_varmixedint_example_1 o{INT32_MIN, UINT64_MAX, UINT32_MAX, INT64_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} \ No newline at end of file From f8c132b4d0a7ea85340e7280cb073d4faa61daf0 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Mon, 20 Nov 2023 21:16:11 +0800 Subject: [PATCH 02/39] allow serialize normal int as varint encode --- include/ylt/struct_pack/calculate_size.hpp | 46 ++--- include/ylt/struct_pack/packer.hpp | 40 +++-- include/ylt/struct_pack/reflection.hpp | 22 ++- include/ylt/struct_pack/type_calculate.hpp | 6 +- include/ylt/struct_pack/type_id.hpp | 30 ++-- include/ylt/struct_pack/unpacker.hpp | 64 +++---- include/ylt/struct_pack/varint.hpp | 25 ++- .../tests/test_compile_time_calculate.cpp | 69 +++++--- src/struct_pack/tests/test_fast_varint.cpp | 157 +++++++++++++++++- src/struct_pack/tests/test_varint.cpp | 23 +++ test.cpp | 30 ++++ 11 files changed, 399 insertions(+), 113 deletions(-) create mode 100644 test.cpp diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index 389b57f64..1d4b84dd4 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -50,6 +50,14 @@ constexpr size_info inline calculate_one_size(const T &item) { size_info ret{}; if constexpr (id == type_id::monostate_t) { } + else if constexpr (detail::varint_t) { + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // skip it. It has been calculated in parent. + } + else { + ret.total = detail::calculate_varint_size(item); + } + } else if constexpr (std::is_fundamental_v || std::is_enum_v || id == type_id::int128_t || id == type_id::uint128_t || id == type_id::bitset_t) { @@ -58,14 +66,6 @@ constexpr size_info inline calculate_one_size(const T &item) { else if constexpr (is_trivial_view_v) { return calculate_one_size(item.get()); } - else if constexpr (detail::varint_t) { - if constexpr (is_enable_fast_varint_coding(parent_tag)) { - // skip it. It has been calculated in parent. - } - else { - ret.total = detail::calculate_varint_size(item); - } - } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value) { ret.total = sizeof(type); @@ -189,27 +189,27 @@ struct fast_varint_result {}; template constexpr std::size_t STRUCT_PACK_INLINE calculate_fast_varint_count() { if constexpr (sizeof...(Args) == 0) { - return varint_t ? 1 : 0; + return varint_t ? 1 : 0; } else { return calculate_fast_varint_count() + - (varint_t ? 1 : 0); + (varint_t ? 1 : 0); } } template constexpr bool STRUCT_PACK_INLINE has_signed_varint() { if constexpr (sizeof...(Args) == 0) { - if constexpr (varint_t) { - return std::is_signed_v; + if constexpr (varint_t) { + return std::is_signed_v()))>>; } else { return false; } } else { - if constexpr (varint_t) { - return std::is_signed_v || + if constexpr (varint_t) { + return std::is_signed_v()))>> || has_signed_varint(); } else { @@ -221,16 +221,16 @@ constexpr bool STRUCT_PACK_INLINE has_signed_varint() { template constexpr bool STRUCT_PACK_INLINE has_unsigned_varint() { if constexpr (sizeof...(Args) == 0) { - if constexpr (varint_t) { - return std::is_unsigned_v; + if constexpr (varint_t) { + return std::is_unsigned_v()))>>; } else { return false; } } else { - if constexpr (varint_t) { - return std::is_unsigned_v || + if constexpr (varint_t) { + return std::is_unsigned_v()))>> || has_unsigned_varint(); } else { @@ -243,7 +243,7 @@ template constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( const Arg &item, int &non_zero_cnt32, int &non_zero_cnt64, uint64_t &unsigned_max, int64_t &signed_max) { - if (item.get()) { + if (get_varint_value(item)) { if constexpr (sizeof(Arg) == 4) { ++non_zero_cnt32; } @@ -253,14 +253,14 @@ constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( else { static_assert(!sizeof(Arg), "illegal branch"); } - if constexpr (varint_t) { + if constexpr (varint_t) { if constexpr (std::is_unsigned_v< - std::remove_reference_t>) { - unsigned_max = std::max(unsigned_max, item.get()); + std::remove_reference_t>) { + unsigned_max = std::max(unsigned_max, get_varint_value(item)); } else { signed_max = std::max( - signed_max, item.get() > 0 ? item.get() : -(item.get() + 1)); + signed_max, get_varint_value(item) > 0 ? get_varint_value(item) : -(get_varint_value(item) + 1)); } } } diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 9ca5bed80..b735657b5 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -152,16 +152,19 @@ class packer { static constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl(std::bitset &vec, int &i, const Arg &item, uint64_t &unsigned_max, int64_t &signed_max) { - if constexpr (varint_t) { - if (item.get() != 0) { + if constexpr (varint_t) { + if (get_varint_value(item) != 0) { vec[i] = 1; - if constexpr (std::is_unsigned_v< - std::remove_reference_t>) { - unsigned_max = std::max(unsigned_max, item.get()); + if constexpr (std::is_unsigned_v>) { + unsigned_max = + std::max(unsigned_max, get_varint_value(item)); } else { - signed_max = std::max( - signed_max, item.get() > 0 ? item.get() : -(item.get() + 1)); + signed_max = std::max(signed_max, + get_varint_value(item) > 0 + ? get_varint_value(item) + : -(get_varint_value(item) + 1)); } } else { @@ -186,9 +189,10 @@ class packer { template constexpr void STRUCT_PACK_INLINE serialize_one_fast_varint(const Arg &item) { - if constexpr (varint_t) { - if (item.get()) - low_bytes_write_wrapper(writer_, item.get()); + if constexpr (varint_t) { + if (get_varint_value(item)) + low_bytes_write_wrapper( + writer_, get_varint_value(item)); } } @@ -241,6 +245,14 @@ class packer { else if constexpr (std::is_same_v) { // do nothing } + else if constexpr (detail::varint_t) { + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // do nothing + } + else { + detail::serialize_varint(writer_, item); + } + } else if constexpr (std::is_fundamental_v || std::is_enum_v || id == type_id::int128_t || id == type_id::uint128_t) { write_wrapper(writer_, (char *)&item); @@ -270,14 +282,6 @@ class packer { } } } - else if constexpr (detail::varint_t) { - if constexpr (is_enable_fast_varint_coding(parent_tag)) { - // do nothing - } - else { - detail::serialize_varint(writer_, item); - } - } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value && is_little_endian_copyable) { diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index c81d4aa90..e7f328edb 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -755,19 +755,22 @@ template constexpr bool sintable_t = std::is_same_v> || std::is_same_v>; - template - constexpr bool varint_t = varintable_t || sintable_t; + template + constexpr bool varint_t = varintable_t || sintable_t || ((parent_tag&struct_pack::ENCODING_WITH_VARINT) && (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v)); template constexpr inline bool is_trivial_view_v = false; - template + template + constexpr uint64_t get_parent_tag(); + + template struct is_trivial_serializable { private: - template + template static constexpr bool class_visit_helper(std::index_sequence) { return (is_trivial_serializable, - ignore_compatible_field>::value && + ignore_compatible_field,parent_tag_>::value && ...); } static constexpr bool solve() { @@ -777,6 +780,9 @@ template else if constexpr (std::is_abstract_v) { return false; } + else if constexpr (varint_t) { + return false; + } else if constexpr (is_compatible_v || is_trivial_view_v) { return ignore_compatible_field; } @@ -802,8 +808,7 @@ template return false; } else if constexpr (container || optional || is_variant_v || - unique_ptr || expected || container_adapter || - varint_t) { + unique_ptr || expected || container_adapter) { return false; } else if constexpr (pair) { @@ -816,8 +821,9 @@ template return class_visit_helper(std::make_index_sequence>{}); } else if constexpr (std::is_class_v) { + constexpr auto tag = get_parent_tag(); using U = decltype(get_types()); - return class_visit_helper(std::make_index_sequence>{}); + return class_visit_helper(std::make_index_sequence>{}); } else return false; diff --git a/include/ylt/struct_pack/type_calculate.hpp b/include/ylt/struct_pack/type_calculate.hpp index ad0337d3d..86c93d23c 100644 --- a/include/ylt/struct_pack/type_calculate.hpp +++ b/include/ylt/struct_pack/type_calculate.hpp @@ -693,7 +693,7 @@ constexpr bool check_if_add_type_literal() { } } else if constexpr (struct_pack::detail::user_defined_config) { - constexpr auto config = T::sp_config & 0b11; + constexpr auto config = T::struct_pack_config & 0b11; if constexpr (config == sp_config::DEFAULT) { return serialize_static_config::has_type_literal; } @@ -799,8 +799,8 @@ constexpr bool check_if_disable_hash_head_impl() { return true; } } - else if constexpr (struct_pack::detail::user_defined_config_by_ADL) { - constexpr auto config = T::sp_config & 0b11; + else if constexpr (struct_pack::detail::user_defined_config) { + constexpr auto config = T::struct_pack_config & 0b11; if constexpr (config == sp_config::DISABLE_ALL_META_INFO) { return true; } diff --git a/include/ylt/struct_pack/type_id.hpp b/include/ylt/struct_pack/type_id.hpp index 175a33496..a25701774 100644 --- a/include/ylt/struct_pack/type_id.hpp +++ b/include/ylt/struct_pack/type_id.hpp @@ -81,16 +81,20 @@ enum class type_id { template constexpr type_id get_varint_type() { if constexpr (is_enable_fast_varint_coding(parent_tag)) { - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v) { return type_id::fast_vint32_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::fast_vint64_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::fast_vuint32_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::fast_vuint64_t; } else { @@ -98,16 +102,20 @@ constexpr type_id get_varint_type() { } } else { - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v) { return type_id::vint32_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::vint64_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::vuint32_t; } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v || + std::is_same_v) { return type_id::vuint64_t; } else { @@ -269,6 +277,9 @@ constexpr type_id get_type_id() { if constexpr (optional && is_compatible_v) { return type_id::compatible_t; } + else if constexpr (detail::varint_t) { + return get_varint_type(); + } else if constexpr (std::is_enum_v) { return get_integral_type>(); } @@ -283,9 +294,6 @@ constexpr type_id get_type_id() { else if constexpr (std::is_floating_point_v) { return get_floating_point_type(); } - else if constexpr (detail::varint_t) { - return get_varint_type(); - } else if constexpr (std::is_same_v || std::is_same_v || std::is_abstract_v) { return type_id::monostate_t; diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index ccc2dc285..41996751c 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -616,21 +616,24 @@ class unpacker { } } - template + template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_one_fast_varint( std::bitset &vec, int &i, Arg &item) { - if constexpr (varint_t) { - constexpr auto real_width = - std::min(width, sizeof(typename Arg::value_type)); + if constexpr (varint_t) { + constexpr auto real_width = std::min(width, sizeof(Arg)); if (!vec[i++]) return {}; + else if constexpr (!no_skip) { + reader_.ignore(real_width) ? errc{} : errc::no_buffer_space; + } else { bool ec; - if constexpr (std::is_unsigned_v< - std::remove_reference_t>) { - item.get() = 0; - bool ec = low_bytes_read_wrapper(reader_, item.get()); + if constexpr (std::is_unsigned_v>) { + get_varint_value(item) = 0; + bool ec = low_bytes_read_wrapper(reader_, + get_varint_value(item)); if SP_UNLIKELY (!ec) { return errc::no_buffer_space; } @@ -650,18 +653,19 @@ class unpacker { return {}; } } - template + template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_fast_varint_helper( std::bitset &vec, int &i, Arg &item, Args &...items) { - auto ec = deserialize_one_fast_varint(vec, i, item); + auto ec = + deserialize_one_fast_varint(vec, i, item); if constexpr (sizeof...(items)) { if SP_UNLIKELY (ec != errc{}) { return ec; } else { - return deserialize_fast_varint_helper(vec, i, - items...); + return deserialize_fast_varint_helper( + vec, i, items...); } } else { @@ -669,7 +673,7 @@ class unpacker { } } - template + template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_fast_varint(Args &...items) { constexpr auto cnt = calculate_fast_varint_count(); @@ -688,16 +692,20 @@ class unpacker { struct_pack::errc ec; switch (width) { case 0: - ec = deserialize_fast_varint_helper(vec, i, items...); + ec = deserialize_fast_varint_helper(vec, i, + items...); break; case 1: - ec = deserialize_fast_varint_helper(vec, i, items...); + ec = deserialize_fast_varint_helper(vec, i, + items...); break; case 2: - ec = deserialize_fast_varint_helper(vec, i, items...); + ec = deserialize_fast_varint_helper(vec, i, + items...); break; case 3: - ec = deserialize_fast_varint_helper(vec, i, items...); + ec = deserialize_fast_varint_helper(vec, i, + items...); break; default: unreachable(); @@ -737,6 +745,14 @@ class unpacker { else if constexpr (std::is_same_v) { // do nothing } + else if constexpr (detail::varint_t) { + if constexpr (is_enable_fast_varint_coding(parent_tag)) { + // do nothing, we have deserialized it in parent. + } + else { + code = detail::deserialize_varint(reader_, item); + } + } else if constexpr (std::is_fundamental_v || std::is_enum_v || id == type_id::int128_t || id == type_id::uint128_t) { if constexpr (NotSkip) { @@ -790,14 +806,6 @@ class unpacker { deserialize_one(*item); } } - else if constexpr (detail::varint_t) { - if constexpr (is_enable_fast_varint_coding(parent_tag)) { - // do nothing, we have deserialized it in parent. - } - else { - code = detail::deserialize_varint(reader_, item); - } - } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value && is_little_endian_copyable) { @@ -1083,7 +1091,7 @@ class unpacker { constexpr uint64_t tag = get_parent_tag(); if constexpr (is_enable_fast_varint_coding(tag)) { visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - code = deserialize_fast_varint(items...); + code = deserialize_fast_varint(items...); }); if SP_UNLIKELY (code != errc::ok) { return code; diff --git a/include/ylt/struct_pack/varint.hpp b/include/ylt/struct_pack/varint.hpp index 0164beb60..94d7697ce 100644 --- a/include/ylt/struct_pack/varint.hpp +++ b/include/ylt/struct_pack/varint.hpp @@ -229,7 +229,7 @@ STRUCT_PACK_INLINE void serialize_varint(writer& writer_, const T& t) { #endif uint64_t v; if constexpr (sintable_t) { - v = encode_zigzag(t.get()); + v = encode_zigzag(get_varint_value(t)); } else { v = t; @@ -304,6 +304,29 @@ template +const auto& get_varint_value(const T& v) { + if constexpr (varint_t) { + return v.get(); + } + else { + return v; + } +} + +template +auto& get_varint_value(T& v) { + if constexpr (varint_t) { + return v.get(); + } + else { + return v; + } +} + + + } // namespace detail using var_int32_t = detail::sint; using var_int64_t = detail::sint; diff --git a/src/struct_pack/tests/test_compile_time_calculate.cpp b/src/struct_pack/tests/test_compile_time_calculate.cpp index bc9c6fe37..34c67afed 100644 --- a/src/struct_pack/tests/test_compile_time_calculate.cpp +++ b/src/struct_pack/tests/test_compile_time_calculate.cpp @@ -477,34 +477,63 @@ TEST_CASE("test has container") { test_has_container>>()); } -struct fast_varint_example_1 { +struct fast_varint_example_ct1 { var_int32_t a; - var_int32_t b; - var_int32_t c; - var_int32_t d; - bool operator==(const fast_varint_example_1& o) const { - return a == o.a && b == o.b && c == o.c && d == o.d; - } + var_int64_t b; + var_uint32_t c; + var_uint64_t d; static constexpr struct_pack::sp_config struct_pack_config = USE_FAST_VARINT; }; -struct fast_varint_example_2 { +struct fast_varint_example_ct2 { var_int32_t a; - var_int32_t b; - var_int32_t c; - var_int32_t d; - bool operator==(const fast_varint_example_1& o) const { - return a == o.a && b == o.b && c == o.c && d == o.d; - } + var_int64_t b; + var_uint32_t c; + var_uint64_t d; }; - -constexpr auto set_sp_config(fast_varint_example_2*) { +constexpr auto set_sp_config(fast_varint_example_ct2*) { return struct_pack::sp_config::USE_FAST_VARINT; } +struct fast_varint_example_ct3 { + int32_t a; + int64_t b; + uint32_t c; + uint64_t d; + static constexpr auto struct_pack_config = + USE_FAST_VARINT | ENCODING_WITH_VARINT; +}; + +struct fast_varint_example_ct4 { + int32_t a; + int64_t b; + uint32_t c; + uint64_t d; + static constexpr auto struct_pack_config = ENCODING_WITH_VARINT; +}; TEST_CASE("test fast varint tag") { - static_assert( - struct_pack::detail::user_defined_config); - static_assert( - struct_pack::detail::user_defined_config_by_ADL); + using namespace struct_pack::detail; + static_assert(user_defined_config); + static_assert(user_defined_config_by_ADL); + static_assert(user_defined_config); + constexpr auto type_info1 = + struct_pack::get_type_literal(); + constexpr auto type_info2 = + struct_pack::get_type_literal(); + constexpr auto type_info3 = + struct_pack::get_type_literal(); + constexpr auto type_info4 = + struct_pack::get_type_literal(); + constexpr auto fast_varint_info = struct_pack::string_literal{ + {(char)type_id::struct_t, (char)type_id::fast_vint32_t, + (char)type_id::fast_vint64_t, (char)type_id::fast_vuint32_t, + (char)type_id::fast_vuint64_t, (char)type_id::type_end_flag}}; + constexpr auto varint_info = struct_pack::string_literal{ + {(char)type_id::struct_t, (char)type_id::vint32_t, + (char)type_id::vint64_t, (char)type_id::vuint32_t, + (char)type_id::vuint64_t, (char)type_id::type_end_flag}}; + CHECK(type_info1 == type_info2); + CHECK(type_info1 == type_info3); + CHECK(type_info3 != type_info4); + CHECK(type_info1 == fast_varint_info); } \ No newline at end of file diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index f3e754df7..0f428fe7a 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -896,7 +896,7 @@ TEST_CASE("fast varmixedint1 test 14") { } TEST_CASE("fast varmixedint1 test 15") { fast_varmixedint_example_1 o{INT32_MIN, UINT32_MAX, UINT32_MAX, - INT32_MIN - 1ull}; + INT32_MIN - 1ll}; auto buffer = struct_pack::serialize(o); auto result = struct_pack::deserialize(buffer); @@ -914,4 +914,159 @@ TEST_CASE("fast varmixedint1 test 16") { CHECK(result == o); CHECK(buffer.size() == 25); return; +} + +struct varmixedint2 { + int32_t a; + uint64_t b; + uint32_t c; + int64_t d; + bool operator==(const varmixedint2& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } + static constexpr auto struct_pack_config = struct_pack::ENCODING_WITH_VARINT | + struct_pack::USE_FAST_VARINT | + struct_pack::DISABLE_ALL_META_INFO; +}; + +TEST_CASE("fast varmixedint2 test 1") { + varmixedint2 o{0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 1); + return; +} + +TEST_CASE("fast varmixedint2 test 2") { + varmixedint2 o{INT8_MIN, UINT8_MAX, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 5); + return; +} + +TEST_CASE("fast varmixedint2 test 3") { + varmixedint2 o{INT8_MIN - 1, UINT8_MAX, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} + +TEST_CASE("fast varmixedint2 test 5") { + varmixedint2 o{INT8_MIN, UINT8_MAX + 1, UINT8_MAX, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint2 test 6") { + varmixedint2 o{INT8_MIN, UINT8_MAX, UINT8_MAX + 1, INT8_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint2 test 7") { + varmixedint2 o{INT8_MIN, UINT8_MAX, UINT8_MAX, INT8_MIN - 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint2 test 8") { + varmixedint2 o{INT16_MIN, UINT16_MAX, UINT16_MAX, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 9); + return; +} +TEST_CASE("fast varmixedint2 test 9") { + varmixedint2 o{INT16_MIN - 1, UINT16_MAX, UINT16_MAX, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} +TEST_CASE("fast varmixedint2 test 10") { + varmixedint2 o{INT16_MIN, UINT16_MAX + 1, UINT16_MAX, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} +TEST_CASE("fast varmixedint2 test 11") { + varmixedint2 o{INT16_MIN, UINT16_MAX, UINT16_MAX + 1, INT16_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint2 test 12") { + varmixedint2 o{INT16_MIN, UINT16_MAX, UINT16_MAX, INT16_MIN - 1}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint2 test 13") { + varmixedint2 o{INT32_MIN, UINT32_MAX, UINT32_MAX, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 17); + return; +} + +TEST_CASE("fast varmixedint2 test 14") { + varmixedint2 o{INT32_MIN, UINT32_MAX + 1ull, UINT32_MAX, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} +TEST_CASE("fast varmixedint2 test 15") { + varmixedint2 o{INT32_MIN, UINT32_MAX, UINT32_MAX, INT32_MIN - 1ll}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; +} +TEST_CASE("fast varmixedint2 test 16") { + varmixedint2 o{INT32_MIN, UINT64_MAX, UINT32_MAX, INT64_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 25); + return; } \ No newline at end of file diff --git a/src/struct_pack/tests/test_varint.cpp b/src/struct_pack/tests/test_varint.cpp index f40198bb5..18666cee0 100644 --- a/src/struct_pack/tests/test_varint.cpp +++ b/src/struct_pack/tests/test_varint.cpp @@ -3,6 +3,7 @@ #include "doctest.h" #include "test_struct.hpp" +#include "ylt/struct_pack/reflection.hpp" using namespace struct_pack; TEST_CASE("test uint32") { @@ -842,4 +843,26 @@ TEST_CASE("test varint zip size") { CHECK(detail::calculate_payload_size(vec).total == 100); CHECK(detail::calculate_payload_size(vec).size_cnt == 1); } +} + +struct var_int_struct { + int32_t a; + uint32_t b; + int64_t c; + uint64_t d; + bool operator==(const var_int_struct &o) const { + return a == o.a && b == o.b && c == o.c && d == o.d; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::ENCODING_WITH_VARINT; +}; + +TEST_CASE("test varint by normal int") { + var_int_struct v{0, 1, 2, 3}; + auto buffer = struct_pack::serialize(v); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == v); + CHECK(buffer.size() == 4); } \ No newline at end of file diff --git a/test.cpp b/test.cpp new file mode 100644 index 000000000..5b3ed0bd7 --- /dev/null +++ b/test.cpp @@ -0,0 +1,30 @@ +#include +struct test3 { + public: + test3() : x(0.f), y(0.f) {} + + public: + float x, y; +}; + +struct test { + public: + test() : x(0.f), y(0.f) {} + + test(test3 tr3) : x(tr3.x), y(tr3.y) {} + + public: + float x, y; +}; +STRUCT_PACK_REFL(test, x, y) + +struct test2 { + test xxx; +}; +STRUCT_PACK_REFL(test2, xxx) +constexpr auto i=struct_pack::members_count; +static_assert(i==1); +int main() { + test2 ttt; + auto data = struct_pack::serialize(ttt); +} From 143e49fb4e78c2ad364c58ce63e4ad5cbdeeca3a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 11:25:12 +0800 Subject: [PATCH 03/39] fix --- include/ylt/struct_pack/calculate_size.hpp | 26 +++++++++------- include/ylt/struct_pack/endian_wrapper.hpp | 4 +-- include/ylt/struct_pack/packer.hpp | 2 +- include/ylt/struct_pack/reflection.hpp | 5 ++-- include/ylt/struct_pack/unpacker.hpp | 14 ++++----- include/ylt/struct_pack/varint.hpp | 2 -- .../tests/test_compile_time_calculate.cpp | 8 ++--- test.cpp | 30 ------------------- 8 files changed, 32 insertions(+), 59 deletions(-) delete mode 100644 test.cpp diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index 1d4b84dd4..3af32b93b 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -201,7 +201,8 @@ template constexpr bool STRUCT_PACK_INLINE has_signed_varint() { if constexpr (sizeof...(Args) == 0) { if constexpr (varint_t) { - return std::is_signed_v()))>>; + return std::is_signed_v< + remove_cvref_t()))>>; } else { return false; @@ -209,7 +210,8 @@ constexpr bool STRUCT_PACK_INLINE has_signed_varint() { } else { if constexpr (varint_t) { - return std::is_signed_v()))>> || + return std::is_signed_v< + remove_cvref_t()))>> || has_signed_varint(); } else { @@ -222,7 +224,8 @@ template constexpr bool STRUCT_PACK_INLINE has_unsigned_varint() { if constexpr (sizeof...(Args) == 0) { if constexpr (varint_t) { - return std::is_unsigned_v()))>>; + return std::is_unsigned_v< + remove_cvref_t()))>>; } else { return false; @@ -230,7 +233,8 @@ constexpr bool STRUCT_PACK_INLINE has_unsigned_varint() { } else { if constexpr (varint_t) { - return std::is_unsigned_v()))>> || + return std::is_unsigned_v< + remove_cvref_t()))>> || has_unsigned_varint(); } else { @@ -254,13 +258,15 @@ constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( static_assert(!sizeof(Arg), "illegal branch"); } if constexpr (varint_t) { - if constexpr (std::is_unsigned_v< - std::remove_reference_t>) { + if constexpr (std::is_unsigned_v>) { unsigned_max = std::max(unsigned_max, get_varint_value(item)); } else { - signed_max = std::max( - signed_max, get_varint_value(item) > 0 ? get_varint_value(item) : -(get_varint_value(item) + 1)); + signed_max = + std::max(signed_max, get_varint_value(item) > 0 + ? get_varint_value(item) + : -(get_varint_value(item) + 1)); } } } @@ -269,7 +275,7 @@ constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( template constexpr int STRUCT_PACK_INLINE get_fast_varint_width_from_max(uint64_t unsigned_max, int64_t signed_max) { - int width_unsigned, width_signed; + int width_unsigned = 0, width_signed = 0; if constexpr (has_unsigned_varint()) { if SP_LIKELY (unsigned_max <= UINT8_MAX) { width_unsigned = 0; @@ -298,7 +304,7 @@ get_fast_varint_width_from_max(uint64_t unsigned_max, int64_t signed_max) { width_signed = 3; } } - if constexpr (has_signed_varint()&& + if constexpr (has_signed_varint() && has_unsigned_varint()) { return std::max(width_unsigned, width_signed); } diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 063ce5be3..605ebede7 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -235,12 +235,12 @@ template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { static_assert(sizeof(T) >= block_size); if constexpr (is_system_little_endian || block_size == sizeof(T)) { - char* SP_RESTRICT data = (char* )&elem; + char* SP_RESTRICT data = (char*)&elem; return static_cast(reader.read(data, block_size)); } else { char tmp[block_size]; - char* SP_RESTRICT data = (char* )&elem + sizeof(T) - block_size; + char* SP_RESTRICT data = (char*)&elem + sizeof(T) - block_size; bool res = static_cast(reader.read(tmp, block_size)); if SP_UNLIKELY (!res) { return res; diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index b735657b5..7e28a71b1 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -265,7 +265,7 @@ class packer { write_wrapper(writer_, (char *)&has_value); if (has_value) { if constexpr (is_base_class) { - bool is_ok; + bool is_ok{}; uint32_t id = item->get_struct_pack_id(); auto index = search_type_by_md5( item->get_struct_pack_id(), is_ok); diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index e7f328edb..f92779654 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -563,15 +563,14 @@ template #if __cpp_concepts >= 201907L template concept user_defined_config = requires { - std::is_same_v; + Type::struct_pack_config; }; #else template struct user_defined_config_impl : std::false_type {}; template - struct user_defined_config_impl>>> + struct user_defined_config_impl> : std::true_type {}; template diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 41996751c..8546358c1 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -597,7 +597,7 @@ class unpacker { if SP_UNLIKELY (code != struct_pack::errc{}) { return code; } - if constexpr (sizeof...(items)) { + if constexpr (sizeof...(items) > 0) { return deserialize_many( items...); } @@ -628,7 +628,7 @@ class unpacker { reader_.ignore(real_width) ? errc{} : errc::no_buffer_space; } else { - bool ec; + bool ec{}; if constexpr (std::is_unsigned_v>) { get_varint_value(item) = 0; @@ -689,7 +689,7 @@ class unpacker { } std::size_t width = vec[cnt] + vec[cnt + 1] * 2; int i = 0; - struct_pack::errc ec; + struct_pack::errc ec{}; switch (width) { case 0: ec = deserialize_fast_varint_helper(vec, i, @@ -785,9 +785,9 @@ class unpacker { return {}; } if constexpr (is_base_class) { - uint32_t id; + uint32_t id{}; read_wrapper(reader_, (char *)&id); - bool ok; + bool ok{}; auto index = search_type_by_md5(id, ok); if SP_UNLIKELY (!ok) { return errc::invalid_buffer; @@ -831,7 +831,7 @@ class unpacker { } else if constexpr (container) { uint64_t size64 = 0; - bool result; + bool result{}; if constexpr (size_type == 1) { if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; @@ -1137,7 +1137,7 @@ class unpacker { } if constexpr (is_base_class) { uint32_t id = item->get_struct_pack_id(); - bool ok; + bool ok{}; auto index = search_type_by_md5(id, ok); assert(ok); return template_switch; using var_int64_t = detail::sint; diff --git a/src/struct_pack/tests/test_compile_time_calculate.cpp b/src/struct_pack/tests/test_compile_time_calculate.cpp index 34c67afed..e83956053 100644 --- a/src/struct_pack/tests/test_compile_time_calculate.cpp +++ b/src/struct_pack/tests/test_compile_time_calculate.cpp @@ -532,8 +532,8 @@ TEST_CASE("test fast varint tag") { {(char)type_id::struct_t, (char)type_id::vint32_t, (char)type_id::vint64_t, (char)type_id::vuint32_t, (char)type_id::vuint64_t, (char)type_id::type_end_flag}}; - CHECK(type_info1 == type_info2); - CHECK(type_info1 == type_info3); - CHECK(type_info3 != type_info4); - CHECK(type_info1 == fast_varint_info); + static_assert(type_info1 == type_info2); + static_assert(type_info1 == type_info3); + static_assert(type_info3 != type_info4); + static_assert(type_info1 == fast_varint_info); } \ No newline at end of file diff --git a/test.cpp b/test.cpp deleted file mode 100644 index 5b3ed0bd7..000000000 --- a/test.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include -struct test3 { - public: - test3() : x(0.f), y(0.f) {} - - public: - float x, y; -}; - -struct test { - public: - test() : x(0.f), y(0.f) {} - - test(test3 tr3) : x(tr3.x), y(tr3.y) {} - - public: - float x, y; -}; -STRUCT_PACK_REFL(test, x, y) - -struct test2 { - test xxx; -}; -STRUCT_PACK_REFL(test2, xxx) -constexpr auto i=struct_pack::members_count; -static_assert(i==1); -int main() { - test2 ttt; - auto data = struct_pack::serialize(ttt); -} From 2db104afab2c2aada0e07d54ccfe7e3f02ad87a8 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 11:29:16 +0800 Subject: [PATCH 04/39] fix --- include/ylt/struct_pack/packer.hpp | 2 +- include/ylt/struct_pack/unpacker.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 7e28a71b1..af5fb1d03 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -444,7 +444,7 @@ class packer { else if constexpr (unique_ptr) { if (item != nullptr) { if constexpr (is_base_class) { - bool is_ok; + bool is_ok{}; auto index = search_type_by_md5( item->get_struct_pack_id(), is_ok); assert(is_ok); diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 8546358c1..232a9aca0 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -659,7 +659,7 @@ class unpacker { std::bitset &vec, int &i, Arg &item, Args &...items) { auto ec = deserialize_one_fast_varint(vec, i, item); - if constexpr (sizeof...(items)) { + if constexpr (sizeof...(items) > 0) { if SP_UNLIKELY (ec != errc{}) { return ec; } From 869fea3faf13c6b89ee84503ae388f1f7379a1de Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 11:38:34 +0800 Subject: [PATCH 05/39] fix --- include/ylt/struct_pack/calculate_size.hpp | 2 +- include/ylt/struct_pack/packer.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index 3af32b93b..4abdf97aa 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -306,7 +306,7 @@ get_fast_varint_width_from_max(uint64_t unsigned_max, int64_t signed_max) { } if constexpr (has_signed_varint() && has_unsigned_varint()) { - return std::max(width_unsigned, width_signed); + return (std::max)(width_unsigned, width_signed); } else if constexpr (has_signed_varint()) { return width_signed; diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index af5fb1d03..2e3d421d8 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -191,7 +191,7 @@ class packer { constexpr void STRUCT_PACK_INLINE serialize_one_fast_varint(const Arg &item) { if constexpr (varint_t) { if (get_varint_value(item)) - low_bytes_write_wrapper( + low_bytes_write_wrapper<(std::min)(sz, sizeof(Arg))>( writer_, get_varint_value(item)); } } From d277e800c1cf5a508f216abc2085a591c1ef837a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 11:44:46 +0800 Subject: [PATCH 06/39] fix --- include/ylt/struct_pack/unpacker.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 232a9aca0..66a6dd697 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -621,7 +621,7 @@ class unpacker { constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_one_fast_varint( std::bitset &vec, int &i, Arg &item) { if constexpr (varint_t) { - constexpr auto real_width = std::min(width, sizeof(Arg)); + constexpr auto real_width = (std::min)(width, sizeof(Arg)); if (!vec[i++]) return {}; else if constexpr (!no_skip) { From c55e05c6a3694577a154e8dab6e41fddcc0bb4f1 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 11:55:16 +0800 Subject: [PATCH 07/39] fix --- include/ylt/struct_pack/unpacker.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 66a6dd697..2be6dc9d2 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -138,7 +138,7 @@ class unpacker { err_code = deserialize_many<8, UINT64_MAX, true>(t, args...); } else { - static_assert(!sizeof(T), "illegal size width"); + return struct_pack::errc::too_width_size; } break; #else From 414604634e691570a3290238fed969795912d449 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 12:12:39 +0800 Subject: [PATCH 08/39] fix --- include/ylt/struct_pack/packer.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 2e3d421d8..59e9bbf3a 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -540,7 +540,12 @@ STRUCT_PACK_MAY_INLINE void serialize_to(Writer &writer, o.template serialize(args...); break; case 3: - o.template serialize(args...); + if constexpr (sizeof(std::size_t) >= 8) { + o.template serialize(args...); + } + else { + unreachable(); + } break; #else case 1: From a00ff4b6263f519f313e7a282475228748bd4676 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 14:35:32 +0800 Subject: [PATCH 09/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 605ebede7..6bf96ca6f 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -234,7 +234,8 @@ bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { static_assert(sizeof(T) >= block_size); - if constexpr (is_system_little_endian || block_size == sizeof(T)) { + if constexpr (is_system_little_endian || block_size == sizeof(T) || + block_size == 1) { char* SP_RESTRICT data = (char*)&elem; return static_cast(reader.read(data, block_size)); } @@ -254,13 +255,8 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { else if constexpr (block_size == 8) { *(uint64_t*)data = bswap64(*(uint64_t*)tmp); } - else if constexpr (block_size == 16) { - *(uint64_t*)(data + 8) = bswap64(*(uint64_t*)tmp); - *(uint64_t*)data = bswap64(*(uint64_t*)(tmp + 8)); - } else { - static_assert(!sizeof(reader), - "illegal block size(should be 1,2,4,8,16)"); + static_assert(!sizeof(reader), "illegal block size(should be 1,2,4,8)"); } return true; } From 9852956293f67812889e01fb244b8115871006ad Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 14:38:52 +0800 Subject: [PATCH 10/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 45 +++++++++++++--------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 6bf96ca6f..72766ed3f 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -174,7 +174,10 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { } else { const char* SP_RESTRICT data = sizeof(T) - block_size + (const char*)&elem; - if constexpr (block_size == 2) { + if constexpr (block_size == 1) { + writer.write((char*)&data, block_size); + } + else if constexpr (block_size == 2) { auto tmp = bswap16(*(uint16_t*)data); writer.write((char*)&tmp, block_size); } @@ -188,7 +191,7 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { } else { static_assert(!sizeof(writer), - "illegal block size(should be 1,2,4,8,16)"); + "illegal block size(should be 1,2,4,8)"); } } } @@ -234,31 +237,35 @@ bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { static_assert(sizeof(T) >= block_size); - if constexpr (is_system_little_endian || block_size == sizeof(T) || - block_size == 1) { + if constexpr (is_system_little_endian || block_size == sizeof(T)) { char* SP_RESTRICT data = (char*)&elem; return static_cast(reader.read(data, block_size)); } else { - char tmp[block_size]; char* SP_RESTRICT data = (char*)&elem + sizeof(T) - block_size; - bool res = static_cast(reader.read(tmp, block_size)); - if SP_UNLIKELY (!res) { - return res; - } - if constexpr (block_size == 2) { - *(uint16_t*)data = bswap16(*(uint16_t*)tmp); - } - else if constexpr (block_size == 4) { - *(uint32_t*)data = bswap32(*(uint32_t*)tmp); - } - else if constexpr (block_size == 8) { - *(uint64_t*)data = bswap64(*(uint64_t*)tmp); + if constexpr (block_size > 1) { + char tmp[block_size]; + bool res = static_cast(reader.read(tmp, block_size)); + if SP_UNLIKELY (!res) { + return res; + } + if constexpr (block_size == 2) { + *(uint16_t*)data = bswap16(*(uint16_t*)tmp); + } + else if constexpr (block_size == 4) { + *(uint32_t*)data = bswap32(*(uint32_t*)tmp); + } + else if constexpr (block_size == 8) { + *(uint64_t*)data = bswap64(*(uint64_t*)tmp); + } + else { + static_assert(!sizeof(reader), "illegal block size(should be 1,2,4,8)"); + } + return true; } else { - static_assert(!sizeof(reader), "illegal block size(should be 1,2,4,8)"); + reader.read(data, block_size); } - return true; } } }; // namespace struct_pack::detail \ No newline at end of file From 392f0b49cbb0c8cc2dcb7a443e45fdcd9154cd26 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 14:48:52 +0800 Subject: [PATCH 11/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 72766ed3f..8f6bc9f06 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -190,8 +190,7 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { writer.write((char*)&tmp, block_size); } else { - static_assert(!sizeof(writer), - "illegal block size(should be 1,2,4,8)"); + static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8)"); } } } @@ -264,7 +263,7 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { return true; } else { - reader.read(data, block_size); + return reader.read(data, block_size); } } } From 74e28592ea075fa23c46e19833ccf7fa6dcc5c3f Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 14:59:26 +0800 Subject: [PATCH 12/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 8f6bc9f06..8a83e3bf1 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -263,7 +263,8 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { return true; } else { - return reader.read(data, block_size); + bool result = reader.read(data, block_size); + return result; } } } From a96743c9ac3226b22d42fd79cedb4cb69a51165c Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 15:50:39 +0800 Subject: [PATCH 13/39] fix --- include/ylt/struct_pack/unpacker.hpp | 115 ++++++++++++++++++--------- 1 file changed, 79 insertions(+), 36 deletions(-) diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 2be6dc9d2..b5b79db6b 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -111,11 +111,6 @@ class unpacker { data_len_ = reader_.tellg(); } auto &&[err_code, buffer_len] = deserialize_metainfo(); - if constexpr (sizeof(std::size_t) < 8) { - if (size_type_ > sizeof(std::size_t)) { - return errc::too_width_size; - } - } if SP_UNLIKELY (err_code != struct_pack::errc{}) { return err_code; } @@ -142,9 +137,13 @@ class unpacker { } break; #else - case 1: - case 2: + case 3: + if constexpr (sizeof(std::size_t) < 8) { + return struct_pack::errc::too_width_size; + } + case 2: + case 1: err_code = deserialize_many<2, UINT64_MAX, true>(t, args...); break; #endif @@ -191,12 +190,20 @@ class unpacker { err_code = deserialize_many<4, UINT64_MAX, true>(t, args...); break; case 3: - err_code = deserialize_many<8, UINT64_MAX, true>(t, args...); + if constexpr (sizeof(std::size_t) >= 8) { + err_code = deserialize_many<8, UINT64_MAX, true>(t, args...); + } + else { + return struct_pack::errc::too_width_size; + } break; #else - case 1: - case 2: case 3: + if constexpr (sizeof(std::size_t) < 8) { + return struct_pack::errc::too_width_size; + } + case 2: + case 1: err_code = deserialize_many<2, UINT64_MAX, true>(t, args...); break; #endif @@ -245,12 +252,20 @@ class unpacker { err_code = get_field_impl<4, UINT64_MAX, U, I>(field); break; case 3: - err_code = get_field_impl<8, UINT64_MAX, U, I>(field); + if constexpr (sizeof(std::size_t) >= 8) { + err_code = get_field_impl<8, UINT64_MAX, U, I>(field); + } + else { + return struct_pack::errc::too_width_size; + } break; #else - case 1: - case 2: case 3: + if constexpr (sizeof(std::size_t) < 8) { + return struct_pack::errc::too_width_size; + } + case 2: + case 1: err_code = get_field_impl<2, UINT64_MAX, U, I>(field); break; #endif @@ -304,18 +319,27 @@ class unpacker { ...); break; case 3: - ([&] { - err_code = - deserialize_many<8, compatible_version_number[I], true>( - t, args...); - return err_code == errc::ok; - }() && - ...); + if constexpr (sizeof(std::size_t) >= 8) { + ([&] { + err_code = + deserialize_many<8, compatible_version_number[I], true>( + t, args...); + return err_code == errc::ok; + }() && + ...); + } + else { + return struct_pack::errc::too_width_size; + } + break; #else - case 1: - case 2: case 3: + if constexpr (sizeof(std::size_t) < 8) { + return struct_pack::errc::too_width_size; + } + case 2: + case 1: ([&] { err_code = deserialize_many<2, compatible_version_number[I], true>( @@ -373,18 +397,26 @@ class unpacker { ...); break; case 3: - ([&] { - err_code = - get_field_impl<8, compatible_version_number[Is], U, I>( - field); - return err_code == errc::ok; - }() && - ...); + if constexpr (sizeof(std::size_t) >= 8) { + ([&] { + err_code = + get_field_impl<8, compatible_version_number[Is], U, I>( + field); + return err_code == errc::ok; + }() && + ...); + } + else { + return errc::too_width_size; + } break; #else - case 1: - case 2: case 3: + if constexpr (sizeof(std::size_t) < 8) { + return errc::too_width_size; + } + case 2: + case 1: ([&] { err_code = get_field_impl<2, compatible_version_number[Is], U, I>( @@ -399,7 +431,7 @@ class unpacker { } if (size_type_ == UCHAR_MAX) { // reuse size_type_ as a tag that the buffer miss some - // compatible field, whic is legal. + // compatible field, which is legal. err_code = {}; } return err_code; @@ -849,8 +881,14 @@ class unpacker { } } else if constexpr (size_type == 8) { - if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { - return struct_pack::errc::no_buffer_space; + if constexpr (sizeof(std::size_t) < 8) { + if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, + size64)) { + return struct_pack::errc::no_buffer_space; + } + } + else { + static_assert("illegal branch"); } } else { @@ -870,8 +908,13 @@ class unpacker { } break; case 3: - if SP_UNLIKELY (!low_bytes_read_wrapper<8>(reader_, size64)) { - return struct_pack::errc::no_buffer_space; + if constexpr (sizeof(std::size_t) < 8) { + if SP_UNLIKELY (!low_bytes_read_wrapper<8>(reader_, size64)) { + return struct_pack::errc::no_buffer_space; + } + } + else { + unreachable(); } break; default: From 679350f1e739af18b6c34aa5964c085d0eb89e03 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 17:02:21 +0800 Subject: [PATCH 14/39] add more test --- include/ylt/struct_pack/unpacker.hpp | 4 +- src/struct_pack/tests/CMakeLists.txt | 1 + src/struct_pack/tests/test_optimize.cpp | 58 +++++++++++++++++++++++ src/struct_pack/tests/test_serialize.cpp | 60 +++++++++++++++++++++++- 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 src/struct_pack/tests/test_optimize.cpp diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index b5b79db6b..1b8d3cb3b 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -881,7 +881,7 @@ class unpacker { } } else if constexpr (size_type == 8) { - if constexpr (sizeof(std::size_t) < 8) { + if constexpr (sizeof(std::size_t) >= 8) { if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { return struct_pack::errc::no_buffer_space; @@ -908,7 +908,7 @@ class unpacker { } break; case 3: - if constexpr (sizeof(std::size_t) < 8) { + if constexpr (sizeof(std::size_t) >= 8) { if SP_UNLIKELY (!low_bytes_read_wrapper<8>(reader_, size64)) { return struct_pack::errc::no_buffer_space; } diff --git a/src/struct_pack/tests/CMakeLists.txt b/src/struct_pack/tests/CMakeLists.txt index bf06ac3e9..cf513a4f8 100644 --- a/src/struct_pack/tests/CMakeLists.txt +++ b/src/struct_pack/tests/CMakeLists.txt @@ -16,6 +16,7 @@ add_executable(struct_pack_test test_derived.cpp test_cross_platform.cpp test_disable_meta_info.cpp + test_optimize.cpp main.cpp ) add_test(NAME struct_pack_test COMMAND struct_pack_test) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp new file mode 100644 index 000000000..131c17e34 --- /dev/null +++ b/src/struct_pack/tests/test_optimize.cpp @@ -0,0 +1,58 @@ +#define STRUCT_PACK_OPTIMIZE +#include "doctest.h" +#include "ylt/struct_pack.hpp" +TEST_CASE("test width too big") { + SUBCASE("1") { + std::string buffer; + buffer.push_back(0b11000); + auto result = struct_pack::deserialize(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("2") { + std::string buffer; + buffer.push_back(0b11000); + std::size_t len = 0; + auto result = struct_pack::deserialize(buffer, len); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("3") { + std::string buffer; + buffer.push_back(0b11000); + auto result = + struct_pack::get_field, 0>(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("4") { + std::string buffer; + buffer.push_back(0b11); + auto result = struct_pack::deserialize< + std::pair>>(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } +} \ No newline at end of file diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index dabdfc420..1b1d6ac94 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include "ylt/struct_pack/compatible.hpp" #define private public #include @@ -1278,4 +1280,60 @@ TEST_CASE("test dynamic span") { } } } -#endif \ No newline at end of file +#endif + +TEST_CASE("test width too big") { + SUBCASE("1") { + std::string buffer; + buffer.push_back(0b11000); + auto result = struct_pack::deserialize(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("2") { + std::string buffer; + buffer.push_back(0b11000); + std::size_t len = 0; + auto result = struct_pack::deserialize(buffer, len); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("3") { + std::string buffer; + buffer.push_back(0b11000); + auto result = + struct_pack::get_field, 0>(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } + SUBCASE("4") { + std::string buffer; + buffer.push_back(0b11); + auto result = + struct_pack::deserialize>>(buffer); + REQUIRE(!result.has_value()); + if constexpr (SIZE_WIDTH < 8) { + CHECK(result.error() == struct_pack::errc::too_width_size); + } + else { + CHECK(result.error() == struct_pack::errc::no_buffer_space); + } + } +} \ No newline at end of file From e080c155bfe894c159b39a33b5b604d1757bff34 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 17:24:32 +0800 Subject: [PATCH 15/39] fix --- src/struct_pack/tests/test_optimize.cpp | 8 ++++---- src/struct_pack/tests/test_serialize.cpp | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp index 131c17e34..679705d77 100644 --- a/src/struct_pack/tests/test_optimize.cpp +++ b/src/struct_pack/tests/test_optimize.cpp @@ -7,7 +7,7 @@ TEST_CASE("test width too big") { buffer.push_back(0b11000); auto result = struct_pack::deserialize(buffer); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -21,7 +21,7 @@ TEST_CASE("test width too big") { std::size_t len = 0; auto result = struct_pack::deserialize(buffer, len); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -34,7 +34,7 @@ TEST_CASE("test width too big") { buffer.push_back(0b11000); auto result = struct_pack::get_field, 0>(buffer); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -47,7 +47,7 @@ TEST_CASE("test width too big") { buffer.push_back(0b11); auto result = struct_pack::deserialize< std::pair>>(buffer); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 1b1d6ac94..5a5457bdb 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -25,6 +25,7 @@ #include #include #include + #include "ylt/struct_pack/compatible.hpp" #define private public @@ -1288,7 +1289,7 @@ TEST_CASE("test width too big") { buffer.push_back(0b11000); auto result = struct_pack::deserialize(buffer); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -1302,7 +1303,7 @@ TEST_CASE("test width too big") { std::size_t len = 0; auto result = struct_pack::deserialize(buffer, len); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -1315,7 +1316,7 @@ TEST_CASE("test width too big") { buffer.push_back(0b11000); auto result = struct_pack::get_field, 0>(buffer); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } @@ -1326,9 +1327,9 @@ TEST_CASE("test width too big") { SUBCASE("4") { std::string buffer; buffer.push_back(0b11); - auto result = - struct_pack::deserialize>>(buffer); - REQUIRE(!result.has_value()); + auto result = struct_pack::deserialize< + std::pair>>(buffer); + REQUIRE(result.has_value() == false); if constexpr (SIZE_WIDTH < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } From 35df72c62820d64865f61b93853737327ef2e3e7 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 17:32:48 +0800 Subject: [PATCH 16/39] fix --- src/struct_pack/tests/test_serialize.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 5a5457bdb..40ae42f46 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1290,7 +1290,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -1304,7 +1304,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize(buffer, len); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -1317,7 +1317,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::get_field, 0>(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -1330,7 +1330,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize< std::pair>>(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { From 1c6be6b7e69e0a39d37dae803fa25c50ef1aa7ce Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 17:36:17 +0800 Subject: [PATCH 17/39] fix --- src/struct_pack/tests/test_optimize.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp index 679705d77..c628430a8 100644 --- a/src/struct_pack/tests/test_optimize.cpp +++ b/src/struct_pack/tests/test_optimize.cpp @@ -8,7 +8,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -22,7 +22,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize(buffer, len); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -35,7 +35,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::get_field, 0>(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { @@ -48,7 +48,7 @@ TEST_CASE("test width too big") { auto result = struct_pack::deserialize< std::pair>>(buffer); REQUIRE(result.has_value() == false); - if constexpr (SIZE_WIDTH < 8) { + if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); } else { From c136807cfaa455a16f89d14797a3f985a50cf217 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 18:08:35 +0800 Subject: [PATCH 18/39] fix --- src/struct_pack/tests/test_optimize.cpp | 6 ++++-- src/struct_pack/tests/test_serialize.cpp | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp index c628430a8..4563aae15 100644 --- a/src/struct_pack/tests/test_optimize.cpp +++ b/src/struct_pack/tests/test_optimize.cpp @@ -32,8 +32,9 @@ TEST_CASE("test width too big") { SUBCASE("3") { std::string buffer; buffer.push_back(0b11000); - auto result = - struct_pack::get_field, 0>(buffer); + auto result = struct_pack::get_field< + std::pair, 0, struct_pack::DISABLE_ALL_META_INFO>( + buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); @@ -46,6 +47,7 @@ TEST_CASE("test width too big") { std::string buffer; buffer.push_back(0b11); auto result = struct_pack::deserialize< + struct_pack::DISABLE_ALL_META_INFO, std::pair>>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 40ae42f46..b9edcf263 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1315,7 +1315,8 @@ TEST_CASE("test width too big") { std::string buffer; buffer.push_back(0b11000); auto result = - struct_pack::get_field, 0>(buffer); + struct_pack::get_field, 0, + struct_pack::DISABLE_ALL_META_INFO>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); @@ -1328,6 +1329,7 @@ TEST_CASE("test width too big") { std::string buffer; buffer.push_back(0b11); auto result = struct_pack::deserialize< + struct_pack::DISABLE_ALL_META_INFO, std::pair>>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { From 219ed9f0682556614923799f2f9852723fa6881a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Tue, 21 Nov 2023 18:11:58 +0800 Subject: [PATCH 19/39] fix --- src/struct_pack/tests/test_serialize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index b9edcf263..a68cf7987 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1284,7 +1284,7 @@ TEST_CASE("test dynamic span") { #endif TEST_CASE("test width too big") { - SUBCASE("1") { + SUBCASE("12") { std::string buffer; buffer.push_back(0b11000); auto result = struct_pack::deserialize Date: Tue, 21 Nov 2023 18:12:21 +0800 Subject: [PATCH 20/39] fix --- src/struct_pack/tests/test_optimize.cpp | 6 +++--- src/struct_pack/tests/test_serialize.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp index 4563aae15..79687aeda 100644 --- a/src/struct_pack/tests/test_optimize.cpp +++ b/src/struct_pack/tests/test_optimize.cpp @@ -32,9 +32,9 @@ TEST_CASE("test width too big") { SUBCASE("3") { std::string buffer; buffer.push_back(0b11000); - auto result = struct_pack::get_field< - std::pair, 0, struct_pack::DISABLE_ALL_META_INFO>( - buffer); + auto result = + struct_pack::get_field, 0, + struct_pack::DISABLE_ALL_META_INFO>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { CHECK(result.error() == struct_pack::errc::too_width_size); diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index a68cf7987..b9edcf263 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1284,7 +1284,7 @@ TEST_CASE("test dynamic span") { #endif TEST_CASE("test width too big") { - SUBCASE("12") { + SUBCASE("1") { std::string buffer; buffer.push_back(0b11000); auto result = struct_pack::deserialize Date: Wed, 22 Nov 2023 11:25:36 +0800 Subject: [PATCH 21/39] fix test case --- src/struct_pack/tests/test_optimize.cpp | 6 +++++- src/struct_pack/tests/test_serialize.cpp | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp index 79687aeda..b786f5922 100644 --- a/src/struct_pack/tests/test_optimize.cpp +++ b/src/struct_pack/tests/test_optimize.cpp @@ -1,3 +1,4 @@ +#include "ylt/struct_pack/derived_helper.hpp" #define STRUCT_PACK_OPTIMIZE #include "doctest.h" #include "ylt/struct_pack.hpp" @@ -45,9 +46,12 @@ TEST_CASE("test width too big") { } SUBCASE("4") { std::string buffer; + using T = std::pair>; + auto code = struct_pack::get_type_code() + 1; + buffer.resize(4); + memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); auto result = struct_pack::deserialize< - struct_pack::DISABLE_ALL_META_INFO, std::pair>>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index b9edcf263..bbe844f3a 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1327,9 +1327,12 @@ TEST_CASE("test width too big") { } SUBCASE("4") { std::string buffer; + using T = std::pair>; + auto code = struct_pack::get_type_code() + 1; + buffer.resize(4); + memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); auto result = struct_pack::deserialize< - struct_pack::DISABLE_ALL_META_INFO, std::pair>>(buffer); REQUIRE(result.has_value() == false); if constexpr (sizeof(std::size_t) < 8) { From 88285e25065ffa5b13d663f756ecc312a108c325 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 22 Nov 2023 12:54:16 +0800 Subject: [PATCH 22/39] fix --- .github/workflows/s390x.yml | 3 +- include/ylt/struct_pack.hpp | 6 +-- src/struct_pack/tests/CMakeLists.txt | 30 ++++-------- src/struct_pack/tests/test_optimize.cpp | 64 ------------------------- 4 files changed, 13 insertions(+), 90 deletions(-) delete mode 100644 src/struct_pack/tests/test_optimize.cpp diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index a5899873a..3edac918d 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -33,4 +33,5 @@ jobs: -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF cmake --build ${{github.workspace}}/build -j cd ${{github.workspace}}/build/output/tests - ./struct_pack_test \ No newline at end of file + ./struct_pack_test + ./struct_pack_test_with_optimize \ No newline at end of file diff --git a/include/ylt/struct_pack.hpp b/include/ylt/struct_pack.hpp index 3d5b2c50a..971874801 100644 --- a/include/ylt/struct_pack.hpp +++ b/include/ylt/struct_pack.hpp @@ -616,7 +616,7 @@ template ())>; expected ret; - auto ec = get_field_to(ret.value(), v); + auto ec = get_field_to(ret.value(), v); if SP_UNLIKELY (ec != struct_pack::errc{}) { ret = unexpected{ec}; } @@ -627,7 +627,7 @@ template [[nodiscard]] STRUCT_PACK_INLINE auto get_field(const char *data, size_t size) { using T_Field = std::tuple_element_t())>; expected ret; - auto ec = get_field_to(ret.value(), data, size); + auto ec = get_field_to(ret.value(), data, size); if SP_UNLIKELY (ec != struct_pack::errc{}) { ret = unexpected{ec}; } @@ -644,7 +644,7 @@ template ())>; expected ret; - auto ec = get_field_to(ret.value(), reader); + auto ec = get_field_to(ret.value(), reader); if SP_UNLIKELY (ec != struct_pack::errc{}) { ret = unexpected{ec}; } diff --git a/src/struct_pack/tests/CMakeLists.txt b/src/struct_pack/tests/CMakeLists.txt index cf513a4f8..6bded7ebf 100644 --- a/src/struct_pack/tests/CMakeLists.txt +++ b/src/struct_pack/tests/CMakeLists.txt @@ -1,31 +1,17 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/output/tests) -add_executable(struct_pack_test - test_serialize.cpp - test_compile_time_calculate.cpp - test_data_struct.cpp - test_data_struct2.cpp - test_tuplet.cpp - test_alignas.cpp - test_pragma_pack.cpp - test_pragma_pack_and_alignas_mix.cpp - test_varint.cpp - test_fast_varint.cpp - test_stream.cpp - test_compatible.cpp - test_non_aggregated_type.cpp - test_derived.cpp - test_cross_platform.cpp - test_disable_meta_info.cpp - test_optimize.cpp - main.cpp - ) +file(GLOB SRCS_PATHS ./*.cpp) +add_executable(struct_pack_test ${SRCS_PATHS}) +add_executable(struct_pack_test_with_optimize ${SRCS_PATHS}) add_test(NAME struct_pack_test COMMAND struct_pack_test) +add_test(NAME struct_pack_test_with_optimize COMMAND struct_pack_test_with_optimize) include (TestBigEndian) TEST_BIG_ENDIAN(IS_BIG_ENDIAN) if(NOT IS_BIG_ENDIAN) target_compile_definitions(struct_pack_test PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE TEST_IN_LITTLE_ENDIAN) +target_compile_definitions(struct_pack_test_with_optimize PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE STRUCT_PACK_OPTIMIZE TEST_IN_LITTLE_ENDIAN ) else() -target_compile_definitions(struct_pack_test PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE) +target_compile_definitions(struct_pack_test PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE ) +target_compile_definitions(struct_pack_test_with_optimize PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE STRUCT_PACK_OPTIMIZE) endif() add_custom_command( TARGET struct_pack_test PRE_BUILD @@ -36,4 +22,4 @@ add_custom_command( TARGET struct_pack_test PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/binary_data - ${CMAKE_BINARY_DIR}/src/struct_pack/tests/binary_data) + ${CMAKE_BINARY_DIR}/src/struct_pack/tests/binary_data) \ No newline at end of file diff --git a/src/struct_pack/tests/test_optimize.cpp b/src/struct_pack/tests/test_optimize.cpp deleted file mode 100644 index b786f5922..000000000 --- a/src/struct_pack/tests/test_optimize.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "ylt/struct_pack/derived_helper.hpp" -#define STRUCT_PACK_OPTIMIZE -#include "doctest.h" -#include "ylt/struct_pack.hpp" -TEST_CASE("test width too big") { - SUBCASE("1") { - std::string buffer; - buffer.push_back(0b11000); - auto result = struct_pack::deserialize(buffer); - REQUIRE(result.has_value() == false); - if constexpr (sizeof(std::size_t) < 8) { - CHECK(result.error() == struct_pack::errc::too_width_size); - } - else { - CHECK(result.error() == struct_pack::errc::no_buffer_space); - } - } - SUBCASE("2") { - std::string buffer; - buffer.push_back(0b11000); - std::size_t len = 0; - auto result = struct_pack::deserialize(buffer, len); - REQUIRE(result.has_value() == false); - if constexpr (sizeof(std::size_t) < 8) { - CHECK(result.error() == struct_pack::errc::too_width_size); - } - else { - CHECK(result.error() == struct_pack::errc::no_buffer_space); - } - } - SUBCASE("3") { - std::string buffer; - buffer.push_back(0b11000); - auto result = - struct_pack::get_field, 0, - struct_pack::DISABLE_ALL_META_INFO>(buffer); - REQUIRE(result.has_value() == false); - if constexpr (sizeof(std::size_t) < 8) { - CHECK(result.error() == struct_pack::errc::too_width_size); - } - else { - CHECK(result.error() == struct_pack::errc::no_buffer_space); - } - } - SUBCASE("4") { - std::string buffer; - using T = std::pair>; - auto code = struct_pack::get_type_code() + 1; - buffer.resize(4); - memcpy(buffer.data(), &code, sizeof(code)); - buffer.push_back(0b11); - auto result = struct_pack::deserialize< - std::pair>>(buffer); - REQUIRE(result.has_value() == false); - if constexpr (sizeof(std::size_t) < 8) { - CHECK(result.error() == struct_pack::errc::too_width_size); - } - else { - CHECK(result.error() == struct_pack::errc::no_buffer_space); - } - } -} \ No newline at end of file From 082bcdbb7d265b6a726a806f3c94e2a8fe9fc8c9 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 22 Nov 2023 15:06:57 +0800 Subject: [PATCH 23/39] fix msvc --- include/ylt/struct_pack/calculate_size.hpp | 15 ++++++++++----- include/ylt/struct_pack/packer.hpp | 15 ++++++++++----- include/ylt/struct_pack/unpacker.hpp | 19 +++++++++++++------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index 4abdf97aa..94ab6a717 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -163,12 +163,17 @@ constexpr size_info inline calculate_one_size(const T &item) { else { constexpr uint64_t tag = get_parent_tag(); if constexpr (is_enable_fast_varint_coding(tag)) { - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - ret.total += calculate_fast_varint_size(items...); - }); + ret.total += + visit_members(item, [](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 + return calculate_fast_varint_size(items...); + }); } - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - ret += calculate_payload_size(items...); + ret += visit_members(item, [](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 + return calculate_payload_size(items...); }); } } diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 59e9bbf3a..0881e1f37 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -408,7 +408,7 @@ class packer { else if constexpr ((is_trivial_serializable::value && !is_little_endian_copyable) || is_trivial_serializable::value) { - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + visit_members(item, [this](auto &&...items) CONSTEXPR_INLINE_LAMBDA { int i = 1; ((serialize_one(items), write_padding(align::padding_size[i++])), @@ -418,11 +418,16 @@ class packer { else { constexpr uint64_t tag = get_parent_tag(); if constexpr (is_enable_fast_varint_coding(tag)) { - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - serialize_fast_varint(items...); - }); + visit_members( + item, [this](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 + serialize_fast_varint(items...); + }); } - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + visit_members(item, [this](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 serialize_many(items...); }); } diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 1b8d3cb3b..3b9b48741 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -1133,16 +1133,23 @@ class unpacker { else { constexpr uint64_t tag = get_parent_tag(); if constexpr (is_enable_fast_varint_coding(tag)) { - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - code = deserialize_fast_varint(items...); - }); + code = visit_members( + item, [this](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 + return deserialize_fast_varint(items...); + }); if SP_UNLIKELY (code != errc::ok) { return code; } } - visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { - code = deserialize_many(items...); - }); + code = visit_members( + item, [this](auto &&...items) CONSTEXPR_INLINE_LAMBDA { + constexpr uint64_t tag = + get_parent_tag(); // to pass msvc with c++17 + return deserialize_many( + items...); + }); } } else { From 697621441e5bd3b2f3f5ede6ca45058cad2d0ba3 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 22 Nov 2023 17:46:23 +0800 Subject: [PATCH 24/39] fix conversation --- include/ylt/struct_pack/calculate_size.hpp | 47 ++++++++++++++-------- include/ylt/struct_pack/packer.hpp | 7 +++- include/ylt/struct_pack/unpacker.hpp | 11 +++-- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/include/ylt/struct_pack/calculate_size.hpp b/include/ylt/struct_pack/calculate_size.hpp index 94ab6a717..f9133da36 100644 --- a/include/ylt/struct_pack/calculate_size.hpp +++ b/include/ylt/struct_pack/calculate_size.hpp @@ -15,10 +15,6 @@ */ #pragma once -#include -#include -#include - #include "alignment.hpp" #include "marco.h" #include "reflection.hpp" @@ -27,7 +23,6 @@ #include "type_trait.hpp" #include "util.h" #include "varint.hpp" -#include "ylt/struct_pack/type_calculate.hpp" namespace struct_pack { @@ -248,21 +243,41 @@ constexpr bool STRUCT_PACK_INLINE has_unsigned_varint() { } } -template -constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( - const Arg &item, int &non_zero_cnt32, int &non_zero_cnt64, - uint64_t &unsigned_max, int64_t &signed_max) { - if (get_varint_value(item)) { - if constexpr (sizeof(Arg) == 4) { - ++non_zero_cnt32; - } - else if constexpr (sizeof(Arg) == 8) { - ++non_zero_cnt64; +template +constexpr bool STRUCT_PACK_INLINE has_64bits_varint() { + if constexpr (sizeof...(Args) == 0) { + if constexpr (varint_t) { + return sizeof(Arg) == 8; } else { - static_assert(!sizeof(Arg), "illegal branch"); + return false; } + } + else { if constexpr (varint_t) { + return sizeof(Arg) == 8 || has_64bits_varint(); + } + else { + return has_64bits_varint(); + } + } +} + +template +constexpr void STRUCT_PACK_INLINE get_fast_varint_width_impl( + const Arg &item, int &non_zero_cnt32, int &non_zero_cnt64, + uint64_t &unsigned_max, int64_t &signed_max) { + if constexpr (varint_t) { + if (get_varint_value(item)) { + if constexpr (sizeof(Arg) == 4) { + ++non_zero_cnt32; + } + else if constexpr (sizeof(Arg) == 8) { + ++non_zero_cnt64; + } + else { + static_assert(!sizeof(Arg), "illegal branch"); + } if constexpr (std::is_unsigned_v>) { unsigned_max = std::max(unsigned_max, get_varint_value(item)); diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 0881e1f37..250f814bb 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -221,7 +221,12 @@ class packer { (serialize_one_fast_varint(items), ...); break; case 3: - (serialize_one_fast_varint(items), ...); + if constexpr (has_64bits_varint()) { + (serialize_one_fast_varint(items), ...); + } + else { + unreachable(); + } break; default: unreachable(); diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 3b9b48741..90e39404c 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -656,7 +656,7 @@ class unpacker { constexpr auto real_width = (std::min)(width, sizeof(Arg)); if (!vec[i++]) return {}; - else if constexpr (!no_skip) { + if constexpr (!no_skip) { reader_.ignore(real_width) ? errc{} : errc::no_buffer_space; } else { @@ -736,8 +736,13 @@ class unpacker { items...); break; case 3: - ec = deserialize_fast_varint_helper(vec, i, - items...); + if constexpr (has_64bits_varint()) { + ec = deserialize_fast_varint_helper( + vec, i, items...); + } + else { + return struct_pack::errc::invalid_buffer; + } break; default: unreachable(); From 6205c57d5d67b9615a89ae6843c8c1bb0805283d Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 22 Nov 2023 19:48:52 +0800 Subject: [PATCH 25/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 8a83e3bf1..849844293 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -185,12 +185,8 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { auto tmp = bswap32(*(uint32_t*)data); writer.write((char*)&tmp, block_size); } - else if constexpr (block_size == 8) { - auto tmp = bswap64(*(uint64_t*)data); - writer.write((char*)&tmp, block_size); - } else { - static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8)"); + static_assert(!sizeof(writer), "illegal block size(should be 1,2,4)"); } } } @@ -254,11 +250,8 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { else if constexpr (block_size == 4) { *(uint32_t*)data = bswap32(*(uint32_t*)tmp); } - else if constexpr (block_size == 8) { - *(uint64_t*)data = bswap64(*(uint64_t*)tmp); - } else { - static_assert(!sizeof(reader), "illegal block size(should be 1,2,4,8)"); + static_assert(!sizeof(reader), "illegal block size(should be 1,2,4)"); } return true; } From 7561770c68503ed14d8029f823d4e56c6e20e6e9 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 22 Nov 2023 20:59:31 +0800 Subject: [PATCH 26/39] 1 --- include/ylt/struct_pack/endian_wrapper.hpp | 5 +---- include/ylt/struct_pack/unpacker.hpp | 15 ++++++--------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 849844293..e55cdcb35 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -224,10 +224,7 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { template bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, std::size_t length) { - if SP_UNLIKELY (length >= PTRDIFF_MAX) - unreachable(); - else - return static_cast(reader.read(data, length)); + return static_cast(reader.read(data, length)); } template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 90e39404c..64bd9bea2 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -993,15 +993,12 @@ class unpacker { } else if constexpr (is_little_endian_copyable) { - if SP_UNLIKELY (mem_sz >= PTRDIFF_MAX) - unreachable(); - else { - item.resize(size64); - if SP_UNLIKELY (!read_bytes_array( - reader_, (char *)item.data(), - size64 * sizeof(value_type))) { - return struct_pack::errc::no_buffer_space; - } + + item.resize(size64); + if SP_UNLIKELY (!read_bytes_array( + reader_, (char *)item.data(), + size64 * sizeof(value_type))) { + return struct_pack::errc::no_buffer_space; } } else { From 73012631ce0a74a9e12ab2535277f9efe7935a44 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 12:01:08 +0800 Subject: [PATCH 27/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 10 +++++----- include/ylt/struct_pack/unpacker.hpp | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index e55cdcb35..77f80fde7 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -169,13 +169,13 @@ template void low_bytes_write_wrapper(writer_t& writer, const T& elem) { static_assert(sizeof(T) >= block_size); if constexpr (is_system_little_endian || block_size == sizeof(T)) { - const char* SP_RESTRICT data = (const char*)&elem; + const char* data = (const char*)&elem; writer.write(data, block_size); } else { - const char* SP_RESTRICT data = sizeof(T) - block_size + (const char*)&elem; + const char* data = sizeof(T) - block_size + (const char*)&elem; if constexpr (block_size == 1) { - writer.write((char*)&data, block_size); + writer.write(data, block_size); } else if constexpr (block_size == 2) { auto tmp = bswap16(*(uint16_t*)data); @@ -230,11 +230,11 @@ template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { static_assert(sizeof(T) >= block_size); if constexpr (is_system_little_endian || block_size == sizeof(T)) { - char* SP_RESTRICT data = (char*)&elem; + char* data = (char*)&elem; return static_cast(reader.read(data, block_size)); } else { - char* SP_RESTRICT data = (char*)&elem + sizeof(T) - block_size; + char* data = (char*)&elem + sizeof(T) - block_size; if constexpr (block_size > 1) { char tmp[block_size]; bool res = static_cast(reader.read(tmp, block_size)); diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 64bd9bea2..340514416 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -509,7 +509,7 @@ class unpacker { } } else { - return {errc::too_width_size, data_len}; + return {errc::too_width_size, 0}; } break; default: @@ -993,7 +993,6 @@ class unpacker { } else if constexpr (is_little_endian_copyable) { - item.resize(size64); if SP_UNLIKELY (!read_bytes_array( reader_, (char *)item.data(), From a9bf8c761251845fc91fd6636fd7051205457c1c Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 14:24:19 +0800 Subject: [PATCH 28/39] fix --- src/struct_pack/tests/test_fast_varint.cpp | 5 ++++- src/struct_pack/tests/test_serialize.cpp | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index 0f428fe7a..aecd5f10a 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -155,7 +155,10 @@ TEST_CASE("fast varint test 1") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value()); - CHECK(result == o); + CHECK(result->a.get() == o.a.get()); + CHECK(result->b.get() == o.b.get()); + CHECK(result->c.get() == o.c.get()); + CHECK(result->d.get() == o.d.get()); CHECK(buffer.size() == 4); return; } diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index bbe844f3a..970362787 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1330,6 +1330,9 @@ TEST_CASE("test width too big") { using T = std::pair>; auto code = struct_pack::get_type_code() + 1; buffer.resize(4); +#ifndef TEST_IN_LITTLE_ENDIAN + code = bswap64(code); +#endif memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); auto result = struct_pack::deserialize< From 3d7e16a921589a06ac429ac190e2287e1c3cb835 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 14:27:15 +0800 Subject: [PATCH 29/39] fix --- src/struct_pack/tests/test_serialize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 970362787..1e1dde1a2 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1331,7 +1331,7 @@ TEST_CASE("test width too big") { auto code = struct_pack::get_type_code() + 1; buffer.resize(4); #ifndef TEST_IN_LITTLE_ENDIAN - code = bswap64(code); + code = struct_pack::detail::bswap64(code); #endif memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); From badc415decb8432c9b6f9875d34301c4695baa44 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 14:31:57 +0800 Subject: [PATCH 30/39] fix --- src/struct_pack/tests/test_serialize.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 1e1dde1a2..29c91b4c3 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -27,6 +27,7 @@ #include #include "ylt/struct_pack/compatible.hpp" +#include "ylt/struct_pack/endian_wrapper.hpp" #define private public #include @@ -1330,9 +1331,9 @@ TEST_CASE("test width too big") { using T = std::pair>; auto code = struct_pack::get_type_code() + 1; buffer.resize(4); -#ifndef TEST_IN_LITTLE_ENDIAN - code = struct_pack::detail::bswap64(code); -#endif + if constexpr (!struct_pack::detail::is_system_little_endian) { + code = struct_pack::detail::bswap64(code); + } memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); auto result = struct_pack::deserialize< From 334d524cc0f31db302a536533ba17804d231c945 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 15:44:30 +0800 Subject: [PATCH 31/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 3 +-- src/struct_pack/tests/test_fast_varint.cpp | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 77f80fde7..30dcf67bb 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -253,8 +253,7 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { return true; } else { - bool result = reader.read(data, block_size); - return result; + return static_cast(reader.read(data, block_size)); } } } diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index aecd5f10a..461530358 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -155,11 +155,14 @@ TEST_CASE("fast varint test 1") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value()); + CHECK(buffer.size() == 4); + CHECK(buffer[1] != '\0'); + CHECK(buffer[2] != '\0'); + CHECK(buffer[3] != '\0'); CHECK(result->a.get() == o.a.get()); CHECK(result->b.get() == o.b.get()); CHECK(result->c.get() == o.c.get()); CHECK(result->d.get() == o.d.get()); - CHECK(buffer.size() == 4); return; } From ea765353cad5b33c4ccb79ebbce9d49ad0756c14 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 16:11:26 +0800 Subject: [PATCH 32/39] test --- src/struct_pack/tests/test_fast_varint.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index 461530358..c880a01e9 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -156,9 +156,11 @@ TEST_CASE("fast varint test 1") { fast_varint_example_1>(buffer); REQUIRE(result.has_value()); CHECK(buffer.size() == 4); + CHECK(buffer[0] != '\0'); CHECK(buffer[1] != '\0'); CHECK(buffer[2] != '\0'); CHECK(buffer[3] != '\0'); + printf("bin: %d,%d,%d,%d\n", buffer[0], buffer[1], buffer[2], buffer[3]); CHECK(result->a.get() == o.a.get()); CHECK(result->b.get() == o.b.get()); CHECK(result->c.get() == o.c.get()); From 1f6b5dc0ed69a32793dee4f12dccfce6b4a64a1a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 17:28:56 +0800 Subject: [PATCH 33/39] fix little endian --- include/ylt/struct_pack/packer.hpp | 19 +++--- include/ylt/struct_pack/unpacker.hpp | 16 ++--- src/struct_pack/tests/test_fast_varint.cpp | 69 +++++++++++++++++++--- 3 files changed, 77 insertions(+), 27 deletions(-) diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 250f814bb..8fdc42c9b 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -150,11 +150,11 @@ class packer { template static constexpr void STRUCT_PACK_INLINE - get_fast_varint_width_impl(std::bitset &vec, int &i, const Arg &item, + get_fast_varint_width_impl(char (&vec)[sz], unsigned int &i, const Arg &item, uint64_t &unsigned_max, int64_t &signed_max) { if constexpr (varint_t) { if (get_varint_value(item) != 0) { - vec[i] = 1; + vec[i / 8] |= (0b1 << (i % 8)); if constexpr (std::is_unsigned_v>) { unsigned_max = @@ -167,19 +167,16 @@ class packer { : -(get_varint_value(item) + 1)); } } - else { - vec[i] = 0; - } ++i; } } template static constexpr int STRUCT_PACK_INLINE - get_fast_varint_width(std::bitset &vec, const Args &...items) { + get_fast_varint_width(char (&vec)[sz], const Args &...items) { uint64_t unsigned_max = 0; int64_t signed_max = 0; - int i = 0; + unsigned int i = 0; (get_fast_varint_width_impl(vec, i, items, unsigned_max, signed_max), ...); @@ -205,11 +202,11 @@ class packer { return; } else { - std::bitset vec; + char vec[bitset_size]{}; auto width = get_fast_varint_width(vec, items...); - vec[cnt] = width & 0b1; - vec[cnt + 1] = width & 0b10; - write_bytes_array(writer_, (char *)&vec, bitset_size); + vec[cnt / 8] |= (width & 0b1) << (cnt % 8); + vec[(cnt + 1) / 8] |= (!!(width & 0b10)) << ((cnt + 1) % 8); + write_bytes_array(writer_, vec, bitset_size); switch (width) { case 0: (serialize_one_fast_varint(items), ...); diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 340514416..7a3ba20ff 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -651,10 +651,11 @@ class unpacker { template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_one_fast_varint( - std::bitset &vec, int &i, Arg &item) { + char (&vec)[bitset_width], unsigned int &i, Arg &item) { if constexpr (varint_t) { constexpr auto real_width = (std::min)(width, sizeof(Arg)); - if (!vec[i++]) + auto index = i++; + if (!(vec[index / 8] & (0b1 << (index % 8)))) return {}; if constexpr (!no_skip) { reader_.ignore(real_width) ? errc{} : errc::no_buffer_space; @@ -688,7 +689,7 @@ class unpacker { template constexpr struct_pack::errc STRUCT_PACK_INLINE deserialize_fast_varint_helper( - std::bitset &vec, int &i, Arg &item, Args &...items) { + char (&vec)[bitset_width], unsigned int &i, Arg &item, Args &...items) { auto ec = deserialize_one_fast_varint(vec, i, item); if constexpr (sizeof...(items) > 0) { @@ -715,12 +716,13 @@ class unpacker { return {}; } else { - std::bitset vec; - if (auto ec = read_bytes_array(reader_, (char *)&vec, bitset_size); !ec) { + char vec[bitset_size]; + if (auto ec = read_bytes_array(reader_, vec, bitset_size); !ec) { return errc::no_buffer_space; } - std::size_t width = vec[cnt] + vec[cnt + 1] * 2; - int i = 0; + int width = !!(vec[cnt / 8] & (0b1 << (cnt % 8))) + + !!(vec[(cnt + 1) / 8] & (0b1 << ((cnt + 1) % 8))) * 2; + unsigned int i = 0; struct_pack::errc ec{}; switch (width) { case 0: diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index c880a01e9..5235e553b 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -155,16 +155,8 @@ TEST_CASE("fast varint test 1") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value()); + CHECK(result == o); CHECK(buffer.size() == 4); - CHECK(buffer[0] != '\0'); - CHECK(buffer[1] != '\0'); - CHECK(buffer[2] != '\0'); - CHECK(buffer[3] != '\0'); - printf("bin: %d,%d,%d,%d\n", buffer[0], buffer[1], buffer[2], buffer[3]); - CHECK(result->a.get() == o.a.get()); - CHECK(result->b.get() == o.b.get()); - CHECK(result->c.get() == o.c.get()); - CHECK(result->d.get() == o.d.get()); return; } @@ -1077,4 +1069,63 @@ TEST_CASE("fast varmixedint2 test 16") { CHECK(result == o); CHECK(buffer.size() == 25); return; +} + +struct six_int { + int a, b, c, d, e, f; + bool operator==(const six_int& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && f == o.f; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT | + struct_pack::sp_config::ENCODING_WITH_VARINT; +}; + +TEST_CASE("six int test") { + six_int o{INT32_MIN, 2435, INT32_MAX, 0, 0, INT32_MIN}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + return; +} + +struct seven_int { + int a, b, c, d, e, f, g; + bool operator==(const seven_int& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && + f == o.f && g == o.g; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT | + struct_pack::sp_config::ENCODING_WITH_VARINT; +}; + +TEST_CASE("seven int test") { + seven_int o{INT32_MIN, 21314, INT32_MAX, 0, 0, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + return; +} + +struct eight_int { + int a, b, c, d, e, f, g, h; + bool operator==(const eight_int& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && + f == o.f && g == o.g && h == o.h; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT | + struct_pack::sp_config::ENCODING_WITH_VARINT; +}; + +TEST_CASE("seven int test") { + eight_int o{INT32_MIN, 521, INT32_MAX, 0, 0, INT32_MIN, 0, 2123}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + return; } \ No newline at end of file From 637143db8361ab6e9dfeca4f02e44af499107fa9 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 17:50:51 +0800 Subject: [PATCH 34/39] test --- src/struct_pack/tests/test_fast_varint.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index 5235e553b..c82d8ce49 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -254,7 +254,10 @@ TEST_CASE("fast varint test 9") { auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value()); - CHECK(result == o); + CHECK(result->a == o.a); + CHECK(result->b == o.b); + CHECK(result->c == o.c); + CHECK(result->d == o.d); CHECK(buffer.size() == 13); return; } From e3345bab97d5d28a398813cb42d99576441a37c0 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 18:53:50 +0800 Subject: [PATCH 35/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 4 ++-- include/ylt/struct_pack/packer.hpp | 2 +- include/ylt/struct_pack/unpacker.hpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 30dcf67bb..df999f6ff 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -168,7 +168,7 @@ void write_bytes_array(writer_t& writer, const char* data, std::size_t length) { template void low_bytes_write_wrapper(writer_t& writer, const T& elem) { static_assert(sizeof(T) >= block_size); - if constexpr (is_system_little_endian || block_size == sizeof(T)) { + if constexpr (is_system_little_endian) { const char* data = (const char*)&elem; writer.write(data, block_size); } @@ -229,7 +229,7 @@ bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, template bool low_bytes_read_wrapper(reader_t& reader, T& elem) { static_assert(sizeof(T) >= block_size); - if constexpr (is_system_little_endian || block_size == sizeof(T)) { + if constexpr (is_system_little_endian) { char* data = (char*)&elem; return static_cast(reader.read(data, block_size)); } diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 8fdc42c9b..4cd02e791 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -100,7 +100,7 @@ class packer { } if constexpr (hash_head % 2) { // has more metainfo auto metainfo = info.metainfo(); - auto sz = info.size(); + std::size_t sz = info.size(); write_wrapper(writer_, (char *)&metainfo); if constexpr (serialize_static_config::has_compatible) { switch (metainfo & 0b11) { diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 7a3ba20ff..112655af5 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -489,7 +489,7 @@ class unpacker { deserialize_compatible(unsigned compatible_sz_len) { constexpr std::size_t sz[] = {0, 2, 4, 8}; auto len_sz = sz[compatible_sz_len]; - uint64_t data_len = 0; + std::size_t data_len = 0; bool result; switch (compatible_sz_len) { case 1: @@ -869,7 +869,7 @@ class unpacker { } } else if constexpr (container) { - uint64_t size64 = 0; + std::size_t size64 = 0; bool result{}; if constexpr (size_type == 1) { if SP_UNLIKELY (!low_bytes_read_wrapper(reader_, size64)) { From 5187bb565f96f74f02fef10d0acc190967b0964b Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 19:07:20 +0800 Subject: [PATCH 36/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index df999f6ff..bb6d9950d 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -172,6 +172,9 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { const char* data = (const char*)&elem; writer.write(data, block_size); } + else if constexpr (block_size == sizeof(T)) { + write_wrapper(writer, &elem); + } else { const char* data = sizeof(T) - block_size + (const char*)&elem; if constexpr (block_size == 1) { @@ -233,6 +236,9 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { char* data = (char*)&elem; return static_cast(reader.read(data, block_size)); } + else if constexpr (block_size == sizeof(T)) { + return read_wrapper(reader, &elem); + } else { char* data = (char*)&elem + sizeof(T) - block_size; if constexpr (block_size > 1) { From a27b74a228ed6dbd142a62b15f3ea0d79fcf27b9 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 19:45:01 +0800 Subject: [PATCH 37/39] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index bb6d9950d..0b4037cf5 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -173,7 +173,7 @@ void low_bytes_write_wrapper(writer_t& writer, const T& elem) { writer.write(data, block_size); } else if constexpr (block_size == sizeof(T)) { - write_wrapper(writer, &elem); + write_wrapper(writer, (const char*)&elem); } else { const char* data = sizeof(T) - block_size + (const char*)&elem; @@ -237,7 +237,7 @@ bool low_bytes_read_wrapper(reader_t& reader, T& elem) { return static_cast(reader.read(data, block_size)); } else if constexpr (block_size == sizeof(T)) { - return read_wrapper(reader, &elem); + return read_wrapper(reader, (char*)&elem); } else { char* data = (char*)&elem + sizeof(T) - block_size; From 462b3080eba3a3820c49f41025b3de6570d4a8c5 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 20:00:55 +0800 Subject: [PATCH 38/39] fix --- src/struct_pack/tests/test_serialize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 29c91b4c3..9bf0de4c3 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1332,7 +1332,7 @@ TEST_CASE("test width too big") { auto code = struct_pack::get_type_code() + 1; buffer.resize(4); if constexpr (!struct_pack::detail::is_system_little_endian) { - code = struct_pack::detail::bswap64(code); + code = struct_pack::detail::bswap32(code); } memcpy(buffer.data(), &code, sizeof(code)); buffer.push_back(0b11); From ecef80472ef8371bc6937653a1785772dc05336b Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 23 Nov 2023 20:21:24 +0800 Subject: [PATCH 39/39] add more test --- src/struct_pack/tests/test_fast_varint.cpp | 60 +++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/struct_pack/tests/test_fast_varint.cpp b/src/struct_pack/tests/test_fast_varint.cpp index c82d8ce49..1858e5d58 100644 --- a/src/struct_pack/tests/test_fast_varint.cpp +++ b/src/struct_pack/tests/test_fast_varint.cpp @@ -1124,11 +1124,67 @@ struct eight_int { struct_pack::sp_config::ENCODING_WITH_VARINT; }; -TEST_CASE("seven int test") { +TEST_CASE("eight int test") { eight_int o{INT32_MIN, 521, INT32_MAX, 0, 0, INT32_MIN, 0, 2123}; auto buffer = struct_pack::serialize(o); auto result = struct_pack::deserialize(buffer); REQUIRE(result.has_value()); CHECK(result == o); return; -} \ No newline at end of file +} + +TEST_CASE("vector test") { + eight_int o{INT32_MIN, 521, INT32_MAX, 0, 0, INT32_MIN, 0, 2123}; + std::vector vec; + vec.resize(100, o); + auto buffer = struct_pack::serialize(vec); + auto result = struct_pack::deserialize>(buffer); + REQUIRE(result.has_value()); + CHECK(result == vec); + return; +} + +struct mixed_fast_varint { + int a, b, c; + std::string d; + int e, f, g, h; + bool operator==(const mixed_fast_varint& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && + f == o.f && g == o.g && h == o.h; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT | + struct_pack::sp_config::ENCODING_WITH_VARINT; +}; + +TEST_CASE("test mixed 1") { + mixed_fast_varint o{INT32_MIN, 521, INT32_MAX, "Hello", 0, 0, INT32_MIN, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + return; +} + +struct mixed_fast_varint2 { + struct_pack::var_int64_t a, b, c; + std::string d; + int e, f, g, h; + bool operator==(const mixed_fast_varint2& o) const { + return a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && + f == o.f && g == o.g && h == o.h; + } + static constexpr auto struct_pack_config = + struct_pack::sp_config::USE_FAST_VARINT | + struct_pack::sp_config::DISABLE_ALL_META_INFO; +}; + +TEST_CASE("test mixed 2") { + mixed_fast_varint2 o{0, 0, 0, "Hello", 0, 0, 0, 0}; + auto buffer = struct_pack::serialize(o); + auto result = struct_pack::deserialize(buffer); + REQUIRE(result.has_value()); + CHECK(result == o); + CHECK(buffer.size() == 24); + return; +}