Skip to content

Commit

Permalink
add pb_base to improve pb_writer; improve pb_reder [struct_pb]
Browse files Browse the repository at this point in the history
  • Loading branch information
@bbbgan
bbbgan committed May 10, 2024
1 parent a0221e9 commit f357e50
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 91 deletions.
102 changes: 57 additions & 45 deletions benchmark/pb_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ class ScopedTimer {
};

void bench(int Count) {
stpb::BaseTypeMsg base_type_st{std::numeric_limits<int32_t>::max(),
stpb::BaseTypeMsg base_type_st{{},
std::numeric_limits<int32_t>::max(),
std::numeric_limits<int64_t>::max(),
std::numeric_limits<uint32_t>::max(),
std::numeric_limits<uint64_t>::max(),
Expand All @@ -39,7 +40,8 @@ void bench(int Count) {
pb::BaseTypeMsg base_type_msg;
SetBaseTypeMsg(base_type_st, base_type_msg);

stpb::IguanaTypeMsg iguana_type_st{{std::numeric_limits<int32_t>::max()},
stpb::IguanaTypeMsg iguana_type_st{{},
{std::numeric_limits<int32_t>::max()},
{std::numeric_limits<int64_t>::max()},
{std::numeric_limits<uint32_t>::max()},
{std::numeric_limits<uint64_t>::max()},
Expand All @@ -49,6 +51,7 @@ void bench(int Count) {
SetIguanaTypeMsg(iguana_type_st, iguana_type_msg);

stpb::RepeatBaseTypeMsg re_base_type_st{
{},
{std::numeric_limits<uint32_t>::max(),
std::numeric_limits<uint32_t>::min()},
{std::numeric_limits<uint64_t>::max(),
Expand All @@ -65,31 +68,39 @@ void bench(int Count) {
SetRepeatBaseTypeMsg(re_base_type_st, re_base_type_msg);

stpb::RepeatIguanaTypeMsg re_iguana_type_st{
{{0}, {1}, {3}}, {{4}, {5}, {6}}, {{7}, {8}, {9}},
{{10}, {11}, {12}}, {{13}, {14}, {15}}, {},
{},
{{0}, {1}, {3}},
{{4}, {5}, {6}},
{{7}, {8}, {9}},
{{10}, {11}, {12}},
{{13}, {14}, {15}},
{},
};
pb::RepeatIguanaTypeMsg re_iguana_type_msg;
SetRepeatIguanaTypeMsg(re_iguana_type_st, re_iguana_type_msg);

stpb::NestedMsg nest_st{
/* base_msg */ {100, 200, 300, 400, 31.4f, 62.8, false, "World",
stpb::Enum::BAZ},
{},
/* base_msg */
{{}, 100, 200, 300, 400, 31.4f, 62.8, false, "World", stpb::Enum::BAZ},
/* repeat_base_msg */
{{1, 2, 3, 4, 5.5f, 6.6, true, "Hello", stpb::Enum::FOO},
{7, 8, 9, 10, 11.11f, 12.12, false, "Hi", stpb::Enum::BAR}},
/* iguana_type_msg */ {{100}, {200}, {300}, {400}, {31}, {32}},
{{{}, 1, 2, 3, 4, 5.5f, 6.6, true, "Hello", stpb::Enum::FOO},
{{}, 7, 8, 9, 10, 11.11f, 12.12, false, "Hi", stpb::Enum::BAR}},
/* iguana_type_msg */ {{}, {100}, {200}, {300}, {400}, {31}, {32}},
/* repeat_iguna_msg */
{{{1}, {2}, {3}}, {{4}, {5}, {6}}, {{7}, {8}, {9}}},
{{}, {{1}, {2}, {3}}, {{4}, {5}, {6}}, {{7}, {8}, {9}}},
/* repeat_repeat_base_msg */
{{{1, 2, 3},
{{{},
{1, 2, 3},
{4, 5, 6},
{7, 8, 9},
{10, 11, 12},
{13.1, 14.2, 15.3},
{16.4, 17.5, 18.6},
{"a", "b", "c"},
{stpb::Enum::FOO, stpb::Enum::BAR, stpb::Enum::BAZ}},
{{19, 20, 21},
{{},
{19, 20, 21},
{22, 23, 24},
{25, 26, 27},
{28, 29, 30},
Expand All @@ -105,12 +116,13 @@ void bench(int Count) {
map_st.sfix64_str_map.emplace(iguana::sfixed64_t{20}, "twenty");

map_st.str_iguana_type_msg_map.emplace(
"first", stpb::IguanaTypeMsg{{10}, {20}, {30}, {40}, {50}, {60}});
"first", stpb::IguanaTypeMsg{{}, {10}, {20}, {30}, {40}, {50}, {60}});
map_st.str_iguana_type_msg_map.emplace(
"second", stpb::IguanaTypeMsg{{11}, {21}, {31}, {41}, {51}, {61}});
"second", stpb::IguanaTypeMsg{{}, {11}, {21}, {31}, {41}, {51}, {61}});

map_st.int_repeat_base_msg_map.emplace(
1, stpb::RepeatBaseTypeMsg{{1, 2},
1, stpb::RepeatBaseTypeMsg{{},
{1, 2},
{3, 4},
{5, 6},
{7, 8},
Expand All @@ -119,7 +131,8 @@ void bench(int Count) {
{"one", "two"},
{stpb::Enum::FOO, stpb::Enum::BAR}});
map_st.int_repeat_base_msg_map.emplace(
2, stpb::RepeatBaseTypeMsg{{2, 3},
2, stpb::RepeatBaseTypeMsg{{},
{2, 3},
{4, 5},
{6, 7},
{8, 9},
Expand All @@ -130,11 +143,36 @@ void bench(int Count) {
pb::MapMsg map_msg{};
SetMapMsg(map_st, map_msg);

stpb::BaseTypeMsg baseTypeMsg{
100, 200, 300, 400, 31.4f, 62.8, false, "World", stpb::Enum::BAZ};
stpb::BaseOneofMsg base_one_of_st{123, baseTypeMsg, 456.78};
stpb::BaseTypeMsg baseTypeMsg{{}, 100, 200, 300, 400,
31.4f, 62.8, false, "World", stpb::Enum::BAZ};
stpb::BaseOneofMsg base_one_of_st{{}, 123, baseTypeMsg, 456.78};
pb::BaseOneofMsg base_one_of_msg;
SetBaseOneofMsg(base_one_of_st, base_one_of_msg);
{
ScopedTimer timer("struct_pb serialize");
for (int i = 0; i < Count; ++i) {
std::string str_base_st_ss;
iguana::to_pb(base_type_st, str_base_st_ss);

std::string str_iguana_st_ss;
iguana::to_pb(iguana_type_st, str_iguana_st_ss);

std::string re_base_type_st_ss;
iguana::to_pb(re_base_type_st, re_base_type_st_ss);

std::string re_iguana_type_st_ss;
iguana::to_pb(re_iguana_type_st, re_iguana_type_st_ss);

std::string nest_st_ss;
iguana::to_pb(nest_st, nest_st_ss);

std::string map_st_ss;
iguana::to_pb(map_st, map_st_ss);

std::string base_one_of_st_ss;
iguana::to_pb(base_one_of_st, base_one_of_st_ss);
}
}

{
ScopedTimer timer("protobuf serialize");
Expand Down Expand Up @@ -166,32 +204,6 @@ void bench(int Count) {
}
}

{
ScopedTimer timer("struct_pb serialize");
for (int i = 0; i < Count; ++i) {
std::string str_base_st_ss;
iguana::to_pb(base_type_st, str_base_st_ss);

std::string str_iguana_st_ss;
iguana::to_pb(iguana_type_st, str_iguana_st_ss);

std::string re_base_type_st_ss;
iguana::to_pb(re_base_type_st, re_base_type_st_ss);

std::string re_iguana_type_st_ss;
iguana::to_pb(re_iguana_type_st, re_iguana_type_st_ss);

std::string nest_st_ss;
iguana::to_pb(nest_st, nest_st_ss);

std::string map_st_ss;
iguana::to_pb(map_st, map_st_ss);

std::string base_one_of_st_ss;
iguana::to_pb(base_one_of_st, base_one_of_st_ss);
}
}

std::string str_base_st_ss;
iguana::to_pb(base_type_st, str_base_st_ss);

Expand Down
74 changes: 38 additions & 36 deletions iguana/pb_reader.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
namespace iguana {

template <typename T>
inline void from_pb(T& t, std::string_view pb_str);
IGUANA_INLINE void from_pb(T& t, std::string_view pb_str);

namespace detail {

template <typename T>
inline void from_pb_impl(T& val, std::string_view& pb_str,
uint32_t field_no = 0);
IGUANA_INLINE void from_pb_impl(T& val, std::string_view& pb_str,
uint32_t field_no = 0);

template <typename T>
inline void decode_pair_value(T& val, std::string_view& pb_str) {
IGUANA_INLINE void decode_pair_value(T& val, std::string_view& pb_str) {
size_t pos;
uint32_t key = detail::decode_varint(pb_str, pos);
pb_str = pb_str.substr(pos);
Expand All @@ -25,15 +25,17 @@ inline void decode_pair_value(T& val, std::string_view& pb_str) {
}

template <typename T>
inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
IGUANA_INLINE void from_pb_impl(T& val, std::string_view& pb_str,
uint32_t field_no) {
size_t pos = 0;
if constexpr (is_reflection_v<T>) {
size_t pos;
uint32_t size = detail::decode_varint(pb_str, pos);
pb_str = pb_str.substr(pos);
if (pb_str.size() < size) {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
if (pb_str.size() < size)
IGUANA_UNLIKELY {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
if (size == 0) {
return;
}
Expand Down Expand Up @@ -66,9 +68,11 @@ inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
size_t pos;
uint32_t size = detail::decode_varint(pb_str, pos);
pb_str = pb_str.substr(pos);
if (pb_str.size() < size) {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
if (pb_str.size() < size)
IGUANA_UNLIKELY {
throw std::invalid_argument(
"Invalid fixed int value: too few bytes.");
}
using item_type = typename T::value_type;
size_t start = pb_str.size();

Expand All @@ -88,10 +92,11 @@ inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
size_t pos;
uint32_t size = detail::decode_varint(pb_str, pos);
pb_str = pb_str.substr(pos);
if (pb_str.size() < size) {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}

if (pb_str.size() < size)
IGUANA_UNLIKELY {
throw std::invalid_argument(
"Invalid fixed int value: too few bytes.");
}
item_type item = {};
decode_pair_value(item.first, pb_str);
decode_pair_value(item.second, pb_str);
Expand All @@ -100,13 +105,11 @@ inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
if (pb_str.empty()) {
break;
}

uint32_t key = detail::decode_varint(pb_str, pos);
uint32_t field_number = key >> 3;
if (field_number != field_no) {
break;
}

pb_str = pb_str.substr(pos);
}
}
Expand All @@ -116,38 +119,40 @@ inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
}
else if constexpr (detail::is_signed_varint_v<T>) {
constexpr size_t len = sizeof(typename T::value_type);

uint64_t temp = detail::decode_varint(pb_str, pos);
if constexpr (len == 8) {
val.val = detail::decode_zigzag(temp);
}
else {
val.val = detail::decode_zigzag((uint32_t)temp);
val.val = detail::decode_zigzag(static_cast<uint32_t>(temp));
}
pb_str = pb_str.substr(pos);
}
else if constexpr (detail::is_fixed_v<T>) {
constexpr size_t size = sizeof(typename T::value_type);
if (pb_str.size() < size) {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
if (pb_str.size() < size)
IGUANA_UNLIKELY {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
memcpy(&(val.val), pb_str.data(), size);
pb_str = pb_str.substr(size);
}
else if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
constexpr size_t size = sizeof(T);
if (pb_str.size() < size) {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
if (pb_str.size() < size)
IGUANA_UNLIKELY {
throw std::invalid_argument("Invalid fixed int value: too few bytes.");
}
memcpy(&(val), pb_str.data(), size);
pb_str = pb_str.substr(size);
}
else if constexpr (std::is_same_v<T, std::string> ||
std::is_same_v<T, std::string_view>) {
size_t size = detail::decode_varint(pb_str, pos);
if (pb_str.size() < pos + size) {
throw std::invalid_argument("Invalid string value: too few bytes.");
}
if (pb_str.size() < pos + size)
IGUANA_UNLIKELY {
throw std::invalid_argument("Invalid string value: too few bytes.");
}
if constexpr (std::is_same_v<T, std::string_view>) {
val = std::string_view(pb_str.data() + pos, size);
}
Expand All @@ -172,18 +177,15 @@ inline void from_pb_impl(T& val, std::string_view& pb_str, uint32_t field_no) {
}

template <typename T, typename Field>
inline void parse_oneof(T& t, const Field& f, std::string_view& pb_str) {
IGUANA_INLINE void parse_oneof(T& t, const Field& f, std::string_view& pb_str) {
using item_type = typename std::decay_t<Field>::value_type;
item_type item{};
from_pb_impl(item, pb_str, f.field_no);
t = std::move(item);
from_pb_impl(t.template emplace<item_type>(), pb_str, f.field_no);
}
} // namespace detail

template <typename T>
inline void from_pb(T& t, std::string_view pb_str) {
IGUANA_INLINE void from_pb(T& t, std::string_view pb_str) {
size_t pos = 0;
// TODO: for_each parse
while (!pb_str.empty()) {
uint32_t key = detail::decode_varint(pb_str, pos);
WireType wire_type = static_cast<WireType>(key & 0b0111);
Expand All @@ -193,9 +195,9 @@ inline void from_pb(T& t, std::string_view pb_str) {
constexpr static auto map = get_members<T>();
auto& member = map.at(field_number);
std::visit(
[&t, &pb_str, wire_type](auto& val) {
[&t, &pb_str, wire_type](auto& val) IGUANA__INLINE_LAMBDA {
using value_type = typename std::decay_t<decltype(val)>::value_type;
if (wire_type != detail::get_wire_type<value_type>()) {
if (wire_type != detail::get_wire_type<value_type>()) IGUANA_UNLIKELY {
throw std::runtime_error("unmatched wire_type");
}
using v_type = std::decay_t<decltype(val.value(t))>;
Expand Down
22 changes: 20 additions & 2 deletions iguana/pb_util.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ enum class WireType : uint32_t {
Unknown
};

struct pb_base {
size_t cache_size;
};

template <typename T>
constexpr bool inherits_from_pb_base_v = std::is_base_of_v<pb_base, T>;

namespace detail {
template <typename T>
constexpr bool is_fixed_v =
Expand Down Expand Up @@ -373,7 +380,13 @@ IGUANA_INLINE size_t pb_key_value_size(T&& t) {
}
},
std::make_index_sequence<SIZE>{});
get_set_size_cache(t) = len;
if constexpr (inherits_from_pb_base_v<value_type>) {
t.cache_size = len;
}
else {
get_set_size_cache(t) = len;
}

if constexpr (key_size == 0) {
// for top level
return len;
Expand Down Expand Up @@ -448,7 +461,12 @@ template <typename T>
IGUANA_INLINE size_t pb_value_size(T&& t) {
using value_type = std::remove_const_t<std::remove_reference_t<T>>;
if constexpr (is_reflection_v<value_type>) {
return get_set_size_cache(t);
if constexpr (inherits_from_pb_base_v<value_type>) {
return t.cache_size;
}
else {
return get_set_size_cache(t);
}
}
else if constexpr (is_sequence_container<value_type>::value) {
using item_type = typename value_type::value_type;
Expand Down
Loading

0 comments on commit f357e50

Please sign in to comment.