Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Producer/Consumer table binary support #801

Merged
merged 14 commits into from
Jul 13, 2023
6 changes: 3 additions & 3 deletions common/consumerstatetable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,16 +68,16 @@ void ConsumerStateTable::pops(std::deque<KeyOpFieldsValuesTuple> &vkco, const st

auto& ctx = ctx0->element[ie];
assert(ctx->element[0]->type == REDIS_REPLY_STRING);
std::string key = ctx->element[0]->str;
std::string key(ctx->element[0]->str, ctx->element[0]->len);
kfvKey(kco) = key;

assert(ctx->element[1]->type == REDIS_REPLY_ARRAY);
auto ctx1 = ctx->element[1];
for (size_t i = 0; i < ctx1->elements / 2; i++)
{
FieldValueTuple e;
fvField(e) = ctx1->element[i * 2]->str;
fvValue(e) = ctx1->element[i * 2 + 1]->str;
fvField(e).assign(ctx1->element[i * 2]->str, ctx1->element[i * 2]->len);
fvValue(e).assign(ctx1->element[i * 2 + 1]->str, ctx1->element[i * 2 + 1]->len);
values.push_back(e);
}

Expand Down
8 changes: 6 additions & 2 deletions common/luatable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,12 @@ bool LuaTable::get(const vector<string> &luaKeys, vector<FieldValueTuple> &value
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
liuh-80 marked this conversation as resolved.
Show resolved Hide resolved
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );
Pterosaur marked this conversation as resolved.
Show resolved Hide resolved

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
RedisReply r(m_db.get(), command, REDIS_REPLY_ARRAY);
redisReply *reply = r.getContext();

Expand Down Expand Up @@ -112,10 +114,12 @@ bool LuaTable::hget(const vector<string> &luaKeys, const string &field, string &
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
RedisReply r(m_db.get(), command);
redisReply *reply = r.getContext();

Expand Down
24 changes: 18 additions & 6 deletions common/producerstatetable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,12 @@ void ProducerStateTable::set(const string &key, const vector<FieldValueTuple> &v
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(command, REDIS_REPLY_NIL);
if (!m_buffered)
{
Expand Down Expand Up @@ -174,10 +176,12 @@ void ProducerStateTable::del(const string &key, const string &op /*= DEL_COMMAND
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(command, REDIS_REPLY_NIL);
if (!m_buffered)
{
Expand Down Expand Up @@ -227,10 +231,12 @@ void ProducerStateTable::set(const std::vector<KeyOpFieldsValuesTuple>& values)
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(command, REDIS_REPLY_NIL);
if (!m_buffered)
{
Expand Down Expand Up @@ -268,10 +274,12 @@ void ProducerStateTable::del(const std::vector<std::string>& keys)
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(command, REDIS_REPLY_NIL);
if (!m_buffered)
{
Expand Down Expand Up @@ -310,10 +318,12 @@ void ProducerStateTable::clear()
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand cmd;
cmd.formatArgv((int)args1.size(), &args1[0], NULL);
cmd.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(cmd, REDIS_REPLY_NIL);
m_pipe->flush();
}
Expand Down Expand Up @@ -469,10 +479,12 @@ void ProducerStateTable::apply_temp_view()
// Transform data structure
vector<const char *> args1;
transform(args.begin(), args.end(), back_inserter(args1), [](const string &s) { return s.c_str(); } );
vector<size_t> lengths;
transform(args.begin(), args.end(), back_inserter(lengths), [](const string &s) { return s.length(); } );

// Invoke redis command
RedisCommand command;
command.formatArgv((int)args1.size(), &args1[0], NULL);
command.formatArgv((int)args1.size(), &args1[0], lengths.data());
m_pipe->push(command, REDIS_REPLY_NIL);
m_pipe->flush();

Expand Down
8 changes: 7 additions & 1 deletion common/redisapi.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,15 @@ static inline std::set<std::string> runRedisScript(RedisContext &ctx, const std:
args.end(),
std::back_inserter(c_args),
[](const std::string& s) { return s.c_str(); } );
std::vector<size_t> lengths;
transform(
Pterosaur marked this conversation as resolved.
Show resolved Hide resolved
args.begin(),
args.end(),
std::back_inserter(lengths),
[](const std::string &s) { return s.length(); } );

RedisCommand command;
command.formatArgv(static_cast<int>(c_args.size()), c_args.data(), NULL);
command.formatArgv(static_cast<int>(c_args.size()), c_args.data(), lengths.data());

std::set<std::string> ret;
try
Expand Down
19 changes: 13 additions & 6 deletions common/rediscommand.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ using namespace std;
namespace swss {

RedisCommand::RedisCommand()
: temp(NULL)
: temp(NULL),
len(0)
{
}

Expand All @@ -26,7 +27,7 @@ void RedisCommand::format(const char *fmt, ...)

va_list ap;
va_start(ap, fmt);
int len = redisvFormatCommand(&temp, fmt, ap);
len = redisvFormatCommand(&temp, fmt, ap);
va_end(ap);
if (len == -1) {
throw std::bad_alloc();
Expand All @@ -43,7 +44,7 @@ void RedisCommand::formatArgv(int argc, const char **argv, const size_t *argvlen
temp = nullptr;
}

int len = redisFormatCommandArgv(&temp, argc, argv, argvlen);
len = redisFormatCommandArgv(&temp, argc, argv, argvlen);
if (len == -1) {
throw std::bad_alloc();
}
Expand All @@ -52,11 +53,13 @@ void RedisCommand::formatArgv(int argc, const char **argv, const size_t *argvlen
void RedisCommand::format(const vector<string> &commands)
{
vector<const char*> args;
vector<size_t> lens;
for (auto& command : commands)
{
args.push_back(command.c_str());
lens.push_back(command.size());
}
formatArgv(static_cast<int>(args.size()), args.data(), NULL);
formatArgv(static_cast<int>(args.size()), args.data(), lens.data());
}

/* Format HSET key multiple field value command */
Expand Down Expand Up @@ -97,11 +100,13 @@ void RedisCommand::formatHDEL(const std::string& key, const std::vector<std::str
if (fields.empty()) throw std::invalid_argument("empty values");

std::vector<const char *> args = {"HDEL", key.c_str()};
std::vector<size_t> lens = {4, key.size()};
for (const std::string &f : fields)
{
args.push_back(f.c_str());
lens.push_back(f.size());
}
formatArgv(static_cast<int>(args.size()), args.data(), NULL);
formatArgv(static_cast<int>(args.size()), args.data(), lens.data());
}

/* Format EXPIRE key field command */
Expand Down Expand Up @@ -129,7 +134,9 @@ const char *RedisCommand::c_str() const

size_t RedisCommand::length() const
{
return strlen(temp);
if (len <= 0)
return 0;
return static_cast<size_t>(len);
}

}
1 change: 1 addition & 0 deletions common/rediscommand.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class RedisCommand {

private:
char *temp;
int len;
Pterosaur marked this conversation as resolved.
Show resolved Hide resolved
};

template<typename InputIterator>
Expand Down
56 changes: 56 additions & 0 deletions pyext/swsscommon.i
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,62 @@
temp = SWIG_NewPointerObj(*$1, SWIGTYPE_p_swss__Selectable, 0);
SWIG_Python_AppendOutput($result, temp);
}

%typemap(in, fragment="SWIG_AsPtr_std_string")
const std::vector<std::pair< std::string,std::string >,std::allocator< std::pair< std::string,std::string > > > &
(std::vector< std::pair< std::string,std::string >,std::allocator< std::pair< std::string,std::string > > > temp,
int res) {
res = SWIG_OK;
for (int i = 0; i < PySequence_Length($input); ++i) {
temp.push_back(std::pair< std::string,std::string >());
PyObject *item = PySequence_GetItem($input, i);
if (!PyTuple_Check(item) || PyTuple_Size(item) != 2) {
SWIG_fail;
}
PyObject *key = PyTuple_GetItem(item, 0);
PyObject *value = PyTuple_GetItem(item, 1);
std::string *ptr = (std::string *)0;
if (PyBytes_Check(key)) {
temp.back().first.assign(PyBytes_AsString(key), PyBytes_Size(key));
} else if (SWIG_AsPtr_std_string(key, &ptr)) {
temp.back().first = *ptr;
} else {
SWIG_fail;
}
if (PyBytes_Check(value)) {
temp.back().second.assign(PyBytes_AsString(value), PyBytes_Size(value));
} else if (SWIG_AsPtr_std_string(value, &ptr)) {
temp.back().second = *ptr;
} else {
SWIG_fail;
}
}
$1 = &temp;
}

%typemap(typecheck) const std::vector< std::pair< std::string,std::string >,std::allocator< std::pair< std::string,std::string > > > &{
$1 = 1;
for (int i = 0; i < PySequence_Length($input); ++i) {
PyObject *item = PySequence_GetItem($input, i);
if (!PyTuple_Check(item) || PyTuple_Size(item) != 2) {
$1 = 0;
break;
}
PyObject *key = PyTuple_GetItem(item, 0);
PyObject *value = PyTuple_GetItem(item, 1);
if (!PyBytes_Check(key)
&& !PyUnicode_Check(key)
&& !PyString_Check(key)
&& !PyBytes_Check(value)
&& !PyUnicode_Check(value)
&& !PyString_Check(value)) {
$1 = 0;
break;
}
}
}


#endif

#ifdef SWIGGO
Expand Down