Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow to scan keys with the suffix #2222

Open
wants to merge 2 commits into
base: unstable
Choose a base branch
from
Open

Allow to scan keys with the suffix #2222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7aaf96d
add scan substring,suffix
Apr 4, 2024
b714528
add scan_false_limit in all scan
Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ version.h
.cache

compactdb
test
testdb

build
Expand Down
9 changes: 6 additions & 3 deletions src/commands/cmd_hash.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*
*/

#include <string>

#include "commander.h"
#include "commands/command_parser.h"
#include "error_constants.h"
Expand Down Expand Up @@ -367,12 +369,13 @@ class CommandHScan : public CommandSubkeyScanBase {
std::vector<std::string> fields;
std::vector<std::string> values;
auto key_name = srv->GetKeyNameFromCursor(cursor_, CursorType::kTypeHash);
auto s = hash_db.Scan(key_, key_name, limit_, prefix_, &fields, &values);
ScanConfig scan_config(limit_,srv->GetConfig()->max_scan_num,prefix_);
auto s = hash_db.Scan(key_, key_name, &fields, scan_config, *match_mode_, &values);
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}

auto cursor = GetNextCursor(srv, fields, CursorType::kTypeHash);
auto &end_cursor = hash_db.end_cursor;
const auto &cursor = srv->GenerateCursorFromKeyName(end_cursor, CursorType::kTypeHash);
std::vector<std::string> entries;
entries.reserve(2 * fields.size());
for (size_t i = 0; i < fields.size(); i++) {
Expand Down
3 changes: 2 additions & 1 deletion src/commands/cmd_server.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -840,7 +840,8 @@ class CommandScan : public CommandScanBase {

std::vector<std::string> keys;
std::string end_key;
auto s = redis_db.Scan(key_name, limit_, prefix_, &keys, &end_key, type_);
ScanConfig scan_config(limit_, srv->GetConfig()->max_scan_num, prefix_);
auto s = redis_db.Scan(key_name, &keys, scan_config, *match_mode_,&end_key, type_);
if (!s.ok()) {
return {Status::RedisExecErr, s.ToString()};
}
Expand Down
6 changes: 3 additions & 3 deletions src/commands/cmd_set.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,12 +427,12 @@ class CommandSScan : public CommandSubkeyScanBase {
redis::Set set_db(srv->storage, conn->GetNamespace());
std::vector<std::string> members;
auto key_name = srv->GetKeyNameFromCursor(cursor_, CursorType::kTypeSet);
auto s = set_db.Scan(key_, key_name, limit_, prefix_, &members);
ScanConfig scan_config(limit_, srv->GetConfig()->max_scan_num, prefix_);
auto s = set_db.Scan(key_, key_name, &members, scan_config, *match_mode_);
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}

*output = CommandScanBase::GenerateOutput(srv, conn, members, CursorType::kTypeSet);
*output = CommandScanBase::GenerateOutput(srv, conn, members, set_db.end_cursor);
return Status::OK();
}
};
Expand Down
6 changes: 3 additions & 3 deletions src/commands/cmd_zset.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1350,12 +1350,12 @@ class CommandZScan : public CommandSubkeyScanBase {
std::vector<std::string> members;
std::vector<double> scores;
auto key_name = srv->GetKeyNameFromCursor(cursor_, CursorType::kTypeZSet);
auto s = zset_db.Scan(key_, key_name, limit_, prefix_, &members, &scores);
ScanConfig scan_config(limit_, srv->GetConfig()->max_scan_num, prefix_);
auto s = zset_db.Scan(key_, key_name, &members, scan_config, *match_mode_, &scores);
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}

auto cursor = GetNextCursor(srv, members, CursorType::kTypeZSet);
auto cursor = srv->GenerateCursorFromKeyName(zset_db.end_cursor, CursorType::kTypeZSet);
std::vector<std::string> entries;
entries.reserve(2 * members.size());
for (size_t i = 0; i < members.size(); i++) {
Expand Down
25 changes: 14 additions & 11 deletions src/commands/scan_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

#pragma once

#include <vector>

#include "commander.h"
#include "commands/command_parser.h"
#include "error_constants.h"
Expand All @@ -39,14 +41,20 @@ class CommandScanBase : public Commander {

return ParseAdditionalFlags<true>(parser);
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please don't introduce random changes.

template <bool IsScan, typename Parser>
Status ParseAdditionalFlags(Parser &parser) {
while (parser.Good()) {
if (parser.EatEqICase("match")) {
prefix_ = GET_OR_RET(parser.TakeStr());
if (!prefix_.empty() && prefix_.back() == '*') {
if (prefix_.size() >= 2 && prefix_.front() == '*' && prefix_.back() == '*') {
prefix_ = prefix_.substr(1, prefix_.size() - 2);
match_mode_ = std::make_unique<SubStringMatchType>();
} else if (!prefix_.empty() && prefix_.back() == '*') {
prefix_ = prefix_.substr(0, prefix_.size() - 1);
match_mode_ = std::make_unique<PerfixMatchType>();
} else if (!prefix_.empty() && prefix_.front() == '*') {
prefix_ = prefix_.substr(1, prefix_.size() - 1);
match_mode_ = std::make_unique<SubStringMatchType>();
} else {
return {Status::RedisParseErr, "currently only key prefix matching is supported"};
}
Expand Down Expand Up @@ -80,16 +88,10 @@ class CommandScanBase : public Commander {
}
}

std::string GenerateOutput(Server *srv, const Connection *conn, const std::vector<std::string> &keys,
CursorType cursor_type) const {
static std::string GenerateOutput(Server *srv, const Connection *conn, const std::vector<std::string> &keys,
const std::string &end_cursor) {
std::vector<std::string> list;
if (keys.size() == static_cast<size_t>(limit_)) {
auto end_cursor = srv->GenerateCursorFromKeyName(keys.back(), cursor_type);
list.emplace_back(redis::BulkString(end_cursor));
} else {
list.emplace_back(redis::BulkString("0"));
}

list.emplace_back(end_cursor);
list.emplace_back(ArrayOfBulkStrings(keys));

return redis::Array(list);
Expand All @@ -100,6 +102,7 @@ class CommandScanBase : public Commander {
std::string prefix_;
int limit_ = 20;
RedisType type_ = kRedisNone;
std::unique_ptr<BaseMatchType> match_mode_;
};

class CommandSubkeyScanBase : public CommandScanBase {
Expand Down
2 changes: 2 additions & 0 deletions src/config/config.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <strings.h>

#include <algorithm>
#include <climits>
#include <cstring>
#include <fstream>
#include <iostream>
Expand Down Expand Up @@ -152,6 +153,7 @@ Config::Config() {
{"pidfile", true, new StringField(&pidfile, kDefaultPidfile)},
{"max-io-mb", false, new IntField(&max_io_mb, 0, 0, INT_MAX)},
{"max-bitmap-to-string-mb", false, new IntField(&max_bitmap_to_string_mb, 16, 0, INT_MAX)},
{"max-scan-num", false, new IntField(&max_scan_num, 2, 0, INT_MAX)},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems 2 is too smaller to be as the max_scan_num?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's just for test easy，jihuayu say they will determine the final value.

{"max-db-size", false, new IntField(&max_db_size, 0, 0, INT_MAX)},
{"max-replication-mb", false, new IntField(&max_replication_mb, 0, 0, INT_MAX)},
{"supervised", true, new EnumField<SupervisedMode>(&supervised_mode, supervised_modes, kSupervisedNone)},
Expand Down
14 changes: 14 additions & 0 deletions src/config/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include "config_type.h"
Expand Down Expand Up @@ -114,6 +115,7 @@ struct Config {
int max_replication_mb = 0;
int max_io_mb = 0;
int max_bitmap_to_string_mb = 16;
int max_scan_num = 2;
bool master_use_repl_port = false;
bool purge_backup_on_fullsync = false;
bool auto_resize_block_and_sst = true;
Expand Down Expand Up @@ -260,3 +262,15 @@ struct Config {
bool checkFieldValueIsDefault(const std::string &key, const std::string &value) const;
Status finish();
};

struct ScanConfig{
public:
ScanConfig(uint64_t scan_matched_limt_value, uint64_t scan_mismatched_limt_value,const std::string& match_str_value)
: scan_matched_limt(scan_matched_limt_value),
scan_mismatched_limt(scan_mismatched_limt_value),
match_str(match_str_value) {}

uint64_t scan_matched_limt;
uint64_t scan_mismatched_limt;
const std::string& match_str;
};
Loading