From d533bec1eeb471458f19bb55f9d8c2cfdbe2dd69 Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 30 Apr 2024 12:41:04 +1000 Subject: [PATCH 1/5] pmproxy: rename configuration keys specific to Redis With Redis no longer open source, there's a transition away from it in all Linux distributions now underway. Here we switch pmproxy/pmseries/pmsearch configuration file entries specifically named 'redis' in some way to a more general 'key server' naming convention (there's multiple options now). Backward compatibility is kept though, we'll just no longer document those entries. --- src/libpcp_web/src/load.c | 3 ++- src/libpcp_web/src/query.c | 6 ++++-- src/libpcp_web/src/schema.c | 3 ++- src/libpcp_web/src/search.c | 3 ++- src/libpcp_web/src/slots.c | 17 ++++++++++----- src/pmproxy/pmproxy.conf | 24 ++++++++++----------- src/pmproxy/src/pmproxy.c | 43 +++++++++++++++++-------------------- src/pmsearch/pmsearch.c | 23 ++++++++++---------- src/pmseries/pmseries.c | 33 ++++++++++++++-------------- 9 files changed, 83 insertions(+), 72 deletions(-) diff --git a/src/libpcp_web/src/load.c b/src/libpcp_web/src/load.c index 09cfa6ccc7..2da8ecc801 100644 --- a/src/libpcp_web/src/load.c +++ b/src/libpcp_web/src/load.c @@ -1038,7 +1038,8 @@ connect_redis_source_service(seriesLoadBaton *baton) baton->slots = data->slots; series_load_end_phase(baton); } else { - option = pmIniFileLookup(data->config, "redis", "enabled"); + if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) + option = pmIniFileLookup(data->config, "redis", "enabled"); if (option && strcmp(option, "false") == 0) { baton->error = -ENOTSUP; } else { diff --git a/src/libpcp_web/src/query.c b/src/libpcp_web/src/query.c index bf4309e861..22fd5c7f08 100644 --- a/src/libpcp_web/src/query.c +++ b/src/libpcp_web/src/query.c @@ -5125,7 +5125,8 @@ series_query_services(void *arg) baton->slots = data->slots; series_query_end_phase(baton); } else { - option = pmIniFileLookup(data->config, "redis", "enabled"); + if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) + option = pmIniFileLookup(data->config, "redis", "enabled"); if (option && strcmp(option, "false") == 0) baton->error = -ENOTSUP; else @@ -6140,7 +6141,8 @@ series_lookup_services(void *arg) baton->slots = data->slots; series_query_end_phase(baton); } else { - option = pmIniFileLookup(data->config, "redis", "enabled"); + if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) + option = pmIniFileLookup(data->config, "redis", "enabled"); if (option && strcmp(option, "false") == 0) baton->error = -ENOTSUP; else diff --git a/src/libpcp_web/src/schema.c b/src/libpcp_web/src/schema.c index b7efd8b8f9..948142b9db 100644 --- a/src/libpcp_web/src/schema.c +++ b/src/libpcp_web/src/schema.c @@ -1681,7 +1681,8 @@ pmSeriesSetup(pmSeriesModule *module, void *arg) module->on_setup(arg); data->shareslots = 1; } else { - option = pmIniFileLookup(data->config, "redis", "enabled"); + if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) + option = pmIniFileLookup(data->config, "redis", "enabled"); if (option && strcmp(option, "false") == 0) return -ENOTSUP; diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 7bb0a1fe9e..6012224ed2 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -1073,7 +1073,8 @@ pmSearchSetup(pmSearchModule *module, void *arg) module->on_setup(arg); data->shareslots = 1; } else { - option = pmIniFileLookup(data->config, "redis", "enabled"); + if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) + option = pmIniFileLookup(data->config, "redis", "enabled"); if (option && strcmp(option, "false") == 0) return -ENOTSUP; diff --git a/src/libpcp_web/src/slots.c b/src/libpcp_web/src/slots.c index cd80a08be3..f8b5684a34 100644 --- a/src/libpcp_web/src/slots.c +++ b/src/libpcp_web/src/slots.c @@ -182,16 +182,22 @@ redisSlotsInit(dict *config, void *events) return NULL; } - servers = pmIniFileLookup(config, "redis", "servers"); + servers = pmIniFileLookup(config, "keys", "servers"); + if (servers == NULL) + servers = pmIniFileLookup(config, "redis", "servers"); if (servers == NULL) servers = pmIniFileLookup(config, "pmseries", "servers"); if (servers == NULL) servers = def_servers = sdsnew(default_server); - username = pmIniFileLookup(config, "redis", "username"); + username = pmIniFileLookup(config, "keys", "username"); + if (username == NULL) + username = pmIniFileLookup(config, "redis", "username"); if (username == NULL) username = pmIniFileLookup(config, "pmseries", "auth.username"); - password = pmIniFileLookup(config, "redis", "password"); + password = pmIniFileLookup(config, "keys", "password"); + if (password == NULL) + password = pmIniFileLookup(config, "redis", "password"); if (password == NULL) password = pmIniFileLookup(config, "pmseries", "auth.password"); @@ -372,13 +378,14 @@ redisSlotsConnect(dict *config, redisSlotsFlags flags, redisSlots *slots; sds enabled, msg; - enabled = pmIniFileLookup(config, "redis", "enabled"); + if (!(enabled = pmIniFileLookup(config, "resp", "enabled"))) + enabled = pmIniFileLookup(config, "redis", "enabled"); if (enabled && strcmp(enabled, "false") == 0) return NULL; slots = redisSlotsInit(config, events); if (slots == NULL) { - infofmt(msg, "Failed to allocate memory for Redis slots"); + infofmt(msg, "Failed to allocate memory for key server slots"); info(PMLOG_ERROR, msg, arg); sdsfree(msg); return NULL; diff --git a/src/pmproxy/pmproxy.conf b/src/pmproxy/pmproxy.conf index 4cbc1c96af..60efd7dc47 100644 --- a/src/pmproxy/pmproxy.conf +++ b/src/pmproxy/pmproxy.conf @@ -28,16 +28,16 @@ pcp.enabled = true # serve the PCP REST APIs (HTTP) http.enabled = true -# support Redis protocol proxying -redis.enabled = false +# RESP proxying (key server serialisation protocol) +resp.enabled = false # support SSL/TLS protocol wrapping secure.enabled = true ##################################################################### -## settings related to redis-server connections +## settings related to RESP key/value server connections ##################################################################### -[redis] +[keys] # allow completely disabling any/all connection attempts across # all of the pmproxy functionality (when this is set to false). @@ -49,7 +49,7 @@ enabled = true servers = localhost:6379 # authentication with username and password is only supported in -# Redis 6+ (with ACL support), for versions prior to 6 only the +# some versions (with RESP ACL support), for older versions only # password authentication is supported (leave the username value # empty). Commented or empty values disable authentication. #username = @@ -60,7 +60,7 @@ servers = localhost:6379 ##################################################################### [discover] -# propogate archives from pmlogger(1) into Redis for querying +# index pmlogger(1) archive content via key server for querying enabled = true # comma-separated metrics name (globs) to skip during discovery @@ -81,25 +81,25 @@ enabled = true count = 10 ##################################################################### -## settings for fast, scalable time series quering via Redis +## settings for fast, scalable time series quering via a key server ##################################################################### [pmseries] # allow REST API queries of fast, scalable time series enabled = true -# number of elements from scan calls (https://redis.io/commands/scan) +# number of elements from scan calls (https://valkey.io/commands/scan) cursor.count = 256 -# seconds to expire in-core series (https://redis.io/commands/expire) +# seconds to expire in-core series (https://valkey.io/commands/expire) # all metric values of a series (a series represents a specific metric # and host combination) will be removed if there was no update to this # series in the specified amount of time (in seconds) stream.expire = 86400 -# limit number of elements in series (https://redis.io/commands/xadd) -# number of metric values per series (per metric and host) -# this should be retention_time/logging_interval +# limit elements in individual series keys (https://.io/commands/xadd) +# indicates the number of metric values to store per series - i.e. per +# metric and also per host data volumes are considerations here. stream.maxlen = 8640 ##################################################################### diff --git a/src/pmproxy/src/pmproxy.c b/src/pmproxy/src/pmproxy.c index f0ba02d437..2e7c06437c 100644 --- a/src/pmproxy/src/pmproxy.c +++ b/src/pmproxy/src/pmproxy.c @@ -1,7 +1,7 @@ /* - * Copyright (c) 2012-2021 Red Hat. + * Copyright (c) 2012-2021,2024 Red Hat. * Copyright (c) 2002 Silicon Graphics, Inc. All Rights Reserved. - * + * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation; either version 2.1 of the License, or @@ -83,8 +83,10 @@ static pmLongOptions longopts[] = { { "interface", 1, 'i', "ADDR", "accept connections on this IP address" }, { "port", 1, 'p', "PORT", "accept connections on this port" }, { "socket", 1, 's', "PATH", "Unix domain socket file [default $PCP_RUN_DIR/pmproxy.socket]" }, - { "redisport", 1, 'r', "PORT", "Connect to Redis instance on this TCP/IP port (implies --timeseries)" }, - { "redishost", 1, 'h', "HOST", "Connect to Redis instance on this host name (implies --timeseries)" }, + { "keyport", 1, 'r', "PORT", "Connect to key server on this TCP/IP port (implies --timeseries)" }, + { "keyhost", 1, 'h', "HOST", "Connect to key server on this host name (implies --timeseries)" }, + { "redisport", 1, 'r', "PORT", "Backwards-compatibility option, do not use" }, + { "redishost", 1, 'h', "HOST", "Backwards-compatibility option, do not use" }, PMAPI_OPTIONS_HEADER("Diagnostic options"), { "", 1, 'T', "TIME", "terminate after an elapsed time interval" }, { "log", 1, 'l', "PATH", "redirect diagnostics and trace output" }, @@ -104,10 +106,10 @@ ParseOptions(int argc, char *argv[], int *nports, int *maxpending) int sts; int usage = 0; int timeseries = 1; - int redis_port = 6379; - char *redis_host = NULL; - char *endnum; + int keys_port = 6379; + char *keys_host = NULL; const char *inifile = NULL; + char *endnum; sds option; while ((c = pmgetopt_r(argc, argv, &opts)) != EOF) { @@ -154,8 +156,8 @@ ParseOptions(int argc, char *argv[], int *nports, int *maxpending) run_mode = RUN_FOREGROUND; break; - case 'h': /* Redis host name */ - redis_host = opts.optarg; + case 'h': /* key server host name */ + keys_host = opts.optarg; timeseries = 1; break; @@ -192,9 +194,9 @@ ParseOptions(int argc, char *argv[], int *nports, int *maxpending) __pmServerSetFeature(PM_SERVER_FEATURE_CERT_REQD); break; - case 'r': /* Redis port number */ - redis_port = (int)strtol(opts.optarg, NULL, 0); - if (redis_port <= 0) { + case 'r': /* key server port number */ + keys_port = (int)strtol(opts.optarg, NULL, 0); + if (keys_port <= 0) { pmprintf("%s: -r requires a positive value\n", pmGetProgname()); opts.errors++; } @@ -253,8 +255,6 @@ ParseOptions(int argc, char *argv[], int *nports, int *maxpending) pmGetProgname(), inifile? inifile : "pmproxy.conf"); opts.errors++; } else { - int fallback = 0; - /* Extract pmproxy configuration information needed immediately */ if ((option = pmIniFileLookup(config, "pmproxy", "maxpending"))) *maxpending = atoi(option); @@ -263,17 +263,14 @@ ParseOptions(int argc, char *argv[], int *nports, int *maxpending) * Push command line options into the configuration, and ensure * we have some default for attemping Redis server connections. */ - if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) { - if ((option = pmIniFileLookup(config, "pmseries", "servers"))) - fallback = 1; + if ((option = pmIniFileLookup(config, "keys", "servers")) == NULL) { + if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) + option = pmIniFileLookup(config, "pmseries", "servers"); } - if (option == NULL || redis_host != NULL || redis_port != 6379) { + if (option == NULL || keys_host != NULL || keys_port != 6379) { option = sdscatfmt(sdsempty(), "%s:%u", - redis_host? redis_host : "localhost", redis_port); - if (!fallback) - pmIniFileUpdate(config, "redis", "servers", option); - else - pmIniFileUpdate(config, "pmseries", "servers", option); + keys_host? keys_host : "localhost", keys_port); + pmIniFileUpdate(config, "keys", "servers", option); } } diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index ebc8dc4e38..2b553cd627 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -334,8 +334,8 @@ main(int argc, char *argv[]) unsigned int search_count = 0; unsigned int search_offset = 0; const char *inifile = NULL; - const char *redis_host = NULL; - unsigned int redis_port = 6379; /* default Redis port */ + const char *keys_host = NULL; + unsigned int keys_port = 6379; /* default key server port */ search_flags flags = 0; search_data *dp; struct dict *config; @@ -355,8 +355,8 @@ main(int argc, char *argv[]) flags |= PMSEARCH_DOCIDS; break; - case 'h': /* Redis host to connect to */ - redis_host = opts.optarg; + case 'h': /* key server host to connect to */ + keys_host = opts.optarg; break; case 'i': /* report search engine info (metrics) */ @@ -375,8 +375,8 @@ main(int argc, char *argv[]) search_offset = strtoul(opts.optarg, NULL, 10); break; - case 'p': /* Redis port to connect to */ - redis_port = (unsigned int)strtol(opts.optarg, NULL, 10); + case 'p': /* keys server port to connect to */ + keys_port = (unsigned int)strtol(opts.optarg, NULL, 10); break; case 'q': /* command line contains query string */ @@ -428,12 +428,13 @@ main(int argc, char *argv[]) pmIniFileUpdate(config, "pmsearch", "count", option); } - if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) - option = pmIniFileLookup(config, "pmseries", "servers"); - if (option == NULL || redis_host != NULL || redis_port != 6379) { + if ((option = pmIniFileLookup(config, "keys", "servers")) == NULL) + if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) + option = pmIniFileLookup(config, "pmseries", "servers"); + if (option == NULL || keys_host != NULL || keys_port != 6379) { option = sdscatfmt(sdsempty(), "%s:%u", - redis_host? redis_host : "localhost", redis_port); - pmIniFileUpdate(config, "redis", "servers", option); + keys_host? keys_host : "localhost", keys_port); + pmIniFileUpdate(config, "keys", "servers", option); } } diff --git a/src/pmseries/pmseries.c b/src/pmseries/pmseries.c index f648bdfe19..dfc54efafb 100644 --- a/src/pmseries/pmseries.c +++ b/src/pmseries/pmseries.c @@ -871,8 +871,8 @@ on_series_done(int sts, void *arg) arg = entry->arg; func(dp, arg); } else { - /* we're in the middle of an Redis async callback, - schedule freeing the Redis context for later */ + /* we're in the middle of an async callback, + schedule freeing the context for later */ if (!(dp->flags & PMSERIES_NEED_CLOSE)) { dp->flags |= PMSERIES_NEED_CLOSE; pmseries_schedule_close(dp); @@ -1227,8 +1227,8 @@ issid(const char *string) static pmLongOptions longopts[] = { PMAPI_OPTIONS_HEADER("Connection Options"), { "config", 1, 'c', "FILE", "configuration file path"}, - { "host", 1, 'h', "HOST", "connect to Redis using given host name" }, - { "port", 1, 'p', "PORT", "connect to Redis using given TCP/IP port" }, + { "host", 1, 'h', "HOST", "connect to key server using given host name" }, + { "port", 1, 'p', "PORT", "connect to key server using given TCP/IP port" }, PMAPI_OPTIONS_HEADER("General Options"), { "load", 0, 'L', 0, "load time series values and metadata" }, { "query", 0, 'q', 0, "perform a time series query (default)" }, @@ -1271,9 +1271,9 @@ main(int argc, char *argv[]) const char *split = ","; const char *space = " "; const char *inifile = NULL; - const char *redis_host = NULL; + const char *keys_host = NULL; static char tzbuffer[128]; - unsigned int redis_port = 6379; /* default Redis port */ + unsigned int keys_port = 6379; /* default RESP port */ struct dict *config; series_flags flags = 0; series_data *dp; @@ -1310,8 +1310,8 @@ main(int argc, char *argv[]) match = sdsnew(opts.optarg); break; - case 'h': /* Redis host to connect to */ - redis_host = opts.optarg; + case 'h': /* key server to connect to */ + keys_host = opts.optarg; break; case 'i': /* command line contains series identifiers */ @@ -1343,8 +1343,8 @@ main(int argc, char *argv[]) flags |= (PMSERIES_OPT_LABELS|PMSERIES_ONLY_NAMES); break; - case 'p': /* Redis port to connect to */ - redis_port = (unsigned int)strtol(opts.optarg, NULL, 10); + case 'p': /* key server port to connect to */ + keys_port = (unsigned int)strtol(opts.optarg, NULL, 10); break; case 'q': /* command line contains query string */ @@ -1399,14 +1399,15 @@ main(int argc, char *argv[]) } else { /* * Push command line options into the configuration, and ensure - * we have some default for attemping Redis server connections. + * we have some default for attemping key server connections. */ - if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) - option = pmIniFileLookup(config, "pmseries", "servers"); - if (option == NULL || redis_host != NULL || redis_port != 6379) { + if ((option = pmIniFileLookup(config, "keys", "servers")) == NULL) + if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) + option = pmIniFileLookup(config, "pmseries", "servers"); + if (option == NULL || keys_host != NULL || keys_port != 6379) { option = sdscatfmt(sdsempty(), "%s:%u", - redis_host? redis_host : "localhost", redis_port); - pmIniFileUpdate(config, "redis", "servers", option); + keys_host? keys_host : "localhost", keys_port); + pmIniFileUpdate(config, "keys", "servers", option); } } From 6c48d19f92195e27999849391d65a3c6ed9aca8c Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 30 Apr 2024 12:42:10 +1000 Subject: [PATCH 2/5] docs: man page updates transitioning away from Redis With Redis no longer open source, there's a transition away from it in all Linux distributions now underway. Here we switch all man page references to Redis over to a more general 'key server' naming convention. --- man/man1/pcpcompat.1 | 9 +++--- man/man1/pmproxy.1 | 57 ++++++++++++++-------------------- man/man1/pmsearch.1 | 31 +++++++++--------- man/man1/pmseries.1 | 24 +++++++------- man/man3/pmdiscoversetup.3 | 12 ++----- man/man3/pmsearchinfo.3 | 5 ++- man/man3/pmsearchsetup.3 | 3 +- man/man3/pmsearchtextindom.3 | 1 - man/man3/pmsearchtextquery.3 | 1 - man/man3/pmsearchtextsuggest.3 | 1 - man/man3/pmseriessetup.3 | 3 +- man/man3/pmwebapi.3 | 16 +++------- man/man5/labels.conf.5 | 5 ++- 13 files changed, 67 insertions(+), 101 deletions(-) diff --git a/man/man1/pcpcompat.1 b/man/man1/pcpcompat.1 index cde55237c7..9a87bed619 100644 --- a/man/man1/pcpcompat.1 +++ b/man/man1/pcpcompat.1 @@ -98,7 +98,7 @@ as well as the encrypted PCP protocol functionality between and .B PMAPI client tools. -Additionally, the Redis service used by +Additionally, the key-value server service used by .BR pmseries (1) and .B pmproxy @@ -201,7 +201,7 @@ as well as data sources for .BR bpftrace (8) scripts) and .BR pmseries (1) -(fast, scalable Redis-based time series analysis). +(fast, scalable, distributed time series analysis). .SH PMWEBD The .B pmwebd @@ -225,8 +225,8 @@ REST APIs (such as derived metrics, namespace lookups and instance domain profiles), support for the HTTPS protocol, and fast, scalable time series querying using the .BR pmseries (1) -REST API and -.BR redis-server (1). +REST API with a distributed key-value server such as +.BR https://valkey.io/ . .PP The partial Graphite API emulation provided by .B pmwebd @@ -246,7 +246,6 @@ services could use the scalable time series REST APIs described on .BR pmseries (1), .BR pmdabpftrace (1), .BR python (1), -.BR redis-server (1), .BR grafana-server (1), .BR mmv_stats_register (3), .BR LOGIMPORT (3), diff --git a/man/man1/pmproxy.1 b/man/man1/pmproxy.1 index 0b3b125349..f2f831b6ec 100644 --- a/man/man1/pmproxy.1 +++ b/man/man1/pmproxy.1 @@ -36,10 +36,9 @@ acts as a protocol proxy, allowing Performance Co-Pilot (PCP) monitoring clients to connect to one or more .BR pmcd (1) -and/or -.BR redis-server (1) -instances via -.BR pmproxy . +and/or key-value servers (such as \c +.BR https://valkey.io/ ) +indirectly. .PP In its default mode of operation .B pmproxy @@ -54,9 +53,8 @@ real-time access to PCP metrics through the .I /pmapi interfaces, and access to the fast, scalable PCP time series query -capabilities offered in conjunction with a -.BR redis-server (1) -(see +capabilities offered in conjunction with a key-value +server (see .BR pmseries (1) for details) via the .I /query @@ -66,8 +64,8 @@ REST interfaces. can be deployed in a firewall domain, or on a cluster ``head'' node where the IP (Internet Protocol) address of the hosts where .B pmcd -and/or -.B redis-server +and/or a key-value server (such as +.BR https://valkey.io/ ) are running may be unknown to the PCP monitoring clients, but where the IP address of the host running .B pmproxy @@ -78,9 +76,7 @@ host where is running, while there is network connectivity from that host to the hosts of interest where .B pmcd -and/or -.B redis-server -are running. +and/or a key-value server are running. .PP The behaviour of the PCP monitoring clients is controlled by either the .B PMPROXY_HOST @@ -99,9 +95,7 @@ where .B pmproxy is running, and the clients will connect to .B pmcd -or -.B redis-server -indirectly through the protocol proxy services of +or a key-value server indirectly through the protocol proxy services of .BR pmproxy. .SH OPTIONS The available command line options are: @@ -172,8 +166,8 @@ and may be specified. .RE .TP -\f3\-h\f1 \f2host\f1, \f3\-\-redishost\f1=\f2host\f1 -Specify an alternate Redis +\f3\-h\f1 \f2host\f1, \f3\-\-keyhost\f1=\f2host\f1 +Specify an alternate key-value server .I host to connect to for time series querying, overriding any configuration file settings. @@ -235,8 +229,8 @@ Specify an alternate number to listen on for client connections. The default value is 44322. .TP -\f3\-r\f1 \f2port\f1, \f3\-\-redisport\f1=\f2port\f1 -Specify an alternate Redis +\f3\-r\f1 \f2port\f1, \f3\-\-keyport\f1=\f2port\f1 +Specify an alternate key-value server .I port number to connect to for time series querying, overriding any configuration file settings. @@ -256,8 +250,8 @@ This mode of operation will enable the REST APIs, dynamically and automatically detect active system archives being written by .BR pmlogger (1) -and import them into a -.BR redis-server (1), +and import them into a key-value server (such as +.BR https://valkey.io/ ), for fast, scalable time series querying described in .BR pmseries (1). Note that in this mode of operation, @@ -268,8 +262,7 @@ instances. When an archive is first discovered (usually but not limited to .B pmproxy startup), -all metadata is loaded and sent to the configured -.BR redis-server (1) +all metadata is loaded and sent to the configured key-value server however note that only .B new archive metric value data from the tail end of each archive is ingested. @@ -320,22 +313,19 @@ section can be used to explicitly enable or disable each of the different protocols. .PP The -.I [redis] +.I [keys] section allows connection information for one or more backing -.B redis-server -processes to be configured (hostnames and ports). -Note to access multiple (scalable) Redis servers, the +key-value server processes to be configured (hostnames and ports). +Note to access multiple (scalable) key-value servers, the .I servers variable in this section can be a comma-separated list of hostname:port pairs. -Alternatively, it can be a single -.B redis-server -host that will be queried using the "CLUSTER INFO" command to -automatically configure multiple backing hosts, described at -.BR https://redis.io/topics/cluster-spec . +Alternatively, it can be a single key-value server host +that will be queried using the "CLUSTER INFO" command to +automatically configure multiple backing hosts. .PP In earlier versions of PCP (before 6) an alternative configuration -setting section was used for this purpose \- Redis +setting section was used for this purpose \- key-value .I servers were specified in the .I [pmseries] @@ -534,7 +524,6 @@ For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). .BR pmdbg (1), .BR pmlogger (1), .BR pmseries (1), -.BR redis-server (1), .BR PMAPI (3), .BR PMWEBAPI (3), .BR pmGetOptions (3), diff --git a/man/man1/pmsearch.1 b/man/man1/pmsearch.1 index 763092512d..84359d1fd5 100644 --- a/man/man1/pmsearch.1 +++ b/man/man1/pmsearch.1 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2024 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -42,22 +42,21 @@ and help text from metrics, instance domains and instances. .PP It makes use of capabilities of the Performance Co-Pilot (PCP) .BR pmproxy (1) -service, the Redis distributed data store, and the RediSearch -module. +service, the Valkey distributed key-value store and associated +ValkeySearch module. .PP Note that in order to use these services, it is .B mandatory that .B pmproxy -is communicating with a -.BR redis-server (1) +is communicating with a Valkey key-value server that has the -.I redisearch.so +.I valkey-search.so module loaded. When configured to do so, .B pmproxy will then automatically index PCP metric names, instance names, -metric and instance domain help text into the RediSearch store, +metric and instance domain help text into the ValkeySearch store, from PCP archives that it discovers locally. Refer to .BR pmlogger (1) @@ -67,11 +66,10 @@ for further details. .PP By default .B pmsearch -communicates with a local -.BR redis-server (1), +communicates with a local key-value server however the \fB\-h\fR and \fB\-p\fR options can be used to -specify an alternate Redis instance. -If this instance is a node of a Redis cluster, all +specify an alternate Valket instance. +If this instance is a node of a Valkey cluster, all other instances in the cluster will be discovered and used automatically. .SH OPTIONS @@ -90,7 +88,7 @@ Documents are identified using a unique SHA-1 hash which is always displayed in a 40-hexdigit human readable form. .TP \fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR -Connect to the Redis server at +Connect to the key-value server at .IR host , rather than the one the localhost. .TP @@ -112,7 +110,7 @@ search results at most. Paginated results from given offset. .TP \fB\-p\fR \fIport\fR, \fB\-\-port\fR=\fIport\fR -Connect to the Redis server at +Connect to the key-value server at .IR port , rather than the default .BR 6379 . @@ -165,11 +163,10 @@ For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). .SH SEE ALSO .BR PCPIntro (1), .BR pmproxy (1), -.BR pmlogger (1), -.BR pmlogger_daily (1) +.BR pmlogger (1) and -.BR redis-server (1). +.BR pmlogger_daily (1). .\" control lines for scripts/man-spell -.\" +ok+ RediSearch SHA hexdigit idletime redisearch +.\" +ok+ SHA Valkey ValkeySearch hexdigit idletime .\" +ok+ stat {from /proc/stat} diff --git a/man/man1/pmseries.1 b/man/man1/pmseries.1 index bce7769936..fab2fc09a8 100644 --- a/man/man1/pmseries.1 +++ b/man/man1/pmseries.1 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2018-2020,2022-2023 Red Hat. +.\" Copyright (c) 2018-2020,2022-2024 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -40,23 +40,24 @@ .B pmseries displays various types of information about performance metrics available through the scalable timeseries facilities of the -Performance Co-Pilot (PCP) and the Redis distributed data store. +Performance Co-Pilot (PCP) and a distributed key-value data store +such as +.BR https://valkey/io/ .. .PP By default .B pmseries -communicates with a local -.BR redis-server (1), +communicates with a local key-value server however the \fB\-h\fR and \fB\-p\fR options can be used to -specify an alternate Redis instance. -If this instance is a node of a Redis cluster, all +specify an alternate key-value server. +If this instance is a node within a cluster, all other instances in the cluster will be discovered and used automatically. .PP .B pmseries runs in several different modes \- either querying timeseries identifiers, metadata or values (already -stored in Redis), or manually loading timeseries -into Redis. +stored in the key-value store), or manually loading +timeseries. The latter mode is generally only used for loading previously collected (inactive) archives, since @@ -739,15 +740,15 @@ Specify the file to use. .TP \fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR -Connect Redis server at +Connect to the key-value server at .IR host , rather than the one the localhost. .TP \fB\-L\fR, \fB\-\-load\fR -Load timeseries metadata and data into the Redis cluster. +Load timeseries metadata and data into the key-value cluster. .TP \fB\-p\fR \fIport\fR, \fB\-\-port\fR=\fIport\fR -Connect Redis server at +Connect to the key-value server at .IR port , rather than the default \fB6379\fR. .TP @@ -846,7 +847,6 @@ For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). .BR pmcd (1), .BR pminfo (1), .BR pmproxy (1), -.BR redis-server (1), .BR access (2), .BR PMAPI (3), .BR PMWEBAPI (3), diff --git a/man/man3/pmdiscoversetup.3 b/man/man3/pmdiscoversetup.3 index 16de0c0694..bdd35756ff 100644 --- a/man/man3/pmdiscoversetup.3 +++ b/man/man3/pmdiscoversetup.3 @@ -77,9 +77,7 @@ are created, written to, compressed and eventually deleted. The underlying archive life-cycle is normally managed by the PCP archive management infrastructure (see .BR pmlogger_daily (1)). Discovered archives that are active (currently being written) are "log-tailed" to extract near live/real-time -performance data which is then passed off via registered callbacks for further processing, e.g. to add the data to a -.BR redis-server (1) -instance. +performance data which is then passed off via registered callbacks for further processing, e.g. to add the data to a key-value server. Archives that are compressed or inactive are tracked/monitored but .B not log-tailed - this is because compressed archives never grow and so log-tailing @@ -89,9 +87,7 @@ See the option in .BR pmseries (1) for a supported mechanism for ingesting previously collected (inctive) -PCP archives, whether compressed or not, into a -.BR redis-server (1) -instance. +PCP archives, whether compressed or not, into a key-value server. .P The .B pmDiscover @@ -157,9 +153,7 @@ and as new data is written to active archive volume and metadata files. In the case of .BR pmproxy (1), callbacks are registered to capture performance data, which is then sent to -.BR redis-server (1) -as the database back-end for fast scalable time-series queries by clients -of the +a key-value server for fast scalable time-series queries by clients of the .BR PMWEBAPI (3) REST API services. Such clients include the diff --git a/man/man3/pmsearchinfo.3 b/man/man3/pmsearchinfo.3 index 6ea5dd6524..d71e64ac97 100644 --- a/man/man3/pmsearchinfo.3 +++ b/man/man3/pmsearchinfo.3 @@ -27,7 +27,7 @@ cc ... \-lpcp_web .SH DESCRIPTION Diagnostic metrics related to the Performance Co-Pilot (PCP) fast, scalable and full-text capable search services. .PP -Currently, as all full-text related services are using single RediSearch index, diagnostic metrics are requestable by passing in +Currently, as all full-text related services are using single ValkeySearch index, diagnostic metrics are requestable by passing in .IR key with a value of .BR `text' . @@ -42,11 +42,10 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmsearch (1), -.BR redis-server (1), .BR pmSearchSetup (3), .BR PMAPI (3) and .BR PMWEBAPI (3). .\" control lines for scripts/man-spell -.\" +ok+ RediSearch on_metrics {from callback} +.\" +ok+ ValkeySearch on_metrics {from callback} .\" +ok+ sds {from typedef char *sds in pcp/sds.h} diff --git a/man/man3/pmsearchsetup.3 b/man/man3/pmsearchsetup.3 index 66ac7f0e7d..3ffec9cf29 100644 --- a/man/man3/pmsearchsetup.3 +++ b/man/man3/pmsearchsetup.3 @@ -105,7 +105,7 @@ failure (negative PMAPI code) of the operation. .PP The helper functions .B pmSearchSetSlots -(redis), +(key-value server), .B pmSearchSetEventLoop (libuv), .B pmSearchSetConfiguration @@ -134,7 +134,6 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmsearch (1), -.BR redis-server (1), .BR mmv_stats_registry (3), .BR pmSearchInfo (3), .BR pmSearchTextInDom (3), diff --git a/man/man3/pmsearchtextindom.3 b/man/man3/pmsearchtextindom.3 index 954bcfac6a..ff4d8aa2b7 100644 --- a/man/man3/pmsearchtextindom.3 +++ b/man/man3/pmsearchtextindom.3 @@ -52,7 +52,6 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmsearch (1), -.BR redis-server (1), .BR pmSearchSetup (3), .BR PMAPI (3) and diff --git a/man/man3/pmsearchtextquery.3 b/man/man3/pmsearchtextquery.3 index 70e7f46d42..c424b4cf9d 100644 --- a/man/man3/pmsearchtextquery.3 +++ b/man/man3/pmsearchtextquery.3 @@ -62,7 +62,6 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmsearch (1), -.BR redis-server (1), .BR pmSearchSetup (3), .BR PMAPI (3) and diff --git a/man/man3/pmsearchtextsuggest.3 b/man/man3/pmsearchtextsuggest.3 index 69f2ff3561..307895db6a 100644 --- a/man/man3/pmsearchtextsuggest.3 +++ b/man/man3/pmsearchtextsuggest.3 @@ -53,7 +53,6 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmsearch (1), -.BR redis-server (1), .BR pmSearchSetup (3), .BR PMAPI (3) and diff --git a/man/man3/pmseriessetup.3 b/man/man3/pmseriessetup.3 index 2e1569f671..46f39359e4 100644 --- a/man/man3/pmseriessetup.3 +++ b/man/man3/pmseriessetup.3 @@ -158,7 +158,7 @@ failure (negative PMAPI code) of the operation. .PP The helper functions .B pmSeriesSetSlots -(redis), +(key-value server), .B pmSeriesSetEventLoop (libuv), .B pmSeriesSetConfiguration @@ -189,7 +189,6 @@ On failure a negative PMAPI error code is returned. .BR pmproxy (1), .BR pmlogger (1), .BR pmseries (1), -.BR redis-server (1), .BR mmv_stats_registry (3), .BR pmSeriesDescs (3), .BR pmSeriesQuery (3), diff --git a/man/man3/pmwebapi.3 b/man/man3/pmwebapi.3 index b6177faee2..01accff53e 100644 --- a/man/man3/pmwebapi.3 +++ b/man/man3/pmwebapi.3 @@ -74,8 +74,7 @@ provided by default. The examples in the scalable time series section use historical data recorded by the .BR pmlogger (1) -service, in conjunction with a local -.BR redis-server (1). +service, in conjunction with a local key-value server/ .SH OPEN METRICS Exporting of live performance metrics in an Open Metrics compatible format (as described at @@ -530,9 +529,7 @@ client string Request identifier sent back with response .TE .PP Load time series performance data from the specified source -into the -.BR redis-server -cache. +into the key-value server cache. This request is equivalent to the \fBpmseries\fR \-\fBl\fR/\-\-\fBload\fR option. .SAMPLE @@ -552,10 +549,8 @@ suitable for a metric search engine, and in a fashion suited to efficient querying by any number of web applications. .PP In order to use this functionality, the optional -.I RediSearch -module must be loaded in the -.B redis-server -at the time +.I ValkeySearch +module must be loaded in the key-value server at the time .B pmproxy is started, such that metrics, instances and help text it discovers can be automatically indexed. @@ -1238,7 +1233,6 @@ header is added to all REST API responses. .BR pmlogger (1), .BR pmproxy (1), .BR pmseries (1), -.BR redis-server (1), .BR PCPIntro (3) and .BR PMAPI (3) @@ -1248,7 +1242,7 @@ and .\" +ok+ offsets_per_term_avg bytes_per_record_avg score_index_size_mb .\" +ok+ records_per_doc_avg skip_index_size_mb filesys_blocksize .\" +ok+ inverted_cap_ovh inverted_cap_mb inverted_sz_mb halt_wakeup -.\" +ok+ polltimeout proc_nprocs halt_exits read_bytes read_merge RediSearch +.\" +ok+ polltimeout proc_nprocs halt_exits read_bytes read_merge ValkeySearch .\" +ok+ domainname appversion prometheus machineid blocksize .\" +ok+ buddyinfo cebcbadc helptext slabinfo blktotal instname .\" +ok+ hostspec avactive wakeups pernode physmem filesys blkread oneline diff --git a/man/man5/labels.conf.5 b/man/man5/labels.conf.5 index c4aa085a42..6fd42d30cd 100644 --- a/man/man5/labels.conf.5 +++ b/man/man5/labels.conf.5 @@ -35,8 +35,8 @@ respectively providing the labels in real-time and then discovering recorded (archive) labels and indexing these for .BR pmseries (1) -queries using a -.BR redis-server (1). +queries using a distributed key-value service such as +.BR https://valkey.io/ . .PP The .IR /etc/pcp/labels.conf @@ -66,7 +66,6 @@ hash of the file. .BR pmcd (1), .BR pmproxy (1), .BR pmseries (1), -.BR redis-server (1), .BR PMAPI (3), .BR pmLookupLabels (3) and From a1531b4bb165f8a1c3a4cab213bbb5bf5d2ce984 Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 30 Apr 2024 15:56:05 +1000 Subject: [PATCH 3/5] docs: update the books to refer to key servers more generically --- docs/HowTos/scaling/decentralized.svg | 6 +- .../scaling/federated-pmlogger-farm.svg | 6 +- docs/HowTos/scaling/index.rst | 56 +++++++++---------- docs/HowTos/scaling/pmlogger-farm.svg | 6 +- docs/QG/EncryptedConnections.rst | 2 +- docs/QG/QuickReferenceGuide.rst | 14 ++--- docs/UAG/IntroductionToPcp.rst | 30 +++++----- docs/UAG/TimeSeriesQuerying.rst | 21 +++---- docs/refs.rst | 2 +- docs/specs/openapi.yaml | 6 +- src/external/README | 2 +- 11 files changed, 76 insertions(+), 75 deletions(-) diff --git a/docs/HowTos/scaling/decentralized.svg b/docs/HowTos/scaling/decentralized.svg index 00492578b5..28daae8952 100644 --- a/docs/HowTos/scaling/decentralized.svg +++ b/docs/HowTos/scaling/decentralized.svg @@ -587,7 +587,7 @@ Redis Instance + style="display: inline-block; font-size: 10px; font-family: sans-serif; color: #000000; line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; word-wrap: normal; ">Key Server @@ -603,11 +603,11 @@ sodipodi:role="line" id="tspan669" x="565" - y="291">RedisKeyInstance + y="303.5">Server Redis Instance + style="display: inline-block; font-size: 10px; font-family: sans-serif; color: #000000; line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; word-wrap: normal; ">Key Server @@ -534,11 +534,11 @@ sodipodi:role="line" id="tspan1332" x="845" - y="511">RedisKeyInstance + y="523.5">Server `_ for more details). -Another viable option is to deploy a Redis cluster in the cloud, or to utilize a managed Redis cluster from a cloud vendor. +The default deployment is a standalone Valkey key server on the localhost. +However, key servers can optionally run in a highly-available and highly scalable *clustered* fashion, where data is sharded across multiple hosts. +Another option is to deploy a key server cluster in the cloud, or to utilize a managed cluster from a cloud vendor. .. note:: @@ -120,10 +120,10 @@ To control the logging interval, update the control file located at ``/etc/pcp/p To configure which metrics should be logged, run ``pmlogconf /var/lib/pcp/config/pmlogger/``. To specify retention settings, i.e. when to purge old PCP archives, update the ``/etc/sysconfig/pmlogger_timers`` file and specify ``PMLOGGER_DAILY_PARAMS="-E -k X"``, where ``X`` is the amount of days to keep PCP archives. -Redis ------ +Key Server +---------- -The `pmproxy(1)`_ daemon sends logged metrics from `pmlogger(1)`_ to a Redis instance. +The `pmproxy(1)`_ daemon sends logged metrics from `pmlogger(1)`_ to a key server such as `Valkey `_. To update the logging interval or the logged metrics, see the section above. Two options are available to specify the retention settings in the pmproxy configuration file located at ``/etc/pcp/pmproxy/pmproxy.conf``: @@ -139,13 +139,13 @@ Centralized logging (pmlogger farm) The following results were gathered on a :ref:`pmlogger farm` deployment, with a default **pcp-zeroconf 5.3.2** installation (version 5.3.1-3 on RHEL), where each remote host is an identical container instance running `pmcd(1)`_ on a server with 64 CPU cores, 376 GB RAM and 1 disk attached (as mentioned above, 64 CPUs increases per-CPU metric volume). -The Redis server is co-located on the same host as pmlogger and pmproxy, and ``proc`` metrics of remote nodes are *not* included. +The key server is co-located on the same host as pmlogger and pmproxy, and ``proc`` metrics of remote nodes are *not* included. The memory values refer to the RSS (Resident Set Size) value. **10s logging interval:** +-----------+----------------+----------+------------------+---------+--------------+---------+-----------+-----------------+-------------+ -| Number of | PCP Archives | pmlogger | pmlogger Network | pmproxy | Redis Memory | pmproxy | Disk IOPS | Disk Throughput | Disk | +| Number of | PCP Archives | pmlogger | pmlogger Network | pmproxy | keys Memory | pmproxy | Disk IOPS | Disk Throughput | Disk | | | | | | | | | | | | | Hosts | Storage p. Day | Memory | per Day (In) | Memory | per Day | CPU% | (write) | (write) | Utilization | +===========+================+==========+==================+=========+==============+=========+===========+=================+=============+ @@ -157,7 +157,7 @@ The memory values refer to the RSS (Resident Set Size) value. **60s logging interval:** +-----------+----------------+----------+------------------+---------+--------------+ -| Number of | PCP Archives | pmlogger | pmlogger Network | pmproxy | Redis Memory | +| Number of | PCP Archives | pmlogger | pmlogger Network | pmproxy | keys Memory | | | | | | | | | Hosts | Storage p. Day | Memory | per Day (In) | Memory | per Day | +===========+================+==========+==================+=========+==============+ @@ -168,7 +168,7 @@ The memory values refer to the RSS (Resident Set Size) value. | 100 | 271 MB | 1049 MB | 3.48 MB | 9 GB | 5.3 GB | +-----------+----------------+----------+------------------+---------+--------------+ -**Note:** pmproxy queues Redis requests and employs Redis pipelining to speed up Redis queries. +**Note:** pmproxy queues key server requests and employs pipelining to speed up queries. This can result in bursts of high memory usage. There are plans to optimize memory usage in future versions of PCP (`#1341 `_). For further troubleshooting, please see the `High memory usage`_ section in the troubleshooting chapter. @@ -179,19 +179,19 @@ Federated setup (multiple pmlogger farms) The following results were observed with a :ref:`federated setup` consisting of three :ref:`pmlogger farms`, where each pmlogger farm was monitoring 100 remote hosts, i.e. 300 hosts in total. -The setup of the pmlogger farms was identical to the configuration above (60s logging interval), except that the Redis servers were operating in cluster mode. +The setup of the pmlogger farms was identical to the configuration above (60s logging interval), except that the key servers were operating in cluster mode. -+----------------+----------+-------------------+---------+--------------+ -| PCP Archives | pmlogger | Network | pmproxy | Redis Memory | -| | | | | | -| Storage p. Day | Memory | per Day (In/Out) | Memory | per Day | -+================+==========+===================+=========+==============+ -| 277 MB | 1058 MB | 15.6 MB / 12.3 MB | 6-8 GB | 5.5 GB | -+----------------+----------+-------------------+---------+--------------+ ++----------------+----------+-------------------+---------+-------------+ +| PCP Archives | pmlogger | Network | pmproxy | keys Memory | +| | | | | | +| Storage p. Day | Memory | per Day (In/Out) | Memory | per Day | ++================+==========+===================+=========+=============+ +| 277 MB | 1058 MB | 15.6 MB / 12.3 MB | 6-8 GB | 5.5 GB | ++----------------+----------+-------------------+---------+-------------+ **Note:** All values are per host. -The network bandwidth is higher due to the inter-node communication of the Redis cluster. +The network bandwidth is higher due to the inter-node communication of the key server cluster. Troubleshooting *************** @@ -200,8 +200,8 @@ High memory usage ----------------- To troubleshoot high memory usage, please run ``pmrep :pmproxy`` and observe the *inflight* column. -This column shows how many Redis requests are in-flight, i.e. they are queued (or sent) and no reply was received so far. -A high number indicates that a) the pmproxy process is busy processing new PCP archives and doesn't have spare CPU cycles to process Redis requests and responses or b) the Redis node (or cluster) is overloaded and cannot process incoming requests on time. +This column shows how many key server requests are in-flight, i.e. they are queued (or sent) and no reply was received so far. +A high number indicates that a) the pmproxy process is busy processing new PCP archives and doesn't have spare CPU cycles to process key server requests and responses or b) the key server (or cluster) is overloaded and cannot process incoming requests on time. .. note:: diff --git a/docs/HowTos/scaling/pmlogger-farm.svg b/docs/HowTos/scaling/pmlogger-farm.svg index deecd2b40c..94bd3885e2 100644 --- a/docs/HowTos/scaling/pmlogger-farm.svg +++ b/docs/HowTos/scaling/pmlogger-farm.svg @@ -494,7 +494,7 @@ Redis Instance + style="display: inline-block; font-size: 10px; font-family: sans-serif; color: #000000; line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; word-wrap: normal; ">Key Server @@ -510,11 +510,11 @@ sodipodi:role="line" id="tspan1328" x="769" - y="205">RedisKeyInstance + id="tspan667">Server `__) -works with local pmlogger and `Redis `__ servers to -allow fast, scalable performance queries spanning multiple hosts. +works with pmlogger and key servers such as `Valkey `__ +to allow fast, scalable performance queries spanning multiple hosts. +-----------------------------------------------------------------------+ | To enable and start metrics series collection:: | | | -| # systemctl enable --now pmlogger pmproxy redis | +| # systemctl enable --now pmlogger pmproxy valkey | +-----------------------------------------------------------------------+ -Redis can be run standalone or in large, highly available setups. It is -also provided as a scalable service by many cloud vendors. +Key servers can be run standalone or in large, highly available setups. +They are also provided as a scalable service by many cloud vendors. The metrics indexing process is designed to spread data across multiple -Redis nodes for improved query performance, so adding more nodes can +key servers for improved query performance, so adding more servers can significantly improve performance. Examples of the @@ -622,7 +622,7 @@ accessing PCP performance metrics over HTTP. +-----------------------------------------------------------------------+ | To install the PCP REST APIs service:: | | | -| # systemctl enable --now pmproxy redis | +| # systemctl enable --now pmproxy valkey | +-----------------------------------------------------------------------+ After installing the PCP REST API services as described above, install diff --git a/docs/UAG/IntroductionToPcp.rst b/docs/UAG/IntroductionToPcp.rst index f2252ff233..a0a7e70e71 100644 --- a/docs/UAG/IntroductionToPcp.rst +++ b/docs/UAG/IntroductionToPcp.rst @@ -313,15 +313,23 @@ The following tools provide the principal services for the PCP end-user with an **pcp-atop** -Full-screen monitor of the load on a system from a kernel, hardware and processes point of view. It is modeled on the Linux **atop(1)** tool (`home page `_) and provides a showcase for the variety of data available using PCP services and the Python scripting interfaces. +Full-screen monitor of the load on a system from a kernel, hardware and processes point of view. It is modeled on the Linux **atop(1)** tool (`home page `_) and supports both live and historical analysis. + +**pcp-dstat** + +Line-based analysis tool, showing arbitrary metrics through a configurable plugin system. This tool was created to continue the original **dstat(1)** utility (`home page `_) and provides a showcase for the variety of data available using PCP services and the Python scripting interfaces. It also supports both live and historical analysis. + +**pcp-htop** + +Beautiful, classic, live monitor of system activity. It is part of the **htop(1)** ecosystem (`home page `_) and allows any PCP metric to be displayed through a configurable system of plugins. **pmchart** Strip chart tool for arbitrary performance metrics. Interactive graphical utility that can display multiple charts simultaneously, from multiple hosts or set of archives, aligned on a unified time axis (X-axis), or on multiple tabs. -**pcp-collectl** +**pmcheck** -Statistics collection tool with good coverage of a number of Linux kernel subsystems, with the everything-in-one-tool approach pioneered by **sar(1)**. It is modeled on the Linux **collectl(1)** utility (`home page `_) and provides another example of use of the Python scripting interfaces to build more complex functionality with relative ease, with PCP as a foundation. +Probe a system and report on possible missing configuration that could improve the analysis capabilities. For example, if a local database metric agent exists (but is not installed) and the database is actively running, it informs the user. **pmrep** @@ -351,6 +359,10 @@ Calculates and reports various statistical summaries of the performance metric v Probes for performance metric availability, values, and instances. +**pmseries** + +Scalable, multi-host time series querying utility. Command line interface to the same query language available in grafana-pcp. + **pmstat** Provides a text-based display of metrics that summarize the performance of one or more systems at a high level. @@ -480,10 +492,6 @@ Extracts performance metrics from the systemd and journald services. Exports transaction performance metrics from application processes that use the **pcp_trace** library. -**pmdavmware** - -Extracts performance metrics from a VMWare virtualization host. - **pmdaweblog** Scans Web-server logs to extract metrics characterizing. @@ -492,7 +500,7 @@ Scans Web-server logs to extract metrics characterizing. Extracts performance metrics from the Linux kernel XFS filesystem implementation. -**pmdumplog** +**pmlogdump** Displays selected state information, control data, and metric values from a set of PCP archives created by **pmlogger**. @@ -518,11 +526,7 @@ Creates PCP archives of performance metrics over time. Many tools accept these P **pmproxy** -Provides REST APIs, archive discovery, and both PCP and Redis protocol proxying when executing PCP or Redis client tools through a network firewall system. - -**pmtrace** - -Provides a simple command line interface to the trace PMDA and its associated **pcp_trace** library. +Provides REST APIs, archive discovery, and both PCP and RESP proxying when executing PCP or key server (e.g. Valkey) tools through a network firewall system. Operational and Infrastructure Support ======================================= diff --git a/docs/UAG/TimeSeriesQuerying.rst b/docs/UAG/TimeSeriesQuerying.rst index 8099e19290..ec5758159e 100644 --- a/docs/UAG/TimeSeriesQuerying.rst +++ b/docs/UAG/TimeSeriesQuerying.rst @@ -23,19 +23,16 @@ Section 9.6, “`Timeseries Options`_”, explains the various timeseries option Section 9.7, “`PCP Environment`_”, describes environment variables used to parameterize the file and directory names used by PCP. -Section 9.8, “`PCP Grafana Plugin`_”, explains the PCP Redis data source and lays out the path to the PCP Grafana Plugin. +Section 9.8, “`PCP Grafana Plugin`_”, explains the PCP data source and lays out the path to using the PCP Grafana Plugin. Introduction to pmseries ************************* -**pmseries** displays various types of information about performance metrics available through the scalable timeseries facilities of the Performance -Co-Pilot (PCP) using the `Redis `_ distributed data store. +**pmseries** displays various types of information about performance metrics available through the scalable timeseries facilities of the Performance Co-Pilot (PCP) using a distributed key-value server such as `Valkey `_ -By default **pmseries** communicates with a local redis-server(1), however the **-h** and **-p** options can be used to specify an alternate Redis -instance. If this instance is a node of a Redis cluster, all other instances in the cluster will be discovered and used automatically. +By default **pmseries** communicates with a local key-value server, however the **-h** and **-p** options can be used to specify an alternate server. If this instance is a node of a cluster, all other instances in the cluster will be discovered and used automatically. -**pmseries** runs in several different modes - either querying timeseries identifiers, metadata or values (already stored in Redis), or manually loading -timeseries into Redis. The latter mode is seldom used, however, since `pmproxy(1) `_ will automatically +**pmseries** runs in several different modes - either querying timeseries identifiers, metadata or values (already stored), or manually loading timeseries into a key-value server. The latter mode is seldom used, however, since `pmproxy(1) `_ will automatically perform this function for local `pmlogger(1) `_ instances, when running in its default time series mode. Without command line options specifying otherwise, **pmseries** will issue a timeseries query to find matching timeseries and values. All timeseries are @@ -427,9 +424,9 @@ The available command line options, in addition to timeseries metadata and sourc options Description =============================================== ===================================================================================================================== **-c** *config* , **--config** = *config* | Specify the *config* file to use. -**-h** *host* , **--host** = *host* | Connect Redis server at *host*, rather than the one the localhost. -**-L** , **--load** | Load timeseries metadata and data into the Redis cluster. -**-p** *port* , **--port** = *port* | Connect Redis server at *port*, rather than the default **6379** . +**-h** *host* , **--host** = *host* | Connect key server at *host*, rather than the one the localhost. +**-L** , **--load** | Load timeseries metadata and data into the key server. +**-p** *port* , **--port** = *port* | Connect key server at *port*, rather than the default **6379** . **-q** , **--query** | Perform a timeseries query. This is the default action. **-t** , **--times** | Report time stamps numerically (in milliseconds) instead of the default human readable form. **-v** , **--values** | Report all of the known values for given *label* name(s). @@ -490,10 +487,10 @@ For environment variables affecting PCP tools, see `pmGetOptions(3) `_ and `Performance Co-Pilot `_ (PCP), allowing PCP metric data to be presented in Grafana panels, such as graphs, tables, heatmaps, etc. Under the hood, the data source makes REST API query requests to the PCP `pmproxy(1) `_ service, which can be running either -locally or on a remote host. The pmproxy daemon can be local or remote and uses the Redis time-series database (local or remote) for persistent storage. +locally or on a remote host. The pmproxy daemon can be local or remote and uses a key-value server (local or remote) for persistent storage. For more information on PCP Grafana Plugin, visit `PCP Grafana Plugin Documentation `_ . diff --git a/docs/refs.rst b/docs/refs.rst index 5a459c6bef..a3837c0eb6 100644 --- a/docs/refs.rst +++ b/docs/refs.rst @@ -2,4 +2,4 @@ .. _pmlogger(1): https://man7.org/linux/man-pages/man1/pmlogger.1.html .. _pmlogrewrite(1): https://man7.org/linux/man-pages/man1/pmlogrewrite.1.html .. _pmproxy(1): https://man7.org/linux/man-pages/man1/pmproxy.1.html -.. _Redis: https://redis.io/ +.. _Valkey: https://valkey.io/ diff --git a/docs/specs/openapi.yaml b/docs/specs/openapi.yaml index bcebdd69ea..f037530b61 100644 --- a/docs/specs/openapi.yaml +++ b/docs/specs/openapi.yaml @@ -11,7 +11,7 @@ info: ```bash /api/endpoint?parameter1=value1¶meter2=value2 ``` - Examples in all following sections use the [curl](https://man7.org/linux/man-pages/man1/curl.1.html)(1) command line utility with a local [pmproxy](https://pcp.io/man/man1/pmproxy.1.html)(1) server listening on port 44322 (default port). The [pmjson](https://pcp.io/man/man1/pmjson.1.html)(1) utility is used to neatly format any JSON output, as opposed to the compact (minimal whitespace) form provided by default. The examples in the scalable time series section use historical data recorded by the [pmlogger](https://pcp.io/man/man1/pmlogger.1.html)(1) service, in conjunction with a local [redis-server](https://man7.org/linux/man-pages/man1/redis-server.1.html)(1). + Examples in all following sections use the [curl](https://man7.org/linux/man-pages/man1/curl.1.html)(1) command line utility with a local [pmproxy](https://pcp.io/man/man1/pmproxy.1.html)(1) server listening on port 44322 (default port). The [pmjson](https://pcp.io/man/man1/pmjson.1.html)(1) utility is used to neatly format any JSON output, as opposed to the compact (minimal whitespace) form provided by default. The examples in the scalable time series section use historical data recorded by the [pmlogger](https://pcp.io/man/man1/pmlogger.1.html)(1) service, in conjunction with a local key-value server](https://valkey.io/). version: "" servers: - url: http://localhost:44322/ @@ -28,7 +28,7 @@ tags: description: | The full text search capabilities provided by the [pmsearch](https://pcp.io/man/man1/pmsearch.1.html)(1) command are also available through a REST API. These queries provide access to an index over performance metric names, instances, instance domains and help text, suitable for a metric search engine, and in a fashion suited to efficient querying by any number of web applications. - In order to use this functionality, the optional *RediSearch* module must be loaded in the **redis-server** at the time **pmproxy** is started, such that metrics, instances and help text it discovers can be automatically indexed. + In order to use this functionality, the optional *ValkeySearch* module must be loaded in the **valkey-server** at the time **pmproxy** is started, such that metrics, instances and help text it discovers can be automatically indexed. - name: PMAPI HOST SERVICES description: | The live performance collection facilities available from [pmcd](https://pcp.io/man/man1/pmcd.1.html)(1) can also be accessed through a REST API. @@ -479,7 +479,7 @@ paths: summary: Load time series performance data description: | It loads time series performance data from the specified source into the - **redis-server** cache. This request is equivalent to the **pmseries - l /-- load** option. + key server cache. This request is equivalent to the **pmseries - l /-- load** option. ```bash $ curl -s http://localhost:44322/series/load&expr={source.name:"/var/log/pcp/pmlogger/acme"} ``` diff --git a/src/external/README b/src/external/README index f0d2b7075c..66b7494c93 100644 --- a/src/external/README +++ b/src/external/README @@ -15,7 +15,7 @@ fnmatch implementation for Windows builds only - freebsd/master/lib/libc/gen/fnmatch.c crc16, dict, sha1, sha256 (hash and string helpers) - - https://github.com/redis/redis/blob/6.0/src + - https://github.com/valkey-io/valkey/tree/unstable/src Callback functions on main libuv thread - https://github.com/litesync/uv_callback From 4b0a70a7817173915b0bf79d38d3918048cd6128 Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 6 Aug 2024 14:50:05 +1000 Subject: [PATCH 4/5] qa: transition from redis to allowing different key servers Currently supporting both Valkey and Redis depending on the operating system version in use, but over time we can expect more of the former and less of the latter. --- qa/1211 | 17 +- qa/1211.out | 2 +- qa/1212 | 2 +- qa/1214 | 39 ++--- qa/1214.out | 12 +- qa/1355 | 39 ++--- qa/1355.out | 10 +- qa/1360 | 14 +- qa/1379 | 2 +- qa/1433 | 2 +- qa/1442.out | 32 ++-- qa/1457 | 4 +- qa/1458 | 2 +- qa/1573 | 4 +- qa/1574 | 23 +-- qa/1574.out | 32 ++-- qa/1600 | 27 +-- qa/1601 | 23 +-- qa/1602 | 74 ++++---- qa/1602.out | 18 +- qa/1603 | 20 +-- qa/1603.out | 4 +- qa/1604 | 21 +-- qa/1626 | 2 +- qa/1661 | 2 +- qa/1662 | 27 +-- qa/1662.out | 2 +- qa/1689 | 11 +- qa/1689.out | 18 +- qa/1690 | 17 +- qa/1691 | 21 +-- qa/1695 | 17 +- qa/1696 | 4 +- qa/1697 | 24 +-- qa/1773 | 25 +-- qa/1824 | 2 +- qa/1850 | 23 +-- qa/1850.out | 4 +- qa/1871 | 32 ++-- qa/1872 | 41 ++--- qa/1874 | 27 +-- qa/1875 | 51 +++--- qa/1875.out | 10 +- qa/1877 | 12 +- qa/1877.out | 6 +- qa/1886 | 21 +-- qa/1886.out | 4 +- qa/1906 | 20 +-- qa/1906.out | 4 +- qa/1984 | 2 +- qa/913 | 2 +- qa/GNUmakefile | 2 +- qa/check-vars | 10 +- qa/common.check | 160 +++++++++++++----- qa/common.filter | 4 +- qa/common.keys | 127 ++++++++++++++ qa/common.redis | 105 ------------ qa/src/GNUlocaldefs | 2 +- ...s_proxy.python => key_server_proxy.python} | 4 +- src/libpcp_web/src/load.c | 2 +- src/libpcp_web/src/slots.c | 24 +-- src/libpcp_web/src/slots.h | 11 +- src/pmproxy/pmproxy.conf | 2 +- 63 files changed, 717 insertions(+), 590 deletions(-) create mode 100644 qa/common.keys delete mode 100644 qa/common.redis rename qa/src/{redis_proxy.python => key_server_proxy.python} (92%) diff --git a/qa/1211 b/qa/1211 index b80579fb44..f0cb1b17c3 100755 --- a/qa/1211 +++ b/qa/1211 @@ -13,12 +13,13 @@ path="" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series _cleanup() { - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -26,8 +27,8 @@ _cleanup() status=1 # failure is the default! hostname=`hostname` -redisport=`_find_free_port` -options="-p $redisport" +key_server_port=`_find_free_port` +options="-p $key_server_port" $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 @@ -53,13 +54,13 @@ _sort_context() _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping "$redisport" -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping "$key_server_port" +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port echo "Verify basic load from proc archive" pmseries $options --load "{source.path: \"$here/archives/proc\"}" | _filter_source diff --git a/qa/1211.out b/qa/1211.out index 1c1a4bb317..96d5543a5f 100644 --- a/qa/1211.out +++ b/qa/1211.out @@ -1,5 +1,5 @@ QA output created by 1211 -Start test Redis server ... +Start test key server ... PING PONG diff --git a/qa/1212 b/qa/1212 index 8f50a3140b..01dfdb99fe 100755 --- a/qa/1212 +++ b/qa/1212 @@ -14,7 +14,7 @@ path="" . ./common.filter . ./common.check -_check_redis_server +_check_key_server _cleanup() { diff --git a/qa/1214 b/qa/1214 index feb2381cbf..ac1687e6a4 100755 --- a/qa/1214 +++ b/qa/1214 @@ -12,18 +12,19 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_valgrind _check_series -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { echo cd $here - echo == Shutdown local redis and pmproxy ... + echo == Shutdown local key server and pmproxy ... [ -n "$proxypid" ] && kill -TERM "$proxypid" - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown echo finished $sudo rm -rf $tmp $tmp.* } @@ -34,11 +35,11 @@ $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 # real QA test starts here -echo "=== Start test redis-server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport +echo "=== Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port echo echo "=== Start test pmproxy ..." proxyport=`_find_free_port` @@ -47,10 +48,10 @@ cat > $tmp.conf <> $seq.full echo - echo == pmseries query for $metric to redis directly - pmseries -p $redisport $metric | tee -a $here/$seq.full > $tmp.out - redis_nseries=`wc -l < $tmp.out` - if [ -z "$redis_nseries" -o "$redis_nseries" -le 0 ] + echo == pmseries query for $metric to key server directly + pmseries -p $key_server_port $metric | tee -a $here/$seq.full > $tmp.out + key_server_nseries=`wc -l < $tmp.out` + if [ -z "$key_server_nseries" -o "$key_server_nseries" -le 0 ] then - echo pmseries -p $redisport $metric failed + echo pmseries -p $key_server_port $metric failed status=1 fi echo - echo == pmseries query for $metric to redis via pmproxy + echo == pmseries query for $metric to key server via pmproxy pmseries -p $proxyport $metric | tee -a $here/$seq.full > $tmp.out proxy_nseries=`wc -l < $tmp.out` if [ -z "$proxy_nseries" -o "$proxy_nseries" -le 0 ] @@ -89,9 +90,9 @@ do status=2 fi - if [ "$redis_nseries" -ne "$proxy_nseries" ] + if [ "$key_server_nseries" -ne "$proxy_nseries" ] then - echo redis=$redis_nseries differs to pmproxy=$proxy_nseries + echo keyserver=$key_server_nseries differs to pmproxy=$proxy_nseries status=3 fi @@ -100,7 +101,7 @@ do if [ $status -ne 0 ] then [ $status -eq 1 ] && $_valgrind_clean_assert \ - pmseries -p $redisport $metric >> $seq.full 2>&1 + pmseries -p $key_server_port $metric >> $seq.full 2>&1 [ $status -ge 2 ] && $_valgrind_clean_assert \ pmseries -p $proxyport $metric >> $seq.full 2>&1 fi diff --git a/qa/1214.out b/qa/1214.out index 0cac9f2510..8882a482f0 100644 --- a/qa/1214.out +++ b/qa/1214.out @@ -1,18 +1,18 @@ QA output created by 1214 -=== Start test redis-server ... +=== Start test key server ... PING PONG === Start test pmproxy ... == load test data for verification -== pmseries query for kernel.all.load to redis directly +== pmseries query for kernel.all.load to key server directly -== pmseries query for kernel.all.load to redis via pmproxy +== pmseries query for kernel.all.load to key server via pmproxy -== pmseries query for disk.dev.read to redis directly +== pmseries query for disk.dev.read to key server directly -== pmseries query for disk.dev.read to redis via pmproxy +== pmseries query for disk.dev.read to key server via pmproxy -== Shutdown local redis and pmproxy ... +== Shutdown local key server and pmproxy ... finished diff --git a/qa/1355 b/qa/1355 index 93ca399ace..29e62d0d43 100755 --- a/qa/1355 +++ b/qa/1355 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1355 -# Exercise pmproxy Redis reconnect. +# Exercise pmproxy key server reconnect. # # Copyright (c) 2021 Red Hat. All Rights Reserved. # @@ -12,16 +12,17 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { cd $here [ -n "$tail_pid" ] && $signal -s TERM $tail_pid [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$redisproxy_pid" ] && $signal -s TERM $redisproxy_pid - [ -n "$redis_pid" ] && $signal -s TERM $redis_pid + [ -n "$keysproxy_pid" ] && $signal -s TERM $keysproxy_pid + [ -n "$keys_pid" ] && $signal -s TERM $keys_pid $sudo rm -rf $tmp $tmp.* } @@ -41,20 +42,20 @@ _filter_pmproxy_log() # real QA test starts here echo "Start Redis server" -redisport=`_find_free_port` -redis-server --port $redisport --save "" > $tmp.redis.log 2>&1 & -redis_pid=$! -_wait_for_port $redisport +key_server_port=`_find_free_port` +$key_server --port $key_server_port --save "" > $tmp.keys.log 2>&1 & +key_server_pid=$! +_wait_for_port $key_server_port -echo "Start Redis proxy to fake loading error for first 3 seconds" -redisproxyport=`_find_free_port` -$python src/redis_proxy.py --loading_delay 3 127.0.0.1:$redisproxyport 127.0.0.1:$redisport > $tmp.redisproxy.log & -redisproxy_pid=$! -_wait_for_port $redisproxyport +echo "Start key server proxy to fake loading error for first 3 seconds" +keysproxyport=`_find_free_port` +$python src/key_server_proxy.py --loading_delay 3 127.0.0.1:$keysproxyport 127.0.0.1:$key_server_port > $tmp.keysproxy.log & +keysproxy_pid=$! +_wait_for_port $keysproxyport echo "Start pmproxy" pmproxyport=`_find_free_port` -pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $redisproxyport -t & +pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $keysproxyport -t & pmproxy_pid=$! _wait_for_port $pmproxyport @@ -67,8 +68,8 @@ tail -n +$lineno $tmp.pmproxy.log | _filter_pmproxy_log lineno=$((`wc -l < $tmp.pmproxy.log` + 1)) echo -echo "Stop redis proxy" -$signal -s TERM $redisproxy_pid +echo "Stop key server proxy" +$signal -s TERM $keysproxy_pid echo echo "== Observe a connection lost error and failed reconnect" @@ -79,9 +80,9 @@ lineno=$((`wc -l < $tmp.pmproxy.log` + 1)) echo echo "Start Redis proxy again (loading error for first 3 seconds)" -$python src/redis_proxy.py --loading_delay 3 127.0.0.1:$redisproxyport 127.0.0.1:$redisport > $tmp.redisproxy.log & -redisproxy_pid=$! -_wait_for_port $redisproxyport +$python src/key_server_proxy.py --loading_delay 3 127.0.0.1:$keysproxyport 127.0.0.1:$key_server_port > $tmp.keysproxy.log & +keysproxy_pid=$! +_wait_for_port $keysproxyport echo echo "== Observe a reconnect" diff --git a/qa/1355.out b/qa/1355.out index f7f3c3150b..3649c619e6 100644 --- a/qa/1355.out +++ b/qa/1355.out @@ -7,13 +7,13 @@ Start pmproxy Cannot connect to Redis: LOADING Redis is loading the dataset in memory Redis slots, schema version setup -Stop redis proxy +Stop key server proxy == Observe a connection lost error and failed reconnect -Lost connection to Redis. -Cannot connect to Redis: Connection refused +Lost connection to key server. +Cannot connect to key server: Connection refused -Start Redis proxy again (loading error for first 3 seconds) +Start key server proxy again (loading error for first 3 seconds) == Observe a reconnect -Redis slots, schema version setup +key server slots, schema version setup diff --git a/qa/1360 b/qa/1360 index e726d8b09a..ae20fbc431 100755 --- a/qa/1360 +++ b/qa/1360 @@ -2,7 +2,7 @@ # PCP QA Test No. 1360 # based on qa/1696 pmproxy REST API load testing ... # pmproxy should start w/out valgrind issues when it refuses to -# connect to older versions of Redis +# connect to older versions of a key server. # # Copyright (c) 2019 Red Hat. # Copyright (c) 2021 Ken McDonell. All Rights Reserved. @@ -12,19 +12,17 @@ seq=`basename $0` echo "QA output created by $seq" # get standard environment, filters and checks -. ./common.product -. ./common.filter -. ./common.check . ./common.python +. ./common.keys _check_python36 # needed by ./src/pmproxy_load_test.python _check_valgrind _check_series -# need older redis version for this test ... -version=`redis-server --version | sed -E 's/.*v=([0-9]).*/\1/g'` +# need older key server version for this test ... +version=`$key_server --version | sed -E 's/.*v=([0-9]).*/\1/g'` [ $version -ge 5 ] && \ - _notrun "Redis version $version needs no backward compatibility testing" + _notrun "Key server version $version needs no backward compatibility testing" _cleanup() { @@ -102,7 +100,7 @@ echo "=== valgrind stdout ===" | tee -a $seq.full cat $tmp.valout | _filter_valgrind echo "=== valgrind stderr ===" | tee -a $seq.full -cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to Redis" | _filter_port +cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to key server" | _filter_port # success, all done status=0 diff --git a/qa/1379 b/qa/1379 index 3af8953651..28fb3a9663 100755 --- a/qa/1379 +++ b/qa/1379 @@ -123,7 +123,7 @@ $sudo cp $tmp.options $PCP_SYSCONF_DIR/pmproxy/pmproxy.options echo "[pmproxy]" >> $tmp.conf echo "pcp.enabled = true" >> $tmp.conf echo "http.enabled = true" >> $tmp.conf -echo "redis.enabled = true" >> $tmp.conf +echo "resp.enabled = true" >> $tmp.conf echo "secure.enabled = true" >> $tmp.conf $sudo cp $tmp.conf $PCP_SYSCONF_DIR/pmproxy/pmproxy.conf cat $tmp.conf >> $here/$seq.full diff --git a/qa/1433 b/qa/1433 index b4e42d3e83..006cb6184b 100755 --- a/qa/1433 +++ b/qa/1433 @@ -13,7 +13,7 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_redis_server +_check_key_server eval `pmconfig -L ipv6 unix_domain_sockets` [ "$ipv6" = "true" ] || _notrun "No IPv6 support" [ "$unix_domain_sockets" = "true" ] || _notrun "No Unix socket support" diff --git a/qa/1442.out b/qa/1442.out index 1b176be5ba..5afa7cf51b 100644 --- a/qa/1442.out +++ b/qa/1442.out @@ -16,67 +16,67 @@ pmproxy.control.files Data Type: 32-bit int InDom: PM_INDOM_NULL 0xffffffff Semantics: instant Units: none -pmproxy.redis.discover.monitored +pmproxy.keys.discover.monitored Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: discrete Units: none -pmproxy.redis.discover.purged +pmproxy.keys.discover.purged Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: discrete Units: none -pmproxy.redis.discover.metadata.callbacks +pmproxy.keys.discover.metadata.callbacks Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.metadata.loops +pmproxy.keys.discover.metadata.loops Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.metadata.decode.desc +pmproxy.keys.discover.metadata.decode.desc Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.metadata.decode.indom +pmproxy.keys.discover.metadata.decode.indom Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.metadata.decode.label +pmproxy.keys.discover.metadata.decode.label Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.metadata.decode.helptext +pmproxy.keys.discover.metadata.decode.helptext Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.callbacks +pmproxy.keys.discover.logvol.callbacks Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.loops +pmproxy.keys.discover.logvol.loops Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.change_vol +pmproxy.keys.discover.logvol.change_vol Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.decode.result +pmproxy.keys.discover.logvol.decode.result Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.decode.result_pmids +pmproxy.keys.discover.logvol.decode.result_pmids Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.decode.mark_record +pmproxy.keys.discover.logvol.decode.mark_record Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.new_contexts +pmproxy.keys.discover.logvol.new_contexts Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none -pmproxy.redis.discover.logvol.get_archive_end_failed +pmproxy.keys.discover.logvol.get_archive_end_failed Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: none diff --git a/qa/1457 b/qa/1457 index dfe448f91c..a6cd404e6e 100755 --- a/qa/1457 +++ b/qa/1457 @@ -10,7 +10,7 @@ echo "QA output created by $seq" . ./common.secure -_check_redis_server +_check_key_server _check_valgrind _check_tls @@ -80,7 +80,7 @@ _setup_tls echo "[pmproxy]" >> $tmp.conf echo "pcp.enabled = true" >> $tmp.conf echo "http.enabled = true" >> $tmp.conf -echo "redis.enabled = true" >> $tmp.conf +echo "resp.enabled = true" >> $tmp.conf echo "secure.enabled = true" >> $tmp.conf echo "private_key = $here/tls.conf/pcp.key" >> $tmp.conf echo "certificates = $here/tls.conf/pcp.crt" >> $tmp.conf diff --git a/qa/1458 b/qa/1458 index 01bd941111..e6854b93de 100755 --- a/qa/1458 +++ b/qa/1458 @@ -117,7 +117,7 @@ openssl req \ -keyout $tmp.key -out $tmp.cert >$seq.full 2>&1 # creates a self-signed (insecure) certificate, so for testing only -echo "[redis]" >> $tmp.conf +echo "[keys]" >> $tmp.conf echo "enabled = false" >> $tmp.conf echo "[pmproxy]" >> $tmp.conf echo "http.enabled = true" >> $tmp.conf diff --git a/qa/1573 b/qa/1573 index 7d2c3ff310..6ce2cd7d26 100755 --- a/qa/1573 +++ b/qa/1573 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1573 -# Exercise libpcp_web memory leak without a redis-server. +# Exercise libpcp_web memory leak without a key server. # # Copyright (c) 2020 Red Hat. # @@ -63,7 +63,7 @@ _restore_pmlogger_control echo "[pmproxy]" > $tmp.conf echo "pcp.enabled = true" >> $tmp.conf echo "http.enabled = true" >> $tmp.conf -echo "redis.enabled = true" >> $tmp.conf +echo "resp.enabled = true" >> $tmp.conf echo "secure.enabled = false" >> $tmp.conf echo "[discover]" >> $tmp.conf echo "enabled = true" >> $tmp.conf diff --git a/qa/1574 b/qa/1574 index b16d4cc595..34f02a92d1 100755 --- a/qa/1574 +++ b/qa/1574 @@ -13,13 +13,14 @@ path="" . ./common.product . ./common.filter . ./common.check +. ./common.keys -# This test is not run if we dont have pmseries and redis installed. +# This test is not run if we dont have pmseries and a key server installed. _check_series _cleanup() { - [ -n "$redisport" ] && redis-cli -p $redisport shutdown + [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -39,7 +40,7 @@ _filter_load() _filter_query() { sed \ - -e "s,$redisport,REDISPORT,g" \ + -e "s,$key_server_port,KEYSERVERPORT,g" \ #end } @@ -54,22 +55,22 @@ _sort_context() } # real QA test starts here -redisport=`_find_free_port` +key_server_port=`_find_free_port` _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -args="-p $redisport -Z UTC" +args="-p $key_server_port -Z UTC" query="kernel.all.load[count:2]" -echo "== Load metric data into this redis instance" +echo "== Load metric data into this key server instance" pmseries $args --load "{source.path: \"$here/archives/proc\"}" | _filter_load echo diff --git a/qa/1574.out b/qa/1574.out index c452c45ff7..5627436416 100644 --- a/qa/1574.out +++ b/qa/1574.out @@ -1,9 +1,9 @@ QA output created by 1574 -Start test Redis server ... +Start test key server ... PING PONG -== Load metric data into this redis instance +== Load metric data into this key server instance pmseries: [Info] processed 5 archive records from PATH/archives/proc == Running valgrind on stdev_sample() and stdev_inst() @@ -15,7 +15,7 @@ ef9785c6b5cae109a5ed4fd8a0cbe868a1c576e1 === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC stdev_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC stdev_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -29,7 +29,7 @@ bdb503f3d93a8af54adf9e33131152ab61358d5e === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC stdev_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC stdev_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -44,7 +44,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC max_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC max_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -58,7 +58,7 @@ a9d70abdd37180508d8f2f56450fa9cbf531847c === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC max_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC max_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -73,7 +73,7 @@ dc4d3bd8c07fb952e89e9765392447b26eea6f8f === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC min_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC min_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -87,7 +87,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC min_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC min_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -102,7 +102,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC sum_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC sum_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -116,7 +116,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC sum_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC sum_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -131,7 +131,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC avg_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC avg_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -145,7 +145,7 @@ d5f1f053bb6b11dcc8245a7a792d0537c9f7de6a === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC avg_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC avg_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -160,7 +160,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC avg_sample(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC avg_sample(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -174,7 +174,7 @@ d5f1f053bb6b11dcc8245a7a792d0537c9f7de6a === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC avg_inst(kernel.all.load[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC avg_inst(kernel.all.load[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -188,7 +188,7 @@ ERROR SUMMARY: 0 errors from 0 contexts ... === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC kernel.all.uptime[count:2]\ *\ 2 +Command: pmseries -p KEYSERVERPORT -Z UTC kernel.all.uptime[count:2]\ *\ 2 LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks @@ -201,7 +201,7 @@ feb9ff0f4ca9c6fff384a5ef545afc0781840ff3 === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: pmseries -p REDISPORT -Z UTC max(kernel.all.uptime[count:2])\ *\ max(kernel.all.uptime[count:2]) +Command: pmseries -p KEYSERVERPORT -Z UTC max(kernel.all.uptime[count:2])\ *\ max(kernel.all.uptime[count:2]) LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks diff --git a/qa/1600 b/qa/1600 index 5002cf96a1..7ce3004a95 100755 --- a/qa/1600 +++ b/qa/1600 @@ -12,13 +12,14 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series _cleanup() { [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown cd $here if $need_restore then @@ -143,15 +144,15 @@ echo '{"latitude": -25.28496, "longitude": 152.87886}' > $tmp.labels $sudo cp $tmp.labels $PCP_SYSCONF_DIR/labels/pcpqa.location _sighup_pmcd -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port # prepare logging location and config export PCP_ARCHIVE_DIR=$tmp.log/pmlogger @@ -186,7 +187,7 @@ pmsleep 0.5 # time for pmlogger to start logging # start pmproxy proxyport=`_find_free_port` -proxyopts="-f -A -p $proxyport -r $redisport -U $username" +proxyopts="-f -A -p $proxyport -r $key_server_port -U $username" #proxyopts="$pmproxyopts -Dseries,discovery" pmproxy -c $tmp.pmproxy.conf -x $seq.full -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! @@ -196,8 +197,8 @@ pmproxy_pid=$! pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec pmsleep 0.5 # time for pmproxy to start discovering -$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '/pmlogger|/pmproxy|/redis' >> $seq.full -redis-cli $options keys pcp:* >> $seq.full +$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '/pmlogger|/pmproxy|/valkey|/redis' >> $seq.full +$keys_cli $options keys pcp:* >> $seq.full ls -lR $PCP_ARCHIVE_DIR >> $seq.full cat $tmp.pmproxy.log >> $seq.full @@ -254,8 +255,8 @@ pmcd_wait -h localhost@localhost:$proxyport -v -t 1sec # pmproxy still up? echo "== log " >> $seq.full cat $tmp.pmproxy.log >> $seq.full -echo "== redis-server log " >> $seq.full -cat $tmp.redis >> $seq.full +echo "== key server log " >> $seq.full +cat $tmp.keys >> $seq.full echo "== all done" | tee -a $seq.full diff --git a/qa/1601 b/qa/1601 index 7782265635..243b5ca033 100755 --- a/qa/1601 +++ b/qa/1601 @@ -12,6 +12,7 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series @@ -19,7 +20,7 @@ _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown if $need_restore then need_restore=false @@ -70,22 +71,22 @@ $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -# import some well-known test data into Redis +# import some well-known test data into the key server pmseries $options --load "$here/archives/proc" | _filter_source # start pmproxy proxyport=`_find_free_port` -proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af +proxyopts="-p $proxyport -r $key_server_port -t" # -Dseries,http,af pmproxy -f -U $username -x $seq.full -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! @@ -93,7 +94,7 @@ pmproxy_pid=$! pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq.full -redis-cli $options keys pcp:* >> $seq.full +$keys_cli $options keys pcp:* >> $seq.full cat $tmp.pmproxy.log >> $seq.full series1=`pmseries $options disk.all.read` diff --git a/qa/1602 b/qa/1602 index bfae140b46..7e494e2b15 100755 --- a/qa/1602 +++ b/qa/1602 @@ -1,8 +1,8 @@ #!/bin/sh # PCP QA Test No. 1602 -# Exercise Redis protocol proxying using redis-cli(1). +# Exercise RESP proxying using key server command line tools. # -# Copyright (c) 2019 Red Hat. +# Copyright (c) 2019,2024 Red Hat. # seq=`basename $0` @@ -12,10 +12,10 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check -. ./common.redis +. ./common.keys _check_series -_check_redis_server_version_offline +_check_key_server_version_offline # only restart pmproxy if it was running before the QA test starts restart_pmproxy=false @@ -25,7 +25,7 @@ _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - _redis_cluster_3nodes_stop + _key_server_cluster_3nodes_stop if $need_restore then need_restore=false @@ -49,14 +49,14 @@ need_restore=false $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 -_filter_redis() +_filter_key_server() { sed \ -e "s,:$port1>,:PORT1>,g" \ #end } -_filter_redis_err() +_filter_key_server_err() { sed \ -e 's;ERR syntax error;expected error;' \ @@ -71,11 +71,11 @@ _save_config $PCP_SYSCONF_DIR/pmproxy $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -# start Redis cluster -_redis_cluster_3nodes_start -echo "redis_node1_port=$redis_node1_port" >>$seq.full -echo "redis_node2_port=$redis_node2_port" >>$seq.full -echo "redis_node3_port=$redis_node3_port" >>$seq.full +# start cluster +_key_server_cluster_3nodes_start +echo "key_server_node1_port=$key_server_node1_port" >>$seq.full +echo "key_server_node2_port=$key_server_node2_port" >>$seq.full +echo "key_server_node3_port=$key_server_node3_port" >>$seq.full echo echo "== start pmproxy" @@ -83,9 +83,9 @@ echo "== start pmproxy" cat >$tmp.conf <>$seq.full echo "pmproxy_pid=$pmproxy_pid" >>$seq.full _wait_for_pmproxy $proxyport $tmp.pmproxy.log -which netstat >/dev/null && netstat -l | grep -E "$redis_node1_port|$redis_node2_port|$redis_node3_port|$proxyport" >>$seq.full -_check_redis_ping $proxyport +which netstat >/dev/null && netstat -l | grep -E "$key_server_node1_port|$key_server_node2_port|$key_server_node3_port|$proxyport" >>$seq.full +_check_key_server_ping $proxyport echo # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec -# check redis protocol proxying -echo "== redis-cli set two keys" -redis-cli -p $proxyport set pcpqa:$seq:one 1 -redis-cli -p $proxyport set pcpqa:$seq:ten 10 +# check key_server protocol proxying +echo "== key server cli set two keys" +$keys_cli -p $proxyport set pcpqa:$seq:one 1 +$keys_cli -p $proxyport set pcpqa:$seq:ten 10 echo # check pcp protocol proxying @@ -112,34 +112,34 @@ echo "== pminfo get two metrics" pminfo -f -h localhost@localhost:$proxyport sample.long.one sample.long.ten echo -# check redis protocol proxying -echo "== redis-cli get two keys" -redis-cli -p $proxyport get pcpqa:$seq:one -redis-cli -p $proxyport get pcpqa:$seq:ten +# check resp protocol proxying +echo "== key server cli get two keys" +$keys_cli -p $proxyport get pcpqa:$seq:one +$keys_cli -p $proxyport get pcpqa:$seq:ten echo -# check redis protocol error handling -echo "== redis-cli empty key set" -redis-cli -p $proxyport keys no-such-key | _filter_redis_err +# check resp protocol error handling +echo "== keys_cli empty key set" +$keys_cli -p $proxyport keys no-such-key | _filter_key_server_err echo -echo "== redis-cli handle errors 1" -redis-cli -p $proxyport get foo bar baz | _filter_redis_err +echo "== keys_cli handle errors 1" +$keys_cli -p $proxyport get foo bar baz | _filter_key_server_err echo -echo "== redis-cli handle errors 2" -redis-cli -p $proxyport set foo bar baz | _filter_redis_err +echo "== keys_cli handle errors 2" +$keys_cli -p $proxyport set foo bar baz | _filter_key_server_err echo -echo "== redis-cli nonexistant key" -redis-cli -p $proxyport get abc +echo "== keys_cli nonexistant key" +$keys_cli -p $proxyport get abc echo # check key distribution -echo "== redis cluster key distribution" +echo "== key server cluster key distribution" echo "== node1" -redis-cli -p $redis_node1_port keys pcpqa:* +$keys_cli -p $key_server_node1_port keys pcpqa:* echo "== node2" -redis-cli -p $redis_node2_port keys pcpqa:* +$keys_cli -p $key_server_node2_port keys pcpqa:* echo "== node3" -redis-cli -p $redis_node3_port keys pcpqa:* +$keys_cli -p $key_server_node3_port keys pcpqa:* echo cat $tmp.conf >> $seq.full diff --git a/qa/1602.out b/qa/1602.out index a6e6591515..647a3ec52b 100644 --- a/qa/1602.out +++ b/qa/1602.out @@ -1,5 +1,5 @@ QA output created by 1602 -== Start a Redis cluster +== Start a key server cluster == Start node 1 PONG OK @@ -10,7 +10,7 @@ OK PONG OK -== Create Redis cluster +== Create key server cluster [OK] All nodes agree about slots configuration. [OK] All 16384 slots covered. @@ -26,7 +26,7 @@ cluster_size:3 PING PONG -== redis-cli set two keys +== keys-cli set two keys OK OK @@ -38,25 +38,25 @@ sample.long.one sample.long.ten value 10 -== redis-cli get two keys +== keys_cli get two keys 1 10 -== redis-cli empty key set +== keys_cli empty key set -== redis-cli handle errors 1 +== keys_cli handle errors 1 expected error -== redis-cli handle errors 2 +== keys_cli handle errors 2 expected error -== redis-cli nonexistant key +== keys-cli nonexistant key -== redis cluster key distribution +== key server cluster key distribution == node1 == node2 diff --git a/qa/1603 b/qa/1603 index 1ab72c8d68..506c2ddffd 100755 --- a/qa/1603 +++ b/qa/1603 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1603 -# Exercise Redis cluster mode support in libpcp_web with pmseries. +# Exercise key server cluster mode support in libpcp_web with pmseries. # # Copyright (c) 2021 Red Hat. # @@ -12,16 +12,16 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check -. ./common.redis +. ./common.keys _check_series -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - _redis_cluster_3nodes_stop + _key_server_cluster_3nodes_stop if $need_restore then @@ -52,14 +52,14 @@ _save_config $PCP_SYSCONF_DIR/pmproxy $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -_redis_cluster_3nodes_start +_key_server_cluster_3nodes_start cat >$tmp.conf </dev/null 2>&1 || _notrun "jq not installed" @@ -20,7 +21,7 @@ _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown if $need_restore then need_restore=false @@ -60,22 +61,22 @@ _save_config $PCP_SYSCONF_DIR/pmproxy $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -# import some well-known test data into Redis +# import some well-known test data into the key server pmseries $options --load "$here/archives/bozo-disk" | _filter_source # start pmproxy proxyport=`_find_free_port` -proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af +proxyopts="-p $proxyport -r $key_server_port -t" # -Dseries,http,af pmproxy -f -U $username -x $seq.full -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! diff --git a/qa/1626 b/qa/1626 index a9be420f8e..2aeb2f3409 100755 --- a/qa/1626 +++ b/qa/1626 @@ -13,7 +13,7 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_redis_server +_check_key_server which curl >/dev/null 2>&1 || _notrun needs curl _cleanup() diff --git a/qa/1661 b/qa/1661 index bd8f748d83..d05fef53e6 100755 --- a/qa/1661 +++ b/qa/1661 @@ -15,7 +15,7 @@ echo "QA output created by $seq" [ $PCP_PLATFORM = linux ] || _notrun "Test only runs on Linux" which pmproxy >/dev/null 2>&1 || _notrun "No pmproxy binary installed" -_check_redis_server +_check_key_server _cleanup() { diff --git a/qa/1662 b/qa/1662 index ca60000eb6..1f7d066de6 100755 --- a/qa/1662 +++ b/qa/1662 @@ -14,10 +14,11 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_valgrind _check_series -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { @@ -25,7 +26,7 @@ _cleanup() && $signal -TERM $pmlogger_pid test -n "$pmlogger2_pid" -a -d "/proc/$pmlogger2_pid" \ && $signal -TERM $pmlogger2_pid - test -n "$REDISPORT" && redis-cli -p $REDISPORT shutdown + test -n "$key_server_port" && $keys_cli -p $key_server_port shutdown cd $here $sudo rm -rf $tmp $tmp.* } @@ -39,7 +40,7 @@ _filter() -e "s,-c $tmp.conf ,-c CONFFILE ,g" \ -e "s,-U $username ,-U USERNAME ,g" \ -e "s,-l $tmp.pmproxy,-l LOGFILE,g" \ - -e "s,-r $REDISPORT,-r REDISPORT,g" \ + -e "s,-r $key_server_port,-r KEYSPORT,g" \ -e "s,-p $PROXYPORT,-p PROXYPORT,g" \ -e "s,-s $PROXYSOCK,-s PROXYSOCK,g" \ | _filter_valgrind_possibly @@ -58,18 +59,18 @@ mkdir $dir # # Test plan is as follows: -# - create a local Redis setup to catch discovered data +# - create a local key server to catch discovered data # - create a directory to house freshly created archives # - run pmlogger in frequent-writes+volume-switching mode, # background-writing archives below new local directory # - run valgrind+pmproxy for a while, see what it uncovers # -REDISPORT=`_find_free_port` -src/time_stamp $tmp.stamp 'redis start' >>$seq.full -redis-server --port $REDISPORT --save "" > $tmp.redis 2>&1 & -_check_redis_ping $REDISPORT -src/time_stamp $tmp.stamp 'redis active' >>$seq.full +key_server_port=`_find_free_port` +src/time_stamp $tmp.stamp 'key server start' >>$seq.full +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +src/time_stamp $tmp.stamp 'key server active' >>$seq.full cat > $tmp.sample << EOF2 log advisory on default { sample } @@ -84,7 +85,7 @@ src/time_stamp $tmp.stamp 'pmlogger active' >>$seq.full PROXYPORT=`_find_free_port` PROXYSOCK=$tmp.pmproxy.sock cat > $tmp.conf << EOF -[redis] +[keys] enabled = true [pmseries] enabled = true @@ -98,12 +99,12 @@ src/time_stamp $tmp.stamp 'pmproxy start' >>$seq.full #grind_extra="--track-origins=yes --read-var-info=yes --read-inline-info=yes" _run_valgrind $PCP_BINADM_DIR/pmproxy \ -f -t -A -c $tmp.conf -U $username -T 10sec -l $tmp.pmproxy \ - -r $REDISPORT -p $PROXYPORT -s $PROXYSOCK \ + -r $key_server_port -p $PROXYPORT -s $PROXYSOCK \ | _filter src/time_stamp $tmp.stamp 'pmproxy exited' >>$seq.full -echo "=== redis log ===" >> $seq.full -cat $tmp.redis >> $seq.full +echo "=== key server log ===" >> $seq.full +cat $tmp.keys >> $seq.full echo "=== pmlogger log ===" >> $seq.full cat $tmp.pmlogger >> $seq.full echo "=== pmlogger2 log ===" >> $seq.full diff --git a/qa/1662.out b/qa/1662.out index 267664c9ba..a8a0aca280 100644 --- a/qa/1662.out +++ b/qa/1662.out @@ -5,7 +5,7 @@ PONG === std err === === filtered valgrind report === Memcheck, a memory error detector -Command: PCP_BINADM_DIR/pmproxy -f -t -A -c CONFFILE -U USERNAME -T 10sec -l LOGFILE -r REDISPORT -p PROXYPORT -s PROXYSOCK +Command: PCP_BINADM_DIR/pmproxy -f -t -A -c CONFFILE -U USERNAME -T 10sec -l LOGFILE -r KEYSPORT -p PROXYPORT -s PROXYSOCK LEAK SUMMARY: definitely lost: 0 bytes in 0 blocks indirectly lost: 0 bytes in 0 blocks diff --git a/qa/1689 b/qa/1689 index 93a43c2b0c..45947f191f 100755 --- a/qa/1689 +++ b/qa/1689 @@ -12,13 +12,10 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -# need both redis and pmproxy running for this test -_check_redis_server -[ -x $PCP_BINADM_DIR/pmproxy ] || \ - _notrun "need $PCP_BINADM_DIR/pmproxy" - -pgrep pmproxy >/dev/null 2>&1 || \ - _notrun "pmproxy not running" +# need both a key server and pmproxy running for this test +[ -x $PCP_BINADM_DIR/pmproxy ] || _notrun "need $PCP_BINADM_DIR/pmproxy" +pgrep pmproxy >/dev/null 2>&1 || _notrun "pmproxy not running" +_check_key_server status=0 # success is the default! $sudo rm -rf $tmp.* diff --git a/qa/1689.out b/qa/1689.out index 043a0a25de..940ea4a8ab 100644 --- a/qa/1689.out +++ b/qa/1689.out @@ -231,55 +231,55 @@ pmproxy.pid PMID: 4.1.0 [PID for the current process] Help: Process identifier for the current process -pmproxy.redis.requests.error PMID: 4.2.2 [number of request errors] +pmproxy.keys.requests.error PMID: 4.2.2 [number of request errors] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: count Help: Total number of Redis request errors -pmproxy.redis.requests.inflight.bytes PMID: 4.2.7 [bytes allocated for inflight requests] +pmproxy.keys.requests.inflight.bytes PMID: 4.2.7 [bytes allocated for inflight requests] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: instant Units: byte Help: Memory currently allocated for inflight Redis requests -pmproxy.redis.requests.inflight.total PMID: 4.2.6 [inflight requests] +pmproxy.keys.requests.inflight.total PMID: 4.2.6 [inflight requests] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: instant Units: count Help: Total number of inflight Redis requests -pmproxy.redis.requests.total PMID: 4.2.1 [number of requests] +pmproxy.keys.requests.total PMID: 4.2.1 [number of requests] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: count Help: Total number of Redis requests sent -pmproxy.redis.requests.total_bytes PMID: 4.2.8 [total bytes sent for requests] +pmproxy.keys.requests.total_bytes PMID: 4.2.8 [total bytes sent for requests] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: byte Help: Cumulative count of bytes sent for all Redis requests -pmproxy.redis.responses.error PMID: 4.2.4 [number of error responses] +pmproxy.keys.responses.error PMID: 4.2.4 [number of error responses] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: count Help: Total number of Redis error responses received -pmproxy.redis.responses.time PMID: 4.2.5 [total time for responses] +pmproxy.keys.responses.time PMID: 4.2.5 [total time for responses] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: microsec Help: Cumulative time taken to receive all Redis responses -pmproxy.redis.responses.total PMID: 4.2.3 [number of responses] +pmproxy.keys.responses.total PMID: 4.2.3 [number of responses] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: count Help: Total number of Redis responses received -pmproxy.redis.responses.total_bytes PMID: 4.2.9 [total bytes received in responses] +pmproxy.keys.responses.total_bytes PMID: 4.2.9 [total bytes received in responses] Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff Semantics: counter Units: byte Help: diff --git a/qa/1690 b/qa/1690 index 5ff0addcc1..e174bbdf2d 100755 --- a/qa/1690 +++ b/qa/1690 @@ -11,12 +11,13 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series _cleanup() { - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -24,8 +25,8 @@ _cleanup() status=1 # failure is the default! hostname=`pmhostname` -redisport=`_find_free_port` -options="-p $redisport" +key_server_port=`_find_free_port` +options="-p $key_server_port" $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 @@ -42,13 +43,13 @@ _filter_source() _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port echo "Load first archive" pmseries $options --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source diff --git a/qa/1691 b/qa/1691 index 394e59624b..6507d29dc0 100755 --- a/qa/1691 +++ b/qa/1691 @@ -11,20 +11,21 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series _cleanup() { - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown cd $here $sudo rm -rf $tmp $tmp.* } status=1 # failure is the default! hostname=`pmhostname` -redisport=`_find_free_port` -options="-p $redisport" +key_server_port=`_find_free_port` +options="-p $key_server_port" $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 @@ -46,12 +47,12 @@ cat > $tmp.ignore.conf < $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port -_check_redis_server_version $redisport +_check_key_server_version $key_server_port echo && echo "Load archive using default config" pmseries $options -c $tmp.default.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source @@ -60,8 +61,8 @@ echo && echo "Query series kernel.*" pmseries $options -m `pmseries $options 'kernel.*'` -echo && echo "Clear Redis DB" -redis-cli $options flushall +echo && echo "Clear key server DB" +$keys_cli $options flushall echo && echo "Load archive using: exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user" pmseries $options -c $tmp.ignore.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source diff --git a/qa/1695 b/qa/1695 index a2ad0794b1..e3506c5d71 100755 --- a/qa/1695 +++ b/qa/1695 @@ -12,13 +12,14 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_valgrind _cleanup() { cd $here - [ -n "$redisport" ] && redis-cli -p $redisport shutdown + [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown $sudo rm -rf $tmp $tmp.* } @@ -32,7 +33,7 @@ cat < $tmp.conf [pmproxy] pcp.enabled = true http.enabled = true -redis.enabled = false +resp.enabled = false [discover] enabled = false [pmsearch] @@ -51,11 +52,11 @@ _filter_port() # real QA test starts here -redisport=`_find_free_port` -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server_version $redisport +key_server_port=`_find_free_port` +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server_version $key_server_port # start pmproxy port=`_find_free_port` @@ -63,7 +64,7 @@ mkdir -p $tmp.pmproxy/pmproxy export PCP_RUN_DIR=$tmp.pmproxy export PCP_TMP_DIR=$tmp.pmproxy -$_valgrind_clean_assert pmproxy -f -r $redisport -p $port -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr & +$_valgrind_clean_assert pmproxy -f -r $key_server_port -p $port -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr & pid=$! echo "valgrind pid: $pid" >>$seq.full diff --git a/qa/1696 b/qa/1696 index ccb9658f35..bdf7ac8552 100755 --- a/qa/1696 +++ b/qa/1696 @@ -18,7 +18,7 @@ echo "QA output created by $seq" _check_python36 # needed by ./src/pmproxy_load_test.python _check_valgrind _check_series -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { @@ -95,7 +95,7 @@ echo "=== valgrind stdout ===" | tee -a $seq.full cat $tmp.valout | _filter_valgrind echo "=== valgrind stderr ===" | tee -a $seq.full -cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to Redis" | _filter_port +cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to key server" | _filter_port # success, all done status=0 diff --git a/qa/1697 b/qa/1697 index 2c68c74706..8c1d599ebf 100755 --- a/qa/1697 +++ b/qa/1697 @@ -18,12 +18,12 @@ echo "QA output created by $seq" _check_valgrind _check_series -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { cd $here - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown $sudo rm -rf $tmp $tmp.* } @@ -58,17 +58,17 @@ _filter_proxyport() } # real QA test starts here -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -# import some well-known test data into Redis +# import some well-known test data into the key server pmseries $options --load "$here/archives/proc" | _filter_source # start pmproxy @@ -76,7 +76,7 @@ mkdir -p $tmp.pmproxy/pmproxy export PCP_RUN_DIR=$tmp.pmproxy export PCP_TMP_DIR=$tmp.pmproxy proxyport=`_find_free_port` -$_valgrind_clean_assert pmproxy -f -p $proxyport -r $redisport -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr & +$_valgrind_clean_assert pmproxy -f -p $proxyport -r $key_server_port -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr & pid=$! # valgrind takes awhile to fire up @@ -127,7 +127,7 @@ echo "=== valgrind stdout ===" | tee -a $seq.full cat $tmp.valout | _filter_valgrind echo "=== valgrind stderr ===" | tee -a $seq.full -cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to Redis" | _filter_proxyport +cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to key server" | _filter_proxyport # success, all done status=0 diff --git a/qa/1773 b/qa/1773 index 3f48901ce4..26546b65c2 100755 --- a/qa/1773 +++ b/qa/1773 @@ -13,6 +13,7 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series @@ -20,7 +21,7 @@ _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown if $need_restore then need_restore=false @@ -95,17 +96,17 @@ need_restore=true mkdir -p $tmp.farm tar -C $tmp.farm -xf archives/farm.tar.xz -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -# import multi-host test data into Redis +# import multi-host test data into key server for node in node80 node81; do pmseries $options --load $tmp.farm/farm/$node/20201124.0 | _filter_source done @@ -113,7 +114,7 @@ done # start pmproxy proxyport=`_find_free_port` echo "proxyport=$proxyport" >>$seq.full -proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af +proxyopts="-p $proxyport -r $key_server_port -t" # -Dseries,http,af pmproxy -f -U $username -x $tmp.err -Dseries,query -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! @@ -122,12 +123,12 @@ pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec if [ -f $tmp.err ] then - echo "Arrgh: errors from Redis server startup ..." + echo "Arrgh: errors from key server startup ..." cat $tmp.err fi $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq.full -redis-cli $options keys pcp:* >>$seq.full +$keys_cli $options keys pcp:* >>$seq.full cat $tmp.pmproxy.log >> $seq.full series1=`pmseries $options kernel.all.load` diff --git a/qa/1824 b/qa/1824 index e49070f293..a9606ac7bd 100755 --- a/qa/1824 +++ b/qa/1824 @@ -18,7 +18,7 @@ fi . ./common.filter . ./common.check -_check_redis_server +_check_key_server _cleanup() { diff --git a/qa/1850 b/qa/1850 index f0bc6fddb6..495a55f0eb 100755 --- a/qa/1850 +++ b/qa/1850 @@ -12,13 +12,14 @@ path="" . ./common.product . ./common.filter . ./common.check +. ./common.keys -# This test is not run if we dont have pmseries and redis installed. +# This test is not run if we dont have pmseries and key server installed. _check_series _cleanup() { - [ -n "$redisport" ] && redis-cli -p $redisport shutdown + [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -50,27 +51,27 @@ _sort_context() mkdir -p $tmp.farm tar -C $tmp.farm -xf archives/farm.tar.xz -redisport=`_find_free_port` +key_server_port=`_find_free_port` _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -args="-p $redisport -Z UTC" +args="-p $key_server_port -Z UTC" -echo "== Load metric data into this redis instance" +echo "== Load metric data into this key server instance" for node in node80 node81; do pmseries $args --load "{source.path: \"$tmp.farm/farm/$node/20201124\"}" | _filter_source done pmsleep 0.2 -# redis-cli -p $redisport --scan | LC_COLLATE=POSIX sort +# $keys_cli -p $key_server_port --scan | LC_COLLATE=POSIX sort echo echo "== extract descriptor for kernel.all.pswitch for each host" diff --git a/qa/1850.out b/qa/1850.out index 5b34b3cde8..87bc814ec1 100644 --- a/qa/1850.out +++ b/qa/1850.out @@ -1,9 +1,9 @@ QA output created by 1850 -Start test Redis server ... +Start test key server ... PING PONG -== Load metric data into this redis instance +== Load metric data into this key server instance pmseries: [Info] processed 40 archive records from PATH/node80/20201124 pmseries: [Info] processed 40 archive records from PATH/node81/20201124 diff --git a/qa/1871 b/qa/1871 index 10b26085bd..7b4cb07c9c 100755 --- a/qa/1871 +++ b/qa/1871 @@ -15,11 +15,11 @@ echo "QA output created by $seq" _check_series _check_search -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { - [ -n "$port" ] && redis-cli $port shutdown + [ -n "$port" ] && $keys_cli $port shutdown cd $here $sudo rm -rf $tmp $tmp.* } @@ -29,16 +29,18 @@ $sudo rm -rf $tmp $tmp.* $seq.full trap "_cleanup; exit \$status" 0 1 2 3 15 # real QA test starts here -echo "=== Start test Redis server ===" -redisport=`_find_free_port` -redispath=`_find_redis_modules` -redisearch="$redispath/redisearch.$DSO_SUFFIX" +echo "=== Start test key server ===" +key_server_port=`_find_free_port` +key_server_name=`_find_key_server_name` +key_server_path=`_find_key_server_modules` +key_server_search=`_find_key_server_search` +keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" # workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -sudo chmod +x $redisearch -# RediSearch module is only accessible by the redis user -sudo -u redis redis-server --port $redisport --save "" --loadmodule $redisearch > $tmp.redis 2>&1 & -port="-p $redisport" -_check_redis_ping $redisport +sudo chmod +x $keysearch +# module is only accessible by the associated key server user account +sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & +port="-p $key_server_port" +_check_key_server_ping $key_server_port echo echo "=== Load an empty index ===" @@ -73,8 +75,10 @@ pmsearch $port -i -C | _filter_info echo "=== Extra reporting - timing, score, hits, doc IDs ===" _filter_extras() { - # need to filter out score, since it seems that Redisearch indexes records asynchronously and therefore they may be - # delay before "score" "stabilizes" until all records are process - this may take longer then adding of records itself + # need to filter out score, since it seems the search module indexes + # records asynchronously and therefore there may be a delay before + # "score" "stabilizes" until all records are processed - this may + # take longer then the adding of the records themselves sed \ -e 's/\(Score:\) 0\.[0-9][0-9]*/\1 0.XX/g' \ -e 's/\([0-9][0-9]* hit in\) 0\.[0-9][0-9]* seconds/\1 0.X seconds/g' \ @@ -93,7 +97,7 @@ echo "=== Indom search ===" pmsearch $port -n 29.3 | _filter_extras -cat $tmp.redis >> $seq.full +cat $tmp.keys >> $seq.full # success, all done status=0 diff --git a/qa/1872 b/qa/1872 index 49f90c0180..49e4ea6e88 100755 --- a/qa/1872 +++ b/qa/1872 @@ -12,16 +12,17 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series _check_search -_check_redis_server_version_offline +_check_key_server_version_offline _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$port" ] && redis-cli $port shutdown + [ -n "$port" ] && $keys_cli $port shutdown if $need_restore then need_restore=false @@ -67,31 +68,33 @@ $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -echo "=== Start test Redis server ===" -redisport=`_find_free_port` -redispath=`_find_redis_modules` -redisearch="$redispath/redisearch.$DSO_SUFFIX" +echo "=== Start test key server ===" +key_server_port=`_find_free_port` +key_server_name=`_find_key_server_name` +key_server_path=`_find_key_server_modules` +key_server_search=`_find_key_server_search` +keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" # workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -sudo chmod +x $redisearch -# RediSearch module is only accessible by the redis user -sudo -u redis redis-server --port $redisport --save "" --loadmodule $redisearch > $tmp.redis 2>&1 & -_check_redis_ping $redisport -port="-p $redisport" +sudo chmod +x $keysearch +# search module is only accessible by the key server user account +sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +port="-p $key_server_port" echo cat > $tmp.conf <>$seq.full -proxyopts="-p $proxyport -r $redisport -c $tmp.conf" # -Dseries,http,af +proxyopts="-p $proxyport -r $key_server_port -c $tmp.conf" # -Dseries,http,af pmproxy -f -U $username -x $seq.full -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec -# have to wait until everything is properly indexed in Redis +# have to wait until everything is properly indexed in the key server pmsleep 10s $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq.full -redis-cli $port keys pcp:text >> $seq.full +$keys_cli $port keys pcp:text >> $seq.full cat $tmp.pmproxy.log >> $seq.full echo "=== /search/text - query for '99' 'random' 'interesting' 'result' ===" @@ -350,7 +353,7 @@ echo "$url" >> $seq.full curl --get --silent "$url" | tee -a $seq.full | pmjson | _filter_info -cat $tmp.redis >> $seq.full +cat $tmp.keys >> $seq.full cat $tmp.pmproxy.log >> $seq.full # success, all done diff --git a/qa/1874 b/qa/1874 index 9ad0c46f2f..91b68eb484 100755 --- a/qa/1874 +++ b/qa/1874 @@ -13,6 +13,7 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series @@ -20,7 +21,7 @@ _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$options" ] && redis-cli $options shutdown + [ -n "$options" ] && $keys_cli $options shutdown if $need_restore then need_restore=false @@ -71,24 +72,24 @@ $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* $sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* need_restore=true -echo "Start test Redis server ..." -redisport=`_find_free_port` -options="-p $redisport" -echo "redisport=$redisport" >>$seq.full -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +key_server_port=`_find_free_port` +options="-p $key_server_port" +echo "key_server_port=$key_server_port" >>$seq.full +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -# import some well-known test data into Redis +# import some well-known test data into the key server pmseries $options --load "$here/archives/proc" | _filter_source # start pmproxy proxyport=`_find_free_port` echo "proxyport=$proxyport" >>$seq.full -proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af +proxyopts="-p $proxyport -r $key_server_port -t" # -Dseries,http,af pmproxy -f -U $username -x $tmp.err -Dseries,query -l $tmp.pmproxy.log $proxyopts & pmproxy_pid=$! @@ -97,12 +98,12 @@ pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec if [ -f $tmp.err ] then - echo "Arrgh: errors from Redis server startup ..." + echo "Arrgh: errors from key server startup ..." cat $tmp.err fi $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq.full -redis-cli $options keys pcp:* >> $seq.full +$keys_cli $options keys pcp:* >> $seq.full cat $tmp.pmproxy.log >> $seq.full # Sanity check diff --git a/qa/1875 b/qa/1875 index c2151322c1..d6b82c1e20 100755 --- a/qa/1875 +++ b/qa/1875 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1875 -# Exercise pmseries & pmproxy with Redis authentication +# Exercise pmseries & pmproxy with key server authentication # # Copyright (c) 2021 Red Hat. All Rights Reserved. # @@ -12,16 +12,17 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check +. ./common.keys _check_series -redis_version_major=`redis-server --version | sed -E 's/.*v=([0-9]).*/\1/g'` -[ $redis_version_major -ge 6 ] || _notrun "Redis version too old (this test requires Redis v6+)" +key_server_version_major=`$key_server --version | sed -E 's/.*v=([0-9]).*/\1/g'` +[ $key_server_version_major -ge 6 ] || _notrun "Key server version too old (this test requires v6+)" _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$redis_pid" ] && $signal -s TERM $redis_pid + [ -n "$keyserver_pid" ] && $signal -s TERM $keyserver_pid $sudo rm -rf $tmp $tmp.* } @@ -35,17 +36,17 @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # real QA test starts here -echo "Start Redis server with password auth ..." -redisport=`_find_free_port` -redis-server --port $redisport --save "" --requirepass 'top$secret' > $tmp.redis 2>&1 & -redis_pid=$! +echo "Start key server with password auth ..." +key_server_port=`_find_free_port` +$key_server --port $key_server_port --save "" --requirepass 'top$secret' > $tmp.keys 2>&1 & +key_server_pid=$! pmsleep 0.125 echo echo "=== password auth enabled, no password provided" | tee -a $seq.full cat < $tmp.conf EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full | grep -o "Cannot connect" +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full | grep -o "Cannot connect" echo echo "=== password auth enabled, wrong password provided" | tee -a $seq.full @@ -54,7 +55,7 @@ cat < $tmp.conf enabled = true auth.password = topsecret EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" echo echo "=== password auth enabled, correct password provided" | tee -a $seq.full @@ -63,35 +64,35 @@ cat < $tmp.conf enabled = true auth.password = top\$secret EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" || echo OK +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" || echo OK echo echo "=== password auth enabled, correct password provided (pmproxy)" | tee -a $seq.full pmproxyport=`_find_free_port` -pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $redisport -t -c $tmp.conf & +pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $key_server_port -t -c $tmp.conf & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$pmproxyport -v -t 5sec -grep -o "Redis slots, schema version setup" $tmp.pmproxy.log +grep -o "Key server slots, schema version setup" $tmp.pmproxy.log $signal -s TERM $pmproxy_pid echo -echo "Stop redis server with password auth ..." -$signal -s TERM $redis_pid +echo "Stop key server with password auth ..." +$signal -s TERM $key_server_pid -echo "Start Redis server with ACL auth ..." -redisport=`_find_free_port` -redis-server --port $redisport --save "" --user alice on '>top$secret' '~*' '+@all' --user default off > $tmp.redis 2>&1 & -redis_pid=$! +echo "Start key server with ACL auth ..." +key_server_port=`_find_free_port` +$key_server --port $key_server_port --save "" --user alice on '>top$secret' '~*' '+@all' --user default off > $tmp.keys 2>&1 & +key_server_pid=$! pmsleep 0.125 echo echo "=== ACL auth enabled, no username/password provided" | tee -a $seq.full cat < $tmp.conf EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full | grep -o "Cannot connect" +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full | grep -o "Cannot connect" echo echo "=== ACL auth enabled, unknown username provided" | tee -a $seq.full @@ -101,7 +102,7 @@ enabled = true auth.username = max auth.password = topsecret EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" echo echo "=== ACL auth enabled, correct username, wrong password provided" | tee -a $seq.full @@ -111,7 +112,7 @@ enabled = true auth.username = alice auth.password = topsecret EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" echo echo "=== ACL auth enabled, correct username and password provided" | tee -a $seq.full @@ -121,16 +122,16 @@ enabled = true auth.username = alice auth.password = top\$secret EOF -pmseries -p $redisport -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" || echo OK +pmseries -p $key_server_port -c $tmp.conf disk.dev.read 2>&1 | tee -a $seq.full| grep -o "Cannot connect" || echo OK echo echo "=== ACL auth enabled, correct username and password provided (pmproxy)" | tee -a $seq.full pmproxyport=`_find_free_port` -pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $redisport -t -c $tmp.conf & +pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -r $key_server_port -t -c $tmp.conf & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$pmproxyport -v -t 5sec -grep -o "Redis slots, schema version setup" $tmp.pmproxy.log +grep -o "Key server slots, schema version setup" $tmp.pmproxy.log # success, all done diff --git a/qa/1875.out b/qa/1875.out index c192fef2ac..7e61aa458a 100644 --- a/qa/1875.out +++ b/qa/1875.out @@ -1,5 +1,5 @@ QA output created by 1875 -Start Redis server with password auth ... +Start key server with password auth ... === password auth enabled, no password provided Cannot connect @@ -11,10 +11,10 @@ Cannot connect OK === password auth enabled, correct password provided (pmproxy) -Redis slots, schema version setup +Key server slots, schema version setup -Stop redis server with password auth ... -Start Redis server with ACL auth ... +Stop key server with password auth ... +Start key server with ACL auth ... === ACL auth enabled, no username/password provided Cannot connect @@ -29,4 +29,4 @@ Cannot connect OK === ACL auth enabled, correct username and password provided (pmproxy) -Redis slots, schema version setup +Key server slots, schema version setup diff --git a/qa/1877 b/qa/1877 index e7d6209a46..e1cb49f11a 100755 --- a/qa/1877 +++ b/qa/1877 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1877 -# Verify that pmproxy does not connect to Redis if it's not required +# Verify that pmproxy does not connect to key server if not required # # Copyright (c) 2021 Red Hat. All Rights Reserved. # @@ -13,7 +13,7 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_redis_server +_check_key_server _cleanup() { @@ -33,20 +33,20 @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # real QA test starts here echo -echo "=== empty configuration file, Redis not required" +echo "=== empty configuration file, key server not required" echo > $tmp.pmproxy.conf pmproxyport=`_find_free_port` pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -s $tmp.pmproxy.socket -c $tmp.pmproxy.conf & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$pmproxyport -v -t 5sec -grep -o "Redis.*" $tmp.pmproxy.log +grep -o "Key.*" $tmp.pmproxy.log cat $tmp.pmproxy.log >>$seq.full $signal -s TERM $pmproxy_pid pmproxy_pid="" echo -echo "=== configuration file with pmseries REST API enabled, Redis is required" +echo "=== configuration file with pmseries REST API enabled, key server is required" cat < $tmp.pmproxy.conf [pmseries] enabled = true @@ -56,7 +56,7 @@ pmproxy -f -U $username -x $tmp.err -l $tmp.pmproxy.log -p $pmproxyport -s $tmp. pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$pmproxyport -v -t 5sec -grep -o "Redis.*" $tmp.pmproxy.log +grep -o "Key.*" $tmp.pmproxy.log cat $tmp.pmproxy.log >>$seq.full $signal -s TERM $pmproxy_pid pmproxy_pid="" diff --git a/qa/1877.out b/qa/1877.out index eeb1fa183b..29e0da3a33 100644 --- a/qa/1877.out +++ b/qa/1877.out @@ -1,6 +1,6 @@ QA output created by 1877 -=== empty configuration file, Redis not required +=== empty configuration file, key server not required -=== configuration file with pmseries REST API enabled, Redis is required -Redis slots, schema version setup +=== configuration file with pmseries REST API enabled, key server is required +Key server slots, schema version setup diff --git a/qa/1886 b/qa/1886 index ef829cedf9..0a6d762975 100755 --- a/qa/1886 +++ b/qa/1886 @@ -12,13 +12,14 @@ path="" . ./common.product . ./common.filter . ./common.check +. ./common.keys -# This test is not run if we dont have pmseries and redis installed. +# This test is not run if we dont have pmseries and a key server installed. _check_series _cleanup() { - [ -n "$redisport" ] && redis-cli -p $redisport shutdown + [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -46,21 +47,21 @@ _sort_context() } # real QA test starts here -redisport=`_find_free_port` +key_server_port=`_find_free_port` _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -args="-p $redisport -Z UTC" +args="-p $key_server_port -Z UTC" -echo "== Load metric data into this redis instance" +echo "== Load metric data into this key server instance" pmseries $args --load "{source.path: \"$here/archives/proc\"}" | _filter_source echo;echo "== Verify rate() function for a singular metric" diff --git a/qa/1886.out b/qa/1886.out index f736da0fc1..5e10ae2014 100644 --- a/qa/1886.out +++ b/qa/1886.out @@ -1,9 +1,9 @@ QA output created by 1886 -Start test Redis server ... +Start test key server ... PING PONG -== Load metric data into this redis instance +== Load metric data into this key server instance pmseries: [Info] processed 5 archive records from PATH/archives/proc == Verify rate() function for a singular metric diff --git a/qa/1906 b/qa/1906 index 17ca77fe8e..1a0f286a91 100755 --- a/qa/1906 +++ b/qa/1906 @@ -14,12 +14,12 @@ path="" . ./common.filter . ./common.check -# This test is not run if we dont have pmseries and redis installed. +# This test is not run if we dont have pmseries and a key server installed. _check_series _cleanup() { - [ -n "$redisport" ] && redis-cli -p $redisport shutdown + [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown _restore_config $PCP_SYSCONF_DIR/pmseries cd $here $sudo rm -rf $tmp $tmp.* @@ -47,21 +47,21 @@ _sort_context() } # real QA test starts here -redisport=`_find_free_port` +key_server_port=`_find_free_port` _save_config $PCP_SYSCONF_DIR/pmseries $sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -echo "Start test Redis server ..." -redis-server --port $redisport --save "" > $tmp.redis 2>&1 & -_check_redis_ping $redisport -_check_redis_server $redisport +echo "Start test key server ..." +$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 & +_check_key_server_ping $key_server_port +_check_key_server $key_server_port echo -_check_redis_server_version $redisport +_check_key_server_version $key_server_port -args="-p $redisport -Z UTC" +args="-p $key_server_port -Z UTC" -echo "== Load metric data into this redis instance" +echo "== Load metric data into this key server instance" pmseries $args --load "{source.path: \"$here/archives/proc\"}" | _filter_source diff --git a/qa/1906.out b/qa/1906.out index 4b92517843..0fba485f6d 100644 --- a/qa/1906.out +++ b/qa/1906.out @@ -1,9 +1,9 @@ QA output created by 1906 -Start test Redis server ... +Start test key server ... PING PONG -== Load metric data into this redis instance +== Load metric data into this key server instance pmseries: [Info] processed 5 archive records from PATH/archives/proc == Verify multiplication operator for a non-singular metric diff --git a/qa/1984 b/qa/1984 index f41cd204e0..19a2bd657d 100755 --- a/qa/1984 +++ b/qa/1984 @@ -14,8 +14,8 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_redis_server [ -d $PCP_PMDAS_DIR/redis ] || _notrun "Redis PMDA is not installed" +which redis-server >/dev/null 2>&1 || _notrun "No redis-server binary found" _cleanup() { diff --git a/qa/913 b/qa/913 index 7988bf2590..fee6d63bdc 100755 --- a/qa/913 +++ b/qa/913 @@ -13,7 +13,7 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -[ -d $PCP_PMDAS_DIR/redis ] || _notrun "redis.io PMDA is not installed" +[ -d $PCP_PMDAS_DIR/redis ] || _notrun "Redis PMDA is not installed" redis_port=6379 $PCP_BINADM_DIR/telnet-probe -c localhost $redis_port \ || _notrun "Noone home on default Redis port $redis_port" diff --git a/qa/GNUmakefile b/qa/GNUmakefile index d83673ba0e..56107a64f7 100644 --- a/qa/GNUmakefile +++ b/qa/GNUmakefile @@ -81,7 +81,7 @@ COMMON = common.check common.config common.filter \ common.qt common.containers \ common.discovery common.compress \ common.python common.openmetrics \ - common.bcc common.bpf common.bpftrace common.redis + common.bcc common.bpf common.bpftrace common.keys OTHERS = group triaged qa_hosts.primary README sanity.coverage \ $(shell echo valgrind-suppress* helgrind-suppress*) diff --git a/qa/check-vars b/qa/check-vars index 7c8d1e5b86..a764e47ab9 100755 --- a/qa/check-vars +++ b/qa/check-vars @@ -63,10 +63,12 @@ docker # $PCP_DOCKER_PROG unless $docker set earlier podman # $PCP_PODMAN_PROG unless $podman set earlier # -- common.python python # alias for $PCP_PYTHOON_PROG (this week's Python interpreter) -# -- common.redis -redis_node1_port # returned from _redis_cluster_3nodes_start() -redis_node2_port # returned from _redis_cluster_3nodes_start() -redis_node3_port # returned from _redis_cluster_3nodes_start() +# -- common.keys +keys_cli # command line client interface for accessing keys +key_server # command line server binary serving RESP protocol +key_server_node1_port # returned from _key_server_cluster_3nodes_start() +key_server_node2_port # returned from _key_server_cluster_3nodes_start() +key_server_node3_port # returned from _key_server_cluster_3nodes_start() End-of-File # these ones are exempted for various reasons diff --git a/qa/common.check b/qa/common.check index 5a1e186d7c..1fd593cc07 100644 --- a/qa/common.check +++ b/qa/common.check @@ -1623,97 +1623,175 @@ _check_64bit_platform() esac } -# check we have pmseries, redis-cli and redis-server installed. -# Note: use _check_redis_server to test if redis is running locally. +# check we have pmseries and a key server -cli and -server binary installed. +# Note: use _check_key_server to test if a key server is running locally. # _check_series() { which pmseries >/dev/null 2>&1 || \ _notrun "pmseries command line utility not installed" - which redis-cli >/dev/null 2>&1 || \ - _notrun "Redis command line utility not installed" - which redis-server >/dev/null 2>&1 || \ - _notrun "Redis server utility not installed" + + if which valkey-cli >/dev/null 2>&1 + then + key_server=valkey-server + elif which redis-cli >/dev/null 2>&1 + then + key_server=redis-server + else + _notrun "No key server command line interface found" + fi + + which $key_server >/dev/null 2>&1 || _notrun "No $key_server binary found" } -# check redis-server version -# ... assumes _check_series already done and redis-server is running +# check key server version +# ... assumes _check_series already done and a key server is running # -_check_redis_server_version() +_check_key_server_version() { - eval __`redis-cli -p "$1" info server | - grep redis_version: | + if which valkey-cli >/dev/null 2>&1 + then + keys_cli=valkey-cli + keys_ver=valkey_version + elif which redis-cli >/dev/null 2>&1 + then + keys_cli=redis-cli + keys_ver=redis_version + else + _notrun "No key server command line interface found" + fi + eval __`$keys_cli -p "$1" info server | + grep "$keys_ver:" | sed -e 's/:/=/g' -e 's/\..*//g'` - [ $__redis_version -ge 5 ] || _notrun "Redis server @ port $1: version $__redis_version too old" + [ $__key_server_version -ge 5 ] || _notrun "Key server @ port $1: version $__key_server_version too old" } -# check redis-server version without running Redis server -_check_redis_server_version_offline() +# check key server version without a running key server +_check_key_server_version_offline() { - __redis_version=`redis-server --version | sed -E 's/.*v=([0-9]).*/\1/g'` - [ $__redis_version -ge 5 ] || _notrun "Redis version $__redis_version too old" + if which valkey-server >/dev/null 2>&1 + then + key_server=valkey-server + elif which redis-server >/dev/null 2>&1 + then + key_server=redis-server + else + _notrun "No key server binary found" + fi + __key_server_version=`$key_server --version | sed -E 's/.*v=([0-9]).*/\1/g'` + [ $__key_server_version -ge 5 ] || _notrun "Key server version $__key_server_version too old" } -# Check redis server is installed and running locally and the version +# Check a key server is installed and running locally and the version # is not too old. Optional port parameter defaults to 6379 # -_check_redis_server() +_check_key_server() { _check_series - __redisport=6379 - [ -n "$1" ] && __redisport="$1" - redis-cli -p $__redisport ping >/dev/null - [ $? -eq 0 ] || _notrun "Redis server not running locally" - _check_redis_server_version $__redisport + + if which valkey-cli >/dev/null 2>&1 + then + keys_cli=valkey-cli + elif which redis-cli >/dev/null 2>&1 + then + keys_cli=redis-cli + else + _notrun "No key server command line interface found" + fi + + __key_server_port=6379 + [ -n "$1" ] && __key_server_port="$1" + $keys_cli -p $__key_server_port ping >/dev/null + [ $? -eq 0 ] || _notrun "Key server not running locally" + _check_key_server_version $__key_server_port } -# be prepared to "ping" the Redis server a few times until we see +# be prepared to "ping" the key server a few times until we see # the expected response # -_check_redis_ping() +_check_key_server_ping() { if [ $# -ne 1 ] then - echo "_check_redis_ping: botch: needs port# as only argument" + echo "_check_key_server_ping: botch: needs port# as only argument" return fi + + if which valkey-cli >/dev/null 2>&1 + then + keys_cli=valkey-cli + elif which redis-cli >/dev/null 2>&1 + then + keys_cli=redis-cli + else + _notrun "No key server command line interface found" + fi + echo "PING" __i=0 while [ $__i -lt 10 ] do - redis-cli -p $1 ping >$tmp.redis.tmp 2>&1 - if grep PONG $tmp.redis.tmp + $keys_cli -p $1 ping >$tmp.keys.tmp 2>&1 + if grep PONG $tmp.keys.tmp then - rm -f $tmp.redis.tmp + rm -f $tmp.keys.tmp break fi pmsleep 0.125 __i=`expr $__i + 1` done - [ -f $tmp.redis.tmp ] && cat $tmp.redis.tmp + [ -f $tmp.keys.tmp ] && cat $tmp.keys.tmp } -# Check if the RediSearch module is installed +# Check if the key server search module is installed _check_search() { which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch command line utility not installed" - __redispath=`_find_redis_modules` - __redisearch="$__redispath/redisearch.$DSO_SUFFIX" - # Redis modules path is not accessibly by the pcpqa user on Fedora/RHEL - sudo test -f $__redisearch || \ - _notrun "RediSearch module is not installed" + __key_search=`_find_key_server_search` + __key_server_path=`_find_key_server_modules` + __key_server_search="$__key_server_path/$__key_search.$DSO_SUFFIX" + # module path is not accessibly by the pcpqa user on Fedora/RHEL + sudo test -f $__key_server_search || \ + _notrun "$__key_search module is not installed" +} + +_find_key_server_search() +{ + if which valkey-server >/dev/null 2>&1 + then + echo valkeysearch + elif which redis-server >/dev/null 2>&1 + then + echo redisearch + else + echo + fi +} + +_find_key_server_name() +{ + if which valkey-server >/dev/null 2>&1 + then + echo valkey + elif which redis-server >/dev/null 2>&1 + then + echo redis + else + echo + fi } -_find_redis_modules() +_find_key_server_modules() { - # scan through a list of known locations for Redis modules - if sudo test -d "$PCP_LIB_DIR/redis/modules" + __key_server_name=`_find_key_server_name` + # scan through a list of known locations for modules + if sudo test -d "$PCP_LIB_DIR/$__key_server_name/modules" then - echo "$PCP_LIB_DIR/redis/modules" + echo "$PCP_LIB_DIR/$__key_server_name/modules" else - echo "$PCP_LIB32_DIR/redis/modules" + echo "$PCP_LIB32_DIR/$__key_server_name/modules" fi } diff --git a/qa/common.filter b/qa/common.filter index 323679fce4..9af53f47c8 100644 --- a/qa/common.filter +++ b/qa/common.filter @@ -161,10 +161,10 @@ _filter_pmproxy_log() -e 's/pmproxy([1-9][0-9]*) Error:/pmproxy(PID) Error:/' \ -e '/^pmproxy: Signalled (signal[=][1-9][0-9]*).*/d' \ -e '/^pmproxy: disabled time series,.*(missing)/d' \ - -e '/Error: unsupported redis server (got v/d' \ + -e '/Error: unsupported key server (got v/d' \ -e '/Info: TLS configured in pmproxy.conf/d' \ -e '/Failed to create avahi client:/d' \ - -e '/Info: connected to .*redis.*/d' \ + -e '/Info: connected to .*key server.*/d' \ -e '/Info: .* setup$/d' \ -e '/Info: OpenSSL /d' \ -e '/Using cipher /d' \ diff --git a/qa/common.keys b/qa/common.keys new file mode 100644 index 0000000000..998bdfa3fc --- /dev/null +++ b/qa/common.keys @@ -0,0 +1,127 @@ +# +# Common shell routines for testing remote key servers (incl. clustering) +# +# Copyright (c) 2021,2024 Red Hat. +# + +# wait for local server on given port +_wait_for_key_server() +{ + __port=6379 # default + [ $# -eq 1 ] && __port=$1 + __n=1 + while [ $__n -lt 10 ]; do + $keys_cli -p $__port ping >/dev/null 2>&1 && return 0 + pmsleep 0.25 + __n=`expr $__n + 1` + done + echo _wait_for_key_server on port $__port FAILED + return 1 +} + +_key_server_cluster_3nodes_start() +{ + echo "== Start a key server cluster" + echo "== Start node 1" + key_server_node1_port=`_find_free_port` + $key_server --port $key_server_node1_port --save "" --cluster-enabled yes --cluster-config-file $tmp.keys1.cluster.conf > $tmp.keys1.out 2>&1 & + __key_server_node1_pid=$! + echo "key_server_node1_port=$key_server_node1_port" >>$seq.full + echo "__key_server_node1_pid=$__key_server_node1_pid" >>$seq.full + _wait_for_key_server $key_server_node1_port + $keys_cli -p $key_server_node1_port ping + $keys_cli -p $key_server_node1_port flushall + + echo "== Start node 2" + key_server_node2_port=`_find_free_port` + $key_server --port $key_server_node2_port --save "" --cluster-enabled yes --cluster-config-file $tmp.keys2.cluster.conf > $tmp.keys2.out 2>&1 & + __key_server_node2_pid=$! + echo "key_server_node2_port=$key_server_node2_port" >>$seq.full + echo "__key_server_node2_pid=$__key_server_node2_pid" >>$seq.full + _wait_for_key_server $key_server_node2_port + $keys_cli -p $key_server_node2_port ping + $keys_cli -p $key_server_node2_port flushall + + echo "== Start node 3" + key_server_node3_port=`_find_free_port` + $key_server --port $key_server_node3_port --save "" --cluster-enabled yes --cluster-config-file $tmp.keys3.cluster.conf > $tmp.keys3.out 2>&1 & + __key_server_node3_pid=$! + echo "key_server_node3_port=$key_server_node3_port" >>$seq.full + echo "__key_server_node3_pid=$__key_server_node3_pid" >>$seq.full + _wait_for_key_server $key_server_node3_port + $keys_cli -p $key_server_node3_port ping + $keys_cli -p $key_server_node3_port flushall + + echo + echo "== Create key server cluster" + # --cluster-yes is not supported <= 5.0.3 (RHEL 7), use workaround instead + yes yes | $keys_cli --cluster create 127.0.0.1:$key_server_node1_port 127.0.0.1:$key_server_node2_port 127.0.0.1:$key_server_node3_port \ + | sed 's/\x1b\[[0-9;]*m//g' \ + | tee -a $seq.full \ + | grep '[OK]' + + echo + echo "== Wait for cluster to get ready" + for __i in `seq 1 30` + do + if $keys_cli -p $key_server_node1_port cluster info | grep -q "cluster_state:ok" && echo "OK" + then + __cluster_ready=yes + break + fi + pmsleep 2 + done + if [ -z "$__cluster_ready" ] + then + echo "Key server cluster didn't get ready in 60 seconds:" + echo $keys_cli -p $key_server_node1_port cluster info + $keys_cli -p $key_server_node1_port cluster info + exit 1 + fi + + echo + echo "== Cluster status" + $keys_cli -p $key_server_node1_port cluster info | tr -d '\r' | grep -E "cluster_(state|known_nodes|size)" +} + +_key_server_cluster_3nodes_stop() +{ + printf "\n== Key server Node 1 keys:\n" >> $seq.full + $keys_cli -p $key_server_node1_port keys '*' >> $seq.full + printf "\n== Key server Node 1 log:\n" >> $seq.full + cat $tmp.keys1.out >> $seq.full + + printf "\n== Key server Node 2 keys:\n" >> $seq.full + $keys_cli -p $key_server_node2_port keys '*' >> $seq.full + printf "\n== Key server Node 2 log:\n" >> $seq.full + cat $tmp.keys2.out >> $seq.full + + printf "\n== Key server Node 3 keys:\n" >> $seq.full + $keys_cli -p $key_server_node3_port keys '*' >> $seq.full + printf "\n== Key server Node 3 log:\n" >> $seq.full + cat $tmp.keys3.out >> $seq.full + + [ -n "$key_server_node1_port" ] && $keys_cli -p $key_server_node1_port shutdown + [ -n "$key_server_node2_port" ] && $keys_cli -p $key_server_node2_port shutdown + [ -n "$key_server_node3_port" ] && $keys_cli -p $key_server_node3_port shutdown +} + +if which valkey-cli >/dev/null 2>&1 +then + keys_cli=valkey-cli +elif which redis-cli >/dev/null 2>&1 +then + keys_cli=redis-cli +else + _notrun "No key server command line interface found" +fi + +if which valkey-server >/dev/null 2>&1 +then + key_server=valkey-server +elif which redis-server >/dev/null 2>&1 +then + key_server=redis-server +else + _notrun "No key server binary found" +fi diff --git a/qa/common.redis b/qa/common.redis deleted file mode 100644 index 8a07218837..0000000000 --- a/qa/common.redis +++ /dev/null @@ -1,105 +0,0 @@ -# -# Common shell routines for testing Redis (cluster) -# -# Copyright (c) 2021 Red Hat. -# - -# wait for local redis on given port -_wait_for_redis() -{ - __port=6379 # default - [ $# -eq 1 ] && __port=$1 - __n=1 - while [ $__n -lt 10 ]; do - redis-cli -p $__port ping >/dev/null 2>&1 && return 0 - pmsleep 0.25 - __n=`expr $__n + 1` - done - echo _wait_for_redis on port $__port FAILED - return 1 -} - -_redis_cluster_3nodes_start() -{ - echo "== Start a Redis cluster" - echo "== Start node 1" - redis_node1_port=`_find_free_port` - redis-server --port $redis_node1_port --save "" --cluster-enabled yes --cluster-config-file $tmp.redis1.cluster.conf > $tmp.redis1.out 2>&1 & - __redis_node1_pid=$! - echo "redis_node1_port=$redis_node1_port" >>$seq.full - echo "__redis_node1_pid=$__redis_node1_pid" >>$seq.full - _wait_for_redis $redis_node1_port - redis-cli -p $redis_node1_port ping - redis-cli -p $redis_node1_port flushall - - echo "== Start node 2" - redis_node2_port=`_find_free_port` - redis-server --port $redis_node2_port --save "" --cluster-enabled yes --cluster-config-file $tmp.redis2.cluster.conf > $tmp.redis2.out 2>&1 & - __redis_node2_pid=$! - echo "redis_node2_port=$redis_node2_port" >>$seq.full - echo "__redis_node2_pid=$__redis_node2_pid" >>$seq.full - _wait_for_redis $redis_node2_port - redis-cli -p $redis_node2_port ping - redis-cli -p $redis_node2_port flushall - - echo "== Start node 3" - redis_node3_port=`_find_free_port` - redis-server --port $redis_node3_port --save "" --cluster-enabled yes --cluster-config-file $tmp.redis3.cluster.conf > $tmp.redis3.out 2>&1 & - __redis_node3_pid=$! - echo "redis_node3_port=$redis_node3_port" >>$seq.full - echo "__redis_node3_pid=$__redis_node3_pid" >>$seq.full - _wait_for_redis $redis_node3_port - redis-cli -p $redis_node3_port ping - redis-cli -p $redis_node3_port flushall - - echo - echo "== Create Redis cluster" - # --cluster-yes is not supported in Redis 5.0.3 (RHEL 7), use workaround instead - yes yes | redis-cli --cluster create 127.0.0.1:$redis_node1_port 127.0.0.1:$redis_node2_port 127.0.0.1:$redis_node3_port \ - | sed 's/\x1b\[[0-9;]*m//g' \ - | tee -a $seq.full \ - | grep '[OK]' - - echo - echo "== Wait for cluster to get ready" - for __i in `seq 1 30` - do - if redis-cli -p $redis_node1_port cluster info | grep -q "cluster_state:ok" && echo "OK"; then - __cluster_ready=yes - break - fi - pmsleep 2 - done - if [ -z "$__cluster_ready" ]; then - echo "Redis cluster didn't get ready in 60 seconds:" - echo redis-cli -p $redis_node1_port cluster info - redis-cli -p $redis_node1_port cluster info - exit 1 - fi - - echo - echo "== Cluster status" - redis-cli -p $redis_node1_port cluster info | tr -d '\r' | grep -E "cluster_(state|known_nodes|size)" -} - -_redis_cluster_3nodes_stop() -{ - printf "\n== Redis Node 1 keys:\n" >> $seq.full - redis-cli -p $redis_node1_port keys '*' >> $seq.full - printf "\n== Redis Node 1 log:\n" >> $seq.full - cat $tmp.redis1.out >> $seq.full - - printf "\n== Redis Node 2 keys:\n" >> $seq.full - redis-cli -p $redis_node2_port keys '*' >> $seq.full - printf "\n== Redis Node 2 log:\n" >> $seq.full - cat $tmp.redis2.out >> $seq.full - - printf "\n== Redis Node 3 keys:\n" >> $seq.full - redis-cli -p $redis_node3_port keys '*' >> $seq.full - printf "\n== Redis Node 3 log:\n" >> $seq.full - cat $tmp.redis3.out >> $seq.full - - [ -n "$redis_node1_port" ] && redis-cli -p $redis_node1_port shutdown - [ -n "$redis_node2_port" ] && redis-cli -p $redis_node2_port shutdown - [ -n "$redis_node3_port" ] && redis-cli -p $redis_node3_port shutdown -} diff --git a/qa/src/GNUlocaldefs b/qa/src/GNUlocaldefs index c697a7e64e..eac34ab75a 100644 --- a/qa/src/GNUlocaldefs +++ b/qa/src/GNUlocaldefs @@ -153,7 +153,7 @@ PYTHONFILES = \ mergelabels.python mergelabelsets.python \ bcc_version_check.python sort_xml.python labelsets.python \ labelsets_memleak.python labels_changing.python \ - bcc_netproc.python redis_proxy.python pythonserver.python + bcc_netproc.python key_server_proxy.python pythonserver.python # not installed: PYFILES = $(shell echo $(PYTHONFILES) | sed -e 's/\.python/.py/g') else diff --git a/qa/src/redis_proxy.python b/qa/src/key_server_proxy.python similarity index 92% rename from qa/src/redis_proxy.python rename to qa/src/key_server_proxy.python index 5137c73a0d..3b8ce32a63 100755 --- a/qa/src/redis_proxy.python +++ b/qa/src/key_server_proxy.python @@ -26,8 +26,8 @@ async def pipe(reader, writer, args, local_writer): and args.loading_delay and time.time() < args.loading_delay ): - # https://github.com/redis/redis/blob/6.2.6/src/server.c#L2508-L2509 - data = b"-LOADING Redis is loading the dataset in memory\r\n" + # https://github.com/valkey-io/valkey/blob/unstable/src/server.c#L1789 + data = b"-LOADING Valkey is loading the dataset in memory\r\n" print("<", data) local_writer.write(data) else: diff --git a/src/libpcp_web/src/load.c b/src/libpcp_web/src/load.c index 2da8ecc801..ab76bc20b2 100644 --- a/src/libpcp_web/src/load.c +++ b/src/libpcp_web/src/load.c @@ -498,7 +498,7 @@ series_cache_update(seriesLoadBaton *baton, struct dict *exclude) if (write_meta) get_instance_metadata(baton, metric->desc.indom, write_inst); - /* initiate writes to backend caching servers (Redis) */ + /* initiate writes to backend caching key servers */ server_cache_metric(baton, metric, timestamp, write_meta, write_data); } diff --git a/src/libpcp_web/src/slots.c b/src/libpcp_web/src/slots.c index f8b5684a34..a3913ba4d1 100644 --- a/src/libpcp_web/src/slots.c +++ b/src/libpcp_web/src/slots.c @@ -52,7 +52,7 @@ redis_disconnect_callback(const redisAsyncContext *redis, int status) { if (status == REDIS_OK) { if (pmDebugOptions.series) - fprintf(stderr, "Disconnected from redis on %s:%d\n", + fprintf(stderr, "Disconnected from key server on %s:%d\n", redis->c.tcp.host, redis->c.tcp.port); } else if (pmDebugOptions.series) { if (redis->c.connection_type == REDIS_CONN_UNIX) @@ -498,20 +498,22 @@ redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) * hiredis, update the disconnect callback to return the cluster context and * use this new disconnect callback instead of the conditional below. * - * Register a Redis disconnect if: - * * Redis returns an I/O error. In this case errno is also set, but + * Register a server disconnect if: + * * The server returns an I/O error. In this case errno is also set, but * there are lots of different error codes for connection failures (for example * ECONNRESET, ENETUNREACH, ENETDOWN, ...) - defensively assume all require a * reconnect - * * Redis returns the "LOADING Redis is loading the dataset in memory" error - * * ignore any errors for Redis requests pre-dating the latest (current) Redis - * connection (to handle the case where a Redis callback returns after a new + * * Server returns the "LOADING ... is loading the dataset in memory" error + * * Ignore any errors for server requests pre-dating the latest (current) + * connection (to handle the case where a callback returns after a new * connection was already established) - * * ignore any errors if the state is already set to SLOTS_DISCONNECTED - * * ignore errors if cluster mode is enabled (for now, will change in a later release) + * * Ignore any errors if the state is already set to SLOTS_DISCONNECTED + * * Ignore errors if cluster mode is enabled. */ if (((reply == NULL && c->err == REDIS_ERR_IO) || - (reply != NULL && reply->type == REDIS_REPLY_ERROR && strcmp(reply->str, REDIS_ELOADING) == 0)) && + (reply != NULL && reply->type == REDIS_REPLY_ERROR && + (strncmp(reply->str, KEYS_ELOADING, strlen(KEYS_ELOADING)) == 0 && + strstr(reply->str, KEYS_ELOADDATA) != NULL))) && srd->conn_seq == srd->slots->conn_seq && srd->slots->state != SLOTS_DISCONNECTED && srd->slots->cluster == 0) { @@ -524,9 +526,9 @@ redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) } /* - * Submit an arbitrary request to a (set of) Redis instance(s). + * Submit an arbitrary request to a (set of) key server instance(s). * The given key is used to determine the slot used, as per the - * cluster specification - https://redis.io/topics/cluster-spec + * cluster specification - https://valkey.io/topics/cluster-spec * * Serves mainly as a wrapper to redisClusterAsyncFormattedCommand * including debug output and error handling diff --git a/src/libpcp_web/src/slots.h b/src/libpcp_web/src/slots.h index 832789e51c..4e65bab402 100644 --- a/src/libpcp_web/src/slots.h +++ b/src/libpcp_web/src/slots.h @@ -24,9 +24,14 @@ #define SLOTMASK (MAXSLOTS-1) #define SLOTS_PHASES 5 -/* Unfortunately there is no error code for these errors to match */ -#define REDIS_ELOADING "LOADING Redis is loading the dataset in memory" -#define REDIS_ENOCLUSTER "ERR This instance has cluster support disabled" +/* + * Unfortunately there is no error code for these errors to match + * In LOADING case, the full error contains the key server name - + * so here we split the check into the two guaranteed substrings. + */ +#define KEYS_ELOADING "LOADING" +#define KEYS_ELOADDATA "loading the dataset in memory" +#define KEYS_ENOCLUSTER "ERR This instance has cluster support disabled" typedef enum redisSlotsFlags { SLOTS_NONE = 0, diff --git a/src/pmproxy/pmproxy.conf b/src/pmproxy/pmproxy.conf index 60efd7dc47..19659660a7 100644 --- a/src/pmproxy/pmproxy.conf +++ b/src/pmproxy/pmproxy.conf @@ -97,7 +97,7 @@ cursor.count = 256 # series in the specified amount of time (in seconds) stream.expire = 86400 -# limit elements in individual series keys (https://.io/commands/xadd) +# limit elements in individual series keys (https://valkey.io/commands/xadd) # indicates the number of metric values to store per series - i.e. per # metric and also per host data volumes are considerations here. stream.maxlen = 8640 From 51c5f0be9bba763bc0a693c63b3bc6c2f16123a1 Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 6 Aug 2024 15:26:48 +1000 Subject: [PATCH 5/5] libpcp_web: on-going efforts toward Valkey support --- src/libpcp_web/src/GNUmakefile | 4 +- src/libpcp_web/src/discover.c | 4 +- src/libpcp_web/src/discover.h | 4 +- src/libpcp_web/src/{redis.c => keys.c} | 8 +- src/libpcp_web/src/{redis.h => keys.h} | 31 +- src/libpcp_web/src/load.c | 34 +- src/libpcp_web/src/maps.c | 42 +- src/libpcp_web/src/maps.h | 54 +-- src/libpcp_web/src/query.c | 612 ++++++++++++------------- src/libpcp_web/src/schema.c | 566 +++++++++++------------ src/libpcp_web/src/search.c | 472 +++++++++---------- src/libpcp_web/src/search.h | 16 +- src/libpcp_web/src/slots.c | 332 +++++++------- src/libpcp_web/src/slots.h | 118 ++--- src/libpcp_web/src/util.c | 6 +- src/pmproxy/src/{redis.c => keys.c} | 164 +++---- src/pmproxy/src/search.c | 2 +- src/pmproxy/src/series.c | 2 +- src/pmproxy/src/server.c | 22 +- src/pmproxy/src/server.h | 36 +- 20 files changed, 1274 insertions(+), 1255 deletions(-) rename src/libpcp_web/src/{redis.c => keys.c} (94%) rename src/libpcp_web/src/{redis.h => keys.h} (64%) rename src/pmproxy/src/{redis.c => keys.c} (53%) diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index 1f95c75881..9c39d061b3 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -38,12 +38,12 @@ HIREDIS_CLUSTER_XFILES = $(HIREDIS_CLUSTER_HFILES) $(HIREDIS_CLUSTER_CFILES) CFILES = jsmn.c http_client.c http_parser.c siphash.c \ query.c schema.c load.c sha1.c util.c slots.c \ - redis.c dict.c maps.c batons.c encoding.c \ + keys.c dict.c maps.c batons.c encoding.c \ search.c json_helpers.c config.c \ $(HIREDIS_CFILES) $(HIREDIS_CLUSTER_CFILES) $(INIH_CFILES) HFILES = jsmn.h http_client.h http_parser.h zmalloc.h \ query.h schema.h load.h sha1.h util.h slots.h \ - redis.h dict.h maps.h batons.h encoding.h \ + keys.h dict.h maps.h batons.h encoding.h \ search.h discover.h private.h \ $(HIREDIS_HFILES) $(HIREDIS_CLUSTER_HFILES) $(INIH_HFILES) YFILES = query_parser.y diff --git a/src/libpcp_web/src/discover.c b/src/libpcp_web/src/discover.c index da6bbce569..bfde7d2187 100644 --- a/src/libpcp_web/src/discover.c +++ b/src/libpcp_web/src/discover.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Red Hat. + * Copyright (c) 2018-2022,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -1727,7 +1727,7 @@ changed_callback(pmDiscover *p) time(&now); if ((p->flags & PM_DISCOVER_FLAGS_NEW) == 0) { if (now - p->lastcb < throttle || - redisSlotsInflightRequests(data->slots) > 1000000) { + keySlotsInflightRequests(data->slots) > 1000000) { mmv_inc(data->map, data->metrics[DISCOVER_THROTTLE_CALLBACKS]); return; /* throttled */ } diff --git a/src/libpcp_web/src/discover.h b/src/libpcp_web/src/discover.h index 503c49aa21..f450995ee0 100644 --- a/src/libpcp_web/src/discover.h +++ b/src/libpcp_web/src/discover.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Red Hat. + * Copyright (c) 2018-2022,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -149,7 +149,7 @@ typedef struct discoverModuleData { struct dict *config; /* configuration dict */ uv_loop_t *events; /* event library loop */ - redisSlots *slots; /* server slots data */ + keySlots *slots; /* key server slots data */ unsigned int exclude_names; /* exclude metric names */ sds *patterns; /* metric name patterns */ diff --git a/src/libpcp_web/src/redis.c b/src/libpcp_web/src/keys.c similarity index 94% rename from src/libpcp_web/src/redis.c rename to src/libpcp_web/src/keys.c index cddcb9cefd..bfb58323e8 100644 --- a/src/libpcp_web/src/redis.c +++ b/src/libpcp_web/src/keys.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, Red Hat. + * Copyright (c) 2017-2020,2024, Red Hat. * * All rights reserved. * @@ -28,14 +28,14 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -#include "redis.h" +#include "keys.h" const char * -redis_reply_type(redisReply *reply) +resp_reply_type(respReply *reply) { if (reply == NULL) return "none"; - switch (reply->type) { + switch (resp->type) { case REDIS_REPLY_STRING: return "string"; case REDIS_REPLY_ARRAY: diff --git a/src/libpcp_web/src/redis.h b/src/libpcp_web/src/keys.h similarity index 64% rename from src/libpcp_web/src/redis.h rename to src/libpcp_web/src/keys.h index a91355f873..3ad4bae9cc 100644 --- a/src/libpcp_web/src/redis.h +++ b/src/libpcp_web/src/keys.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Red Hat. + * Copyright (c) 2017-2020,2024 Red Hat. * * All rights reserved. * @@ -27,17 +27,30 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef SERIES_REDIS_H -#define SERIES_REDIS_H +#ifndef RESP_KEYS_H +#define RESP_KEYS_H #include #include -extern const char *redis_reply_type(redisReply *); -extern int redisAsyncEnableKeepAlive(redisAsyncContext *); +#define RESP_OK REDIS_OK +#define RESP_CONN_UNIX REDIS_CONN_UNIX -#define REDIS_ENOCLUSTER "ERR This instance has cluster support disabled" -#define REDIS_ESTREAMXADD "ERR The ID specified in XADD is equal or smaller than the target stream top item" -#define REDIS_EDROPINDEX "Index already exists. Drop it first!" /* RediSearch */ +#define RESP_REPLY_STRING REDIS_REPLY_STRING +#define RESP_REPLY_ARRAY REDIS_REPLY_ARRAY +#define RESP_REPLY_INTEGER REDIS_REPLY_INTEGER +#define RESP_REPLY_NIL REDIS_REPLY_NIL +#define RESP_REPLY_STATUS REDIS_REPLY_STATUS +#define RESP_REPLY_ERROR REDIS_REPLY_ERROR -#endif /* SERIES_REDIS_H */ +#define respReply redisReply +#define keysAsyncContext redisAsyncContext + +extern const char *resp_reply_type(respReply *); +extern int keysAsyncEnableKeepAlive(keysAsyncContext *); + +#define KEYS_ENOCLUSTER "ERR This instance has cluster support disabled" +#define KEYS_ESTREAMXADD "ERR The ID specified in XADD is equal or smaller than the target stream top item" +#define KEYS_EDROPINDEX "Index already exists. Drop it first!" /* search module */ + +#endif /* RESP_KEYS_H */ diff --git a/src/libpcp_web/src/load.c b/src/libpcp_web/src/load.c index ab76bc20b2..b70dbac9a7 100644 --- a/src/libpcp_web/src/load.c +++ b/src/libpcp_web/src/load.c @@ -22,7 +22,7 @@ #include "util.h" void initSeriesLoadBaton(seriesLoadBaton *, void *, pmSeriesFlags, - pmLogInfoCallBack, pmSeriesDoneCallBack, redisSlots *, void *); + pmLogInfoCallBack, pmSeriesDoneCallBack, keySlots *, void *); void freeSeriesLoadBaton(seriesLoadBaton *); void initSeriesGetContext(seriesGetContext *, void *); @@ -34,7 +34,7 @@ static void server_cache_window(void *); static void server_cache_source(seriesLoadBaton *baton) { - redis_series_source(baton->slots, baton); + keys_series_source(baton->slots, baton); } /* cache information about this metric (values/metadata) */ @@ -42,14 +42,14 @@ static void server_cache_metric(seriesLoadBaton *baton, metric_t *metric, sds timestamp, int meta, int data) { - redis_series_metric(baton->slots, metric, timestamp, meta, data, baton); + keys_series_metric(baton->slots, metric, timestamp, meta, data, baton); } /* cache a mark record (discontinuity) for metrics from this source */ static void server_cache_mark(seriesLoadBaton *baton, sds timestamp, int data) { - redis_series_mark(baton->slots, timestamp, data, baton); + keys_series_mark(baton->slots, timestamp, data, baton); } #if 0 @@ -881,11 +881,11 @@ series_string_mapping_callback(void *arg) } static void -series_string_mapping(seriesLoadBaton *baton, redisMap *mapping, +series_string_mapping(seriesLoadBaton *baton, keyMap *mapping, unsigned char *hash, sds string) { /* completes immediately (fills hash), but may issue async I/O too */ - redisGetMap(baton->slots, mapping, hash, string, + keyGetMap(baton->slots, mapping, hash, string, series_string_mapping_callback, baton->info, baton->userdata, baton); } @@ -1024,11 +1024,11 @@ connect_pmapi_source_service(void *arg) } static void -connect_redis_source_service(seriesLoadBaton *baton) +connect_keys_source_service(seriesLoadBaton *baton) { pmSeriesModule *module = (pmSeriesModule *)baton->module; seriesModuleData *data = getSeriesModuleData(module); - redisSlotsFlags flags; + keySlotsFlags flags; sds option; /* attempt to re-use existing slots connections */ @@ -1039,7 +1039,7 @@ connect_redis_source_service(seriesLoadBaton *baton) series_load_end_phase(baton); } else { if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); + option = pmIniFileLookup(data->config, "redis", "enabled"); // compat if (option && strcmp(option, "false") == 0) { baton->error = -ENOTSUP; } else { @@ -1047,7 +1047,7 @@ connect_redis_source_service(seriesLoadBaton *baton) if ((baton->flags & PM_SERIES_FLAG_TEXT)) flags |= SLOTS_SEARCH; baton->slots = data->slots = - redisSlotsConnect( + keySlotsConnect( data->config, flags, baton->info, series_load_end_phase, baton->userdata, data->events, (void *)baton); @@ -1067,19 +1067,19 @@ setup_pmapi_source_service(void *arg) } static void -setup_redis_source_service(void *arg) +setup_keys_source_service(void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesBatonCheckMagic(baton, MAGIC_LOAD, "setup_redis_source_service"); - seriesBatonReferences(baton, 1, "setup_redis_source_service"); + seriesBatonCheckMagic(baton, MAGIC_LOAD, "setup_keys_source_service"); + seriesBatonReferences(baton, 1, "setup_keys_source_service"); - connect_redis_source_service(baton); + connect_keys_source_service(baton); } void initSeriesLoadBaton(seriesLoadBaton *baton, void *module, pmSeriesFlags flags, - pmLogInfoCallBack info, pmSeriesDoneCallBack done, redisSlots *slots, + pmLogInfoCallBack info, pmSeriesDoneCallBack done, keySlots *slots, void *userdata) { initSeriesBatonMagic(baton, MAGIC_LOAD); @@ -1158,9 +1158,9 @@ series_load(pmSeriesSettings *settings, /* ordering of async operations */ i = 0; baton->current = &baton->phases[i]; - /* Coverity CID341699 - split pmapi and redis setup to avoid use after free */ + /* Coverity CID341699 - split pmapi and keys setup to avoid use after free */ baton->phases[i++].func = setup_pmapi_source_service; - baton->phases[i++].func = setup_redis_source_service; + baton->phases[i++].func = setup_keys_source_service; /* assign source/host string map (series_source_mapping) */ baton->phases[i++].func = series_source_mapping; /* write source info into schema (series_cache_source) */ diff --git a/src/libpcp_web/src/maps.c b/src/libpcp_web/src/maps.c index 96e5c7f4e4..15a9091792 100644 --- a/src/libpcp_web/src/maps.c +++ b/src/libpcp_web/src/maps.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019,2022 Red Hat. + * Copyright (c) 2017-2019,2022,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -19,10 +19,10 @@ #include "maps.h" /* reverse hash mapping of all SHA1 hashes to strings */ -redisMap *instmap; -redisMap *namesmap; -redisMap *labelsmap; -redisMap *contextmap; +keyMap *instmap; +keyMap *namesmap; +keyMap *labelsmap; +keyMap *contextmap; static uint64_t intHashCallBack(const void *key) @@ -122,7 +122,7 @@ dictType sdsDictCallBacks = { }; void -redisMapsInit(void) +keyMapsInit(void) { if (instmap == NULL) instmap = dictCreate(&sdsDictCallBacks, @@ -139,44 +139,44 @@ redisMapsInit(void) } void -redisMapsClose(void) +keyMapsClose(void) { if (instmap) { - sdsfree(redisMapName(instmap)); + sdsfree(keyMapName(instmap)); dictRelease(instmap); instmap = NULL; } if (namesmap) { - sdsfree(redisMapName(namesmap)); + sdsfree(keyMapName(namesmap)); dictRelease(namesmap); namesmap = NULL; } if (labelsmap) { - sdsfree(redisMapName(labelsmap)); + sdsfree(keyMapName(labelsmap)); dictRelease(labelsmap); labelsmap = NULL; } if (contextmap) { - sdsfree(redisMapName(contextmap)); + sdsfree(keyMapName(contextmap)); dictRelease(contextmap); contextmap = NULL; } } sds -redisMapName(redisMap *map) +keyMapName(keyMap *map) { return (sds)map->privdata; } -redisMap * -redisMapCreate(sds name) +keyMap * +keyMapCreate(sds name) { return dictCreate(&sdsDictCallBacks, (void *)name); } -redisMapEntry * -redisMapLookup(redisMap *map, sds key) +keyMapEntry * +keyMapLookup(keyMap *map, sds key) { if (map) return dictFind(map, key); @@ -184,9 +184,9 @@ redisMapLookup(redisMap *map, sds key) } void -redisMapInsert(redisMap *map, sds key, sds value) +keyMapInsert(keyMap *map, sds key, sds value) { - redisMapEntry *entry = redisMapLookup(map, key); + keyMapEntry *entry = keyMapLookup(map, key); if (entry) { /* fix for Coverity CID323605 Resource Leak */ @@ -196,14 +196,14 @@ redisMapInsert(redisMap *map, sds key, sds value) } sds -redisMapValue(redisMapEntry *entry) +keyMapValue(keyMapEntry *entry) { return (sds)dictGetVal(entry); } void -redisMapRelease(redisMap *map) +keyMapRelease(keyMap *map) { - sdsfree(redisMapName(map)); + sdsfree(keyMapName(map)); dictRelease(map); } diff --git a/src/libpcp_web/src/maps.h b/src/libpcp_web/src/maps.h index 700570aaaf..e3463fbdbc 100644 --- a/src/libpcp_web/src/maps.h +++ b/src/libpcp_web/src/maps.h @@ -1,6 +1,6 @@ /* - * Copyright (c) 2017-2018,2022 Red Hat. - * + * Copyright (c) 2017-2018,2022,2024 Red Hat. + * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation; either version 2.1 of the License, or @@ -18,50 +18,50 @@ #include "dict.h" #include "batons.h" -struct redisSlots; -typedef dict redisMap; -typedef dictEntry redisMapEntry; +struct keySlots; +typedef dict keyMap; +typedef dictEntry keyMapEntry; /* * Mapping SHA1 hashes (identifiers) to sds strings. */ -extern redisMap *instmap; -extern redisMap *namesmap; -extern redisMap *labelsmap; -extern redisMap *contextmap; +extern keyMap *instmap; +extern keyMap *namesmap; +extern keyMap *labelsmap; +extern keyMap *contextmap; -extern redisMap *redisMapCreate(sds); -extern redisMapEntry *redisMapLookup(redisMap *, sds); -extern sds redisMapValue(redisMapEntry *); -extern void redisMapInsert(redisMap *, sds, sds); +extern keyMap *keyMapCreate(sds); +extern keyMapEntry *keyMapLookup(keyMap *, sds); +extern sds keyMapValue(keyMapEntry *); +extern void keyMapInsert(keyMap *, sds, sds); /* * Helper utilities and data structures */ -extern void redisMapsInit(void); -extern void redisMapsClose(void); -extern sds redisMapName(redisMap *); -extern void redisMapRelease(redisMap *); +extern void keyMapsInit(void); +extern void keyMapsClose(void); +extern sds keyMapName(keyMap *); +extern void keyMapRelease(keyMap *); /* * Asynchronous mapping response helpers */ -typedef void (*redisInfoCallBack)(pmLogLevel, sds, void *); -typedef void (*redisDoneCallBack)(void *); +typedef void (*keysInfoCallBack)(pmLogLevel, sds, void *); +typedef void (*keysDoneCallBack)(void *); -typedef struct redisMapBaton { +typedef struct keyMapBaton { seriesBatonMagic magic; /* MAGIC_MAPPING */ - redisMap *mapping; + keyMap *mapping; sds mapKey; /* 20-byte SHA1 */ sds mapStr; /* string value */ - struct redisSlots *slots; - redisDoneCallBack mapped; - redisInfoCallBack info; + struct keySlots *slots; + keysDoneCallBack mapped; + keysInfoCallBack info; void *userdata; void *arg; -} redisMapBaton; +} keyMapBaton; -extern void redisGetMap(struct redisSlots *, redisMap *, unsigned char *, - sds, redisDoneCallBack, redisInfoCallBack, void *, void *); +extern void keyGetMap(struct keySlots *, keyMap *, unsigned char *, + sds, keysDoneCallBack, keysInfoCallBack, void *, void *); #endif /* SERIES_MAPS_H */ diff --git a/src/libpcp_web/src/query.c b/src/libpcp_web/src/query.c index 22fd5c7f08..97b28b13b0 100644 --- a/src/libpcp_web/src/query.c +++ b/src/libpcp_web/src/query.c @@ -33,7 +33,7 @@ typedef struct seriesGetLabelMap { seriesBatonMagic header; /* MAGIC_LABELMAP */ - redisMap *map; + keyMap *map; sds series; sds name; sds mapID; @@ -42,7 +42,7 @@ typedef struct seriesGetLabelMap { } seriesGetLabelMap; typedef struct seriesGetLookup { - redisMap *map; + keyMap *map; pmSeriesStringCallBack func; sds pattern; unsigned int nseries; @@ -62,7 +62,7 @@ typedef struct seriesQueryBaton { pmSeriesCallBacks *callbacks; pmLogInfoCallBack info; void *userdata; - redisSlots *slots; + keySlots *slots; int error; seriesGetLookup lookup; seriesGetQuery query; @@ -72,14 +72,14 @@ static void series_pattern_match(seriesQueryBaton *, node_t *); static int series_union(series_set_t *, series_set_t *); static int series_intersect(series_set_t *, series_set_t *); static int series_calculate(node_t *, int, void *); -static void series_redis_hash_expression(seriesQueryBaton *, char *, int); +static void series_keys_hash_expression(seriesQueryBaton *, char *, int); static void series_node_get_metric_name(seriesQueryBaton *, seriesGetSID *, series_sample_set_t *); static void series_node_get_desc(seriesQueryBaton *, sds, series_sample_set_t *); static void series_lookup_services(void *); static void series_lookup_mapping(void *); static void series_lookup_finished(void *); static void series_query_mapping(void *arg); -static void series_instances_reply_callback(redisClusterAsyncContext *, void *, void *); +static void series_instances_reply_callback(keyClusterAsyncContext *, void *, void *); sds cursorcount; /* number of elements in each SCAN call */ @@ -158,7 +158,7 @@ freeSeriesGetQuery(seriesQueryBaton *baton) static void initSeriesGetLabelMap(seriesGetLabelMap *value, sds series, sds name, - redisMap *map, sds mapID, sds mapKey, void *baton) + keyMap *map, sds mapID, sds mapKey, void *baton) { initSeriesBatonMagic(value, MAGIC_LABELMAP); value->map = map; @@ -174,7 +174,7 @@ freeSeriesGetLabelMap(seriesGetLabelMap *value) { seriesBatonCheckMagic(value, MAGIC_LABELMAP, "freeSeriesGetLabelMap"); - redisMapRelease(value->map); + keyMapRelease(value->map); sdsfree(value->series); sdsfree(value->name); sdsfree(value->mapID); @@ -227,7 +227,7 @@ initSeriesQueryBaton(seriesQueryBaton *baton, static void initSeriesGetLookup(seriesQueryBaton *baton, int nseries, sds *series, - pmSeriesStringCallBack func, redisMap *map) + pmSeriesStringCallBack func, keyMap *map) { seriesGetSID *sid; unsigned int i; @@ -385,7 +385,7 @@ node_subtype(node_t *np) static int extract_string(seriesQueryBaton *baton, pmSID series, - redisReply *reply, sds *string, const char *message) + respReply *reply, sds *string, const char *message) { sds msg; @@ -394,24 +394,24 @@ extract_string(seriesQueryBaton *baton, pmSID series, return 0; } infofmt(msg, "expected string result for %s of series %s (got %s)", - message, series, redis_reply_type(reply)); + message, series, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); return -EINVAL; } static int extract_mapping(seriesQueryBaton *baton, pmSID series, - redisReply *reply, sds *string, const char *message) + respReply *reply, sds *string, const char *message) { - redisMapEntry *entry; + keyMapEntry *entry; sds msg, key; if (reply->type == REDIS_REPLY_STRING) { key = sdsnewlen(reply->str, reply->len); - entry = redisMapLookup(baton->lookup.map, key); + entry = keyMapLookup(baton->lookup.map, key); sdsfree(key); if (entry != NULL) { - key = redisMapValue(entry); + key = keyMapValue(entry); *string = sdscpylen(*string, key, sdslen(key)); return 0; } @@ -426,14 +426,14 @@ extract_mapping(seriesQueryBaton *baton, pmSID series, static int extract_sha1(seriesQueryBaton *baton, pmSID series, - redisReply *reply, sds *sha, const char *message) + respReply *reply, sds *sha, const char *message) { sds msg; char hashbuf[42]; if (reply->type != REDIS_REPLY_STRING) { infofmt(msg, "expected string result for \"%s\" of series %s got %s", - message, series, redis_reply_type(reply)); + message, series, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); return -EINVAL; } @@ -453,7 +453,7 @@ extract_sha1(seriesQueryBaton *baton, pmSID series, */ static int series_instance_reply(seriesQueryBaton *baton, sds series, - pmSeriesValue *value, int nelements, redisReply **elements) + pmSeriesValue *value, int nelements, respReply **elements) { char hashbuf[42]; sds inst; @@ -486,7 +486,7 @@ series_instance_reply(seriesQueryBaton *baton, sds series, static int extract_time(seriesQueryBaton *baton, pmSID series, - redisReply *reply, sds *stamp, pmTimespec *ts) + respReply *reply, sds *stamp, pmTimespec *ts) { sds msg, val; char *point = NULL; @@ -566,10 +566,10 @@ use_next_sample(seriesSampling *sampling) static void series_values_reply(seriesQueryBaton *baton, sds series, - int nsamples, redisReply **samples, void *arg) + int nsamples, respReply **samples, void *arg) { seriesSampling sampling = {0}; - redisReply *reply, *sample, **elements; + respReply *reply, *sample, **elements; timing_t *tp = &baton->query.timing; int n, sts, next, nelements; sds msg, save_timestamp; @@ -597,7 +597,7 @@ series_values_reply(seriesQueryBaton *baton, sds series, reply = elements[1]; if (reply->type != REDIS_REPLY_ARRAY) { infofmt(msg, "expected value array for series %s %s (type=%s)", - series, XRANGE, redis_reply_type(reply)); + series, XRANGE, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; break; @@ -684,17 +684,17 @@ series_values_reply(seriesQueryBaton *baton, sds series, } /* - * Save the series hash identifiers contained in a Redis response + * Save the series hash identifiers contained in a RESP response * for all series that are not already in this nodes set (union). * Used at the leaves of the query tree, then merged result sets * are propagated upward. */ static int -node_series_reply(seriesQueryBaton *baton, node_t *np, int nelements, redisReply **elements) +node_series_reply(seriesQueryBaton *baton, node_t *np, int nelements, respReply **elements) { series_set_t set; unsigned char *series; - redisReply *reply; + respReply *reply; char hashbuf[42]; sds msg; int i, sts = 0; @@ -723,7 +723,7 @@ node_series_reply(seriesQueryBaton *baton, node_t *np, int nelements, redisReply } else { infofmt(msg, "expected string in %s set \"%s\" (type=%s)", node_subtype(np->left), np->left->key, - redis_reply_type(reply)); + resp_reply_type(reply)); batoninfo(baton, PMLOG_REQUEST, msg); sts = -EPROTO; } @@ -927,13 +927,13 @@ string_pattern_match(node_t *np, sds pattern, char *string, int length) /* * Add a node subtree representing glob (N_GLOB) pattern matches. * Each of these matches are then further evaluated (as if N_EQ). - * Response format is described at https://redis.io/commands/scan + * Response format is described at https://valkey.io/commands/scan */ static int -node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, int nelements, - redisReply **elements) +node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, + int nelements, respReply **elements) { - redisReply *reply, *r; + respReply *reply, *r; sds msg, key, pattern, *matches; char buffer[42]; size_t bytes; @@ -950,7 +950,7 @@ node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, int ne reply = elements[0]; if (!reply || reply->type != REDIS_REPLY_STRING) { infofmt(msg, "expected integer cursor result from %s (got %s)", - HSCAN, reply ? redis_reply_type(reply) : "null"); + HSCAN, reply ? resp_reply_type(reply) : "null"); batoninfo(baton, PMLOG_RESPONSE, msg); return -EPROTO; } @@ -959,7 +959,7 @@ node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, int ne reply = elements[1]; if (!reply || reply->type != REDIS_REPLY_ARRAY) { infofmt(msg, "expected array of results from %s (got %s)", - HSCAN, reply ? redis_reply_type(reply) : "null"); + HSCAN, reply ? resp_reply_type(reply) : "null"); batoninfo(baton, PMLOG_RESPONSE, msg); return -EPROTO; } @@ -978,7 +978,7 @@ node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, int ne r = reply->element[i]; if (r->type != REDIS_REPLY_STRING) { infofmt(msg, "expected only string results from %s (type=%s)", - HSCAN, redis_reply_type(r)); + HSCAN, resp_reply_type(r)); batoninfo(baton, PMLOG_REQUEST, msg); return -EPROTO; } @@ -1033,11 +1033,11 @@ node_pattern_reply(seriesQueryBaton *baton, node_t *np, const char *name, int ne static void series_prepare_maps_pattern_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { node_t *np = (node_t *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; - redisReply *reply = r; + respReply *reply = r; const char *name; node_t *left; sds msg; @@ -1049,7 +1049,7 @@ series_prepare_maps_pattern_reply( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array for %s key \"%s\" (type=%s)", - node_subtype(left), left->key, redis_reply_type(reply)); + node_subtype(left), left->key, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else { @@ -1071,21 +1071,21 @@ series_pattern_match(seriesQueryBaton *baton, node_t *np) seriesBatonReference(baton, "series_pattern_match"); cur = sdscatfmt(sdsempty(), "%U", np->cursor); key = sdsdup(np->left->key); - cmd = redis_command(5); - cmd = redis_param_str(cmd, HSCAN, HSCAN_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_sds(cmd, cur); /* cursor */ - cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1); - cmd = redis_param_sds(cmd, cursorcount); + cmd = key_command(5); + cmd = key_param_str(cmd, HSCAN, HSCAN_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_sds(cmd, cur); /* cursor */ + cmd = key_param_str(cmd, "COUNT", sizeof("COUNT")-1); + cmd = key_param_sds(cmd, cursorcount); sdsfree(cur); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_prepare_maps_pattern_reply, np); sdsfree(cmd); } /* - * Map human names to internal Redis identifiers. + * Map human names to internal key identifiers. */ static int series_prepare_maps(seriesQueryBaton *baton, node_t *np, int level) @@ -1160,11 +1160,11 @@ series_node_value(node_t *np) static void series_prepare_smembers_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { node_t *np = (node_t *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; - redisReply *reply = r; + respReply *reply = r; sds msg; int sts; @@ -1173,7 +1173,7 @@ series_prepare_smembers_reply( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array for %s set \"%s\" (type=%s)", node_subtype(np->left), np->right->value, - redis_reply_type(reply)); + resp_reply_type(reply)); batoninfo(baton, PMLOG_CORRUPT, msg); baton->error = -EPROTO; } else { @@ -1195,20 +1195,20 @@ series_prepare_smembers(seriesQueryBaton *baton, sds kp, node_t *np) { sds cmd; - cmd = redis_command(2); - cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); - cmd = redis_param_sds(cmd, kp); - redisSlotsRequest(baton->slots, cmd, + cmd = key_command(2); + cmd = key_param_str(cmd, SMEMBERS, SMEMBERS_LEN); + cmd = key_param_sds(cmd, kp); + keySlotsRequest(baton->slots, cmd, series_prepare_smembers_reply, np); sdsfree(cmd); } static void series_hmset_function_desc_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_hmset_function_desc_callback"); checkStatusReplyOK(baton->info, baton->userdata, c, reply, @@ -1223,34 +1223,34 @@ series_hmset_function_desc(seriesQueryBaton *baton, sds key, pmSeriesDesc *desc) seriesBatonReference(baton, "series_hmset_function_desc"); - cmd = redis_command(14); - cmd = redis_param_str(cmd, HMSET, HMSET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "indom", sizeof("indom")-1); - cmd = redis_param_sds(cmd, desc->indom); - cmd = redis_param_str(cmd, "pmid", sizeof("pmid")-1); - cmd = redis_param_sds(cmd, desc->pmid); - cmd = redis_param_str(cmd, "semantics", sizeof("semantics")-1); - cmd = redis_param_sds(cmd, desc->semantics); - cmd = redis_param_str(cmd, "source", sizeof("source")-1); + cmd = key_command(14); + cmd = key_param_str(cmd, HMSET, HMSET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "indom", sizeof("indom")-1); + cmd = key_param_sds(cmd, desc->indom); + cmd = key_param_str(cmd, "pmid", sizeof("pmid")-1); + cmd = key_param_sds(cmd, desc->pmid); + cmd = key_param_str(cmd, "semantics", sizeof("semantics")-1); + cmd = key_param_sds(cmd, desc->semantics); + cmd = key_param_str(cmd, "source", sizeof("source")-1); /* for fabricated SIDs, this is a binary string of 20 zero (NULL) bytes */ - cmd = redis_param_sds(cmd, desc->source); - cmd = redis_param_str(cmd, "type", sizeof("type")-1); - cmd = redis_param_sds(cmd, desc->type); - cmd = redis_param_str(cmd, "units", sizeof("units")-1); - cmd = redis_param_sds(cmd, desc->units); + cmd = key_param_sds(cmd, desc->source); + cmd = key_param_str(cmd, "type", sizeof("type")-1); + cmd = key_param_sds(cmd, desc->type); + cmd = key_param_str(cmd, "units", sizeof("units")-1); + cmd = key_param_sds(cmd, desc->units); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_hmset_function_desc_callback, baton); sdsfree(cmd); } static void series_hmset_function_expr_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_hmset_function_expr_callback"); checkStatusReplyOK(baton->info, baton->userdata, c, reply, @@ -1265,15 +1265,15 @@ series_hmset_function_expr(seriesQueryBaton *baton, sds key, sds expr) seriesBatonReference(baton, "series_hmset_function_expr"); - cmd = redis_command(4); - cmd = redis_param_str(cmd, HMSET, HMSET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "query", sizeof("query")-1); - cmd = redis_param_sds(cmd, expr); + cmd = key_command(4); + cmd = key_param_str(cmd, HMSET, HMSET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "query", sizeof("query")-1); + cmd = key_param_sds(cmd, expr); sdsfree(expr); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, series_hmset_function_expr_callback, baton); + keySlotsRequest(baton->slots, cmd, series_hmset_function_expr_callback, baton); sdsfree(cmd); } @@ -1435,14 +1435,14 @@ on_series_solve_inst_value(pmSID sid, pmSeriesValue *value, void *arg) seriesBatonReference(sidbat, "on_series_solve_inst_value"); key = sdscatfmt(sdsempty(), "pcp:inst:series:%S", value->series); - cmd = redis_command(5); - cmd = redis_param_str(cmd, HMGET, HMGET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "inst", sizeof("inst")-1); - cmd = redis_param_str(cmd, "name", sizeof("name")-1); - cmd = redis_param_str(cmd, "source", sizeof("source")-1); + cmd = key_command(5); + cmd = key_param_str(cmd, HMGET, HMGET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "inst", sizeof("inst")-1); + cmd = key_param_str(cmd, "name", sizeof("name")-1); + cmd = key_param_str(cmd, "source", sizeof("source")-1); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_instances_reply_callback, sidbat); sdsfree(cmd); return 0; @@ -1504,11 +1504,11 @@ series_solve_sid_expr(pmSeriesSettings *settings, pmSeriesExpr *expr, void *arg) } static void -series_query_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) +series_query_expr_reply(keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; pmSeriesExpr expr = {0}; sds msg; int sts; @@ -1517,7 +1517,7 @@ series_query_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_query_expr_reply"); if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array of one string element (got %zu) from series %s %s (type=%s)", - reply->elements, sid->name, HMGET, redis_reply_type(reply)); + reply->elements, sid->name, HMGET, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); freeSeriesGetSID(sid); } else if (reply->element[0]->type == REDIS_REPLY_STRING) { @@ -1536,11 +1536,11 @@ series_query_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) static void series_prepare_time_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; seriesGetSID *expr; sds key, exprcmd; sds msg; @@ -1549,7 +1549,7 @@ series_prepare_time_reply( seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_prepare_time_reply"); if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from %s XSTREAM values (type=%s)", - sid->name, redis_reply_type(reply)); + sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else { @@ -1558,10 +1558,10 @@ series_prepare_time_reply( series_values_reply(baton, sid->name, reply->elements, reply->element, arg); } else { /* Handle fabricated/expression SID in /series/values : - * - get the expr for sid->name from redis. In the callback for that, - * parse the expr and then solve the expression tree with timing from - * this series query baton. Then merge the values in the reply elements. - * TODO (maybe) - also get the desc and check source hash is zero. + * get expr for sid->name from the server. In the callback for that, + * parse the expr and then solve the expression tree with timing from + * this series query baton. Then merge the values in the reply elements. + * TODO (maybe) - also get the desc and check source hash is zero. */ if (pmDebugOptions.series) fprintf(stderr, "series_prepare_time_reply: sid %s is fabricated\n", sid->name); @@ -1570,12 +1570,12 @@ series_prepare_time_reply( seriesBatonReference(baton, "series_query_expr_reply"); key = sdscatfmt(sdsempty(), "pcp:expr:series:%S", expr->name); - exprcmd = redis_command(3); - exprcmd = redis_param_str(exprcmd, HMGET, HMGET_LEN); - exprcmd = redis_param_sds(exprcmd, key); - exprcmd = redis_param_str(exprcmd, "query", sizeof("query")-1); + exprcmd = key_command(3); + exprcmd = key_param_str(exprcmd, HMGET, HMGET_LEN); + exprcmd = key_param_sds(exprcmd, key); + exprcmd = key_param_str(exprcmd, "query", sizeof("query")-1); sdsfree(key); - redisSlotsRequest(baton->slots, exprcmd, series_query_expr_reply, expr); + keySlotsRequest(baton->slots, exprcmd, series_query_expr_reply, expr); sdsfree(exprcmd); } } @@ -1638,21 +1638,21 @@ series_prepare_time(seriesQueryBaton *baton, series_set_t *result) /* X[REV]RANGE key t1 t2 [count N] */ if (reverse) { - cmd = redis_command(6); - cmd = redis_param_str(cmd, XREVRANGE, XREVRANGE_LEN); + cmd = key_command(6); + cmd = key_param_str(cmd, XREVRANGE, XREVRANGE_LEN); } else { - cmd = redis_command(4); - cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN); + cmd = key_command(4); + cmd = key_param_str(cmd, XRANGE, XRANGE_LEN); } - cmd = redis_param_sds(cmd, key); - cmd = redis_param_sds(cmd, start); - cmd = redis_param_sds(cmd, end); + cmd = key_param_sds(cmd, key); + cmd = key_param_sds(cmd, start); + cmd = key_param_sds(cmd, end); if (reverse) { - cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1); - cmd = redis_param_str(cmd, revbuf, revlen); + cmd = key_param_str(cmd, "COUNT", sizeof("COUNT")-1); + cmd = key_param_str(cmd, revbuf, revlen); } sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_prepare_time_reply, sid); sdsfree(cmd); } @@ -1745,7 +1745,7 @@ series_query_report_matches(void *arg) * function operation. */ if (has_function != 0) - series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf)); + series_key_hash_expression(baton, hashbuf, sizeof(hashbuf)); series_report_set(baton, baton->query.root); series_query_end_phase(baton); @@ -1792,7 +1792,7 @@ series_query_expr(void *arg) static int series_instance_store_to_node(seriesQueryBaton *baton, sds series, - pmSeriesValue *value, int nelements, redisReply **elements, node_t *np, int idx_sample) + pmSeriesValue *value, int nelements, respReply **elements, node_t *np, int idx_sample) { char hashbuf[42]; sds inst; @@ -1836,10 +1836,10 @@ series_instance_store_to_node(seriesQueryBaton *baton, sds series, /* Do something like memcpy */ static void series_values_store_to_node(seriesQueryBaton *baton, sds series, - int nsamples, redisReply **samples, node_t *np) + int nsamples, respReply **samples, node_t *np) { seriesSampling sampling = {0}; - redisReply *reply, *sample, **elements; + respReply *reply, *sample, **elements; timing_t *tp = &baton->query.timing; int i, sts, next, nelements; int idx_series = np->value_set.num_series; @@ -1869,7 +1869,7 @@ series_values_store_to_node(seriesQueryBaton *baton, sds series, reply = elements[1]; if (reply->type != REDIS_REPLY_ARRAY) { infofmt(msg, "expected value array for series %s %s (type=%s)", - series, XRANGE, redis_reply_type(reply)); + series, XRANGE, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; break; @@ -1965,7 +1965,7 @@ series_values_store_to_node(seriesQueryBaton *baton, sds series, static int extract_series_desc(seriesQueryBaton *baton, pmSID series, - int nelements, redisReply **elements, pmSeriesDesc *desc) + int nelements, respReply **elements, pmSeriesDesc *desc) { sds msg; @@ -2004,7 +2004,7 @@ extract_series_desc(seriesQueryBaton *baton, pmSID series, static int extract_series_node_desc(seriesQueryBaton *baton, pmSID series, - int nelements, redisReply **elements, pmSeriesDesc *desc) + int nelements, respReply **elements, pmSeriesDesc *desc) { sds msg; @@ -2029,10 +2029,10 @@ extract_series_node_desc(seriesQueryBaton *baton, pmSID series, static void series_node_get_desc_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { series_sample_set_t *sample_set = (series_sample_set_t *) arg; - redisReply *reply = r; + respReply *reply = r; int sts; pmSeriesDesc *desc = &sample_set->series_desc; seriesQueryBaton *baton = (seriesQueryBaton *)sample_set->baton; @@ -2049,7 +2049,7 @@ series_node_get_desc_reply( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array type from series %s %s (type=%s)", - sample_set->sid->name, HMGET, redis_reply_type(reply)); + sample_set->sid->name, HMGET, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else if ((sts = extract_series_node_desc(baton, sample_set->sid->name, @@ -2067,25 +2067,25 @@ series_node_get_desc(seriesQueryBaton *baton, sds sid_name, series_sample_set_t seriesBatonReference(baton, "series_node_get_desc"); key = sdscatfmt(sdsempty(), "pcp:desc:series:%S", sid_name); - cmd = redis_command(7); - cmd = redis_param_str(cmd, HMGET, HMGET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "indom", sizeof("indom")-1); - cmd = redis_param_str(cmd, "semantics", sizeof("semantics")-1); - cmd = redis_param_str(cmd, "source", sizeof("source")-1); - cmd = redis_param_str(cmd, "type", sizeof("type")-1); - cmd = redis_param_str(cmd, "units", sizeof("units")-1); + cmd = key_command(7); + cmd = key_param_str(cmd, HMGET, HMGET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "indom", sizeof("indom")-1); + cmd = key_param_str(cmd, "semantics", sizeof("semantics")-1); + cmd = key_param_str(cmd, "source", sizeof("source")-1); + cmd = key_param_str(cmd, "type", sizeof("type")-1); + cmd = key_param_str(cmd, "units", sizeof("units")-1); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, series_node_get_desc_reply, sample_set); + keySlotsRequest(baton->slots, cmd, series_node_get_desc_reply, sample_set); sdsfree(cmd); } static int series_store_metric_name(seriesQueryBaton *baton, series_sample_set_t *sample_set, - sds series, int nelements, redisReply **elements) + sds series, int nelements, respReply **elements) { - redisMapEntry *entry; - redisReply *reply; + keyMapEntry *entry; + respReply *reply; sds msg, key; unsigned int i; int sts = 0; @@ -2096,8 +2096,8 @@ series_store_metric_name(seriesQueryBaton *baton, series_sample_set_t *sample_se if (reply->type == REDIS_REPLY_STRING) { sdsclear(key); key = sdscatlen(key, reply->str, reply->len); - if ((entry = redisMapLookup(namesmap, key)) != NULL) { - sample_set->metric_name = redisMapValue(entry); + if ((entry = keyMapLookup(namesmap, key)) != NULL) { + sample_set->metric_name = keyMapValue(entry); } else { infofmt(msg, "%s - timeseries string map", series); batoninfo(baton, PMLOG_CORRUPT, msg); @@ -2105,7 +2105,7 @@ series_store_metric_name(seriesQueryBaton *baton, series_sample_set_t *sample_se } } else { infofmt(msg, "expected string in %s set (type=%s)", - series, redis_reply_type(reply)); + series, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); sts = -EPROTO; } @@ -2117,11 +2117,11 @@ series_store_metric_name(seriesQueryBaton *baton, series_sample_set_t *sample_se static void series_node_get_metric_name_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { series_sample_set_t *sample_set = (series_sample_set_t *) arg; seriesQueryBaton *baton = (seriesQueryBaton *)sample_set->baton; - redisReply *reply = r; + respReply *reply = r; int sts; sds msg; @@ -2130,7 +2130,7 @@ series_node_get_metric_name_reply( /* unpack - extract names for this source via context name map */ if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from %s %s (type=%s)", - SMEMBERS, sample_set->sid->name, redis_reply_type(reply)); + SMEMBERS, sample_set->sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else if ((sts = series_store_metric_name(baton, sample_set, sample_set->sid->name, @@ -2148,11 +2148,11 @@ series_node_get_metric_name( seriesBatonReference(baton, "series_node_get_metric_name"); key = sdscatfmt(sdsempty(), "pcp:metric.name:series:%S", sid->name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, SMEMBERS, SMEMBERS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_node_get_metric_name_reply, sample_set); sdsfree(cmd); } @@ -2162,11 +2162,11 @@ series_node_get_metric_name( */ static void series_node_prepare_time_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { node_t *np = (node_t *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; - redisReply *reply = r; + respReply *reply = r; sds msg; int idx = np->value_set.num_series; seriesGetSID *sid = np->value_set.series_values[idx].sid; @@ -2182,7 +2182,7 @@ series_node_prepare_time_reply( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from %s XSTREAM values (type=%s)", - sid->name, redis_reply_type(reply)); + sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else { @@ -2261,24 +2261,24 @@ series_node_prepare_time(seriesQueryBaton *baton, series_set_t *query_series_set /* X[REV]RANGE key t1 t2 [count N] */ if (reverse) { - cmd = redis_command(6); - cmd = redis_param_str(cmd, XREVRANGE, XREVRANGE_LEN); + cmd = key_command(6); + cmd = key_param_str(cmd, XREVRANGE, XREVRANGE_LEN); } else { - cmd = redis_command(4); - cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN); + cmd = key_command(4); + cmd = key_param_str(cmd, XRANGE, XRANGE_LEN); } - cmd = redis_param_sds(cmd, key); - cmd = redis_param_sds(cmd, start); - cmd = redis_param_sds(cmd, end); + cmd = key_param_sds(cmd, key); + cmd = key_param_sds(cmd, start); + cmd = key_param_sds(cmd, end); if (reverse) { - cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1); - cmd = redis_param_str(cmd, revbuf, revlen); + cmd = key_param_str(cmd, "COUNT", sizeof("COUNT")-1); + cmd = key_param_str(cmd, revbuf, revlen); } sdsfree(key); np->value_set.series_values[i].baton = baton; np->value_set.series_values[i].sid = sid; /* Note: np->series_set.num_series is not equal to nseries in this function */ - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_node_prepare_time_reply, np); sdsfree(cmd); @@ -4922,7 +4922,7 @@ series_compatibility_convert( } static void -series_redis_hash_expression(seriesQueryBaton *baton, char *hashbuf, int len_hashbuf) +series_key_hash_expression(seriesQueryBaton *baton, char *hashbuf, int len_hashbuf) { unsigned char hash[20]; sds key, msg; @@ -5060,7 +5060,7 @@ series_query_funcs_report_values(void *arg) * function operation. */ if (has_function) - series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf)); + series_key_hash_expression(baton, hashbuf, sizeof(hashbuf)); /* time series values saved in root node so report them directly. */ series_node_values_report(baton, baton->query.root); @@ -5126,12 +5126,12 @@ series_query_services(void *arg) series_query_end_phase(baton); } else { if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); + option = pmIniFileLookup(data->config, "redis", "enabled"); // compat if (option && strcmp(option, "false") == 0) baton->error = -ENOTSUP; else baton->slots = data->slots = - redisSlotsConnect( + keySlotsConnect( data->config, 1, baton->info, series_query_end_phase, baton->userdata, data->events, (void *)baton); @@ -5190,9 +5190,9 @@ series_solve(pmSeriesSettings *settings, /* build a reverse hash mapping */ static void -reverse_map(seriesQueryBaton *baton, redisMap *map, int nkeys, redisReply **elements) +reverse_map(seriesQueryBaton *baton, keyMap *map, int nkeys, respReply **elements) { - redisReply *name, *hash; + respReply *name, *hash; sds msg, key, val; unsigned int i; @@ -5203,17 +5203,17 @@ reverse_map(seriesQueryBaton *baton, redisMap *map, int nkeys, redisReply **elem if (hash->type == REDIS_REPLY_STRING) { key = sdsnewlen(hash->str, hash->len); val = sdsnewlen(name->str, name->len); - redisMapInsert(map, key, val); + keyMapInsert(map, key, val); sdsfree(key); // map has keyDup set } else { infofmt(msg, "expected string key for hashmap (type=%s)", - redis_reply_type(hash)); + key_reply_type(hash)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EINVAL; } } else { infofmt(msg, "expected string name for hashmap (type=%s)", - redis_reply_type(name)); + key_reply_type(name)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EINVAL; } @@ -5221,11 +5221,11 @@ reverse_map(seriesQueryBaton *baton, redisMap *map, int nkeys, redisReply **elem } static void -series_map_lookup_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) +series_map_lookup_expr_reply(keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; sds query; sds msg; int sts; @@ -5234,7 +5234,7 @@ series_map_lookup_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_map_lookup_expr_reply"); if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY || reply->elements == 0)) { infofmt(msg, "expected array of one string element (got %zu) from series %s %s (type=%s)", - reply->elements, sid->name, HMGET, redis_reply_type(reply)); + reply->elements, sid->name, HMGET, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); } else if (reply->element[0]->type == REDIS_REPLY_STRING) { query = sdsempty(); @@ -5261,12 +5261,12 @@ series_map_lookup_expr(seriesQueryBaton *baton, sds series, void *arg) seriesBatonReference(baton, "series_map_lookup_expr"); key = sdscatfmt(sdsempty(), "pcp:expr:series:%S", sidexpr->name); - exprcmd = redis_command(3); - exprcmd = redis_param_str(exprcmd, HMGET, HMGET_LEN); - exprcmd = redis_param_sds(exprcmd, key); - exprcmd = redis_param_str(exprcmd, "query", sizeof("query")-1); + exprcmd = key_command(3); + exprcmd = key_param_str(exprcmd, HMGET, HMGET_LEN); + exprcmd = key_param_sds(exprcmd, key); + exprcmd = key_param_str(exprcmd, "query", sizeof("query")-1); sdsfree(key); - redisSlotsRequest(baton->slots, exprcmd, series_map_lookup_expr_reply, sidexpr); + keySlotsRequest(baton->slots, exprcmd, series_map_lookup_expr_reply, sidexpr); sdsfree(exprcmd); } @@ -5275,10 +5275,10 @@ series_map_lookup_expr(seriesQueryBaton *baton, sds series, void *arg) */ static int series_map_reply(seriesQueryBaton *baton, sds series, - int nelements, redisReply **elements) + int nelements, respReply **elements) { - redisMapEntry *entry; - redisReply *reply; + keyMapEntry *entry; + respReply *reply; sds msg, key; unsigned int i; int sts = 0; @@ -5296,8 +5296,8 @@ series_map_reply(seriesQueryBaton *baton, sds series, if (reply->type == REDIS_REPLY_STRING) { sdsclear(key); key = sdscatlen(key, reply->str, reply->len); - if ((entry = redisMapLookup(baton->lookup.map, key)) != NULL) - baton->lookup.func(series, redisMapValue(entry), baton->userdata); + if ((entry = keyMapLookup(baton->lookup.map, key)) != NULL) + baton->lookup.func(series, keyMapValue(entry), baton->userdata); else { infofmt(msg, "%s - timeseries string map", series); batoninfo(baton, PMLOG_CORRUPT, msg); @@ -5305,7 +5305,7 @@ series_map_reply(seriesQueryBaton *baton, sds series, } } else { infofmt(msg, "expected string in %s set (type=%s)", - series, redis_reply_type(reply)); + series, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); sts = -EPROTO; } @@ -5318,11 +5318,11 @@ series_map_reply(seriesQueryBaton *baton, sds series, static void series_map_keys_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesQueryBaton *baton = (seriesQueryBaton *)arg; - redisReply *reply = r; - redisReply *child; + respReply *reply = r; + respReply *child; sds val, msg; unsigned int i; @@ -5339,7 +5339,7 @@ series_map_keys_callback( baton->lookup.func(NULL, val, baton->userdata); } else { infofmt(msg, "bad response for string map %s (%s)", - HVALS, redis_reply_type(child)); + HVALS, key_reply_type(child)); batoninfo(baton, PMLOG_RESPONSE, msg); sdsfree(val); baton->error = -EINVAL; @@ -5348,7 +5348,7 @@ series_map_keys_callback( sdsfree(val); } else { infofmt(msg, "expected array from string map %s (reply=%s)", - HVALS, redis_reply_type(reply)); + HVALS, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -5363,11 +5363,11 @@ series_map_keys(seriesQueryBaton *baton, const char *name) sds cmd, key; key = sdscatfmt(sdsempty(), "pcp:map:%s", name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HVALS, HVALS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, HVALS, HVALS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_map_keys_callback, baton); sdsfree(cmd); return 0; @@ -5375,12 +5375,12 @@ series_map_keys(seriesQueryBaton *baton, const char *name) static void series_label_value_reply( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesGetLabelMap *value = (seriesGetLabelMap *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)value->baton; - redisReply *reply = r; - redisMapEntry *entry; + respReply *reply = r; + keyMapEntry *entry; pmSeriesLabel label; sds msg; @@ -5391,16 +5391,16 @@ series_label_value_reply( reverse_map(baton, value->map, reply->elements, reply->element); } else { infofmt(msg, "expected array from %s %s.%s.value (type=%s)", HGETALL, - "pcp:map:label", value->mapID, redis_reply_type(reply)); + "pcp:map:label", value->mapID, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } if (baton->error == 0) { label.name = value->name; - if ((entry = redisMapLookup(value->map, value->mapID)) == NULL) + if ((entry = keyMapLookup(value->map, value->mapID)) == NULL) label.value = sdsnew("null"); else - label.value = redisMapValue(entry); + label.value = keyMapValue(entry); baton->callbacks->on_labelmap(value->series, &label, baton->userdata); @@ -5418,12 +5418,12 @@ series_label_value_reply( static int series_label_reply(seriesQueryBaton *baton, sds series, - int nelements, redisReply **elements) + int nelements, respReply **elements) { seriesGetLabelMap *labelmap; - redisMapEntry *entry; - redisReply *reply; - redisMap *vmap; + keyMapEntry *entry; + respReply *reply; + keyMap *vmap; char hashbuf[42]; sds msg, key, cmd, name, vkey, nmapID, vmapID; unsigned int i, index; @@ -5442,7 +5442,7 @@ series_label_reply(seriesQueryBaton *baton, sds series, reply = elements[i]; if (reply->type != REDIS_REPLY_STRING) { infofmt(msg, "expected only string results from %s (type=%s)", - HGETALL, redis_reply_type(reply)); + HGETALL, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); return -EPROTO; } @@ -5461,11 +5461,11 @@ series_label_reply(seriesQueryBaton *baton, sds series, sdsclear(vmapID); vmapID = sdscatlen(vmapID, elements[index+1]->str, elements[index+1]->len); - if ((entry = redisMapLookup(baton->lookup.map, nmapID)) != NULL) { + if ((entry = keyMapLookup(baton->lookup.map, nmapID)) != NULL) { pmwebapi_hash_str((unsigned char *)nmapID, hashbuf, sizeof(hashbuf)); vkey = sdscatfmt(sdsempty(), "label.%s.value", hashbuf); - vmap = redisMapCreate(vkey); - name = redisMapValue(entry); + vmap = keyMapCreate(vkey); + name = keyMapValue(entry); baton->callbacks->on_label(series, name, baton->userdata); @@ -5474,7 +5474,7 @@ series_label_reply(seriesQueryBaton *baton, sds series, batoninfo(baton, PMLOG_ERROR, msg); sts = -ENOMEM; if (vmap) /* Coverity CID308763 */ - redisMapRelease(vmap); + keyMapRelease(vmap); continue; } initSeriesGetLabelMap(labelmap, series, name, vmap, vmapID, vkey, baton); @@ -5483,11 +5483,11 @@ series_label_reply(seriesQueryBaton *baton, sds series, pmwebapi_hash_str((unsigned char *)nmapID, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "pcp:map:label.%s.value", hashbuf); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, HGETALL, HGETALL_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_label_value_reply, labelmap); sdsfree(cmd); } else { @@ -5504,11 +5504,11 @@ series_label_reply(seriesQueryBaton *baton, sds series, static void series_lookup_labels_callback( - redisClusterAsyncContext *c, void *r,void *arg) + keyClusterAsyncContext *c, void *r,void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; sds msg; int sts; @@ -5518,7 +5518,7 @@ series_lookup_labels_callback( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from %s %s:%s (type=%s)", HGETALL, "pcp:labelvalue:series", sid->name, - redis_reply_type(reply)); + key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else if ((sts = series_label_reply(baton, sid->name, @@ -5546,11 +5546,11 @@ series_lookup_labels(void *arg) seriesBatonReference(baton, "series_lookup_labels"); sid = &baton->lookup.series[i]; key = sdscatfmt(sdsempty(), "pcp:labelvalue:series:%S", sid->name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, HGETALL, HGETALL_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_lookup_labels_callback, sid); sdsfree(cmd); } @@ -5574,7 +5574,7 @@ pmSeriesLabels(pmSeriesSettings *settings, int nseries, pmSID *series, void *arg initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_label, labelsmap); if (nseries == 0) - return series_map_keys(baton, redisMapName(baton->lookup.map)); + return series_map_keys(baton, keyMapName(baton->lookup.map)); baton->current = &baton->phases[0]; baton->phases[i++].func = series_lookup_services; @@ -5588,12 +5588,12 @@ pmSeriesLabels(pmSeriesSettings *settings, int nseries, pmSID *series, void *arg static void series_lookup_labelvalues_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; - redisReply *child; + respReply *reply = r; + respReply *child; pmSeriesLabel label; unsigned int i; sds msg; @@ -5612,7 +5612,7 @@ series_lookup_labelvalues_callback( baton->callbacks->on_labelmap(NULL, &label, baton->userdata); } else { infofmt(msg, "bad response for string map %s (%s)", - HVALS, redis_reply_type(child)); + HVALS, key_reply_type(child)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EINVAL; } @@ -5620,7 +5620,7 @@ series_lookup_labelvalues_callback( sdsfree(label.value); } else { infofmt(msg, "expected array from string map %s (reply=%s)", - HVALS, redis_reply_type(reply)); + HVALS, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -5649,11 +5649,11 @@ series_lookup_labelvalues(void *arg) pmwebapi_string_hash(hash, sid->name, sdslen(sid->name)); key = sdscatfmt(sdsempty(), "pcp:map:label.%s.value", pmwebapi_hash_str(hash, buffer, sizeof(buffer))); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HVALS, HVALS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, HVALS, HVALS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_lookup_labelvalues_callback, sid); sdsfree(cmd); } @@ -5686,13 +5686,13 @@ pmSeriesLabelValues(pmSeriesSettings *settings, int nlabels, pmSID *labels, void static void -redis_series_desc_reply( - redisClusterAsyncContext *c, void *r, void *arg) +key_series_desc_reply( + keyClusterAsyncContext *c, void *r, void *arg) { pmSeriesDesc desc; seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; sds msg; int sts; @@ -5705,7 +5705,7 @@ redis_series_desc_reply( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array type from series %s %s (type=%s)", - sid->name, HMGET, redis_reply_type(reply)); + sid->name, HMGET, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -5741,17 +5741,17 @@ series_lookup_desc(void *arg) seriesBatonReference(baton, "series_lookup_desc"); key = sdscatfmt(sdsempty(), "pcp:desc:series:%S", sid->name); - cmd = redis_command(8); - cmd = redis_param_str(cmd, HMGET, HMGET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "indom", sizeof("indom")-1); - cmd = redis_param_str(cmd, "pmid", sizeof("pmid")-1); - cmd = redis_param_str(cmd, "semantics", sizeof("semantics")-1); - cmd = redis_param_str(cmd, "source", sizeof("source")-1); - cmd = redis_param_str(cmd, "type", sizeof("type")-1); - cmd = redis_param_str(cmd, "units", sizeof("units")-1); + cmd = key_command(8); + cmd = key_param_str(cmd, HMGET, HMGET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "indom", sizeof("indom")-1); + cmd = key_param_str(cmd, "pmid", sizeof("pmid")-1); + cmd = key_param_str(cmd, "semantics", sizeof("semantics")-1); + cmd = key_param_str(cmd, "source", sizeof("source")-1); + cmd = key_param_str(cmd, "type", sizeof("type")-1); + cmd = key_param_str(cmd, "units", sizeof("units")-1); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, redis_series_desc_reply, sid); + keySlotsRequest(baton->slots, cmd, key_series_desc_reply, sid); sdsfree(cmd); } } @@ -5785,7 +5785,7 @@ pmSeriesDescs(pmSeriesSettings *settings, int nseries, pmSID *series, void *arg) static int extract_series_inst(seriesQueryBaton *baton, seriesGetSID *sid, - pmSeriesInst *inst, int nelements, redisReply **elements) + pmSeriesInst *inst, int nelements, respReply **elements) { sds msg, series = sid->metric; @@ -5809,11 +5809,11 @@ extract_series_inst(seriesQueryBaton *baton, seriesGetSID *sid, static void series_instances_reply_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; pmSeriesInst inst; sds msg; int sts; @@ -5828,7 +5828,7 @@ series_instances_reply_callback( if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from series %s %s (type=%s)", - HMGET, sid->name, redis_reply_type(reply)); + HMGET, sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -5849,7 +5849,7 @@ series_instances_reply_callback( static void series_instances_reply(seriesQueryBaton *baton, - pmSID series, int nelements, redisReply **elements) + pmSID series, int nelements, respReply **elements) { seriesGetSID *sid; pmSID name = sdsempty(); @@ -5879,14 +5879,14 @@ series_instances_reply(seriesQueryBaton *baton, seriesBatonReference(baton, "series_instances_reply"); key = sdscatfmt(sdsempty(), "pcp:inst:series:%S", sid->name); - cmd = redis_command(5); - cmd = redis_param_str(cmd, HMGET, HMGET_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_str(cmd, "inst", sizeof("inst")-1); - cmd = redis_param_str(cmd, "name", sizeof("name")-1); - cmd = redis_param_str(cmd, "source", sizeof("source")-1); + cmd = key_command(5); + cmd = key_param_str(cmd, HMGET, HMGET_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_str(cmd, "inst", sizeof("inst")-1); + cmd = key_param_str(cmd, "name", sizeof("name")-1); + cmd = key_param_str(cmd, "source", sizeof("source")-1); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_instances_reply_callback, sid); sdsfree(cmd); } @@ -5894,11 +5894,11 @@ series_instances_reply(seriesQueryBaton *baton, } static void -series_inst_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) +series_inst_expr_reply(keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; pmSeriesExpr expr = {0}; sds msg; int sts; @@ -5908,7 +5908,7 @@ series_inst_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array of one string element (got %zu) from series %s %s (type=%s)", - reply->elements, sid->name, HMGET, redis_reply_type(reply)); + reply->elements, sid->name, HMGET, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); } else if (reply->element[0]->type == REDIS_REPLY_STRING) { expr.query = sdsempty(); @@ -5925,11 +5925,11 @@ series_inst_expr_reply(redisClusterAsyncContext *c, void *r, void *arg) void series_lookup_instances_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; seriesGetSID *expr; sds key, exprcmd; sds msg; @@ -5943,9 +5943,9 @@ series_lookup_instances_callback( series_instances_reply(baton, sid->name, reply->elements, reply->element); } else { /* Handle fabricated/expression SID in /series/instances : - * - get the expr for sid->name from redis. In the callback for that, - * parse the expr and then solve the expression tree with timing from - * this series query baton, then merge the instances into the reply. + * get the expr for sid->name from server. In the callback for that, + * parse the expr and then solve the expression tree with timing from + * this series query baton, then merge the instances into the reply. */ if (pmDebugOptions.query) fprintf(stderr, "series_lookup_instances_callback: sid %s is fabricated\n", sid->name); @@ -5954,17 +5954,17 @@ series_lookup_instances_callback( seriesBatonReference(baton, "series_lookup_instances_callback"); key = sdscatfmt(sdsempty(), "pcp:expr:series:%S", expr->name); - exprcmd = redis_command(3); - exprcmd = redis_param_str(exprcmd, HMGET, HMGET_LEN); - exprcmd = redis_param_sds(exprcmd, key); - exprcmd = redis_param_str(exprcmd, "query", sizeof("query")-1); + exprcmd = key_command(3); + exprcmd = key_param_str(exprcmd, HMGET, HMGET_LEN); + exprcmd = key_param_sds(exprcmd, key); + exprcmd = key_param_str(exprcmd, "query", sizeof("query")-1); sdsfree(key); - redisSlotsRequest(baton->slots, exprcmd, series_inst_expr_reply, expr); + keySlotsRequest(baton->slots, exprcmd, series_inst_expr_reply, expr); sdsfree(exprcmd); } } else { infofmt(msg, "expected array from series %s %s (type=%s)", - SMEMBERS, sid->name, redis_reply_type(reply)); + SMEMBERS, sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -5987,11 +5987,11 @@ series_lookup_instances(void *arg) seriesBatonReference(baton, "series_lookup_instances_callback"); sid = &baton->lookup.series[i]; key = sdscatfmt(sdsempty(), "pcp:instances:series:%S", sid->name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, SMEMBERS, SMEMBERS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_lookup_instances_callback, sid); sdsfree(cmd); } @@ -6015,7 +6015,7 @@ pmSeriesInstances(pmSeriesSettings *settings, int nseries, pmSID *series, void * initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_instance, instmap); if (nseries == 0) - return series_map_keys(baton, redisMapName(baton->lookup.map)); + return series_map_keys(baton, keyMapName(baton->lookup.map)); baton->current = &baton->phases[0]; baton->phases[i++].func = series_lookup_services; @@ -6028,21 +6028,21 @@ pmSeriesInstances(pmSeriesSettings *settings, int nseries, pmSID *series, void * } static void -redis_lookup_mapping_callback( - redisClusterAsyncContext *c, void *r, void *arg) +key_lookup_mapping_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesQueryBaton *baton = (seriesQueryBaton *)arg; - redisReply *reply = r; + respReply *reply = r; sds msg; - seriesBatonCheckMagic(baton, MAGIC_QUERY, "redis_lookup_mapping_callback"); + seriesBatonCheckMagic(baton, MAGIC_QUERY, "key_lookup_mapping_callback"); /* unpack - produce reverse map of ids-to-names for each context */ if (LIKELY(reply && reply->type == REDIS_REPLY_ARRAY)) { reverse_map(baton, baton->lookup.map, reply->elements, reply->element); } else { infofmt(msg, "expected array from %s %s (type=%s)", - HGETALL, "pcp:map:context.name", redis_reply_type(reply)); + HGETALL, "pcp:map:context.name", key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -6060,22 +6060,22 @@ series_lookup_mapping(void *arg) seriesBatonCheckCount(baton, "series_lookup_mapping"); seriesBatonReference(baton, "series_lookup_mapping"); - key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->lookup.map)); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); - cmd = redis_param_sds(cmd, key); + key = sdscatfmt(sdsempty(), "pcp:map:%s", keyMapName(baton->lookup.map)); + cmd = key_command(2); + cmd = key_param_str(cmd, HGETALL, HGETALL_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, - redis_lookup_mapping_callback, baton); + keySlotsRequest(baton->slots, cmd, + key_lookup_mapping_callback, baton); sdsfree(cmd); } static void series_query_mapping_callback( - redisClusterAsyncContext *c, void *r, void *arg) + keyClusterAsyncContext *c, void *r, void *arg) { seriesQueryBaton *baton = (seriesQueryBaton *)arg; - redisReply *reply = r; + respReply *reply = r; sds msg; seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_query_mapping_callback"); @@ -6085,7 +6085,7 @@ series_query_mapping_callback( reverse_map(baton, namesmap, reply->elements, reply->element); } else { infofmt(msg, "expected array from %s %s (type=%s)", - HGETALL, "pcp:map:context.name", redis_reply_type(reply)); + HGETALL, "pcp:map:context.name", key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } @@ -6104,11 +6104,11 @@ series_query_mapping(void *arg) seriesBatonReference(baton, "series_query_mapping"); key = sdsnew("pcp:map:metric.name"); - cmd = redis_command(2); - cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, HGETALL, HGETALL_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, + keySlotsRequest(baton->slots, cmd, series_query_mapping_callback, baton); sdsfree(cmd); } @@ -6142,12 +6142,12 @@ series_lookup_services(void *arg) series_query_end_phase(baton); } else { if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); + option = pmIniFileLookup(data->config, "redis", "enabled"); // compat if (option && strcmp(option, "false") == 0) baton->error = -ENOTSUP; else baton->slots = data->slots = - redisSlotsConnect( + keySlotsConnect( data->config, 1, baton->info, series_query_end_phase, baton->userdata, data->events, (void *)baton); @@ -6155,22 +6155,22 @@ series_lookup_services(void *arg) } static void -redis_get_sid_callback( - redisClusterAsyncContext *redis, void *r, void *arg) +key_get_sid_callback( + keyClusterAsyncContext *context, void *r, void *arg) { seriesGetSID *sid = (seriesGetSID *)arg; seriesQueryBaton *baton = (seriesQueryBaton *)sid->baton; - redisReply *reply = r; + respReply *reply = r; sds msg; int sts; - seriesBatonCheckMagic(sid, MAGIC_SID, "redis_get_sid_callback"); - seriesBatonCheckMagic(baton, MAGIC_QUERY, "redis_get_sid_callback"); + seriesBatonCheckMagic(sid, MAGIC_SID, "key_get_sid_callback"); + seriesBatonCheckMagic(baton, MAGIC_QUERY, "key_get_sid_callback"); /* unpack - extract names for this source via context name map */ if (UNLIKELY(reply == NULL || reply->type != REDIS_REPLY_ARRAY)) { infofmt(msg, "expected array from %s %s (type=%s)", - SMEMBERS, sid->name, redis_reply_type(reply)); + SMEMBERS, sid->name, key_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } else if ((sts = series_map_reply(baton, sid->name, @@ -6196,12 +6196,12 @@ series_lookup_sources(void *arg) seriesBatonReference(baton, "series_lookup_sources"); sid = &baton->lookup.series[i]; key = sdscatfmt(sdsempty(), "pcp:context.name:source:%S", sid->name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, SMEMBERS, SMEMBERS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, - redis_get_sid_callback, sid); + keySlotsRequest(baton->slots, cmd, + key_get_sid_callback, sid); sdsfree(cmd); } series_query_end_phase(baton); @@ -6225,7 +6225,7 @@ pmSeriesSources(pmSeriesSettings *settings, int nsources, pmSID *sources, void * initSeriesGetLookup(baton, nsources, sources, settings->callbacks.on_context, contextmap); if (nsources == 0) - return series_map_keys(baton, redisMapName(baton->lookup.map)); + return series_map_keys(baton, keyMapName(baton->lookup.map)); baton->current = &baton->phases[0]; baton->phases[i++].func = series_lookup_services; @@ -6252,12 +6252,12 @@ series_lookup_metrics(void *arg) seriesBatonReference(baton, "series_lookup_metrics"); sid = &baton->lookup.series[i]; key = sdscatfmt(sdsempty(), "pcp:metric.name:series:%S", sid->name); - cmd = redis_command(2); - cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, SMEMBERS, SMEMBERS_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, - redis_get_sid_callback, sid); + keySlotsRequest(baton->slots, cmd, + key_get_sid_callback, sid); sdsfree(cmd); } } @@ -6280,7 +6280,7 @@ pmSeriesMetrics(pmSeriesSettings *settings, int nseries, sds *series, void *arg) initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_metric, namesmap); if (nseries == 0) - return series_map_keys(baton, redisMapName(baton->lookup.map)); + return series_map_keys(baton, keyMapName(baton->lookup.map)); baton->current = &baton->phases[0]; baton->phases[i++].func = series_lookup_services; diff --git a/src/libpcp_web/src/schema.c b/src/libpcp_web/src/schema.c index 948142b9db..5cfbc759f5 100644 --- a/src/libpcp_web/src/schema.c +++ b/src/libpcp_web/src/schema.c @@ -23,7 +23,7 @@ #define STRINGIFY(s) #s #define TO_STRING(s) STRINGIFY(s) #define SERIES_VERSION 2 -#define REDIS_VERSION 5 +#define OLDEST_VERSION 5 extern sds cursorcount; static sds maxstreamlen; @@ -33,8 +33,8 @@ static sds DEFAULT_MAXSTREAMLEN; static sds DEFAULT_STREAMEXPIRE; static void -initRedisSlotsBaton(redisSlotsBaton *baton, - redisInfoCallBack info, redisDoneCallBack done, +initKeySlotsBaton(keySlotsBaton *baton, + keysInfoCallBack info, keysDoneCallBack done, void *userdata, void *events, void *arg) { initSeriesBatonMagic(baton, MAGIC_SLOTS); @@ -46,11 +46,11 @@ initRedisSlotsBaton(redisSlotsBaton *baton, } static void -redis_slots_finished(void *arg) +keys_slots_finished(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_slots_finished"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_slots_finished"); if (baton->error == 0) { baton->slots->state = SLOTS_READY; @@ -61,24 +61,24 @@ redis_slots_finished(void *arg) } void -redis_slots_end_phase(void *arg) +keys_slots_end_phase(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_slots_end_phase"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_slots_end_phase"); if (baton->error == 0) { - seriesPassBaton(&baton->current, baton, "redis_slots_end_phase"); + seriesPassBaton(&baton->current, baton, "keys_slots_end_phase"); } else { /* fail after waiting on outstanding I/O */ - if (seriesBatonDereference(baton, "redis_slots_end_phase")) - redis_slots_finished(baton); + if (seriesBatonDereference(baton, "keys_slots_end_phase")) + keys_slots_finished(baton); } } static void -initRedisMapBaton(redisMapBaton *baton, redisSlots *slots, - redisMap *mapping, sds mapKey, sds mapStr, - redisDoneCallBack on_done, redisInfoCallBack on_info, +initKeyMapBaton(keyMapBaton *baton, keySlots *slots, + keyMap *mapping, sds mapKey, sds mapStr, + keysDoneCallBack on_done, keysInfoCallBack on_info, void *userdata, void *arg) { initSeriesBatonMagic(baton, MAGIC_MAPPING); @@ -93,9 +93,9 @@ initRedisMapBaton(redisMapBaton *baton, redisSlots *slots, } static void -doneRedisMapBaton(redisMapBaton *baton) +doneKeyMapBaton(keyMapBaton *baton) { - seriesBatonCheckMagic(baton, MAGIC_MAPPING, "doneRedisMapBaton"); + seriesBatonCheckMagic(baton, MAGIC_MAPPING, "doneKeyMapBaton"); if (baton->mapped) baton->mapped(baton->arg); sdsfree(baton->mapKey); @@ -104,28 +104,28 @@ doneRedisMapBaton(redisMapBaton *baton) } static void -redis_map_publish_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_map_publish_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisMapBaton *baton = (redisMapBaton *)arg; - redisReply *reply = r; + keyMapBaton *baton = (keyMapBaton *)arg; + respReply *reply = r; - seriesBatonCheckMagic(baton, MAGIC_MAPPING, "redis_map_publish_callback"); + seriesBatonCheckMagic(baton, MAGIC_MAPPING, "keys_map_publish_callback"); checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", PUBLISH, "new %s mapping", - redisMapName(baton->mapping)); - doneRedisMapBaton(baton); + keyMapName(baton->mapping)); + doneKeyMapBaton(baton); } static void -redis_map_publish(redisMapBaton *baton) +keys_map_publish(keyMapBaton *baton) { char hash[42]; sds msg, key, cmd; pmwebapi_hash_str((unsigned char *)baton->mapKey, hash, sizeof(hash)); msg = sdscatfmt(sdsempty(), "%s:%S", hash, baton->mapStr); - key = sdscatfmt(sdsempty(), "pcp:channel:%s", redisMapName(baton->mapping)); + key = sdscatfmt(sdsempty(), "pcp:channel:%s", keyMapName(baton->mapping)); cmd = redis_command(3); cmd = redis_param_str(cmd, PUBLISH, PUBLISH_LEN); cmd = redis_param_sds(cmd, key); @@ -133,38 +133,38 @@ redis_map_publish(redisMapBaton *baton) sdsfree(msg); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, - redis_map_publish_callback, baton); + keySlotsRequest(baton->slots, cmd, + keys_map_publish_callback, baton); sdsfree(cmd); } static void -redis_map_request_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_map_request_callback( + mapClusterAsyncContext *c, void *r, void *arg) { - redisMapBaton *baton = (redisMapBaton *)arg; - redisReply *reply = r; + keyMapBaton *baton = (keyMapBaton *)arg; + respReply *reply = r; int newname; seriesBatonCheckMagic(baton, MAGIC_MAPPING, "redis_map_request_callback"); newname = checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s (%s)", HSET, "string mapping script", - redisMapName(baton->mapping)); + keyMapName(baton->mapping)); /* publish any newly created name mapping */ if (newname > 0) redis_map_publish(baton); else - doneRedisMapBaton(baton); + doneKeyMapBaton(baton); } void -redisMapRequest(redisMapBaton *baton, redisMap *map, sds name, sds value) +keyMapRequest(keyMapBaton *baton, keyMap *map, sds name, sds value) { sds cmd, key; - key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->mapping)); + key = sdscatfmt(sdsempty(), "pcp:map:%s", keyMapName(baton->mapping)); cmd = redis_command(4); cmd = redis_param_str(cmd, HSET, HSET_LEN); cmd = redis_param_sds(cmd, key); @@ -172,23 +172,23 @@ redisMapRequest(redisMapBaton *baton, redisMap *map, sds name, sds value) cmd = redis_param_sds(cmd, value); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, redis_map_request_callback, baton); + keySlotsRequest(baton->slots, cmd, redis_map_request_callback, baton); sdsfree(cmd); } void -redisGetMap(redisSlots *slots, redisMap *mapping, unsigned char *hash, sds mapStr, +redisGetMap(keySlots *slots, keyMap *mapping, unsigned char *hash, sds mapStr, redisDoneCallBack on_done, redisInfoCallBack on_info, void *userdata, void *arg) { - redisMapBaton *baton; - redisMapEntry *entry; + keyMapBaton *baton; + keyMapEntry *entry; sds mapKey; pmwebapi_string_hash(hash, mapStr, sdslen(mapStr)); mapKey = sdsnewlen(hash, 20); - if ((entry = redisMapLookup(mapping, mapKey)) != NULL) { + if ((entry = keyMapLookup(mapping, mapKey)) != NULL) { sdsfree(mapKey); on_done(arg); } else { @@ -198,11 +198,11 @@ redisGetMap(redisSlots *slots, redisMap *mapping, unsigned char *hash, sds mapSt * The caller does not need to wait as we provide the calculated hash * straight away. */ - if ((baton = calloc(1, sizeof(redisMapBaton))) != NULL) { - initRedisMapBaton(baton, slots, mapping, mapKey, mapStr, + if ((baton = calloc(1, sizeof(keyMapBaton))) != NULL) { + initKeyMapBaton(baton, slots, mapping, mapKey, mapStr, on_done, on_info, userdata, arg); - redisMapInsert(mapping, mapKey, sdsdup(mapStr)); - redisMapRequest(baton, mapping, mapKey, mapStr); + keyMapInsert(mapping, mapKey, sdsdup(mapStr)); + keyMapRequest(baton, mapping, mapKey, mapStr); } else { on_done(arg); } @@ -210,43 +210,43 @@ redisGetMap(redisSlots *slots, redisMap *mapping, unsigned char *hash, sds mapSt } static void -redis_source_context_name( - redisClusterAsyncContext *c, void *r, void *arg) +keys_source_context_name( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", SADD, "mapping context to source or host name"); - doneSeriesLoadBaton(baton, "redis_source_context_name"); + doneSeriesLoadBaton(baton, "keys_source_context_name"); } static void -redis_source_location( - redisClusterAsyncContext *c, void *r, void *arg) +keys_source_location( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", GEOADD, "mapping source location"); - doneSeriesLoadBaton(baton, "redis_source_location"); + doneSeriesLoadBaton(baton, "keys_source_location"); } static void -redis_context_name_source( - redisClusterAsyncContext *c, void *r, void *arg) +keys_context_name_source( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", SADD, "mapping source names to context"); - doneSeriesLoadBaton(baton, "redis_context_name_source"); + doneSeriesLoadBaton(baton, "keys_context_name_source"); } void -redis_series_source(redisSlots *slots, void *arg) +keys_series_source(keySlots *slots, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; context_t *context = seriesLoadBatonContext(baton); @@ -259,7 +259,7 @@ redis_series_source(redisSlots *slots, void *arg) * . SADD pcp:source:context.name: * . GEOADD pcp:source:location */ - seriesBatonReferences(baton, 4, "redis_series_source"); + seriesBatonReferences(baton, 4, "keys_series_source"); pmwebapi_hash_str(context->name.id, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "pcp:source:context.name:%s", hashbuf); @@ -268,7 +268,7 @@ redis_series_source(redisSlots *slots, void *arg) cmd = redis_param_sds(cmd, key); cmd = redis_param_sha(cmd, context->name.hash); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_source_context_name, arg); + keySlotsRequest(slots, cmd, keys_source_context_name, arg); sdsfree(cmd); pmwebapi_hash_str(context->hostid, hashbuf, sizeof(hashbuf)); @@ -278,7 +278,7 @@ redis_series_source(redisSlots *slots, void *arg) cmd = redis_param_sds(cmd, key); cmd = redis_param_sha(cmd, context->name.hash); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_source_context_name, arg); + keySlotsRequest(slots, cmd, keys_source_context_name, arg); sdsfree(cmd); pmwebapi_hash_str(context->name.hash, hashbuf, sizeof(hashbuf)); @@ -289,7 +289,7 @@ redis_series_source(redisSlots *slots, void *arg) cmd = redis_param_sha(cmd, context->name.id); cmd = redis_param_sha(cmd, context->hostid); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_context_name_source, arg); + keySlotsRequest(slots, cmd, redis_context_name_source, arg); sdsfree(cmd); key = sdsnew("pcp:source:location"); @@ -304,56 +304,56 @@ redis_series_source(redisSlots *slots, void *arg) sdsfree(val2); sdsfree(val); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_source_location, arg); + keySlotsRequest(slots, cmd, keys_source_location, arg); sdsfree(cmd); } static void -redis_series_inst_name_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_inst_name_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", SADD, "mapping series to inst name"); - doneSeriesLoadBaton(baton, "redis_series_inst_name_callback"); + doneSeriesLoadBaton(baton, "keys_series_inst_name_callback"); } static void -redis_instances_series_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_instances_series_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(baton->info, baton->userdata, c, reply, "%s: %s", SADD, "mapping instance to series"); - doneSeriesLoadBaton(baton, "redis_instances_series_callback"); + doneSeriesLoadBaton(baton, "keys_instances_series_callback"); } static void -redis_series_inst_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_inst_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(baton->info, baton->userdata, c, reply, "%s: %s", HMSET, "setting metric inst"); - doneSeriesLoadBaton(baton, "redis_series_inst_callback"); + doneSeriesLoadBaton(baton, "keys_series_inst_callback"); } void -redis_series_instance(redisSlots *slots, metric_t *metric, instance_t *instance, void *arg) +keys_series_instance(keySlots *slots, metric_t *metric, instance_t *instance, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; char mhashbuf[42], hashbuf[42]; sds cmd, key, val; int i; - seriesBatonCheckMagic(baton, MAGIC_LOAD, "redis_series_instance"); - seriesBatonReferences(baton, 2, "redis_series_instance"); + seriesBatonCheckMagic(baton, MAGIC_LOAD, "keys_series_instance"); + seriesBatonReferences(baton, 2, "keys_series_instance"); assert(instance->name.sds); pmwebapi_hash_str(instance->name.id, hashbuf, sizeof(hashbuf)); @@ -361,7 +361,7 @@ redis_series_instance(redisSlots *slots, metric_t *metric, instance_t *instance, if (pmDebugOptions.series) { pmwebapi_hash_str(metric->names[0].id, mhashbuf, sizeof(mhashbuf)); fprintf(stderr, "%s: loading inst name %s [%s] for metric %s [%s]\n", - "redis_series_instance", instance->name.sds, hashbuf, + "keys_series_instance", instance->name.sds, hashbuf, metric->names[0].sds, mhashbuf); } @@ -372,11 +372,11 @@ redis_series_instance(redisSlots *slots, metric_t *metric, instance_t *instance, sdsfree(key); for (i = 0; i < metric->numnames; i++) cmd = redis_param_sha(cmd, metric->names[i].hash); - redisSlotsRequest(slots, cmd, redis_series_inst_name_callback, arg); + keySlotsRequest(slots, cmd, keys_series_inst_name_callback, arg); sdsfree(cmd); for (i = 0; i < metric->numnames; i++) { - seriesBatonReference(baton, "redis_series_instance"); + seriesBatonReference(baton, "keys_series_instance"); pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "pcp:instances:series:%s", hashbuf); cmd = redis_command(3); @@ -384,7 +384,7 @@ redis_series_instance(redisSlots *slots, metric_t *metric, instance_t *instance, cmd = redis_param_sds(cmd, key); cmd = redis_param_sha(cmd, instance->name.hash); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_instances_series_callback, arg); + keySlotsRequest(slots, cmd, keys_instances_series_callback, arg); sdsfree(cmd); } @@ -402,7 +402,7 @@ redis_series_instance(redisSlots *slots, metric_t *metric, instance_t *instance, cmd = redis_param_sha(cmd, metric->indom->domain->context->name.hash); sdsfree(val); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_series_inst_callback, arg); + keySlotsRequest(slots, cmd, keys_series_inst_callback, arg); sdsfree(cmd); } @@ -413,7 +413,7 @@ label_value_mapping_callback(void *arg) seriesLoadBaton *baton = (seriesLoadBaton *)list->arg; seriesBatonCheckMagic(baton, MAGIC_LOAD, "label_value_mapping_callback"); - redisMapRelease(list->valuemap); + keyMapRelease(list->valuemap); list->valuemap = NULL; doneSeriesLoadBaton(baton, "label_value_mapping_callback"); } @@ -498,7 +498,7 @@ annotate_metric(const pmLabel *label, const char *json, void *arg) pmwebapi_hash_str(list->nameid, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "label.%s.value", hashbuf); - list->valuemap = redisMapCreate(key); + list->valuemap = keyMapCreate(key); redisGetMap(baton->slots, list->valuemap, list->valueid, list->value, @@ -509,56 +509,56 @@ annotate_metric(const pmLabel *label, const char *json, void *arg) } static void -redis_series_labelvalue_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_labelvalue_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(load->info, load->userdata, c, reply, "%s: %s", HMSET, "setting series label value"); - doneSeriesLoadBaton(arg, "redis_series_labelvalue_callback"); + doneSeriesLoadBaton(arg, "keys_series_labelvalue_callback"); } static void -redis_series_maplabelvalue_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_maplabelvalue_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(load->info, load->userdata, c, reply, "%s: %s", HMSET, "setting series map label value"); - doneSeriesLoadBaton(arg, "redis_series_maplabelvalue_callback"); + doneSeriesLoadBaton(arg, "keys_series_maplabelvalue_callback"); } static void -redis_series_labelflags_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_labelflags_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(load->info, load->userdata, c, reply, "%s: %s", HMSET, "setting series label flags"); - doneSeriesLoadBaton(arg, "redis_series_labelflags_callback"); + doneSeriesLoadBaton(arg, "keys_series_labelflags_callback"); } static void -redis_series_label_set_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_label_set_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(load->info, load->userdata, c, reply, "%s %s", SADD, "pcp:series:label.X.value:Y"); - doneSeriesLoadBaton(arg, "redis_series_label_set_callback"); + doneSeriesLoadBaton(arg, "keys_series_label_set_callback"); } static void -redis_series_label(redisSlots *slots, metric_t *metric, char *hash, +keys_series_label(keySlots *slots, metric_t *metric, char *hash, labellist_t *list, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; @@ -566,10 +566,10 @@ redis_series_label(redisSlots *slots, metric_t *metric, char *hash, sds cmd, key, val; int i; - seriesBatonReferences(baton, 3, "redis_series_label"); + seriesBatonReferences(baton, 3, "keys_series_label"); if (list->flags != PM_LABEL_CONTEXT) { - seriesBatonReference(baton, "redis_series_label"); + seriesBatonReference(baton, "keys_series_label"); val = sdscatfmt(sdsempty(), "%I", list->flags); key = sdscatfmt(sdsempty(), "pcp:labelflags:series:%s", hash); @@ -580,8 +580,8 @@ redis_series_label(redisSlots *slots, metric_t *metric, char *hash, cmd = redis_param_sds(cmd, val); sdsfree(val); sdsfree(key); - redisSlotsRequest(slots, cmd, - redis_series_labelflags_callback, arg); + keySlotsRequest(slots, cmd, + keys_series_labelflags_callback, arg); sdsfree(cmd); } @@ -592,8 +592,8 @@ redis_series_label(redisSlots *slots, metric_t *metric, char *hash, cmd = redis_param_sha(cmd, list->nameid); cmd = redis_param_sha(cmd, list->valueid); sdsfree(key); - redisSlotsRequest(slots, cmd, - redis_series_labelvalue_callback, arg); + keySlotsRequest(slots, cmd, + keys_series_labelvalue_callback, arg); sdsfree(cmd); pmwebapi_hash_str(list->nameid, namehash, sizeof(namehash)); @@ -606,8 +606,8 @@ redis_series_label(redisSlots *slots, metric_t *metric, char *hash, cmd = redis_param_sha(cmd, list->valueid); cmd = redis_param_sds(cmd, list->value); sdsfree(key); - redisSlotsRequest(slots, cmd, - redis_series_maplabelvalue_callback, arg); + keySlotsRequest(slots, cmd, + keys_series_maplabelvalue_callback, arg); sdsfree(cmd); key = sdscatfmt(sdsempty(), "pcp:series:label.%s.value:%s", @@ -618,13 +618,13 @@ redis_series_label(redisSlots *slots, metric_t *metric, char *hash, sdsfree(key); for (i = 0; i < metric->numnames; i++) cmd = redis_param_sha(cmd, metric->names[i].hash); - redisSlotsRequest(slots, cmd, - redis_series_label_set_callback, arg); + keySlotsRequest(slots, cmd, + keys_series_label_set_callback, arg); sdsfree(cmd); } static void -redis_series_labelset(redisSlots *slots, metric_t *metric, instance_t *instance, void *arg) +keys_series_labelset(keySlots *slots, metric_t *metric, instance_t *instance, void *arg) { labellist_t *list; char hashbuf[42]; @@ -634,14 +634,14 @@ redis_series_labelset(redisSlots *slots, metric_t *metric, instance_t *instance, pmwebapi_hash_str(instance->name.hash, hashbuf, sizeof(hashbuf)); list = instance->labellist; do { - redis_series_label(slots, metric, hashbuf, list, arg); + keys_series_label(slots, metric, hashbuf, list, arg); } while ((list = list->next) != NULL); } else { for (i = 0; i < metric->numnames; i++) { pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf)); list = metric->labellist; do { - redis_series_label(slots, metric, hashbuf, list, arg); + keys_series_label(slots, metric, hashbuf, list, arg); } while ((list = list->next) != NULL); } } @@ -692,11 +692,11 @@ series_name_mapping_callback(void *arg) doneSeriesLoadBaton(arg, "series_name_mapping_callback"); } -static void redis_series_metadata(context_t *, metric_t *, void *); -static void redis_series_streamed(sds, metric_t *, void *); +static void keys_series_metadata(context_t *, metric_t *, void *); +static void keys_series_streamed(sds, metric_t *, void *); void -redis_series_metric(redisSlots *slots, metric_t *metric, +keys_series_metric(keySlots *slots, metric_t *metric, sds timestamp, int meta, int data, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; @@ -714,7 +714,7 @@ redis_series_metric(redisSlots *slots, metric_t *metric, /* ensure all metric name strings are mapped */ for (i = 0; metric->cached == 0 && i < metric->numnames; i++) { assert(metric->names[i].sds != NULL); - seriesBatonReference(baton, "redis_series_metric"); + seriesBatonReference(baton, "keys_series_metric"); redisGetMap(slots, namesmap, metric->names[i].id, metric->names[i].sds, series_name_mapping_callback, @@ -735,7 +735,7 @@ redis_series_metric(redisSlots *slots, metric_t *metric, continue; } assert(instance->name.sds != NULL); - seriesBatonReference(baton, "redis_series_metric"); + seriesBatonReference(baton, "keys_series_metric"); redisGetMap(slots, instmap, instance->name.id, instance->name.sds, series_name_mapping_callback, @@ -748,66 +748,66 @@ redis_series_metric(redisSlots *slots, metric_t *metric, /* push the metric, instances and any label metadata into the cache */ if (meta || data) - redis_series_metadata(&baton->pmapi.context, metric, baton); + keys_series_metadata(&baton->pmapi.context, metric, baton); /* push values for all instances, no-value or errors into the cache */ if (data) - redis_series_streamed(timestamp, metric, baton); + keys_series_streamed(timestamp, metric, baton); } static void -redis_metric_name_series_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_metric_name_series_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(load->info, load->userdata, c, reply, "%s %s", SADD, "map metric name to series"); - doneSeriesLoadBaton(arg, "redis_metric_name_series_callback"); + doneSeriesLoadBaton(arg, "keys_metric_name_series_callback"); } static void -redis_series_metric_name_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_metric_name_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(load->info, load->userdata, c, reply, "%s: %s", SADD, "map series to metric name"); - doneSeriesLoadBaton(arg, "redis_series_metric_name_callback"); + doneSeriesLoadBaton(arg, "keys_series_metric_name_callback"); } static void -redis_desc_series_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_desc_series_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(load->info, load->userdata, c, reply, "%s: %s", HMSET, "setting metric desc"); - doneSeriesLoadBaton(arg, "redis_desc_series_callback"); + doneSeriesLoadBaton(arg, "keys_desc_series_callback"); } static void -redis_series_source_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_source_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; - redisReply *reply = r; + respReply *reply = r; checkIntegerReply(load->info, load->userdata, c, reply, "%s: %s", SADD, "mapping series to context"); - doneSeriesLoadBaton(arg, "redis_series_source_callback"); + doneSeriesLoadBaton(arg, "keys_series_source_callback"); } static void -redis_series_metadata(context_t *context, metric_t *metric, void *arg) +keys_series_metadata(context_t *context, metric_t *metric, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - redisSlots *slots = baton->slots; + keySlots *slots = baton->slots; instance_t *instance; value_t *value; const char *units, *indom = NULL, *pmid, *sem, *type; @@ -828,7 +828,7 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) for (i = 0; i < metric->numnames; i++) { assert(metric->names[i].sds != NULL); - seriesBatonReferences(baton, 3, "redis_series_metadata names"); + seriesBatonReferences(baton, 3, "keys_series_metadata names"); pmwebapi_hash_str(metric->names[i].id, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "pcp:series:metric.name:%s", hashbuf); @@ -837,8 +837,8 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) cmd = redis_param_sds(cmd, key); cmd = redis_param_sha(cmd, metric->names[i].hash); sdsfree(key); - redisSlotsRequest(slots, cmd, - redis_series_metric_name_callback, arg); + keySlotsRequest(slots, cmd, + keys_series_metric_name_callback, arg); sdsfree(cmd); pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf)); @@ -848,8 +848,8 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) cmd = redis_param_sds(cmd, key); cmd = redis_param_sha(cmd, metric->names[i].id); sdsfree(key); - redisSlotsRequest(slots, cmd, - redis_metric_name_series_callback, arg); + keySlotsRequest(slots, cmd, + keys_metric_name_series_callback, arg); sdsfree(cmd); key = sdscatfmt(sdsempty(), "pcp:desc:series:%s", hashbuf); @@ -869,16 +869,16 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) cmd = redis_param_str(cmd, "units", sizeof("units")-1); cmd = redis_param_str(cmd, units, strlen(units)); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_desc_series_callback, arg); + keySlotsRequest(slots, cmd, keys_desc_series_callback, arg); sdsfree(cmd); if ((baton->flags & PM_SERIES_FLAG_TEXT) && slots->search) - redis_search_text_add(slots, PM_SEARCH_TYPE_METRIC, + keys_search_text_add(slots, PM_SEARCH_TYPE_METRIC, metric->names[i].sds, indom, metric->oneline, metric->helptext, baton); } - seriesBatonReference(baton, "redis_series_metadata"); + seriesBatonReference(baton, "keys_series_metadata"); pmwebapi_hash_str(context->name.id, hashbuf, sizeof(hashbuf)); key = sdscatfmt(sdsempty(), "pcp:series:context.name:%s", hashbuf); @@ -888,7 +888,7 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) sdsfree(key); for (i = 0; i < metric->numnames; i++) cmd = redis_param_sha(cmd, metric->names[i].hash); - redisSlotsRequest(slots, cmd, redis_series_source_callback, arg); + keySlotsRequest(slots, cmd, keys_series_source_callback, arg); sdsfree(cmd); check_instances: @@ -896,13 +896,13 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) (baton->flags & PM_SERIES_FLAG_TEXT) && slots->search) { if (indom == NULL) indom = pmwebapi_indom_str(metric, ibuf, sizeof(ibuf)); - redis_search_text_add(slots, PM_SEARCH_TYPE_INDOM, indom, indom, + keys_search_text_add(slots, PM_SEARCH_TYPE_INDOM, indom, indom, metric->indom->oneline, metric->indom->helptext, baton); } if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) { if (metric->cached == 0) { - redis_series_labelset(slots, metric, NULL, baton); + keys_series_labelset(slots, metric, NULL, baton); metric->cached = 1; } } else { @@ -911,13 +911,13 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) if ((instance = dictFetchValue(metric->indom->insts, &value->inst)) == NULL) continue; if (instance->cached == 0 || metric->cached == 0) { - redis_series_instance(slots, metric, instance, baton); - redis_series_labelset(slots, metric, instance, baton); + keys_series_instance(slots, metric, instance, baton); + keys_series_labelset(slots, metric, instance, baton); if ((baton->flags & PM_SERIES_FLAG_TEXT) && slots->search) { if (indom == NULL) indom = pmwebapi_indom_str(metric, ibuf, sizeof(ibuf)); - redis_search_text_add(slots, PM_SEARCH_TYPE_INST, + keys_search_text_add(slots, PM_SEARCH_TYPE_INST, instance->name.sds, indom, NULL, NULL, baton); } } @@ -929,7 +929,7 @@ redis_series_metadata(context_t *context, metric_t *metric, void *arg) typedef struct redisStreamBaton { seriesBatonMagic header; - redisSlots *slots; + keySlots *slots; sds stamp; char hash[40+1]; redisInfoCallBack info; @@ -938,7 +938,7 @@ typedef struct redisStreamBaton { } redisStreamBaton; static void -initRedisStreamBaton(redisStreamBaton *baton, redisSlots *slots, +initKeyStreamBaton(redisStreamBaton *baton, keySlots *slots, sds stamp, const char *hash, seriesLoadBaton *load) { initSeriesBatonMagic(baton, MAGIC_STREAM); @@ -951,17 +951,17 @@ initRedisStreamBaton(redisStreamBaton *baton, redisSlots *slots, } static void -doneRedisStreamBaton(redisStreamBaton *baton) +doneKeyStreamBaton(redisStreamBaton *baton) { void *load = baton->arg; - seriesBatonCheckMagic(baton, MAGIC_STREAM, "doneRedisStreamBaton"); - seriesBatonCheckMagic(load, MAGIC_LOAD, "doneRedisStreamBaton"); + seriesBatonCheckMagic(baton, MAGIC_STREAM, "doneKeyStreamBaton"); + seriesBatonCheckMagic(load, MAGIC_LOAD, "doneKeyStreamBaton"); sdsfree(baton->stamp); memset(baton, 0, sizeof(*baton)); free(baton); - doneSeriesLoadBaton(load, "doneRedisStreamBaton"); + doneSeriesLoadBaton(load, "doneKeyStreamBaton"); } static sds @@ -1022,15 +1022,15 @@ series_stream_value(sds cmd, sds name, int type, pmAtomValue *avp) } static void -redis_series_stream_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_stream_callback( + keyClusterAsyncContext *c, void *r, void *arg) { redisStreamBaton *baton = (redisStreamBaton *)arg; - redisReply *reply = r; + respReply *reply = r; sds msg; - seriesBatonCheckMagic(baton, MAGIC_STREAM, "redis_series_stream_callback"); - if (testReplyError(reply, REDIS_ESTREAMXADD)) { + seriesBatonCheckMagic(baton, MAGIC_STREAM, "keys_series_stream_callback"); + if (testReplyError(reply, RESP_ESTREAMXADD)) { // duplicate streams can happen when pmproxy is restarted and discovery // re-adds a part of the archive; only show warnings in desperate mode if (UNLIKELY(pmDebugOptions.desperate)) { @@ -1045,21 +1045,21 @@ redis_series_stream_callback( baton->hash, baton->stamp); } - doneRedisStreamBaton(baton); + doneKeyStreamBaton(baton); } static void -redis_series_timer_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_series_timer_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesBatonCheckMagic(baton, MAGIC_LOAD, "redis_series_timer_callback"); - doneSeriesLoadBaton(baton, "redis_series_timer_callback"); + seriesBatonCheckMagic(baton, MAGIC_LOAD, "keys_series_timer_callback"); + doneSeriesLoadBaton(baton, "keys_series_timer_callback"); } static void -redis_series_stream(redisSlots *slots, sds stamp, metric_t *metric, +keys_series_stream(keySlots *slots, sds stamp, metric_t *metric, const char *hash, void *arg) { seriesLoadBaton *load = (seriesLoadBaton *)arg; @@ -1073,8 +1073,8 @@ redis_series_stream(redisSlots *slots, sds stamp, metric_t *metric, batoninfo(load, PMLOG_ERROR, stream); return; } - initRedisStreamBaton(baton, slots, stamp, hash, load); - seriesBatonReferences(load, 2, "redis_series_stream"); + initKeyStreamBaton(baton, slots, stamp, hash, load); + seriesBatonReferences(load, 2, "keys_series_stream"); count = 6; /* XADD key MAXLEN ~ len stamp */ key = sdscatfmt(sdsempty(), "pcp:values:series:%s", hash); @@ -1121,7 +1121,7 @@ redis_series_stream(redisSlots *slots, sds stamp, metric_t *metric, cmd = redis_param_raw(cmd, stream); sdsfree(key); sdsfree(stream); - redisSlotsRequest(slots, cmd, redis_series_stream_callback, baton); + keySlotsRequest(slots, cmd, keys_series_stream_callback, baton); sdsfree(cmd); key = sdscatfmt(sdsempty(), "pcp:values:series:%s", hash); @@ -1130,54 +1130,54 @@ redis_series_stream(redisSlots *slots, sds stamp, metric_t *metric, cmd = redis_param_sds(cmd, key); cmd = redis_param_sds(cmd, streamexpire); sdsfree(key); - redisSlotsRequest(slots, cmd, redis_series_timer_callback, load); + keySlotsRequest(slots, cmd, keys_series_timer_callback, load); sdsfree(cmd); } static void -redis_series_streamed(sds stamp, metric_t *metric, void *arg) +keys_series_streamed(sds stamp, metric_t *metric, void *arg) { seriesLoadBaton *baton= (seriesLoadBaton *)arg; - redisSlots *slots = baton->slots; + keySlots *slots = baton->slots; char hashbuf[42]; int i; for (i = 0; i < metric->numnames; i++) { pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf)); - redis_series_stream(slots, stamp, metric, hashbuf, arg); + keys_series_stream(slots, stamp, metric, hashbuf, arg); } } void -redis_series_mark(redisSlots *redis, sds timestamp, int data, void *arg) +keys_series_mark(keySlots *redis, sds timestamp, int data, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; seriesGetContext *context = &baton->pmapi; - /* TODO: cache mark records in Redis series, then in done callback... */ - doneSeriesGetContext(context, "redis_series_mark"); + /* TODO: cache mark records in keys series, then in done callback... */ + doneSeriesGetContext(context, "keys_series_mark"); } static void -redis_update_version_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_update_version_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_update_version_callback"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_update_version_callback"); checkStatusReplyOK(baton->info, baton->userdata, c, reply, "%s setup", "pcp:version:schema"); - redis_slots_end_phase(baton); + keys_slots_end_phase(baton); } static void -redis_update_version(redisSlotsBaton *baton) +keys_update_version(keySlotsBaton *baton) { sds cmd, key; const char ver[] = TO_STRING(SERIES_VERSION); - seriesBatonReference(baton, "redis_update_version"); + seriesBatonReference(baton, "keys_update_version"); key = sdsnew("pcp:version:schema"); cmd = redis_command(3); @@ -1185,24 +1185,24 @@ redis_update_version(redisSlotsBaton *baton) cmd = redis_param_sds(cmd, key); cmd = redis_param_str(cmd, ver, sizeof(ver)-1); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, redis_update_version_callback, baton); + keySlotsRequest(baton->slots, cmd, keys_update_version_callback, baton); sdsfree(cmd); } static void -redis_load_series_version_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_load_series_version_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; unsigned int version = 0; sds msg; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_series_version_callback"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_series_version_callback"); if (!reply) { baton->version = 0; /* NIL - no version key yet */ - } else if (reply->type == REDIS_REPLY_STRING) { + } else if (reply->type == RESP_REPLY_STRING) { version = (unsigned int)atoi(reply->str); if (version == 0 || version == SERIES_VERSION) { baton->version = version; @@ -1211,12 +1211,12 @@ redis_load_series_version_callback( version, SERIES_VERSION); batoninfo(baton, PMLOG_ERROR, msg); } - } else if (reply->type == REDIS_REPLY_ERROR) { + } else if (reply->type == RESP_REPLY_ERROR) { infofmt(msg, "version check error: %s", reply->str); batoninfo(baton, PMLOG_REQUEST, msg); - } else if (reply->type != REDIS_REPLY_NIL) { + } else if (reply->type != RESP_REPLY_NIL) { infofmt(msg, "unexpected schema version reply type (%s)", - redis_reply_type(reply)); + resp_reply_type(reply)); batoninfo(baton, PMLOG_ERROR, msg); } else { baton->version = 0; /* NIL - no version key yet */ @@ -1225,57 +1225,57 @@ redis_load_series_version_callback( /* set the version when none found (first time through) */ if (version != SERIES_VERSION && baton->version != -1) { /* drop reference from schema version request */ - seriesBatonDereference(baton, "redis_load_series_version_callback"); - redis_update_version(arg); + seriesBatonDereference(baton, "keys_load_series_version_callback"); + keys_update_version(arg); } else { - redis_slots_end_phase(baton); + keys_slots_end_phase(baton); } } static void -redis_load_series_version(void *arg) +keys_load_series_version(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; sds cmd, key; - seriesBatonReference(baton, "redis_load_series_version"); + seriesBatonReference(baton, "keys_load_series_version"); key = sdsnew("pcp:version:schema"); cmd = redis_command(2); cmd = redis_param_str(cmd, GETS, GETS_LEN); cmd = redis_param_sds(cmd, key); sdsfree(key); - redisSlotsRequest(baton->slots, cmd, redis_load_series_version_callback, baton); + keySlotsRequest(baton->slots, cmd, keys_load_series_version_callback, baton); sdsfree(cmd); } static void -redis_load_version_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_load_version_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; unsigned int server_version = 0; size_t l; char *endnum; sds msg; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_version_callback"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_version_callback"); if (!reply) { - /* This situation should not happen, since we can always get server info from redis */ - infofmt(msg, "no redis version reply"); + /* This situation should not happen, we can always get server info */ + infofmt(msg, "no key server version reply"); batoninfo(baton, PMLOG_ERROR, msg); - } else if (reply->type == REDIS_REPLY_STRING) { + } else if (reply->type == RESP_REPLY_STRING) { for (l = 0; l < reply->len; l++) { if (strncmp("redis_version:", reply->str+l, sizeof("redis_version:")-1) == 0) { - l += sizeof("redis_version:")-1; + l += sizeof("redis_version:")-1; // compat server_version = (unsigned int)strtoul(reply->str+l, &endnum, 10); if (*endnum != '.') { - infofmt(msg, "redis server version parse error"); + infofmt(msg, "key server version parse error"); batoninfo(baton, PMLOG_ERROR, msg); - } else if (server_version < REDIS_VERSION) { - infofmt(msg, "unsupported redis server (got v%u, expected v%u or above)", - server_version, REDIS_VERSION); + } else if (server_version < OLDEST_VERSION) { + infofmt(msg, "unsupported key server (got v%u, expected v%u or above)", + server_version, OLDEST_VERSION); batoninfo(baton, PMLOG_ERROR, msg); baton->slots->state = SLOTS_ERR_FATAL; /* set error flag and do not continue with other callbacks of this baton */ @@ -1289,37 +1289,37 @@ redis_load_version_callback( break; } } - } else if (reply->type == REDIS_REPLY_ERROR) { - infofmt(msg, "redis server version check error: %s", reply->str); + } else if (reply->type == RESP_REPLY_ERROR) { + infofmt(msg, "key server version check error: %s", reply->str); batoninfo(baton, PMLOG_REQUEST, msg); } else { - infofmt(msg, "unexpected redis server version reply type (%s)", redis_reply_type(reply)); + infofmt(msg, "unexpected key server version reply type (%s)", resp_reply_type(reply)); batoninfo(baton, PMLOG_ERROR, msg); } - redis_slots_end_phase(baton); + keys_slots_end_phase(baton); } static void -redis_load_version(void *arg) +keys_load_version(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; sds cmd; - seriesBatonReference(baton, "redis_load_version"); + seriesBatonReference(baton, "keys_load_version"); cmd = redis_command(2); cmd = redis_param_str(cmd, INFO, INFO_LEN); cmd = redis_param_str(cmd, "SERVER", sizeof("SERVER")-1); // TODO: check all nodes in cluster - redisSlotsRequestFirstNode(baton->slots, cmd, redis_load_version_callback, baton); + keySlotsRequestFirstNode(baton->slots, cmd, keys_load_version_callback, baton); sdsfree(cmd); } static int -decodeCommandKey(redisSlotsBaton *baton, int index, redisReply *reply) +decodeCommandKey(keySlotsBaton *baton, int index, respReply *reply) { - redisSlots *slots = baton->slots; - redisReply *node; + keySlots *slots = baton->slots; + respReply *node; dictEntry *entry; long long position; sds msg, cmd; @@ -1335,7 +1335,7 @@ decodeCommandKey(redisSlotsBaton *baton, int index, redisReply *reply) * * We care primarily about the command name and position of * the first key, as that key is the one used when selecting - * the Redis server to communicate with for each command, in + * the key server to communicate with for each command, in * a setup with more than one server (cluster or otherwise). */ if (reply->elements < 6) { @@ -1364,94 +1364,94 @@ decodeCommandKey(redisSlotsBaton *baton, int index, redisReply *reply) } static void -redis_load_keymap_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_load_keymap_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; - redisReply *command; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; + respReply *command; sds msg; int i; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_keymap_callback"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_keymap_callback"); if (!reply) { - infofmt(msg, "received NULL reply in redis_load_keymap_callback"); + infofmt(msg, "received NULL reply in keys_load_keymap_callback"); batoninfo(baton, PMLOG_ERROR, msg); } - else if (reply->type == REDIS_REPLY_ARRAY) { + else if (reply->type == RESP_REPLY_ARRAY) { for (i = 0; i < reply->elements; i++) { command = reply->element[i]; if (checkArrayReply(baton->info, baton->userdata, c, command, "%s entry %d", COMMAND, i) == 0) decodeCommandKey(baton, i, command); } - } else if (reply->type == REDIS_REPLY_ERROR) { + } else if (reply->type == RESP_REPLY_ERROR) { infofmt(msg, "command key mapping error: %s", reply->str); batoninfo(baton, PMLOG_REQUEST, msg); - } else if (reply->type != REDIS_REPLY_NIL) { + } else if (reply->type != RESP_REPLY_NIL) { infofmt(msg, "unexpected command reply type (%s)", - redis_reply_type(reply)); + resp_reply_type(reply)); batoninfo(baton, PMLOG_ERROR, msg); } - redis_slots_end_phase(baton); + keys_slots_end_phase(baton); } static void -redis_load_keymap(void *arg) +keys_load_keymap(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; sds cmd; - seriesBatonReference(baton, "redis_load_keymap"); + seriesBatonReference(baton, "keys_load_keymap"); cmd = redis_command(1); cmd = redis_param_str(cmd, COMMAND, COMMAND_LEN); - redisSlotsRequestFirstNode(baton->slots, cmd, redis_load_keymap_callback, baton); + keySlotsRequestFirstNode(baton->slots, cmd, keys_load_keymap_callback, baton); sdsfree(cmd); } void -redisSchemaLoad(redisSlots *slots, redisSlotsFlags flags, - redisInfoCallBack info, redisDoneCallBack done, +keysSchemaLoad(keySlots *slots, keySlotsFlags flags, + keysInfoCallBack info, keysDoneCallBack done, void *userdata, void *events, void *arg) { - redisSlotsBaton *baton; + keySlotsBaton *baton; sds msg; unsigned int i = 0; - baton = (redisSlotsBaton *)calloc(1, sizeof(redisSlotsBaton)); + baton = (keySlotsBaton *)calloc(1, sizeof(keySlotsBaton)); if (baton == NULL) { - infofmt(msg, "Failed to allocate memory for Redis slots baton"); + infofmt(msg, "Failed to allocate memory for key slots baton"); info(PMLOG_ERROR, msg, arg); sdsfree(msg); return; } - initRedisSlotsBaton(baton, info, done, userdata, events, arg); + initKeySlotsBaton(baton, info, done, userdata, events, arg); baton->slots = slots; baton->current = &baton->phases[0]; /* Prepare mapping of commands to key positions if needed */ if (flags & SLOTS_KEYMAP) - baton->phases[i++].func = redis_load_keymap; + baton->phases[i++].func = keys_load_keymap; /* Verify pmseries schema version and create it if needed */ if (flags & SLOTS_VERSION) { - baton->phases[i++].func = redis_load_version; /* Redis v5 */ - baton->phases[i++].func = redis_load_series_version; + baton->phases[i++].func = keys_load_version; /* v5 */ + baton->phases[i++].func = keys_load_series_version; } /* Register the pmsearch schema with RediSearch if needed */ if (flags & SLOTS_SEARCH) { /* if we got a route update means we are in cluster mode */ if (slots->acc->cc->route_version > 0) { - pmNotifyErr(LOG_INFO, "disabling RediSearch because it does not " - "support Redis cluster mode\n"); + pmNotifyErr(LOG_INFO, "disabling search module " + "because it does not support cluster mode\n"); } else { - baton->phases[i++].func = redis_load_search_schema; + baton->phases[i++].func = keys_load_search_schema; } } - baton->phases[i++].func = redis_slots_finished; + baton->phases[i++].func = keys_slots_finished; assert(i <= SLOTS_PHASES); seriesBatonPhases(baton->current, i, baton); } @@ -1470,7 +1470,7 @@ pmSeriesSetSlots(pmSeriesModule *module, void *slots) seriesModuleData *data = getSeriesModuleData(module); if (data) { - data->slots = (redisSlots *)slots; + data->slots = (keySlots *)slots; data->shareslots = 1; return 0; } @@ -1522,7 +1522,7 @@ pmSeriesSetMetricRegistry(pmSeriesModule *module, mmv_registry_t *registry) } static void -redisSeriesInit(struct dict *config) +keysSeriesInit(struct dict *config) { sds option; @@ -1549,7 +1549,7 @@ redisSeriesInit(struct dict *config) } static void -redisSeriesClose(void) +keysSeriesClose(void) { if (DEFAULT_CURSORCOUNT) { sdsfree(DEFAULT_CURSORCOUNT); @@ -1566,19 +1566,19 @@ redisSeriesClose(void) } void -redisGlobalsInit(struct dict *config) +keysGlobalsInit(struct dict *config) { - redisSeriesInit(config); - redisSearchInit(config); - redisMapsInit(); + keysSeriesInit(config); + keysSearchInit(config); + keyMapsInit(); } void -redisGlobalsClose(void) +keysGlobalsClose(void) { - redisSeriesClose(); - redisSearchClose(); - redisMapsClose(); + keysSeriesClose(); + keysSearchClose(); + keyMapsClose(); } static void @@ -1668,32 +1668,32 @@ pmSeriesSetup(pmSeriesModule *module, void *arg) { seriesModuleData *data = getSeriesModuleData(module); sds option; - redisSlotsFlags flags; + keySlotsFlags flags; if (data == NULL) return -ENOMEM; /* create string map caches */ - redisGlobalsInit(data->config); + keysGlobalsInit(data->config); - /* fast path for when Redis has been setup already */ + /* fast path for when key server has been setup already */ if (data->slots) { module->on_setup(arg); data->shareslots = 1; } else { if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); + option = pmIniFileLookup(data->config, "redis", "enabled"); // compat if (option && strcmp(option, "false") == 0) return -ENOTSUP; - /* establish an initial connection to Redis instance(s) */ + /* establish an initial connection to key server instance(s) */ flags = SLOTS_VERSION; option = pmIniFileLookup(data->config, "pmsearch", "enabled"); if (option && strcmp(option, "true") == 0) flags |= SLOTS_SEARCH; - data->slots = redisSlotsConnect( + data->slots = keySlotsConnect( data->config, flags, module->on_info, module->on_setup, arg, data->events, arg); data->shareslots = 0; @@ -1711,13 +1711,13 @@ pmSeriesClose(pmSeriesModule *module) if (data) { if (data->slots && !data->shareslots) - redisSlotsFree(data->slots); + keySlotsFree(data->slots); memset(data, 0, sizeof(seriesModuleData)); free(data); module->privdata = NULL; } - redisGlobalsClose(); + keysGlobalsClose(); } discoverModuleData * @@ -1734,7 +1734,7 @@ pmDiscoverSetSlots(pmDiscoverModule *module, void *slots) discoverModuleData *data = getDiscoverModuleData(module); if (data) { - data->slots = (redisSlots *)slots; + data->slots = (keySlots *)slots; data->shareslots = 1; return 0; } @@ -2008,7 +2008,7 @@ pmDiscoverSetup(pmDiscoverModule *module, pmDiscoverCallBacks *cbs, void *arg) } /* create global string map caches */ - redisGlobalsInit(data->config); + keysGlobalsInit(data->config); if (!logdir) logdir = fallback; @@ -2034,7 +2034,7 @@ pmDiscoverClose(pmDiscoverModule *module) if (discover) { pmDiscoverUnregister(discover->handle); if (discover->slots && !discover->shareslots) - redisSlotsFree(discover->slots); + keySlotsFree(discover->slots); for (i = 0; i < discover->exclude_names; i++) sdsfree(discover->patterns[i]); if (discover->patterns) @@ -2047,5 +2047,5 @@ pmDiscoverClose(pmDiscoverModule *module) free(discover); } - redisGlobalsClose(); + keysGlobalsClose(); } diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 6012224ed2..15b81bdcb8 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Red Hat. + * Copyright (c) 2020-2022,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -24,7 +24,7 @@ static sds DEFAULT_RESULTCOUNT; static unsigned int resultcount; /* converted resultcount_str */ static void -initRedisSearchBaton(redisSearchBaton *baton, redisSlots *slots, +initKeysSearchBaton(keysSearchBaton *baton, keySlots *slots, pmSearchSettings *settings, void *userdata) { initSeriesBatonMagic(baton, MAGIC_SEARCH); @@ -37,10 +37,10 @@ initRedisSearchBaton(redisSearchBaton *baton, redisSlots *slots, } static void -doneRedisSearchBaton(redisSearchBaton *baton) +doneKeysSearchBaton(keysSearchBaton *baton) { - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "doneRedisSearchBaton"); - memset(baton, 0, sizeof(redisSearchBaton)); + seriesBatonCheckMagic(baton, MAGIC_SEARCH, "doneKeysSearchBaton"); + memset(baton, 0, sizeof(keysSearchBaton)); free(baton); } @@ -61,7 +61,7 @@ pmSearchTextTypeStr(pmSearchTextType type) } static sds -redis_search_docid(const char *key, const char *type, const char *name) +keys_search_docid(const char *key, const char *type, const char *name) { unsigned char hash[20]; sds docid = sdsempty(); @@ -74,11 +74,11 @@ redis_search_docid(const char *key, const char *type, const char *name) /* - * This issue isn't fixed in OSS version of RediSearch we are using, + * This issue isn't fixed in version of search module we are using, * https://github.com/RediSearch/RediSearch/issues/748 */ static int -redis_search_is_stopword(sds s) +keys_search_is_stopword(sds s) { size_t i; static const char* const stopwords[] = { @@ -95,7 +95,7 @@ redis_search_is_stopword(sds s) } /* - * Tokenizes text by delimiters described here https://oss.redislabs.com/redisearch/Escaping.html + + * Tokenizes text by delimiters described in search module Escaping.html doc + * ('/' - this one giving troubles to prefix search), * omits tokens of length greated then *min_length* (0 = don't omit any), optionally prepends *prefix* * and appends *suffix* (NULL doesnt get prepended / appended), joins tokens by space back into new sds. @@ -103,7 +103,7 @@ redis_search_is_stopword(sds s) * unfortunately, this doesnt seem to work when searching for indoms and buch of other inconsitencies */ static sds -redis_search_text_prep(sds s, int min_length, char *prefix, char *suffix) +keys_search_text_prep(sds s, int min_length, char *prefix, char *suffix) { static const char *delimiters = ",.<>{}[]\"\':;!@#$%^&*()-+=~/"; size_t len = sdslen(s); @@ -133,7 +133,7 @@ redis_search_text_prep(sds s, int min_length, char *prefix, char *suffix) size_t token_len = sdslen(tokens[i]); if (min_length != 0 && token_len < min_length) continue; - if (redis_search_is_stopword(tokens[i])) + if (keys_search_is_stopword(tokens[i])) continue; if (prefix == NULL && suffix == NULL) formatted_result = sdscatsds(formatted_result, tokens[i]); @@ -172,20 +172,20 @@ redis_search_text_prep(sds s, int min_length, char *prefix, char *suffix) } static void -redis_search_text_add_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_search_text_add_callback( + keyClusterAsyncContext *c, void *r, void *arg) { seriesLoadBaton *baton = (seriesLoadBaton *)arg; seriesGetContext *context = &baton->pmapi; - redisReply *reply = r; + respReply *reply = r; checkStatusReplyOK(baton->info, baton->userdata, c, reply, "%s: %s", FT_ADD, "search text add"); - doneSeriesGetContext(context, "redis_search_text_add_callback"); + doneSeriesGetContext(context, "keys_search_text_add_callback"); } void -redis_search_text_add(redisSlots *slots, pmSearchTextType type, +keys_search_text_add(keySlots *slots, pmSearchTextType type, const char *name, const char *indom, const char *oneline, const char *helptext, void *arg) { @@ -196,12 +196,12 @@ redis_search_text_add(redisSlots *slots, pmSearchTextType type, char buffer[8]; sds cmd, key, docid; - seriesBatonCheckMagic(baton, MAGIC_LOAD, "redis_search_text_add"); + seriesBatonCheckMagic(baton, MAGIC_LOAD, "keys_search_text_add"); if (pmDebugOptions.search) - fprintf(stderr, "%s: %s %s\n", "redis_search_text_add", typestr, name); + fprintf(stderr, "%s: %s %s\n", "keys_search_text_add", typestr, name); - seriesBatonReference(context, "redis_search_text_add"); + seriesBatonReference(context, "keys_search_text_add"); /* * FT.ADD pcp:text 1.0 @@ -218,42 +218,42 @@ redis_search_text_add(redisSlots *slots, pmSearchTextType type, length += 2; if (helptext && *helptext != '\0') length += 2; - cmd = redis_command(length); + cmd = key_command(length); - cmd = redis_param_str(cmd, FT_ADD, FT_ADD_LEN); - cmd = redis_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - docid = redis_search_docid(FT_TEXT_KEY, typestr, name); - cmd = redis_param_sds(cmd, docid); + cmd = key_param_str(cmd, FT_ADD, FT_ADD_LEN); + cmd = key_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); + docid = keys_search_docid(FT_TEXT_KEY, typestr, name); + cmd = key_param_sds(cmd, docid); sdsfree(docid); - cmd = redis_param_str(cmd, "1", 1); + cmd = key_param_str(cmd, "1", 1); - cmd = redis_param_str(cmd, FT_REPLACE, FT_REPLACE_LEN); - cmd = redis_param_str(cmd, FT_PARTIAL, FT_PARTIAL_LEN); + cmd = key_param_str(cmd, FT_REPLACE, FT_REPLACE_LEN); + cmd = key_param_str(cmd, FT_PARTIAL, FT_PARTIAL_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", type); - cmd = redis_param_str(cmd, FT_PAYLOAD, FT_PAYLOAD_LEN); - cmd = redis_param_str(cmd, buffer, length); - - cmd = redis_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = redis_param_str(cmd, name, strlen(name)); - cmd = redis_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = redis_param_str(cmd, typestr, strlen(typestr)); + cmd = key_param_str(cmd, FT_PAYLOAD, FT_PAYLOAD_LEN); + cmd = key_param_str(cmd, buffer, length); + + cmd = key_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, name, strlen(name)); + cmd = key_param_str(cmd, FT_TYPE, FT_TYPE_LEN); + cmd = key_param_str(cmd, typestr, strlen(typestr)); if (indom && *indom != '\0') { - cmd = redis_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = redis_param_str(cmd, indom, strlen(indom)); + cmd = key_param_str(cmd, FT_INDOM, FT_INDOM_LEN); + cmd = key_param_str(cmd, indom, strlen(indom)); } if (oneline && *oneline != '\0') { - cmd = redis_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = redis_param_str(cmd, oneline, strlen(oneline)); + cmd = key_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); + cmd = key_param_str(cmd, oneline, strlen(oneline)); } if (helptext && *helptext != '\0') { - cmd = redis_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = redis_param_str(cmd, helptext, strlen(helptext)); + cmd = key_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + cmd = key_param_str(cmd, helptext, strlen(helptext)); } sdsfree(key); - redisSlotsRequestFirstNode(slots, cmd, redis_search_text_add_callback, arg); + keySlotsRequestFirstNode(slots, cmd, keys_search_text_add_callback, arg); sdsfree(cmd); } @@ -288,7 +288,7 @@ pmSearchDiscoverMetric(pmDiscoverEvent *event, pmInDomStr_r(desc->indom, buffer, sizeof(buffer)); for (i = 0; i < numnames; i++) - redis_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, + keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, names[i], buffer, oneline, helptext, baton); if (oneline) @@ -324,12 +324,12 @@ pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_ONELINE, &oneline); pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_HELP | PM_TEXT_DIRECT, &helptext); - redis_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, + keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, buffer, buffer, oneline, helptext, baton); for (i = 0; i < in->numinst; i++) { if (in->namelist[i] == NULL) continue; - redis_search_text_add(baton->slots, PM_SEARCH_TYPE_INST, + keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INST, in->namelist[i], buffer, NULL, NULL, baton); } if (oneline) @@ -362,33 +362,33 @@ pmSearchDiscoverText(pmDiscoverEvent *event, if ((count = pmNameAll(ident, &metrics)) <= 0) return; for (i = 0; i < count; i++) - redis_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, + keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, metrics[i], NULL, oneline, helptext, baton); free(metrics); } else { /* PM_TEXT_INDOM */ pmInDomStr_r(ident, indom, sizeof(indom)); - redis_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, + keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, indom, indom, oneline, helptext, baton); } } static void -redis_search_info_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_search_info_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; - redisReply *reply = r; - redisReply *child, *value; + keysSearchBaton *baton = (keysSearchBaton *)arg; + respReply *reply = r; + respReply *child, *value; pmSearchMetrics metrics = {0}; int i; sds msg; - if (reply && reply->type == REDIS_REPLY_ARRAY && reply->elements >= 30) { + if (reply && reply->type == RESP_REPLY_ARRAY && reply->elements >= 30) { for (i = 0; i < reply->elements-1; i++) { value = reply->element[i+1]; child = reply->element[i]; - if (child->type != REDIS_REPLY_STRING && - value->type != REDIS_REPLY_STRING) + if (child->type != RESP_REPLY_STRING && + value->type != RESP_REPLY_STRING) continue; else if (strcmp("num_docs", child->str) == 0) metrics.docs = strtoull(value->str, NULL, 0); @@ -420,38 +420,38 @@ redis_search_info_callback( baton->callbacks->on_metrics(&metrics, baton->userdata); } else { infofmt(msg, "expected array from %s (reply=%s)", - FT_INFO, redis_reply_type(reply)); + FT_INFO, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } baton->callbacks->on_done(baton->error, baton->userdata); - doneRedisSearchBaton(baton); + doneKeysSearchBaton(baton); } void -redis_search_info(redisSlots *slots, sds pcpkey, void *arg) +keys_search_info(keySlots *slots, sds pcpkey, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; + keysSearchBaton *baton = (keysSearchBaton *)arg; sds cmd, key; - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "redis_search_info"); - seriesBatonCheckCount(baton, "redis_search_info"); + seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_info"); + seriesBatonCheckCount(baton, "keys_search_info"); if (pmDebugOptions.search) - fprintf(stderr, "%s: search key metrics\n", "redis_search_info"); + fprintf(stderr, "%s: search key metrics\n", "keys_search_info"); - seriesBatonReference(baton, "redis_search_info"); + seriesBatonReference(baton, "keys_search_info"); /* * FT.INFO pcp: */ key = sdscatfmt(sdsempty(), "pcp:%S", pcpkey); - cmd = redis_command(2); - cmd = redis_param_str(cmd, FT_INFO, FT_INFO_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(2); + cmd = key_param_str(cmd, FT_INFO, FT_INFO_LEN); + cmd = key_param_sds(cmd, key); sdsfree(key); - redisSlotsRequestFirstNode(slots, cmd, redis_search_info_callback, arg); + keySlotsRequestFirstNode(slots, cmd, keys_search_info_callback, arg); sdsfree(cmd); } @@ -459,23 +459,23 @@ int pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) { seriesModuleData *data = getSeriesModuleData(&settings->module); - redisSearchBaton *baton; + keysSearchBaton *baton; if (data == NULL) return -ENOMEM; - if ((baton = calloc(1, sizeof(redisSearchBaton))) == NULL) + if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) return -ENOMEM; - initRedisSearchBaton(baton, data->slots, settings, arg); - redis_search_info(data->slots, key, baton); + initKeysSearchBaton(baton, data->slots, settings, arg); + keys_search_info(data->slots, key, baton); return 0; } static void -extract_search_results(redisSearchBaton *baton, - unsigned int total, double timer, redisReply *reply) +extract_search_results(keysSearchBaton *baton, + unsigned int total, double timer, respReply *reply) { pmSearchTextResult result; - redisReply *docid, *score, *payload, *array; + respReply *docid, *score, *payload, *array; int i, j; for (i = 1; i < reply->elements - 3; i += 4) { @@ -483,10 +483,10 @@ extract_search_results(redisSearchBaton *baton, score = reply->element[i+1]; payload = reply->element[i+2]; array = reply->element[i+3]; - if (payload->type != REDIS_REPLY_STRING || - score->type != REDIS_REPLY_STRING || - docid->type != REDIS_REPLY_STRING || - array->type != REDIS_REPLY_ARRAY) { + if (payload->type != RESP_REPLY_STRING || + score->type != RESP_REPLY_STRING || + docid->type != RESP_REPLY_STRING || + array->type != RESP_REPLY_ARRAY) { baton->error = -EPROTO; break; } @@ -499,12 +499,12 @@ extract_search_results(redisSearchBaton *baton, result.score = strtod(score->str, NULL); for (j = 0; j < array->elements; j += 2) { - redisReply *field = array->element[j]; - redisReply *value = array->element[j+1]; + respReply *field = array->element[j]; + respReply *value = array->element[j+1]; - if (field->type != REDIS_REPLY_STRING || - (value->type != REDIS_REPLY_STRING && - value->type != REDIS_REPLY_NIL)) { + if (field->type != RESP_REPLY_STRING || + (value->type != RESP_REPLY_STRING && + value->type != RESP_REPLY_NIL)) { baton->error = -EPROTO; break; } @@ -532,24 +532,24 @@ extract_search_results(redisSearchBaton *baton, } static void -redis_search_text_query_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_search_text_query_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; - redisReply *reply = r; - redisReply *value; + keysSearchBaton *baton = (keysSearchBaton *)arg; + respReply *reply = r; + respReply *value; struct timespec finished; unsigned int total; double timer; sds msg; - if (LIKELY(reply && reply->type == REDIS_REPLY_ARRAY)) { + if (LIKELY(reply && reply->type == RESP_REPLY_ARRAY)) { value = reply->element[0]; if (reply->elements == 1) /* no search results - done! */ /* do nothing */; else if (reply->elements < 2) /* expect total then results */ baton->error = -EPROTO; - else if (value->type != REDIS_REPLY_INTEGER) + else if (value->type != RESP_REPLY_INTEGER) baton->error = -EPROTO; else { pmtimespecNow(&finished); @@ -559,32 +559,32 @@ redis_search_text_query_callback( } } else { infofmt(msg, "expected array from %s (reply=%s)", - FT_SEARCH, redis_reply_type(reply)); + FT_SEARCH, resp_reply_type(reply)); batoninfo(baton, PMLOG_RESPONSE, msg); baton->error = -EPROTO; } baton->callbacks->on_done(baton->error, baton->userdata); - doneRedisSearchBaton(baton); + doneKeysSearchBaton(baton); } static void -redis_search_text_query(redisSlots *slots, pmSearchTextRequest *request, void *arg) +keys_search_text_query(keySlots *slots, pmSearchTextRequest *request, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; + keysSearchBaton *baton = (keysSearchBaton *)arg; const char *typestr; size_t length; char buffer[64]; sds cmd, key, query, base_query; unsigned int types = 0, infields = 0, returns = 0, highlights = 0; - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "redis_search_text_query"); - seriesBatonCheckCount(baton, "redis_search_text_query"); + seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_query"); + seriesBatonCheckCount(baton, "keys_search_text_query"); if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "redis_search_text_query", request->query); + fprintf(stderr, "%s: %s\n", "keys_search_text_query", request->query); - seriesBatonReference(baton, "redis_search_text_query"); + seriesBatonReference(baton, "keys_search_text_query"); types += request->type_metric; types += request->type_indom; @@ -636,9 +636,9 @@ redis_search_text_query(redisSlots *slots, pmSearchTextRequest *request, void *a length += 3 + highlights; length += 2 + 3; - cmd = redis_command(length); - cmd = redis_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = redis_param_sds(cmd, key); + cmd = key_command(length); + cmd = key_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); + cmd = key_param_sds(cmd, key); query = sdscatlen(sdsempty(), "\'", 1); if (types) { @@ -661,72 +661,72 @@ redis_search_text_query(redisSlots *slots, pmSearchTextRequest *request, void *a } query = sdscatlen(query, "} ", 2); } - base_query = redis_search_text_prep(request->query, 0, NULL, NULL); + base_query = keys_search_text_prep(request->query, 0, NULL, NULL); query = sdscatfmt(query, "(%S)=>{$inorder:true}\'", base_query); sdsfree(base_query); - cmd = redis_param_sds(cmd, query); + cmd = key_param_sds(cmd, query); sdsfree(query); - cmd = redis_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = redis_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); + cmd = key_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); + cmd = key_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); if (infields) { - cmd = redis_param_str(cmd, FT_INFIELDS, FT_INFIELDS_LEN); + cmd = key_param_str(cmd, FT_INFIELDS, FT_INFIELDS_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", infields); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); if (request->infields_name) - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); if (request->infields_oneline) - cmd = redis_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); + cmd = key_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); if (request->infields_helptext) - cmd = redis_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + cmd = key_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); } if (returns) { - cmd = redis_param_str(cmd, FT_RETURN, FT_RETURN_LEN); + cmd = key_param_str(cmd, FT_RETURN, FT_RETURN_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", returns); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); if (request->return_name) - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); if (request->return_indom) - cmd = redis_param_str(cmd, FT_INDOM, FT_INDOM_LEN); + cmd = key_param_str(cmd, FT_INDOM, FT_INDOM_LEN); if (request->return_oneline) - cmd = redis_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); + cmd = key_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); if (request->return_helptext) - cmd = redis_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + cmd = key_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); if (request->return_type) - cmd = redis_param_str(cmd, FT_TYPE, FT_TYPE_LEN); + cmd = key_param_str(cmd, FT_TYPE, FT_TYPE_LEN); } if (highlights) { - cmd = redis_param_str(cmd, FT_HIGHLIGHT, FT_HIGHLIGHT_LEN); - cmd = redis_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); + cmd = key_param_str(cmd, FT_HIGHLIGHT, FT_HIGHLIGHT_LEN); + cmd = key_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", highlights); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); if (request->highlight_name) - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); if (request->highlight_oneline) - cmd = redis_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); + cmd = key_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); if (request->highlight_helptext) - cmd = redis_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + cmd = key_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); } - cmd = redis_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = redis_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); + cmd = key_param_str(cmd, FT_SCORER, FT_SCORER_LEN); + cmd = key_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); - cmd = redis_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); + cmd = key_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); if (request->count == 0) { - cmd = redis_param_sds(cmd, resultcount_str); + cmd = key_param_sds(cmd, resultcount_str); request->count = resultcount; } else { length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); } sdsfree(key); - redisSlotsRequestFirstNode(slots, cmd, redis_search_text_query_callback, arg); + keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); sdsfree(cmd); } @@ -734,38 +734,38 @@ int pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { seriesModuleData *data = getSeriesModuleData(&settings->module); - redisSearchBaton *baton; + keysSearchBaton *baton; if (data == NULL) return -ENOMEM; - if ((baton = calloc(1, sizeof(redisSearchBaton))) == NULL) + if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) return -ENOMEM; - initRedisSearchBaton(baton, data->slots, settings, arg); - redis_search_text_query(data->slots, request, baton); + initKeysSearchBaton(baton, data->slots, settings, arg); + keys_search_text_query(data->slots, request, baton); return 0; } static void -redis_search_text_suggest(redisSlots *slots, pmSearchTextRequest *request, void *arg) +keys_search_text_suggest(keySlots *slots, pmSearchTextRequest *request, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; + keysSearchBaton *baton = (keysSearchBaton *)arg; size_t length, prefix_length, fuzzy_length; const char *prefix; char buffer[64]; sds cmd, key, query, prefix_query, fuzzy_query; - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "redis_search_text_suggest"); - seriesBatonCheckCount(baton, "redis_search_text_suggest"); + seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_suggest"); + seriesBatonCheckCount(baton, "keys_search_text_suggest"); if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "redis_search_text_suggest", request->query); + fprintf(stderr, "%s: %s\n", "keys_search_text_suggest", request->query); - seriesBatonReference(baton, "redis_search_text_suggest"); + seriesBatonReference(baton, "keys_search_text_suggest"); /* by default we cannot use prefix search with words of length less than 2 */ - prefix_query = redis_search_text_prep(request->query, 2, NULL, "*"); - fuzzy_query = redis_search_text_prep(request->query, 2, "%", "%"); + prefix_query = keys_search_text_prep(request->query, 2, NULL, "*"); + fuzzy_query = keys_search_text_prep(request->query, 2, "%", "%"); prefix_length = sdslen(prefix_query); fuzzy_length = sdslen(fuzzy_query); @@ -808,31 +808,31 @@ redis_search_text_suggest(redisSlots *slots, pmSearchTextRequest *request, void key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); length = 13; // Resp array size - cmd = redis_command(length); - cmd = redis_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_sds(cmd, query); + cmd = key_command(length); + cmd = key_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_sds(cmd, query); sdsfree(query); - cmd = redis_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = redis_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - cmd = redis_param_str(cmd, FT_RETURN, FT_RETURN_LEN); - cmd = redis_param_str(cmd, "1", 1); - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = redis_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = redis_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); - cmd = redis_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - cmd = redis_param_str(cmd, "0", 1); + cmd = key_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); + cmd = key_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); + cmd = key_param_str(cmd, FT_RETURN, FT_RETURN_LEN); + cmd = key_param_str(cmd, "1", 1); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, FT_SCORER, FT_SCORER_LEN); + cmd = key_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); + cmd = key_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); + cmd = key_param_str(cmd, "0", 1); if (request->count == 0) { - cmd = redis_param_sds(cmd, resultcount_str); + cmd = key_param_sds(cmd, resultcount_str); request->count = resultcount; } else { length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); } sdsfree(key); - redisSlotsRequestFirstNode(slots, cmd, redis_search_text_query_callback, arg); + keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); sdsfree(cmd); } @@ -840,32 +840,32 @@ int pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { seriesModuleData *data = getSeriesModuleData(&settings->module); - redisSearchBaton *baton; + keysSearchBaton *baton; if (data == NULL) return -ENOMEM; - if ((baton = calloc(1, sizeof(redisSearchBaton))) == NULL) + if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) return -ENOMEM; - initRedisSearchBaton(baton, data->slots, settings, arg); - redis_search_text_suggest(data->slots, request, baton); + initKeysSearchBaton(baton, data->slots, settings, arg); + keys_search_text_suggest(data->slots, request, baton); return 0; } static void -redis_search_text_indom(redisSlots *slots, pmSearchTextRequest *request, void *arg) +keys_search_text_indom(keySlots *slots, pmSearchTextRequest *request, void *arg) { - redisSearchBaton *baton = (redisSearchBaton *)arg; + keysSearchBaton *baton = (keysSearchBaton *)arg; size_t length; char buffer[64]; sds cmd, key, query; - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "redis_search_text_indom"); - seriesBatonCheckCount(baton, "redis_search_text_indom"); + seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_indom"); + seriesBatonCheckCount(baton, "keys_search_text_indom"); if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "redis_search_text_indom", request->query); + fprintf(stderr, "%s: %s\n", "keys_search_text_indom", request->query); - seriesBatonReference(baton, "redis_search_text_indom"); + seriesBatonReference(baton, "keys_search_text_indom"); query = sdscatfmt( sdsnewlen("", 0), @@ -882,33 +882,33 @@ redis_search_text_indom(redisSlots *slots, pmSearchTextRequest *request, void *a key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); length = 12; // Resp array size - cmd = redis_command(length); - cmd = redis_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = redis_param_sds(cmd, key); - cmd = redis_param_sds(cmd, query); + cmd = key_command(length); + cmd = key_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); + cmd = key_param_sds(cmd, key); + cmd = key_param_sds(cmd, query); sdsfree(query); - cmd = redis_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = redis_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); + cmd = key_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); + cmd = key_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - cmd = redis_param_str(cmd, FT_SORTBY, FT_SORTBY_LEN); - cmd = redis_param_str(cmd, "2", 1); - cmd = redis_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = redis_param_str(cmd, FT_ASC, FT_ASC_LEN); + cmd = key_param_str(cmd, FT_SORTBY, FT_SORTBY_LEN); + cmd = key_param_str(cmd, "2", 1); + cmd = key_param_str(cmd, FT_TYPE, FT_TYPE_LEN); + cmd = key_param_str(cmd, FT_ASC, FT_ASC_LEN); - cmd = redis_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); + cmd = key_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); if (request->count == 0) { - cmd = redis_param_sds(cmd, resultcount_str); + cmd = key_param_sds(cmd, resultcount_str); request->count = resultcount; } else { length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = redis_param_str(cmd, buffer, length); + cmd = key_param_str(cmd, buffer, length); } sdsfree(key); - redisSlotsRequestFirstNode(slots, cmd, redis_search_text_query_callback, arg); + keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); sdsfree(cmd); } @@ -916,54 +916,54 @@ int pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { seriesModuleData *data = getSeriesModuleData(&settings->module); - redisSearchBaton *baton; + keysSearchBaton *baton; if (data == NULL) return -ENOMEM; - if ((baton = calloc(1, sizeof(redisSearchBaton))) == NULL) + if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) return -ENOMEM; - initRedisSearchBaton(baton, data->slots, settings, arg); - redis_search_text_indom(data->slots, request, baton); + initKeysSearchBaton(baton, data->slots, settings, arg); + keys_search_text_indom(data->slots, request, baton); return 0; } static void -redis_search_schema_callback( - redisClusterAsyncContext *c, void *r, void *arg) +keys_search_schema_callback( + keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; - redisReply *reply = r; + keySlotsBaton *baton = (keySlotsBaton *)arg; + respReply *reply = r; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_search_schema_callback"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_search_schema_callback"); - if (testReplyError(reply, REDIS_EDROPINDEX)) { + if (testReplyError(reply, RESP_EDROPINDEX)) { // index already exists baton->slots->search = 1; } - else if (reply && reply->type == REDIS_REPLY_STATUS && + else if (reply && reply->type == RESP_REPLY_STATUS && (strcmp("OK", reply->str) == 0 || strcmp("QUEUED", reply->str) == 0)) { // index created baton->slots->search = 1; } else { - // probably no RediSearch module installed, ignore silently + // probably no search module installed, ignore silently baton->slots->search = 0; } - redis_slots_end_phase(baton); + keys_slots_end_phase(baton); } void -redis_load_search_schema(void *arg) +keys_load_search_schema(void *arg) { - redisSlotsBaton *baton = (redisSlotsBaton *)arg; + keySlotsBaton *baton = (keySlotsBaton *)arg; sds cmd, key; - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_search_schema"); + seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_search_schema"); if (pmDebugOptions.search && pmDebugOptions.desperate) - fprintf(stderr, "%s: loading schema\n", "redis_search_schema"); + fprintf(stderr, "%s: loading schema\n", "keys_search_schema"); - seriesBatonReference(baton, "redis_load_search_schema"); + seriesBatonReference(baton, "keys_load_search_schema"); /* * FT.CREATE pcp:text SCHEMA @@ -974,37 +974,37 @@ redis_load_search_schema(void *arg) * helptext TEXT WEIGHT 2 */ key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = redis_command(3 + 3 + 5 + 2 + 4 + 4); + cmd = key_command(3 + 3 + 5 + 2 + 4 + 4); - cmd = redis_param_str(cmd, FT_CREATE, FT_CREATE_LEN); - cmd = redis_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = redis_param_str(cmd, FT_SCHEMA, FT_SCHEMA_LEN); + cmd = key_param_str(cmd, FT_CREATE, FT_CREATE_LEN); + cmd = key_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); + cmd = key_param_str(cmd, FT_SCHEMA, FT_SCHEMA_LEN); - cmd = redis_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = redis_param_str(cmd, FT_TAG, FT_TAG_LEN); - cmd = redis_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); + cmd = key_param_str(cmd, FT_TYPE, FT_TYPE_LEN); + cmd = key_param_str(cmd, FT_TAG, FT_TAG_LEN); + cmd = key_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - cmd = redis_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = redis_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = redis_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = redis_param_str(cmd, "9", sizeof("9")-1); - cmd = redis_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); + cmd = key_param_str(cmd, FT_NAME, FT_NAME_LEN); + cmd = key_param_str(cmd, FT_TEXT, FT_TEXT_LEN); + cmd = key_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); + cmd = key_param_str(cmd, "9", sizeof("9")-1); + cmd = key_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - cmd = redis_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = redis_param_str(cmd, FT_TAG, FT_TAG_LEN); + cmd = key_param_str(cmd, FT_INDOM, FT_INDOM_LEN); + cmd = key_param_str(cmd, FT_TAG, FT_TAG_LEN); - cmd = redis_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = redis_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = redis_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = redis_param_str(cmd, "4", sizeof("4")-1); + cmd = key_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); + cmd = key_param_str(cmd, FT_TEXT, FT_TEXT_LEN); + cmd = key_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); + cmd = key_param_str(cmd, "4", sizeof("4")-1); - cmd = redis_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = redis_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = redis_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = redis_param_str(cmd, "2", sizeof("2")-1); + cmd = key_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + cmd = key_param_str(cmd, FT_TEXT, FT_TEXT_LEN); + cmd = key_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); + cmd = key_param_str(cmd, "2", sizeof("2")-1); sdsfree(key); - redisSlotsRequestFirstNode(baton->slots, cmd, redis_search_schema_callback, arg); + keySlotsRequestFirstNode(baton->slots, cmd, keys_search_schema_callback, arg); sdsfree(cmd); } @@ -1033,7 +1033,7 @@ pmSearchSetMetricRegistry(pmSearchModule *module, mmv_registry_t *registry) } void -redisSearchInit(struct dict *config) +keysSearchInit(struct dict *config) { sds option; @@ -1047,7 +1047,7 @@ redisSearchInit(struct dict *config) } void -redisSearchClose(void) +keysSearchClose(void) { if (DEFAULT_RESULTCOUNT) { sdsfree(DEFAULT_RESULTCOUNT); @@ -1060,25 +1060,25 @@ pmSearchSetup(pmSearchModule *module, void *arg) { seriesModuleData *data = getSeriesModuleData(module); sds option; - redisSlotsFlags flags; + keySlotsFlags flags; if (data == NULL) return -ENOMEM; /* create string map caches */ - redisGlobalsInit(data->config); + keysGlobalsInit(data->config); - /* fast path for when Redis has been setup already */ + /* fast path for when key server has been setup already */ if (data->slots) { module->on_setup(arg); data->shareslots = 1; } else { if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); + option = pmIniFileLookup(data->config, "redis", "enabled"); // compat if (option && strcmp(option, "false") == 0) return -ENOTSUP; - /* establish an initial connection to Redis instance(s) */ + /* establish an initial connection to key server instance(s) */ flags = SLOTS_VERSION; option = pmIniFileLookup(data->config, "pmsearch", "enabled"); @@ -1087,8 +1087,8 @@ pmSearchSetup(pmSearchModule *module, void *arg) else return -ENOTSUP; - /* establish an initial connection to Redis instance(s) */ - data->slots = redisSlotsConnect( + /* establish an initial connection to key server instance(s) */ + data->slots = keySlotsConnect( data->config, flags, module->on_info, module->on_setup, arg, data->events, arg); data->shareslots = 0; @@ -1099,7 +1099,7 @@ pmSearchSetup(pmSearchModule *module, void *arg) int pmSearchEnabled(void *arg) { - redisSlots *slots = (redisSlots *)arg; + keySlots *slots = (keySlots *)arg; if (slots) return slots->search > 0 ? 1 : 0; @@ -1113,11 +1113,11 @@ pmSearchClose(pmSearchModule *module) if (search) { if (search->slots && !search->shareslots) - redisSlotsFree(search->slots); + keySlotsFree(search->slots); memset(search, 0, sizeof(*search)); free(search); module->privdata = NULL; } - redisGlobalsClose(); + keysGlobalsClose(); } diff --git a/src/libpcp_web/src/search.h b/src/libpcp_web/src/search.h index fcf639558f..b753f7c00b 100644 --- a/src/libpcp_web/src/search.h +++ b/src/libpcp_web/src/search.h @@ -16,7 +16,7 @@ #include #include -#include "redis.h" +#include "keys.h" #include "private.h" #include "schema.h" #include "slots.h" @@ -83,19 +83,19 @@ #define FT_WITHSCORES "WITHSCORES" #define FT_WITHSCORES_LEN (sizeof(FT_WITHSCORES)-1) -extern void redisSearchInit(struct dict *); -extern void redisSearchClose(void); -extern void redis_load_search_schema(void *); -extern void redis_search_text_add(redisSlots *, pmSearchTextType, +extern void keysSearchInit(struct dict *); +extern void keysSearchClose(void); +extern void keys_load_search_schema(void *); +extern void keys_search_text_add(keySlots *, pmSearchTextType, const char *, const char *, const char *, const char *, void *); /* * Asynchronous search baton structures */ -typedef struct redisSearchBaton { +typedef struct keysSearchBaton { seriesBatonMagic header; /* MAGIC_SEARCH */ - redisSlots *slots; /* Redis server slots */ + keySlots *slots; /* key server slots */ pmSearchFlags flags; int error; void *module; @@ -104,6 +104,6 @@ typedef struct redisSearchBaton { struct timespec started; void *userdata; void *arg; -} redisSearchBaton; +} keysSearchBaton; #endif /* SEARCH_SCHEMA_H */ diff --git a/src/libpcp_web/src/slots.c b/src/libpcp_web/src/slots.c index a3913ba4d1..09e82b4b95 100644 --- a/src/libpcp_web/src/slots.c +++ b/src/libpcp_web/src/slots.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Red Hat. + * Copyright (c) 2017-2021,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -23,49 +23,49 @@ #if defined(HAVE_LIBUV) #include #else -static int redisClusterLibuvAttach() { return REDIS_OK; } +static int keysClusterLibuvAttach() { return RESP_OK; } #endif static char default_server[] = "localhost:6379"; static void -redis_connect_callback(const redisAsyncContext *redis, int status) +key_server_connect_callback(const keysAsyncContext *keys, int status) { - if (status == REDIS_OK) { + if (status == RESP_OK) { if (pmDebugOptions.series) - fprintf(stderr, "Connected to Redis on %s:%d\n", - redis->c.tcp.host, redis->c.tcp.port); - redisAsyncEnableKeepAlive((redisAsyncContext *)redis); - /* TODO: if SSL inject redisSecureConnection() here */ + fprintf(stderr, "Connected to key server on %s:%d\n", + keys->c.tcp.host, keys->c.tcp.port); + keysAsyncEnableKeepAlive((keysAsyncContext *)keys); + /* TODO: client SSL? inject keysSecureConnection() here */ } else if (pmDebugOptions.series) { - if (redis->c.connection_type == REDIS_CONN_UNIX) + if (keys->c.connection_type == RESP_CONN_UNIX) fprintf(stderr, "Connecting to %s failed - %s\n", - redis->c.unix_sock.path, redis->errstr); + keys->c.unix_sock.path, keys->errstr); else fprintf(stderr, "Connecting to %s:%d failed - %s\n", - redis->c.tcp.host, redis->c.tcp.port, redis->errstr); + keys->c.tcp.host, keys->c.tcp.port, keys->errstr); } } static void -redis_disconnect_callback(const redisAsyncContext *redis, int status) +key_server_disconnect_callback(const keysAsyncContext *keys, int status) { if (status == REDIS_OK) { if (pmDebugOptions.series) fprintf(stderr, "Disconnected from key server on %s:%d\n", - redis->c.tcp.host, redis->c.tcp.port); + keys->c.tcp.host, keys->c.tcp.port); } else if (pmDebugOptions.series) { - if (redis->c.connection_type == REDIS_CONN_UNIX) + if (keys->c.connection_type == REDIS_CONN_UNIX) fprintf(stderr, "Disconnecting from %s failed - %s\n", - redis->c.unix_sock.path, redis->errstr); + keys->c.unix_sock.path, keys->errstr); else fprintf(stderr, "Disconnecting from %s:%d failed - %s\n", - redis->c.tcp.host, redis->c.tcp.port, redis->errstr); + keys->c.tcp.host, keys->c.tcp.port, keys->errstr); } } void -redisSlotsSetupMetrics(redisSlots *slots) +keySlotsSetupMetrics(keySlots *slots) { pmAtomValue **table; pmUnits units_count = MMV_UNITS(0, 0, 1, 0, 0, PM_COUNT_ONE); @@ -79,47 +79,47 @@ redisSlotsSetupMetrics(redisSlots *slots) mmv_stats_add_metric(slots->registry, "requests.total", 1, MMV_TYPE_U64, MMV_SEM_COUNTER, units_count, MMV_INDOM_NULL, "number of requests", - "Total number of Redis requests sent"); + "Total number of key server requests sent"); mmv_stats_add_metric(slots->registry, "requests.error", 2, MMV_TYPE_U64, MMV_SEM_COUNTER, units_count, MMV_INDOM_NULL, "number of request errors", - "Total number of Redis request errors"); + "Total number of key server request errors"); mmv_stats_add_metric(slots->registry, "responses.total", 3, MMV_TYPE_U64, MMV_SEM_COUNTER, units_count, MMV_INDOM_NULL, "number of responses", - "Total number of Redis responses received"); + "Total number of key server responses received"); mmv_stats_add_metric(slots->registry, "responses.error", 4, MMV_TYPE_U64, MMV_SEM_COUNTER, units_count, MMV_INDOM_NULL, "number of error responses", - "Total number of Redis error responses received"); + "Total number of key server error responses received"); mmv_stats_add_metric(slots->registry, "responses.time", 5, MMV_TYPE_U64, MMV_SEM_COUNTER, units_us, MMV_INDOM_NULL, "total time for responses", - "Cumulative time taken to receive all Redis responses"); + "Cumulative time taken to receive all key server responses"); mmv_stats_add_metric(slots->registry, "requests.inflight.total", 6, MMV_TYPE_U64, MMV_SEM_INSTANT, units_count, MMV_INDOM_NULL, "inflight requests", - "Total number of inflight Redis requests"); + "Total number of inflight key server requests"); mmv_stats_add_metric(slots->registry, "requests.inflight.bytes", 7, MMV_TYPE_U64, MMV_SEM_INSTANT, units_bytes, MMV_INDOM_NULL, "bytes allocated for inflight requests", - "Memory currently allocated for inflight Redis requests"); + "Memory currently allocated for inflight key server requests"); mmv_stats_add_metric(slots->registry, "requests.total_bytes", 8, MMV_TYPE_U64, MMV_SEM_COUNTER, units_bytes, MMV_INDOM_NULL, "total bytes sent for requests", - "Cumulative count of bytes sent for all Redis requests"); + "Cumulative count of bytes sent for all key server requests"); mmv_stats_add_metric(slots->registry, "responses.total_bytes", 9, MMV_TYPE_U64, MMV_SEM_COUNTER, units_bytes, MMV_INDOM_NULL, "total bytes received in responses", - "Cumulative count of bytes received in Redis responses"); + "Cumulative count of bytes received in key server responses"); slots->map = map = mmv_stats_start(slots->registry); @@ -145,7 +145,7 @@ redisSlotsSetupMetrics(redisSlots *slots) } int -redisSlotsSetMetricRegistry(redisSlots *slots, mmv_registry_t *registry) +keySlotsSetMetricRegistry(keySlots *slots, mmv_registry_t *registry) { if (slots) { slots->registry = registry; @@ -154,10 +154,10 @@ redisSlotsSetMetricRegistry(redisSlots *slots, mmv_registry_t *registry) return -ENOMEM; } -redisSlots * -redisSlotsInit(dict *config, void *events) +keySlots * +keySlotsInit(dict *config, void *events) { - redisSlots *slots; + keySlots *slots; sds servers = NULL; sds def_servers = NULL; sds username = NULL; @@ -166,9 +166,9 @@ redisSlotsInit(dict *config, void *events) struct timeval connection_timeout = {5, 0}; // 5s struct timeval command_timeout = {60, 0}; // 1m - if ((slots = (redisSlots *)calloc(1, sizeof(redisSlots))) == NULL) { - pmNotifyErr(LOG_ERR, "%s: failed to allocate redisSlots\n", - "redisSlotsInit"); + if ((slots = (keySlots *)calloc(1, sizeof(keySlots))) == NULL) { + pmNotifyErr(LOG_ERR, "%s: failed to allocate keySlots\n", + "keySlotsInit"); return NULL; } @@ -177,48 +177,48 @@ redisSlotsInit(dict *config, void *events) slots->keymap = dictCreate(&sdsKeyDictCallBacks, "keymap"); if (slots->keymap == NULL) { pmNotifyErr(LOG_ERR, "%s: failed to allocate keymap\n", - "redisSlotsInit"); + "keySlotsInit"); free(slots); return NULL; } servers = pmIniFileLookup(config, "keys", "servers"); if (servers == NULL) - servers = pmIniFileLookup(config, "redis", "servers"); + servers = pmIniFileLookup(config, "redis", "servers"); // back-compat if (servers == NULL) - servers = pmIniFileLookup(config, "pmseries", "servers"); + servers = pmIniFileLookup(config, "pmseries", "servers"); // back-compat if (servers == NULL) servers = def_servers = sdsnew(default_server); username = pmIniFileLookup(config, "keys", "username"); if (username == NULL) - username = pmIniFileLookup(config, "redis", "username"); + username = pmIniFileLookup(config, "redis", "username"); // back-compat if (username == NULL) username = pmIniFileLookup(config, "pmseries", "auth.username"); password = pmIniFileLookup(config, "keys", "password"); if (password == NULL) - password = pmIniFileLookup(config, "redis", "password"); + password = pmIniFileLookup(config, "redis", "password"); // back-compat if (password == NULL) password = pmIniFileLookup(config, "pmseries", "auth.password"); - if ((slots->acc = redisClusterAsyncContextInit()) == NULL) { + if ((slots->acc = keyClusterAsyncContextInit()) == NULL) { /* Coverity CID370635 */ pmNotifyErr(LOG_ERR, "%s: %s failed\n", - "redisSlotsInit", "redisClusterAsyncContextInit"); + "keySlotsInit", "keyClusterAsyncContextInit"); sdsfree(def_servers); return slots; } if (slots->acc->err) { - pmNotifyErr(LOG_ERR, "%s: %s\n", "redisSlotsInit", slots->acc->errstr); + pmNotifyErr(LOG_ERR, "%s: %s\n", "keySlotsInit", slots->acc->errstr); sdsfree(def_servers); return slots; } - sts = redisClusterSetOptionAddNodes(slots->acc->cc, servers); + sts = keyClusterSetOptionAddNodes(slots->acc->cc, servers); if (sts != REDIS_OK) { - pmNotifyErr(LOG_ERR, "%s: failed to add redis nodes: %s\n", - "redisSlotsInit", slots->acc->cc->errstr); + pmNotifyErr(LOG_ERR, "%s: failed to add key server nodes: %s\n", + "keySlotsInit", slots->acc->cc->errstr); sdsfree(def_servers); return slots; } @@ -230,60 +230,60 @@ redisSlotsInit(dict *config, void *events) * fine */ if (username != NULL && sdslen(username) > 0) { - sts = redisClusterSetOptionUsername(slots->acc->cc, username); + sts = keyClusterSetOptionUsername(slots->acc->cc, username); if (sts != REDIS_OK) { - pmNotifyErr(LOG_ERR, "%s: failed to set Redis username: %s\n", - "redisClusterSetOptionUsername", slots->acc->cc->errstr); + pmNotifyErr(LOG_ERR, "%s: failed to set key server username: %s\n", + "keyClusterSetOptionUsername", slots->acc->cc->errstr); return slots; } } /* - * see note above re empty configuration values - * having only a password set and no username is a valid Redis - * configuration, see https://redis.io/commands/auth + * see note above re empty configuration values having only a password + * set and no username is a valid key server configuration, details: + * https://valkey.io/commands/auth */ if (password != NULL && sdslen(password) > 0) { - sts = redisClusterSetOptionPassword(slots->acc->cc, password); + sts = keyClusterSetOptionPassword(slots->acc->cc, password); if (sts != REDIS_OK) { - pmNotifyErr(LOG_ERR, "%s: failed to set Redis password: %s\n", - "redisClusterSetOptionPassword", slots->acc->cc->errstr); + pmNotifyErr(LOG_ERR, "%s: failed to set key server password: %s\n", + "keyClusterSetOptionPassword", slots->acc->cc->errstr); return slots; } } - sts = redisClusterSetOptionConnectTimeout(slots->acc->cc, connection_timeout); + sts = keyClusterSetOptionConnectTimeout(slots->acc->cc, connection_timeout); if (sts != REDIS_OK) { pmNotifyErr(LOG_ERR, "%s: failed to set connect timeout: %s\n", - "redisSlotsInit", slots->acc->errstr); + "keySlotsInit", slots->acc->errstr); return slots; } - sts = redisClusterSetOptionTimeout(slots->acc->cc, command_timeout); + sts = keyClusterSetOptionTimeout(slots->acc->cc, command_timeout); if (sts != REDIS_OK) { pmNotifyErr(LOG_ERR, "%s: failed to set command timeout: %s\n", - "redisSlotsInit", slots->acc->cc->errstr); + "keySlotsInit", slots->acc->cc->errstr); return slots; } - sts = redisClusterLibuvAttach(slots->acc, slots->events); + sts = keysClusterLibuvAttach(slots->acc, slots->events); if (sts != REDIS_OK) { pmNotifyErr(LOG_ERR, "%s: failed to attach to event loop: %s\n", - "redisSlotsInit", slots->acc->errstr); + "keySlotsInit", slots->acc->errstr); return slots; } - sts = redisClusterAsyncSetConnectCallback(slots->acc, redis_connect_callback); + sts = keyClusterAsyncSetConnectCallback(slots->acc, key_server_connect_callback); if (sts != REDIS_OK) { pmNotifyErr(LOG_ERR, "%s: failed to set connect callback: %s\n", - "redisSlotsInit", slots->acc->errstr); + "keySlotsInit", slots->acc->errstr); return slots; } - sts = redisClusterAsyncSetDisconnectCallback(slots->acc, redis_disconnect_callback); + sts = keyClusterAsyncSetDisconnectCallback(slots->acc, key_server_disconnect_callback); if (sts != REDIS_OK) { pmNotifyErr(LOG_ERR, "%s: failed to set disconnect callback: %s\n", - "redisSlotsInit", slots->acc->errstr); + "keySlotsInit", slots->acc->errstr); return slots; } @@ -292,11 +292,11 @@ redisSlotsInit(dict *config, void *events) /** * despite the name, this function also handles the initial - * connection to Redis + * connection to the key server */ void -redisSlotsReconnect(redisSlots *slots, redisSlotsFlags flags, - redisInfoCallBack info, redisDoneCallBack done, +keySlotsReconnect(keySlots *slots, keySlotsFlags flags, + keysInfoCallBack info, keysDoneCallBack done, void *userdata, void *events, void *arg) { dictIterator *iterator; @@ -310,26 +310,26 @@ redisSlotsReconnect(redisSlots *slots, redisSlotsFlags flags, slots->state = SLOTS_CONNECTING; slots->conn_seq++; - /* reset Redis context in case of reconnect */ + /* reset key server context in case of reconnect */ if (slots->acc->err) { /* reset possible 'Connection refused' error before reconnecting */ slots->acc->err = 0; memset(slots->acc->errstr, '\0', strlen(slots->acc->errstr)); } - redisClusterAsyncDisconnect(slots->acc); + keyClusterAsyncDisconnect(slots->acc); - /* reset redisSlots in case of reconnect */ + /* reset keySlots in case of reconnect */ slots->cluster = 0; slots->search = 0; dictEmpty(slots->keymap, NULL); - sts = redisClusterConnect2(slots->acc->cc); + sts = keyClusterConnect2(slots->acc->cc); if (sts == REDIS_OK) { slots->cluster = 1; } else if (slots->acc->cc->err && strcmp(slots->acc->cc->errstr, REDIS_ENOCLUSTER) == 0) { - /* Redis instance has cluster support disabled */ + /* key server instance has cluster support disabled */ slots->acc->cc->err = 0; memset(slots->acc->cc->errstr, '\0', strlen(slots->acc->cc->errstr)); slots->cluster = 0; @@ -344,7 +344,7 @@ redisSlotsReconnect(redisSlots *slots, redisSlotsFlags flags, if (entry && dictNext(iterator)) { dictReleaseIterator(iterator); pmNotifyErr(LOG_ERR, "%s: more than one node is configured, " - "but cluster mode is disabled", "redisSlotsReconnect"); + "but cluster mode is disabled", "keySlotsReconnect"); slots->state = SLOTS_ERR_FATAL; return; } @@ -352,7 +352,7 @@ redisSlotsReconnect(redisSlots *slots, redisSlotsFlags flags, } else { if (log_connection_errors || pmDebugOptions.desperate) { - pmNotifyErr(LOG_INFO, "Cannot connect to Redis: %s\n", + pmNotifyErr(LOG_INFO, "Cannot connect to key server: %s\n", slots->acc->cc->errstr); log_connection_errors = 0; } @@ -362,28 +362,28 @@ redisSlotsReconnect(redisSlots *slots, redisSlotsFlags flags, slots->state = SLOTS_CONNECTED; log_connection_errors = 1; - redisSchemaLoad(slots, flags, info, done, userdata, events, arg); + keySchemaLoad(slots, flags, info, done, userdata, events, arg); } /** - * this method allocates the redisSlots struct and exists for backwards - * compatibility, the actual connection to Redis happens in - * redisSlotsReconnect() + * this method allocates the keySlots struct and exists for backwards + * compatibility, the actual connection to the key server happens in + * keySlotsReconnect() */ -redisSlots * -redisSlotsConnect(dict *config, redisSlotsFlags flags, - redisInfoCallBack info, redisDoneCallBack done, +keySlots * +keySlotsConnect(dict *config, keySlotsFlags flags, + keysInfoCallBack info, keysDoneCallBack done, void *userdata, void *events, void *arg) { - redisSlots *slots; + keySlots *slots; sds enabled, msg; if (!(enabled = pmIniFileLookup(config, "resp", "enabled"))) - enabled = pmIniFileLookup(config, "redis", "enabled"); + enabled = pmIniFileLookup(config, "redis", "enabled"); // back-compat if (enabled && strcmp(enabled, "false") == 0) return NULL; - slots = redisSlotsInit(config, events); + slots = keySlotsInit(config, events); if (slots == NULL) { infofmt(msg, "Failed to allocate memory for key server slots"); info(PMLOG_ERROR, msg, arg); @@ -391,15 +391,15 @@ redisSlotsConnect(dict *config, redisSlotsFlags flags, return NULL; } - redisSlotsReconnect(slots, flags, info, done, userdata, events, arg); + keySlotsReconnect(slots, flags, info, done, userdata, events, arg); return slots; } void -redisSlotsFree(redisSlots *slots) +keySlotsFree(keySlots *slots) { - redisClusterAsyncDisconnect(slots->acc); - redisClusterAsyncFree(slots->acc); + keyClusterAsyncDisconnect(slots->acc); + keyClusterAsyncFree(slots->acc); dictRelease(slots->keymap); memset(slots, 0, sizeof(*slots)); free(slots); @@ -414,13 +414,13 @@ gettimeusec(void) return (uint64_t)now.tv_sec * 1000000 + (uint64_t)now.tv_usec; } -static redisSlotsReplyData * -redisSlotsReplyDataAlloc(redisSlots *slots, size_t req_size, - redisClusterCallbackFn *callback, void *arg) +static keySlotsReplyData * +keySlotsReplyDataAlloc(keySlots *slots, size_t req_size, + keyClusterCallbackFn *callback, void *arg) { - redisSlotsReplyData *srd; + keySlotsReplyData *srd; - srd = calloc(1, sizeof(redisSlotsReplyData)); + srd = calloc(1, sizeof(keySlotsReplyData)); if (srd == NULL) { return NULL; } @@ -435,13 +435,13 @@ redisSlotsReplyDataAlloc(redisSlots *slots, size_t req_size, } static inline void -redisSlotsReplyDataFree(redisSlotsReplyData *srd) +keySlotsReplyDataFree(keySlotsReplyData *srd) { free(srd); } uint64_t -redisSlotsInflightRequests(redisSlots *slots) +keySlotsInflightRequests(keySlots *slots) { pmAtomValue *atom; @@ -450,10 +450,10 @@ redisSlotsInflightRequests(redisSlots *slots) } static void -redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) +keySlotsReplyCallback(keyClusterAsyncContext *c, void *r, void *arg) { - redisSlotsReplyData *srd = arg; - redisReply *reply = r; + keySlotsReplyData *srd = arg; + respReply *reply = r; void *map = srd->slots->map; if (map) { @@ -484,9 +484,9 @@ redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) /** * handle connection resets * - * Why here and not in redis_disconnect_callback? - * Access to redisSlots->state is required. We cannot save a pointer to - * redisSlots in redisAsyncContext->data, because this member is already + * Why here and not in key_server_disconnect_callback? + * Access to keySlots->state is required. We cannot save a pointer to + * keySlots in keysAsyncContext->data, because this member is already * used by hiredis-cluster * * fwiw, in case one node of a cluster is down, slots->state should not be @@ -517,12 +517,12 @@ redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) srd->conn_seq == srd->slots->conn_seq && srd->slots->state != SLOTS_DISCONNECTED && srd->slots->cluster == 0) { - pmNotifyErr(LOG_ERR, "Lost connection to Redis.\n"); + pmNotifyErr(LOG_ERR, "Lost connection to key server.\n"); srd->slots->state = SLOTS_DISCONNECTED; } srd->callback(c, r, srd->arg); - redisSlotsReplyDataFree(arg); + keySlotsReplyDataFree(arg); } /* @@ -530,42 +530,42 @@ redisSlotsReplyCallback(redisClusterAsyncContext *c, void *r, void *arg) * The given key is used to determine the slot used, as per the * cluster specification - https://valkey.io/topics/cluster-spec * - * Serves mainly as a wrapper to redisClusterAsyncFormattedCommand + * Serves mainly as a wrapper to keyClusterAsyncFormattedCommand * including debug output and error handling */ int -redisSlotsRequest(redisSlots *slots, const sds cmd, - redisClusterCallbackFn *callback, void *arg) +keySlotsRequest(keySlots *slots, const sds cmd, + keyClusterCallbackFn *callback, void *arg) { int sts; uint64_t size; - redisSlotsReplyData *srd; + keySlotsReplyData *srd; /* - * redisSlotsSetupStart() also sends Redis requests, + * keySlotsSetupStart() also sends key server requests, * therefore both SLOTS_CONNECTED and SLOTS_READY states are valid */ if (UNLIKELY(slots->state != SLOTS_CONNECTED && slots->state != SLOTS_READY)) return -ENOTCONN; if (!slots->cluster) - return redisSlotsRequestFirstNode(slots, cmd, callback, arg); + return keySlotsRequestFirstNode(slots, cmd, callback, arg); if (UNLIKELY(pmDebugOptions.desperate)) - fprintf(stderr, "%s: sending raw redis command:\n%s", - "redisSlotsRequest", cmd); + fprintf(stderr, "%s: sending raw key server command:\n%s", + "keySlotsRequest", cmd); size = sdslen(cmd); - if ((srd = redisSlotsReplyDataAlloc(slots, size, callback, arg)) == NULL) { + if ((srd = keySlotsReplyDataAlloc(slots, size, callback, arg)) == NULL) { mmv_inc(slots->map, slots->metrics[SLOT_REQUESTS_ERROR]); pmNotifyErr(LOG_ERR, "%s: failed to allocate reply data (%llu bytes)\n", - "redisSlotsRequest", (unsigned long long)size); + "keySlotsRequest", (unsigned long long)size); return -ENOMEM; } - if ((sts = redisClusterAsyncFormattedCommand(slots->acc, - redisSlotsReplyCallback, srd, cmd, size)) != REDIS_OK) { + if ((sts = keyClusterAsyncFormattedCommand(slots->acc, + keySlotsReplyCallback, srd, cmd, size)) != REDIS_OK) { mmv_inc(slots->map, slots->metrics[SLOT_REQUESTS_ERROR]); - pmNotifyErr(LOG_ERR, "%s: %s (%s)\n", "redisSlotsRequest", + pmNotifyErr(LOG_ERR, "%s: %s (%s)\n", "keySlotsRequest", slots->acc->errstr, cmd); return -ENOMEM; } @@ -579,18 +579,18 @@ redisSlotsRequest(redisSlots *slots, const sds cmd, } int -redisSlotsRequestFirstNode(redisSlots *slots, const sds cmd, - redisClusterCallbackFn *callback, void *arg) +keySlotsRequestFirstNode(keySlots *slots, const sds cmd, + keyClusterCallbackFn *callback, void *arg) { dictIterator *iterator; dictEntry *entry; cluster_node *node; - redisSlotsReplyData *srd; + keySlotsReplyData *srd; uint64_t size; int sts; /* - * redisSlotsSetupStart() also sends Redis requests, + * keySlotsSetupStart() also sends key server requests, * therefore both SLOTS_CONNECTED and SLOTS_READY states are valid */ if (UNLIKELY(slots->state != SLOTS_CONNECTED && slots->state != SLOTS_READY)) @@ -600,29 +600,29 @@ redisSlotsRequestFirstNode(redisSlots *slots, const sds cmd, entry = dictNext(iterator); dictReleaseIterator(iterator); if (!entry) { - pmNotifyErr(LOG_ERR, "%s: No Redis node configured.", - "redisSlotsRequestFirstNode"); + pmNotifyErr(LOG_ERR, "%s: No key server node configured.", + "keySlotsRequestFirstNode"); return REDIS_ERR; } node = dictGetVal(entry); if (UNLIKELY(pmDebugOptions.desperate)) - fprintf(stderr, "%s: sending raw redis command to node %s\n%s", - "redisSlotsRequestFirstNode", node->addr, cmd); + fprintf(stderr, "%s: sending raw key server command to node %s\n%s", + "keySlotsRequestFirstNode", node->addr, cmd); size = sdslen(cmd); - if ((srd = redisSlotsReplyDataAlloc(slots, size, callback, arg)) == NULL) { + if ((srd = keySlotsReplyDataAlloc(slots, size, callback, arg)) == NULL) { mmv_inc(slots->map, slots->metrics[SLOT_REQUESTS_ERROR]); pmNotifyErr(LOG_ERR, "%s: failed to allocate reply data (%llu bytes)", - "redisSlotsRequestFirstNode", (unsigned long long)size); + "keySlotsRequestFirstNode", (unsigned long long)size); return -ENOMEM; } - sts = redisClusterAsyncFormattedCommandToNode(slots->acc, node, - redisSlotsReplyCallback, srd, cmd, size); + sts = keyClusterAsyncFormattedCommandToNode(slots->acc, node, + keySlotsReplyCallback, srd, cmd, size); if (sts != REDIS_OK) { mmv_inc(slots->map, slots->metrics[SLOT_REQUESTS_ERROR]); pmNotifyErr(LOG_ERR, "%s: %s (%s)\n", - "redisSlotsRequestFirstNode", slots->acc->errstr, cmd); + "keySlotsRequestFirstNode", slots->acc->errstr, cmd); return -ENOMEM; } @@ -635,12 +635,12 @@ redisSlotsRequestFirstNode(redisSlots *slots, const sds cmd, } int -redisSlotsProxyConnect(redisSlots *slots, redisInfoCallBack info, - redisReader **readerp, const char *buffer, ssize_t nread, - redisClusterCallbackFn *callback, void *arg) +keySlotsProxyConnect(keySlots *slots, keysInfoCallBack info, + respReader **readerp, const char *buffer, ssize_t nread, + keyClusterCallbackFn *callback, void *arg) { - redisReader *reader = *readerp; - redisReply *reply = NULL; + respReader *reader = *readerp; + respReply *reply = NULL; dictEntry *entry; size_t replyStartPosition; long long position; @@ -649,24 +649,24 @@ redisSlotsProxyConnect(redisSlots *slots, redisInfoCallBack info, int sts; if (!reader && - (reader = *readerp = redisReaderCreate()) == NULL) { - infofmt(msg, "out-of-memory for Redis client reader"); + (reader = *readerp = respReaderCreate()) == NULL) { + infofmt(msg, "out-of-memory for key client reader"); info(PMLOG_REQUEST, msg, arg), sdsfree(msg); return -ENOMEM; } - if (redisReaderFeed(reader, buffer, nread) != REDIS_OK) { - infofmt(msg, "failed to parse Redis protocol request"); + if (respReaderFeed(reader, buffer, nread) != REDIS_OK) { + infofmt(msg, "failed to parse RESP protocol request"); info(PMLOG_REQUEST, msg, arg), sdsfree(msg); return -EPROTO; } - /* parse all Redis requests contained in buffer (Redis pipelining) */ + /* parse all key server requests contained in buffer (pipelining) */ while (1) { replyStartPosition = reader->pos; - sts = redisReaderGetReply(reader, (void **)&reply); - if (sts != REDIS_OK) { - infofmt(msg, "failed to parse Redis protocol request"); + sts = respReaderGetReply(reader, (void **)&reply); + if (sts != RESP_OK) { + infofmt(msg, "failed to parse RESP protocol request"); info(PMLOG_REQUEST, msg, arg), sdsfree(msg); return -EPROTO; } @@ -676,9 +676,9 @@ redisSlotsProxyConnect(redisSlots *slots, redisInfoCallBack info, cmd = NULL; hasKey = 0; - if (reply->type == REDIS_REPLY_ARRAY || - reply->type == REDIS_REPLY_MAP || - reply->type == REDIS_REPLY_SET) + if (reply->type == RESP_REPLY_ARRAY || + reply->type == RESP_REPLY_MAP || + reply->type == RESP_REPLY_SET) cmd = sdsnew(reply->element[0]->str); if (cmd && (entry = dictFind(slots->keymap, cmd)) != NULL) { position = dictGetSignedIntegerVal(entry); @@ -689,13 +689,13 @@ redisSlotsProxyConnect(redisSlots *slots, redisInfoCallBack info, cmd = sdsnewlen(reader->buf + replyStartPosition, reader->pos - replyStartPosition); if (hasKey) - sts = redisSlotsRequest(slots, cmd, callback, arg); + sts = keySlotsRequest(slots, cmd, callback, arg); else - sts = redisSlotsRequestFirstNode(slots, cmd, callback, arg); + sts = keySlotsRequestFirstNode(slots, cmd, callback, arg); sdsfree(cmd); if (sts != REDIS_OK) { - redisReply *errorReply = calloc(1, sizeof(redisReply)); + respReply *errorReply = calloc(1, sizeof(respReply)); errorReply->type = REDIS_REPLY_ERROR; errorReply->str = slots->acc->errstr; errorReply->len = strlen(slots->acc->errstr); @@ -706,43 +706,43 @@ redisSlotsProxyConnect(redisSlots *slots, redisInfoCallBack info, } void -redisSlotsProxyFree(redisReader *reader) +keySlotsProxyFree(respReader *reader) { if (reader) - redisReaderFree(reader); + respReaderFree(reader); } /* - * Helper routines for handling various expected Redis reply types. + * Helper routines for handling various expected RESP reply types. */ int -testReplyError(redisReply *reply, const char *server_message) +testReplyError(respReply *reply, const char *server_message) { return (reply && reply->type == REDIS_REPLY_ERROR && strcmp(reply->str, server_message) == 0); } void -reportReplyError(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, const char *format, va_list argp) +reportReplyError(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, const char *format, va_list argp) { sds msg; msg = sdscatvprintf(sdsempty(), format, argp); if (reply && reply->type == REDIS_REPLY_ERROR) - msg = sdscatfmt(msg, "\nRedis reply error: %s", reply->str); + msg = sdscatfmt(msg, "\nRESP reply error: %s", reply->str); else if (acc->err) - msg = sdscatfmt(msg, "\nRedis acc error: %s", acc->errstr); + msg = sdscatfmt(msg, "\nRESP acc error: %s", acc->errstr); else if (acc->cc->err) - msg = sdscatfmt(msg, "\nRedis cc error: %s", acc->cc->errstr); + msg = sdscatfmt(msg, "\nRESP cc error: %s", acc->cc->errstr); info(PMLOG_RESPONSE, msg, userdata); sdsfree(msg); } int -checkStatusReplyOK(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, const char *format, ...) +checkStatusReplyOK(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, const char *format, ...) { va_list argp; @@ -756,8 +756,8 @@ checkStatusReplyOK(redisInfoCallBack info, void *userdata, } int -checkStreamReplyString(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, sds s, const char *format, ...) +checkStreamReplyString(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, sds s, const char *format, ...) { va_list argp; @@ -770,8 +770,8 @@ checkStreamReplyString(redisInfoCallBack info, void *userdata, } int -checkArrayReply(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, const char *format, ...) +checkArrayReply(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, const char *format, ...) { va_list argp; @@ -784,8 +784,8 @@ checkArrayReply(redisInfoCallBack info, void *userdata, } long long -checkIntegerReply(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, const char *format, ...) +checkIntegerReply(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, const char *format, ...) { va_list argp; @@ -798,8 +798,8 @@ checkIntegerReply(redisInfoCallBack info, void *userdata, } sds -checkStringReply(redisInfoCallBack info, void *userdata, - redisClusterAsyncContext *acc, redisReply *reply, const char *format, ...) +checkStringReply(keysInfoCallBack info, void *userdata, + keyClusterAsyncContext *acc, respReply *reply, const char *format, ...) { va_list argp; diff --git a/src/libpcp_web/src/slots.h b/src/libpcp_web/src/slots.h index 4e65bab402..1cc2fdd60e 100644 --- a/src/libpcp_web/src/slots.h +++ b/src/libpcp_web/src/slots.h @@ -17,7 +17,7 @@ #include #include #include "batons.h" -#include "redis.h" +#include "keys.h" #include "maps.h" #define MAXSLOTS (1 << 14) @@ -33,12 +33,12 @@ #define KEYS_ELOADDATA "loading the dataset in memory" #define KEYS_ENOCLUSTER "ERR This instance has cluster support disabled" -typedef enum redisSlotsFlags { +typedef enum keySlotsFlags { SLOTS_NONE = 0, SLOTS_VERSION = (1 << 0), SLOTS_KEYMAP = (1 << 1), SLOTS_SEARCH = (1 << 2), -} redisSlotsFlags; +} keySlotsFlags; enum { SLOT_REQUESTS_TOTAL, @@ -53,58 +53,58 @@ enum { NUM_SLOT_METRICS }; -typedef enum redisSlotsState { +typedef enum keySlotsState { SLOTS_DISCONNECTED, SLOTS_CONNECTING, SLOTS_CONNECTED, - SLOTS_READY /* Redis version check done, keymap loaded, search schema setup completed */, + SLOTS_READY, /* version check, keymap loaded, search schema done */ SLOTS_ERR_FATAL /* fatal error, do not try to reconnect */ -} redisSlotsState; +} keySlotsState; /* note: this struct persists for reconnects */ -typedef struct redisSlots { - redisClusterAsyncContext *acc; /* cluster context */ - redisSlotsState state; /* connection state */ +typedef struct keySlots { + keyClusterAsyncContext *acc; /* cluster context */ + keySlotsState state; /* connection state */ unsigned int conn_seq; /* connection sequence (incremented for every connection) */ - unsigned int search : 1; /* RediSearch use enabled */ - unsigned int cluster : 1; /* Redis cluster mode enabled */ - redisMap *keymap; /* map command names to key position */ + unsigned int search : 1; /* search module enabled */ + unsigned int cluster : 1; /* cluster mode enabled */ + keyMap *keymap; /* map command names to key position */ void *events; /* libuv event loop */ mmv_registry_t *registry; /* MMV metrics for instrumentation */ void *map; /* MMV mapped metric values handle */ pmAtomValue *metrics[NUM_SLOT_METRICS]; /* direct handle lookup */ -} redisSlots; +} keySlots; -/* wraps the actual Redis callback and data */ -typedef struct redisSlotsReplyData { - redisSlots *slots; +/* wraps the actual callback and data */ +typedef struct keySlotsReplyData { + keySlots *slots; uint64_t start; /* time of the request (usec) */ unsigned int conn_seq; /* connection sequence when this request was issued */ size_t req_size; /* size of request */ - redisClusterCallbackFn *callback; /* actual callback */ + keyClusterCallbackFn *callback; /* actual callback */ void *arg; /* actual callback args */ -} redisSlotsReplyData; - -typedef void (*redisPhase)(redisSlots *, void *); /* phased operations */ - -extern void redisSlotsSetupMetrics(redisSlots *); -extern int redisSlotsSetMetricRegistry(redisSlots *, mmv_registry_t *); -extern redisSlots *redisSlotsInit(dict *, void *); -extern redisSlots *redisSlotsConnect(dict *, redisSlotsFlags, - redisInfoCallBack, redisDoneCallBack, void *, void *, void *); -extern void redisSlotsReconnect(redisSlots *, redisSlotsFlags, - redisInfoCallBack, redisDoneCallBack, void *, void *, void *); -extern uint64_t redisSlotsInflightRequests(redisSlots *); -extern int redisSlotsRequest(redisSlots *, sds, redisClusterCallbackFn *, void *); -extern int redisSlotsRequestFirstNode(redisSlots *slots, const sds cmd, - redisClusterCallbackFn *callback, void *arg); -extern void redisSlotsFree(redisSlots *); - -extern int redisSlotsProxyConnect(redisSlots *, - redisInfoCallBack, redisReader **, const char *, ssize_t, - redisClusterCallbackFn *, void *); -extern void redisSlotsProxyFree(redisReader *); +} keySlotsReplyData; + +typedef void (*keyPhase)(keySlots *, void *); /* phased operations */ + +extern void keySlotsSetupMetrics(keySlots *); +extern int keySlotsSetMetricRegistry(keySlots *, mmv_registry_t *); +extern keySlots *keySlotsInit(dict *, void *); +extern keySlots *keySlotsConnect(dict *, keySlotsFlags, + keysInfoCallBack, keysDoneCallBack, void *, void *, void *); +extern void keySlotsReconnect(keySlots *, keySlotsFlags, + keysInfoCallBack, keysDoneCallBack, void *, void *, void *); +extern uint64_t keysSlotsInflightRequests(keySlots *); +extern int keySlotsRequest(keySlots *, sds, keyClusterCallbackFn *, void *); +extern int keySlotsRequestFirstNode(keySlots *slots, const sds cmd, + keyClusterCallbackFn *callback, void *arg); +extern void keySlotsFree(keySlots *); + +extern int keySlotsProxyConnect(keySlots *, + keysInfoCallBack, respReader **, const char *, ssize_t, + keyClusterCallbackFn *, void *); +extern void keysSlotsProxyFree(respReader *); typedef struct { seriesBatonMagic magic; /* MAGIC_SLOTS */ @@ -112,28 +112,28 @@ typedef struct { seriesBatonPhase phases[SLOTS_PHASES]; int version; int error; - redisSlots *slots; - redisInfoCallBack info; - redisDoneCallBack done; + keySlots *slots; + keysInfoCallBack info; + keysDoneCallBack done; void *userdata; void *arg; -} redisSlotsBaton; - -extern void redis_slots_end_phase(void *); - -/* Redis reply helper routines */ -extern int testReplyError(redisReply *, const char *); -extern void reportReplyError(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, const char *, va_list); -extern int checkStatusReplyOK(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, const char *, ...); -extern int checkStreamReplyString(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, sds, const char *, ...); -extern int checkArrayReply(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, const char *, ...); -extern long long checkIntegerReply(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, const char *, ...); -extern sds checkStringReply(redisInfoCallBack, void *, - redisClusterAsyncContext *, redisReply *, const char *, ...); +} keySlotsBaton; + +extern void key_slots_end_phase(void *); + +/* Key server reply helper routines */ +extern int testReplyError(respReply *, const char *); +extern void reportReplyError(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, const char *, va_list); +extern int checkStatusReplyOK(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, const char *, ...); +extern int checkStreamReplyString(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, sds, const char *, ...); +extern int checkArrayReply(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, const char *, ...); +extern long long checkIntegerReply(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, const char *, ...); +extern sds checkStringReply(keysInfoCallBack, void *, + keyClusterAsyncContext *, respReply *, const char *, ...); #endif /* SLOTS_H */ diff --git a/src/libpcp_web/src/util.c b/src/libpcp_web/src/util.c index 7851c50782..dc808681ec 100644 --- a/src/libpcp_web/src/util.c +++ b/src/libpcp_web/src/util.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Red Hat. + * Copyright (c) 2017-2022,2024 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -976,7 +976,7 @@ pmwebapi_free_instance(instance_t *instance) sdsfree(list->name); sdsfree(list->value); if (list->valuemap) - redisMapRelease(list->valuemap); + keyMapRelease(list->valuemap); list = list->next; } @@ -1109,7 +1109,7 @@ pmwebapi_free_metric(metric_t *metric) sdsfree(list->name); sdsfree(list->value); if (list->valuemap) - redisMapRelease(list->valuemap); + keyMapRelease(list->valuemap); list = list->next; } diff --git a/src/pmproxy/src/redis.c b/src/pmproxy/src/keys.c similarity index 53% rename from src/pmproxy/src/redis.c rename to src/pmproxy/src/keys.c index a80dde391d..8f06d11399 100644 --- a/src/pmproxy/src/redis.c +++ b/src/pmproxy/src/keys.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Red Hat. + * Copyright (c) 2018-2021,2024 Red Hat. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -14,14 +14,14 @@ #include "server.h" #include "discover.h" -#define REDIS_RECONNECT_INTERVAL 2 +#define KEY_SERVER_RECONNECT_INTERVAL 2 static int search_queries; static int series_queries; -static int redis_protocol; +static int key_server_resp; static int archive_discovery; -static pmDiscoverCallBacks redis_series = { +static pmDiscoverCallBacks key_server_series = { .on_source = pmSeriesDiscoverSource, .on_closed = pmSeriesDiscoverClosed, .on_labels = pmSeriesDiscoverLabels, @@ -31,20 +31,20 @@ static pmDiscoverCallBacks redis_series = { .on_text = pmSeriesDiscoverText, }; -static pmDiscoverCallBacks redis_search = { +static pmDiscoverCallBacks key_server_search = { .on_metric = pmSearchDiscoverMetric, .on_indom = pmSearchDiscoverInDom, .on_text = pmSearchDiscoverText, }; -static pmDiscoverSettings redis_discover = { +static pmDiscoverSettings key_server_discover = { .module.on_info = proxylog, }; -static void redis_reconnect_worker(void *); +static void key_server_reconnect_worker(void *); static sds -redisfmt(redisReply *reply) +replyfmt(keyServerReply *reply) { sds c, command = sdsempty(); int i; @@ -53,44 +53,44 @@ redisfmt(redisReply *reply) return command; switch (reply->type) { - case REDIS_REPLY_STRING: + case RESP_REPLY_STRING: command = sdscatfmt(command, "$%U\r\n", (uint64_t)reply->len); command = sdscatlen(command, reply->str, reply->len); return sdscatlen(command, "\r\n", 2); - case REDIS_REPLY_ARRAY: - case REDIS_REPLY_MAP: - case REDIS_REPLY_SET: + case RESP_REPLY_ARRAY: + case RESP_REPLY_MAP: + case RESP_REPLY_SET: c = sdsempty(); for (i = 0; i < reply->elements; i++) { - sds e = redisfmt(reply->element[i]); + sds e = replyfmt(reply->element[i]); c = sdscatsds(c, e); sdsfree(e); } - if (reply->type == REDIS_REPLY_ARRAY) + if (reply->type == RESP_REPLY_ARRAY) command = sdscatfmt(command, "*%U\r\n%S", (uint64_t)reply->elements, c); - else if (reply->type == REDIS_REPLY_MAP) + else if (reply->type == RESP_REPLY_MAP) command = sdscatfmt(command, "%%%U\r\n%S", (uint64_t)reply->elements, c); - else /* (reply->type == REDIS_REPLY_SET) */ + else /* (reply->type == RESP_REPLY_SET) */ command = sdscatfmt(command, "~%U\r\n%S", (uint64_t)reply->elements, c); sdsfree(c); return command; - case REDIS_REPLY_INTEGER: + case RESP_REPLY_INTEGER: return sdscatfmt(command, ":%I\r\n", reply->integer); - case REDIS_REPLY_DOUBLE: + case RESP_REPLY_DOUBLE: command = sdscatlen(command, ",", 1); command = sdscatlen(command, reply->str, reply->len); return sdscatlen(command, "\r\n", 2); - case REDIS_REPLY_STATUS: + case RESP_REPLY_STATUS: command = sdscatlen(command, "+", 1); command = sdscatlen(command, reply->str, reply->len); return sdscatlen(command, "\r\n", 2); - case REDIS_REPLY_ERROR: + case RESP_REPLY_ERROR: command = sdscatlen(command, "-", 1); command = sdscatlen(command, reply->str, reply->len); return sdscatlen(command, "\r\n", 2); - case REDIS_REPLY_BOOL: + case RESP_REPLY_BOOL: return sdscatfmt(command, "#%s\r\n", reply->integer ? "t" : "f"); - case REDIS_REPLY_NIL: + case RESP_REPLY_NIL: return sdscat(command, "$-1\r\n"); default: break; @@ -99,91 +99,91 @@ redisfmt(redisReply *reply) } static void -on_redis_server_reply( - redisClusterAsyncContext *c, void *r, void *arg) +on_key_server_reply( + keysClusterAsyncContext *c, void *r, void *arg) { struct client *client = (struct client *)arg; - redisReply *reply = r; + keysReply *reply = r; (void)c; - client_write(client, redisfmt(reply), NULL); + client_write(client, replyfmt(reply), NULL); } void -on_redis_client_read(struct proxy *proxy, struct client *client, +on_key_client_read(struct proxy *proxy, struct client *client, ssize_t nread, const uv_buf_t *buf) { if (pmDebugOptions.pdu) - fprintf(stderr, "%s: client %p\n", "on_redis_client_read", client); + fprintf(stderr, "%s: client %p\n", "on_key_client_read", client); - if (redis_protocol == 0 || proxy->redisetup == 0 || - redisSlotsProxyConnect(proxy->slots, - proxylog, &client->u.redis.reader, - buf->base, nread, on_redis_server_reply, client) < 0) { + if (key_server_resp == 0 || proxy->keys_setup == 0 || + keySlotsProxyConnect(proxy->slots, + proxylog, &client->u.keys.reader, + buf->base, nread, on_key_server_reply, client) < 0) { client_close(client); } } void -on_redis_client_write(struct client *client) +on_key_server_client_write(struct client *client) { if (pmDebugOptions.pdu) - fprintf(stderr, "%s: client %p\n", "on_redis_client_write", client); + fprintf(stderr, "%s: client %p\n", "on_key_client_write", client); } void -on_redis_client_close(struct client *client) +on_key_client_close(struct client *client) { - redisSlotsProxyFree(client->u.redis.reader); + keySlotsProxyFree(client->u.keys.reader); } static void -on_redis_connected(void *arg) +on_key_server_connected(void *arg) { struct proxy *proxy = (struct proxy *)arg; sds message; - message = sdsnew("Redis slots"); - if (redis_protocol) + message = sdsnew("Key server slots"); + if (key_server_resp) message = sdscat(message, ", command keys"); if ((search_queries = pmSearchEnabled(proxy->slots))) - message = sdscat(message, ", RediSearch"); + message = sdscat(message, ", search"); if (series_queries) message = sdscat(message, ", schema version"); pmNotifyErr(LOG_INFO, "%s setup\n", message); sdsfree(message); - /* Redis was already connected before */ - if (proxy->redisetup == 1) + /* Key server was already connected before */ + if (proxy->keys_setup == 1) return; if (series_queries) { if (search_queries) - redis_series.next = &redis_search; - redis_discover.callbacks = redis_series; + key_server_series.next = &key_server_search; + key_server_discover.callbacks = key_server_series; } else if (search_queries) { - redis_discover.callbacks = redis_search; + key_server_discover.callbacks = key_search_search; } if (archive_discovery && (series_queries || search_queries)) { mmv_registry_t *registry = proxymetrics(proxy, METRICS_DISCOVER); - pmDiscoverSetEventLoop(&redis_discover.module, proxy->events); - pmDiscoverSetConfiguration(&redis_discover.module, proxy->config); - pmDiscoverSetMetricRegistry(&redis_discover.module, registry); - pmDiscoverSetup(&redis_discover.module, &redis_discover.callbacks, proxy); - pmDiscoverSetSlots(&redis_discover.module, proxy->slots); + pmDiscoverSetEventLoop(&key_server_discover.module, proxy->events); + pmDiscoverSetConfiguration(&key_server_discover.module, proxy->config); + pmDiscoverSetMetricRegistry(&key_server_discover.module, registry); + pmDiscoverSetup(&key_server_discover.module, &key_server_discover.callbacks, proxy); + pmDiscoverSetSlots(&key_server_discover.module, proxy->slots); } - proxy->redisetup = 1; + proxy->keys_setup = 1; } -static redisSlotsFlags -get_redis_slots_flags() +static keySlotsFlags +get_key_slots_flags() { - redisSlotsFlags flags = SLOTS_NONE; + keySlotsFlags flags = SLOTS_NONE; - if (redis_protocol) + if (key_server_resp) flags |= SLOTS_KEYMAP; if (series_queries) flags |= SLOTS_VERSION; @@ -194,21 +194,26 @@ get_redis_slots_flags() } /* - * Attempt to establish a Redis connection straight away; + * Attempt to establish a server connection straight away * which is achieved via a timer that expires immediately * during the startup process. */ void -setup_redis_module(struct proxy *proxy) +setup_keys_module(struct proxy *proxy) { sds option; - if ((option = pmIniFileLookup(config, "redis", "enabled")) && + if ((option = pmIniFileLookup(config, "keys", "enabled")) && + (strcmp(option, "false") == 0)) + return; + else if ((option = pmIniFileLookup(config, "redis", "enabled")) && (strcmp(option, "false") == 0)) return; - if ((option = pmIniFileLookup(config, "pmproxy", "redis.enabled"))) - redis_protocol = (strcmp(option, "true") == 0); + if ((option = pmIniFileLookup(config, "pmproxy", "resp.enabled"))) + key_server_resp = (strcmp(option, "true") == 0); + else if ((option = pmIniFileLookup(config, "pmproxy", "redis.enabled"))) + key_server_resp = (strcmp(option, "true") == 0); if ((option = pmIniFileLookup(config, "pmseries", "enabled"))) series_queries = (strcmp(option, "true") == 0); if ((option = pmIniFileLookup(config, "pmsearch", "enabled"))) @@ -216,57 +221,58 @@ setup_redis_module(struct proxy *proxy) if ((option = pmIniFileLookup(config, "discover", "enabled"))) archive_discovery = (strcmp(option, "true") == 0); - if (proxy->slots == NULL && (redis_protocol || series_queries || search_queries || archive_discovery)) { - mmv_registry_t *registry = proxymetrics(proxy, METRICS_REDIS); - redisSlotsFlags flags = get_redis_slots_flags(); + if (proxy->slots == NULL && + (key_server_resp || series_queries || search_queries || archive_discovery)) { + mmv_registry_t *registry = proxymetrics(proxy, METRICS_KEYS); + keySlotsFlags flags = get_key_slots_flags(); - proxy->slots = redisSlotsConnect(proxy->config, - flags, proxylog, on_redis_connected, + proxy->slots = keySlotsConnect(proxy->config, + flags, proxylog, on_key_server_connected, proxy, proxy->events, proxy); - redisSlotsSetMetricRegistry(proxy->slots, registry); - redisSlotsSetupMetrics(proxy->slots); - pmWebTimerRegister(redis_reconnect_worker, proxy); + keySlotsSetMetricRegistry(proxy->slots, registry); + keySlotsSetupMetrics(proxy->slots); + pmWebTimerRegister(key_server_reconnect_worker, proxy); } } static void -redis_reconnect_worker(void *arg) +key_server_reconnect_worker(void *arg) { struct proxy *proxy = (struct proxy *)arg; - static unsigned int wait_sec = REDIS_RECONNECT_INTERVAL; + static unsigned int wait_sec = KEY_SERVER_RECONNECT_INTERVAL; /* wait X seconds, because this timer callback is called every second */ if (wait_sec > 1) { wait_sec--; return; } - wait_sec = REDIS_RECONNECT_INTERVAL; + wait_sec = KEY_SERVER_RECONNECT_INTERVAL; /* - * skip if Redis is disabled or state is not SLOTS_DISCONNECTED + * skip if server is disabled or state is not SLOTS_DISCONNECTED */ if (!proxy->slots || proxy->slots->state != SLOTS_DISCONNECTED) return; if (pmDebugOptions.desperate) - proxylog(PMLOG_INFO, "Trying to connect to Redis ...", arg); + proxylog(PMLOG_INFO, "Trying to connect to key server ...", arg); - redisSlotsFlags flags = get_redis_slots_flags(); - redisSlotsReconnect(proxy->slots, flags, proxylog, on_redis_connected, + keySlotsFlags flags = get_key_slots_flags(); + keySlotsReconnect(proxy->slots, flags, proxylog, on_key_server_connected, proxy, proxy->events, proxy); } void -close_redis_module(struct proxy *proxy) +close_keys_module(struct proxy *proxy) { if (proxy->slots) { - redisSlotsFree(proxy->slots); + keySlotsFree(proxy->slots); proxy->slots = NULL; } if (archive_discovery) - pmDiscoverClose(&redis_discover.module); + pmDiscoverClose(&key_server_discover.module); - proxymetrics_close(proxy, METRICS_REDIS); + proxymetrics_close(proxy, METRICS_KEYS); proxymetrics_close(proxy, METRICS_DISCOVER); } diff --git a/src/pmproxy/src/search.c b/src/pmproxy/src/search.c index 7175a51d9e..cb06c8e2c4 100644 --- a/src/pmproxy/src/search.c +++ b/src/pmproxy/src/search.c @@ -484,7 +484,7 @@ pmsearch_request_done(struct client *client) pmSearchBaton *baton = (pmSearchBaton *)client->u.http.data; int sts; - /* reference to prevent freeing while waiting for a Redis reply callback */ + /* reference to prevent freeing while waiting for a reply callback */ client_get(client); if (client->u.http.parser.status_code) { diff --git a/src/pmproxy/src/series.c b/src/pmproxy/src/series.c index 82f8046852..ba9323c83c 100644 --- a/src/pmproxy/src/series.c +++ b/src/pmproxy/src/series.c @@ -922,7 +922,7 @@ pmseries_request_done(struct client *client) pmSeriesBaton *baton = (pmSeriesBaton *)client->u.http.data; int sts; - /* reference to prevent freeing while waiting for a Redis reply callback */ + /* reference to prevent freeing while waiting for a reply callback */ client_get(client); if (client->u.http.parser.status_code) { diff --git a/src/pmproxy/src/server.c b/src/pmproxy/src/server.c index 5824353ff9..f544210d2e 100644 --- a/src/pmproxy/src/server.c +++ b/src/pmproxy/src/server.c @@ -23,7 +23,7 @@ static struct { } server_metrics[] = { { .group = NULL }, /* METRICS_NOTUSED */ { .group = "server" }, /* METRICS_SERVER */ - { .group = "redis" }, /* METRICS_REDIS */ + { .group = "keys" }, /* METRICS_KEYS */ { .group = "http" }, /* METRICS_HTTP */ { .group = "pcp" }, /* METRICS_PCP */ { .group = "discover" }, /* METRICS_DISCOVER */ @@ -247,10 +247,10 @@ client_put(struct client *client) if (refcount == 0) { if (client->protocol & STREAM_PCP) on_pcp_client_close(client); + if (client->protocol & STREAM_KEYS) + on_key_client_close(client); if (client->protocol & STREAM_HTTP) on_http_client_close(client); - if (client->protocol & STREAM_REDIS) - on_redis_client_close(client); if ((client->protocol & STREAM_SECURE) && client->stream.secure) on_secure_client_close(client); if (client->buffer) @@ -290,10 +290,10 @@ on_client_write(uv_write_t *writer, int status) on_secure_client_write(client); if (client->protocol & STREAM_PCP) on_pcp_client_write(client); + else if (client->protocol & STREAM_KEYS) + on_key_client_write(client); else if (client->protocol & STREAM_HTTP) on_http_client_write(client); - else if (client->protocol & STREAM_REDIS) - on_redis_client_write(client); } sdsfree(request->buffer[0].base); @@ -410,7 +410,7 @@ client_protocol(int key) case '#': /* RESP bool */ case '%': /* RESP map */ case '~': /* RESP set */ - return STREAM_REDIS; + return STREAM_KEYS; case 0x14: /* TLS ChangeCipherSpec */ case 0x15: /* TLS Alert */ case 0x16: /* TLS Handshake */ @@ -432,15 +432,15 @@ on_protocol_read(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) if (nread < 0) return; - if ((client->protocol & (STREAM_PCP|STREAM_HTTP|STREAM_REDIS)) == 0) + if ((client->protocol & (STREAM_PCP|STREAM_HTTP|STREAM_KEYS)) == 0) client->protocol |= client_protocol(*buf->base); if (client->protocol & STREAM_PCP) on_pcp_client_read(proxy, client, nread, buf); + else if (client->protocol & STREAM_KEYS) + on_key_client_read(proxy, client, nread, buf); else if (client->protocol & STREAM_HTTP) on_http_client_read(proxy, client, nread, buf); - else if (client->protocol & STREAM_REDIS) - on_redis_client_read(proxy, client, nread, buf); else { if (pmDebugOptions.af) fprintf(stderr, "%s: unknown protocol key '%c' (0x%x)" @@ -776,7 +776,7 @@ close_proxy(struct proxy *proxy) { close_pcp_module(proxy); close_http_module(proxy); - close_redis_module(proxy); + close_keys_module(proxy); close_secure_module(proxy); } @@ -840,7 +840,7 @@ setup_proxy(uv_timer_t *arg) struct proxy *proxy = (struct proxy *)handle->data; setup_secure_module(proxy); - setup_redis_module(proxy); + setup_keys_module(proxy); setup_http_module(proxy); setup_pcp_module(proxy); } diff --git a/src/pmproxy/src/server.h b/src/pmproxy/src/server.h index 1718290388..b1647442fd 100644 --- a/src/pmproxy/src/server.h +++ b/src/pmproxy/src/server.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019,2021-2023 Red Hat. + * Copyright (c) 2018-2019,2021-2024 Red Hat. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -37,7 +37,7 @@ typedef enum proxy_registry { METRICS_NOTUSED = 0, /* special "next available" MMV cluster */ METRICS_SERVER, - METRICS_REDIS, + METRICS_KEYS, METRICS_HTTP, METRICS_PCP, METRICS_DISCOVER, @@ -84,15 +84,11 @@ typedef struct stream { typedef enum stream_protocol { STREAM_UNKNOWN = 0, STREAM_SECURE = 0x1, - STREAM_REDIS = 0x2, + STREAM_KEYS = 0x2, STREAM_HTTP = 0x4, STREAM_PCP = 0x8, } stream_protocol_t; -typedef struct redis_client { - redisReader *reader; /* RESP request handling state */ -} redis_client_t; - typedef struct http_client { http_parser parser; /* HTTP request parsing state */ struct servlet *servlet; /* servicing current request */ @@ -110,6 +106,10 @@ typedef struct http_client { #endif } http_client_t; +typedef struct key_client { + keyReader *reader; /* RESP request handling state */ +} key_client_t; + typedef struct pcp_client { pcp_proxy_state_t state; unsigned int port : 16; @@ -146,8 +146,8 @@ typedef struct client { secure_client secure; #endif union { - redis_client_t redis; http_client_t http; + key_client_t keys; pcp_client_t pcp; } u; struct proxy *proxy; @@ -163,13 +163,13 @@ typedef struct proxy { struct client *first; /* doubly linked list of clients */ struct server *servers; /* array of tcp/pipe socket servers */ unsigned int nservers; /* count of entries in server array */ - unsigned int redisetup; /* is Redis slots information setup */ + unsigned int keys_setup; /* key server slot information setup */ struct client *pending_writes; #ifdef HAVE_OPENSSL SSL_CTX *ssl; __pmSecureConfig tls; #endif - redisSlots *slots; /* mapping of Redis keys to servers */ + keySlots *slots; /* mapping of key names to servers */ struct servlet *servlets; /* linked list of http URL handlers */ mmv_registry_t *metrics[NUM_REGISTRY]; /* performance metrics */ pmAtomValue *values[NUM_VALUES]; /* local metric values */ @@ -209,16 +209,16 @@ extern void on_secure_client_close(struct client *); #define on_secure_client_close(c) do { (void)(c); } while (0) #endif -extern void on_redis_client_read(struct proxy *, struct client *, - ssize_t, const uv_buf_t *); -extern void on_redis_client_write(struct client *); -extern void on_redis_client_close(struct client *); - extern void on_http_client_read(struct proxy *, struct client *, ssize_t, const uv_buf_t *); extern void on_http_client_write(struct client *); extern void on_http_client_close(struct client *); +extern void on_key_client_read(struct proxy *, struct client *, + ssize_t, const uv_buf_t *); +extern void on_key_client_write(struct client *); +extern void on_key_client_close(struct client *); + extern void on_pcp_client_read(struct proxy *, struct client *, ssize_t, const uv_buf_t *); extern void on_pcp_client_write(struct client *); @@ -234,12 +234,12 @@ extern void close_secure_module(struct proxy *); #define close_secure_module(p) do { (void)(p); } while (0) #endif -extern void setup_redis_module(struct proxy *); -extern void close_redis_module(struct proxy *); - extern void setup_http_module(struct proxy *); extern void close_http_module(struct proxy *); +extern void setup_keys_module(struct proxy *); +extern void close_keys_module(struct proxy *); + extern void setup_pcp_module(struct proxy *); extern void close_pcp_module(struct proxy *);