From 59a83dc1ccb934291c424cd53fe5037b4dcc4cab Mon Sep 17 00:00:00 2001 From: sjahl <636687+sjahl@users.noreply.github.com> Date: Fri, 9 Jun 2023 15:03:29 -0400 Subject: [PATCH 1/4] modify configs for using a clustered redis --- .../browser/base/api.deployment.yaml | 6 ++- deploy/manifests/redis/redis-values.yaml | 31 ++++++++++++ graphql-api/package.json | 2 + graphql-api/src/cache.ts | 36 ++++++++------ graphql-api/src/config.ts | 3 +- yarn.lock | 49 ++++++++++++++++++- 6 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 deploy/manifests/redis/redis-values.yaml diff --git a/deploy/manifests/browser/base/api.deployment.yaml b/deploy/manifests/browser/base/api.deployment.yaml index 4ec3a457f..ddd70015c 100644 --- a/deploy/manifests/browser/base/api.deployment.yaml +++ b/deploy/manifests/browser/base/api.deployment.yaml @@ -31,8 +31,10 @@ spec: secretKeyRef: name: gnomad-es-elastic-user # FIXME: This depends on using "gnomad" as the ES cluster name key: elastic - - name: CACHE_REDIS_URL - value: redis://redis:6379/1 + - name: READ_CACHE_REDIS_URL + value: redis + - name: WRITE_CACHE_REDIS_URL + value: redis - name: RATE_LIMITER_REDIS_URL value: redis://redis:6379/2 - name: TRUST_PROXY diff --git a/deploy/manifests/redis/redis-values.yaml b/deploy/manifests/redis/redis-values.yaml new file mode 100644 index 000000000..9b6e1692b --- /dev/null +++ b/deploy/manifests/redis/redis-values.yaml @@ -0,0 +1,31 @@ +architecture: replication +auth: + enabled: false +sentinel: + enabled: true + resources: + requests: + cpu: '500m' + memory: '1Gi' + limits: + cpu: 1 + memory: '2Gi' +global: + storageClass: premium-rwo +nodeSelector: &nodeselector + cloud.google.com/gke-nodepool: redis +master: + persistence: &podpersistence + size: 100Gi + resources: &podresources + requests: + cpu: 1 + memory: '36Gi' + limits: + cpu: 2 + memory: '36Gi' + nodeSelector: *nodeselector +replica: + persistence: *podpersistence + resources: *podresources + nodeSelector: *nodeselector diff --git a/graphql-api/package.json b/graphql-api/package.json index 4159e6acb..1244287d2 100644 --- a/graphql-api/package.json +++ b/graphql-api/package.json @@ -24,6 +24,7 @@ "graphql": "^15.3.0", "graphql-query-complexity": "^0.7.2", "http-errors": "1.8.0", + "ioredis": "^5.3.2", "lodash": "^4.17.21", "node-fetch": "^2.6.7", "on-finished": "^2.3.0", @@ -33,6 +34,7 @@ "typescript": "^5.0.4" }, "devDependencies": { + "@types/ioredis": "^5.0.0", "fishery": "^2.2.2" } } diff --git a/graphql-api/src/cache.ts b/graphql-api/src/cache.ts index e1980b208..54e9864a3 100644 --- a/graphql-api/src/cache.ts +++ b/graphql-api/src/cache.ts @@ -1,6 +1,6 @@ import { promisify } from 'util' -import redis from 'redis' +import { Redis } from 'ioredis' import config from './config' import logger from './logger' @@ -9,16 +9,18 @@ let fetchCacheValue = () => Promise.resolve(null) let setCacheValue = () => Promise.resolve() let setCacheExpiration = () => Promise.resolve() -if (config.CACHE_REDIS_URL) { - const cacheDb = redis.createClient({ - url: config.CACHE_REDIS_URL, - retry_strategy: - process.env.NODE_ENV === 'development' - ? ({ attempt }: any) => { - logger.info('Retrying connection to cache database') - return Math.min(attempt * 100, 3000) - } - : ({ attempt }: any) => Math.min(attempt * 100, 3000), +if (config.READ_CACHE_REDIS_URL) { + const readCacheDb = new Redis({ + sentinels: [{ host: config.READ_CACHE_REDIS_URL, port: 26379 }], + name: 'mymaster', + role: 'slave', + db: 1, + }) + + const writeCacheDb = new Redis({ + sentinels: [{ host: config.WRITE_CACHE_REDIS_URL, port: 26379 }], + name: 'mymaster', + db: 1, }) const withTimeout = (fn: any, timeout: any) => { @@ -33,10 +35,16 @@ if (config.CACHE_REDIS_URL) { ]) } - fetchCacheValue = withTimeout(promisify(cacheDb.get).bind(cacheDb), config.CACHE_REQUEST_TIMEOUT) - setCacheValue = withTimeout(promisify(cacheDb.set).bind(cacheDb), config.CACHE_REQUEST_TIMEOUT) + fetchCacheValue = withTimeout( + promisify(readCacheDb.get).bind(readCacheDb), + config.CACHE_REQUEST_TIMEOUT + ) + setCacheValue = withTimeout( + promisify(writeCacheDb.set).bind(writeCacheDb), + config.CACHE_REQUEST_TIMEOUT + ) setCacheExpiration = withTimeout( - promisify(cacheDb.expire).bind(cacheDb), + promisify(writeCacheDb.expire).bind(writeCacheDb), config.CACHE_REQUEST_TIMEOUT ) } else { diff --git a/graphql-api/src/config.ts b/graphql-api/src/config.ts index 03335b457..cae7f29ca 100644 --- a/graphql-api/src/config.ts +++ b/graphql-api/src/config.ts @@ -27,7 +27,8 @@ const config: Record = { ELASTICSEARCH_QUEUE_TIMEOUT: JSON.parse(env.ELASTICSEARCH_QUEUE_TIMEOUT || '30') * 1000, ELASTICSEARCH_REQUEST_TIMEOUT: JSON.parse(env.ELASTICSEARCH_REQUEST_TIMEOUT || '60') * 1000, // Cache - CACHE_REDIS_URL: env.CACHE_REDIS_URL, + READ_CACHE_REDIS_URL: env.READ_CACHE_REDIS_URL, + WRITE_CACHE_REDIS_URL: env.WRITE_CACHE_REDIS_URL, CACHE_REQUEST_TIMEOUT: JSON.parse(env.CACHE_REQUEST_TIMEOUT || '15') * 1000, // Web server PORT: JSON.parse(env.PORT || '8000'), diff --git a/yarn.lock b/yarn.lock index 53aab8fd8..9444a6cfb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1470,6 +1470,11 @@ gud "^1.0.0" warning "^4.0.3" +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -2193,6 +2198,13 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.1.tgz#20172f9578b225f6c7da63446f56d4ce108d5a65" integrity sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ== +"@types/ioredis@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-5.0.0.tgz#c1ea7e2f3e2c5a942a27cfee6f62ddcfb23fb3e7" + integrity sha512-zJbJ3FVE17CNl5KXzdeSPtdltc4tMT3TzC6fxQS0sQngkbFZ6h+0uTafsRqu+eSLIugf6Yb0Ea0SUuRr42Nk9g== + dependencies: + ioredis "*" + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" @@ -4010,7 +4022,7 @@ clone-regexp@^2.1.0: dependencies: is-regexp "^2.0.0" -cluster-key-slot@1.1.2: +cluster-key-slot@1.1.2, cluster-key-slot@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== @@ -4724,6 +4736,11 @@ denque@^1.5.0: resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -6774,6 +6791,21 @@ invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +ioredis@*, ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -8086,11 +8118,21 @@ lodash.debounce@^4.0.8: resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + lodash.get@^4.4.2: version "4.4.2" resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ== +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.memoize@4.x: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" @@ -11103,6 +11145,11 @@ stack-utils@^2.0.3: dependencies: escape-string-regexp "^2.0.0" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + state-toggle@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe" From a341e8a62bc3d986681b818722e47a734f132689 Mon Sep 17 00:00:00 2001 From: sjahl <636687+sjahl@users.noreply.github.com> Date: Fri, 27 Oct 2023 09:21:47 -0400 Subject: [PATCH 2/4] document redis installation --- deploy/docs/RedisCache.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/deploy/docs/RedisCache.md b/deploy/docs/RedisCache.md index 23c55d2d5..331c3df3d 100644 --- a/deploy/docs/RedisCache.md +++ b/deploy/docs/RedisCache.md @@ -1,5 +1,26 @@ # Redis Cache +### Deploying the redis cache + +#### Single node + +- The configs in the deploy/manifests/redis folder can be deployed with kustomize: + + ``` + cd deploy/manifests/redis + # ensure your kubectl context is pointed at the desired GKE cluster + kubectl apply -k . + ``` + +#### Clustered + +- Provided in this repository is a Helm values file that can deploy a clustered redis with the bitnami/redis Helm chart: + + ``` + helm repo add bitnami https://charts.bitnami.com/bitnami + helm install redis -f deploy/manifests/redis/redis-values.yaml + ``` + ### Connect to Redis cache - Get the name of the Redis pod. From 907c97aa8803059e6a7c49e5a46871e6f5c86f90 Mon Sep 17 00:00:00 2001 From: sjahl <636687+sjahl@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:43:45 -0400 Subject: [PATCH 3/4] update to ioredis for rate limit db --- .../manifests/browser/base/api.deployment.yaml | 2 +- graphql-api/src/graphql/rate-limiting.ts | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/deploy/manifests/browser/base/api.deployment.yaml b/deploy/manifests/browser/base/api.deployment.yaml index ddd70015c..f5d7d61fc 100644 --- a/deploy/manifests/browser/base/api.deployment.yaml +++ b/deploy/manifests/browser/base/api.deployment.yaml @@ -36,7 +36,7 @@ spec: - name: WRITE_CACHE_REDIS_URL value: redis - name: RATE_LIMITER_REDIS_URL - value: redis://redis:6379/2 + value: redis - name: TRUST_PROXY valueFrom: configMapKeyRef: diff --git a/graphql-api/src/graphql/rate-limiting.ts b/graphql-api/src/graphql/rate-limiting.ts index c39fc4c0c..2881f83ab 100644 --- a/graphql-api/src/graphql/rate-limiting.ts +++ b/graphql-api/src/graphql/rate-limiting.ts @@ -1,17 +1,12 @@ -import redis from 'redis' +import { Redis } from 'ioredis' import config from '../config' import { UserVisibleError } from '../errors' import logger from '../logger' -const rateLimitDb = redis.createClient({ - url: config.RATE_LIMITER_REDIS_URL, - retry_strategy: - process.env.NODE_ENV === 'development' - ? ({ attempt }: any) => { - logger.info('Retrying connection to rate limit database') - return Math.min(attempt * 100, 3000) - } - : ({ attempt }: any) => Math.min(attempt * 100, 3000), +const rateLimitDb = new Redis({ + sentinels: [{ host: config.RATE_LIMITER_REDIS_URL, port: 26379 }], + name: 'mymaster', + db: 2, }) const increaseRateLimitCounter = (key: any, value: any) => { @@ -19,7 +14,7 @@ const increaseRateLimitCounter = (key: any, value: any) => { new Promise((resolve, reject) => { rateLimitDb .multi() - .set([key, 0, 'EX', 59, 'NX']) + .set(key, 0, 'EX', 59, 'NX') .incrby(key, value) .exec((err: any, replies: any) => { if (err) { From 560ce8dc2d136dc97e090deeb8313c783528e71b Mon Sep 17 00:00:00 2001 From: sjahl <636687+sjahl@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:57:03 -0400 Subject: [PATCH 4/4] Support mulpitle connection modes --- .../browser/base/api.deployment.yaml | 12 +++--- deploy/manifests/redis/redis-values.yaml | 1 + graphql-api/src/cache.ts | 38 +++++++++++++------ graphql-api/src/config.ts | 9 +++-- graphql-api/src/graphql/rate-limiting.ts | 20 +++++++--- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/deploy/manifests/browser/base/api.deployment.yaml b/deploy/manifests/browser/base/api.deployment.yaml index f5d7d61fc..3f483d778 100644 --- a/deploy/manifests/browser/base/api.deployment.yaml +++ b/deploy/manifests/browser/base/api.deployment.yaml @@ -31,12 +31,12 @@ spec: secretKeyRef: name: gnomad-es-elastic-user # FIXME: This depends on using "gnomad" as the ES cluster name key: elastic - - name: READ_CACHE_REDIS_URL - value: redis - - name: WRITE_CACHE_REDIS_URL - value: redis - - name: RATE_LIMITER_REDIS_URL - value: redis + - name: REDIS_HOST + value: 'redis' + - name: REDIS_PORT + value: '26379' + - name: REDIS_USE_SENTINEL + value: 'true' - name: TRUST_PROXY valueFrom: configMapKeyRef: diff --git a/deploy/manifests/redis/redis-values.yaml b/deploy/manifests/redis/redis-values.yaml index 9b6e1692b..730991c5a 100644 --- a/deploy/manifests/redis/redis-values.yaml +++ b/deploy/manifests/redis/redis-values.yaml @@ -3,6 +3,7 @@ auth: enabled: false sentinel: enabled: true + masterSet: gnomad resources: requests: cpu: '500m' diff --git a/graphql-api/src/cache.ts b/graphql-api/src/cache.ts index 54e9864a3..be7fb2b64 100644 --- a/graphql-api/src/cache.ts +++ b/graphql-api/src/cache.ts @@ -9,19 +9,33 @@ let fetchCacheValue = () => Promise.resolve(null) let setCacheValue = () => Promise.resolve() let setCacheExpiration = () => Promise.resolve() -if (config.READ_CACHE_REDIS_URL) { - const readCacheDb = new Redis({ - sentinels: [{ host: config.READ_CACHE_REDIS_URL, port: 26379 }], - name: 'mymaster', - role: 'slave', - db: 1, - }) +if (config.REDIS_HOST) { + let readCacheDb + let writeCacheDb + if (config.REDIS_USE_SENTINEL) { + readCacheDb = new Redis({ + sentinels: [{ host: config.REDIS_HOST, port: config.REDIS_PORT }], + name: config.REDIS_GROUP_NAME, + role: 'slave', + db: 1, + }) - const writeCacheDb = new Redis({ - sentinels: [{ host: config.WRITE_CACHE_REDIS_URL, port: 26379 }], - name: 'mymaster', - db: 1, - }) + writeCacheDb = new Redis({ + sentinels: [{ host: config.REDIS_HOST, port: config.REDIS_PORT }], + name: config.REDIS_GROUP_NAME, + db: 1, + }) + } else { + readCacheDb = new Redis({ + host: config.REDIS_HOST, + db: 1, + }) + + writeCacheDb = new Redis({ + host: config.REDIS_HOST, + db: 1, + }) + } const withTimeout = (fn: any, timeout: any) => { return (...args: any[]) => diff --git a/graphql-api/src/config.ts b/graphql-api/src/config.ts index cae7f29ca..a42d11d4a 100644 --- a/graphql-api/src/config.ts +++ b/graphql-api/src/config.ts @@ -27,8 +27,10 @@ const config: Record = { ELASTICSEARCH_QUEUE_TIMEOUT: JSON.parse(env.ELASTICSEARCH_QUEUE_TIMEOUT || '30') * 1000, ELASTICSEARCH_REQUEST_TIMEOUT: JSON.parse(env.ELASTICSEARCH_REQUEST_TIMEOUT || '60') * 1000, // Cache - READ_CACHE_REDIS_URL: env.READ_CACHE_REDIS_URL, - WRITE_CACHE_REDIS_URL: env.WRITE_CACHE_REDIS_URL, + REDIS_GROUP_NAME: env.REDIS_GROUP_NAME || 'gnomad', + REDIS_HOST: env.REDIS_HOST, + REDIS_PORT: JSON.parse(env.REDIS_PORT || '6379'), + REDIS_USE_SENTINEL: env.REDIS_USE_SENTINEL, CACHE_REQUEST_TIMEOUT: JSON.parse(env.CACHE_REQUEST_TIMEOUT || '15') * 1000, // Web server PORT: JSON.parse(env.PORT || '8000'), @@ -39,10 +41,9 @@ const config: Record = { MAX_QUERY_COST: JSON.parse(env.MAX_QUERY_COST || '25'), MAX_QUERY_COST_PER_MINUTE: JSON.parse(env.MAX_QUERY_COST_PER_MINUTE || '100'), MAX_REQUESTS_PER_MINUTE: JSON.parse(env.MAX_REQUESTS_PER_MINUTE || '30'), - RATE_LIMITER_REDIS_URL: env.RATE_LIMITER_REDIS_URL, } -const requiredConfig = ['ELASTICSEARCH_URL', 'RATE_LIMITER_REDIS_URL'] +const requiredConfig = ['ELASTICSEARCH_URL'] for (const setting of requiredConfig) { if (!config[setting]) { diff --git a/graphql-api/src/graphql/rate-limiting.ts b/graphql-api/src/graphql/rate-limiting.ts index 2881f83ab..de5c0e5a7 100644 --- a/graphql-api/src/graphql/rate-limiting.ts +++ b/graphql-api/src/graphql/rate-limiting.ts @@ -3,11 +3,21 @@ import config from '../config' import { UserVisibleError } from '../errors' import logger from '../logger' -const rateLimitDb = new Redis({ - sentinels: [{ host: config.RATE_LIMITER_REDIS_URL, port: 26379 }], - name: 'mymaster', - db: 2, -}) +let rateLimitDb: Redis + +if (config.REDIS_USE_SENTINEL) { + rateLimitDb = new Redis({ + sentinels: [{ host: config.REDIS_HOST, port: config.REDIS_PORT }], + name: config.REDIS_GROUP_NAME, + db: 2, + }) +} else { + rateLimitDb = new Redis({ + host: config.REDIS_HOST, + port: config.REDIS_PORT, + db: 2, + }) +} const increaseRateLimitCounter = (key: any, value: any) => { return Promise.race([