diff --git a/public/v1/components/examples.yaml b/public/v1/components/examples.yaml index 747b0723..81ea5257 100644 --- a/public/v1/components/examples.yaml +++ b/public/v1/components/examples.yaml @@ -75,6 +75,14 @@ components: "packets": 6 } } + createMeasurementPingLocationsMeasurementId: + summary: 'ping: previous measurement id' + value: + { + "type": "ping", + "target": "cdn.jsdelivr.net", + "locations": "1wzMrzLBZfaPoT1c" + } createMeasurementResponse: value: { diff --git a/public/v1/components/schemas.yaml b/public/v1/components/schemas.yaml index 6f4f79d2..8b3331aa 100644 --- a/public/v1/components/schemas.yaml +++ b/public/v1/components/schemas.yaml @@ -76,12 +76,18 @@ components: maximum: 200 default: 1 MeasurementLocations: - type: array - description: | - An array of locations from which to run the measurement. - Each object specifies a location using one or multiple keys. - items: - $ref: 'schemas.yaml#/components/schemas/MeasurementLocationOption' + oneOf: + - type: array + description: | + An array of locations from which to run the measurement. Each object specifies a location using one or multiple keys. + items: + $ref: 'schemas.yaml#/components/schemas/MeasurementLocationOption' + - type: string + description: | + `id` of a previous measurement - its probes are reused for the new measurement and returned in the same order. + Measurement type and options are not reused and need to be specified in the request. + Note that this option only works for the lifetime of the original measurement + and will result in a `422` response for expired or invalid `id` values. MeasurementOptions: anyOf: - $ref: 'schemas.yaml#/components/schemas/MeasurementPingOptions' @@ -571,6 +577,7 @@ components: - continent - region - country + - state - city - asn - network @@ -716,6 +723,7 @@ components: anyOf: - $ref: 'schemas.yaml#/components/schemas/InProgressTestResult' - $ref: 'schemas.yaml#/components/schemas/FailedTestResult' + - $ref: 'schemas.yaml#/components/schemas/OfflineTestResult' - $ref: 'schemas.yaml#/components/schemas/FinishedPingTestResult' - $ref: 'schemas.yaml#/components/schemas/FinishedTracerouteTestResult' - $ref: 'schemas.yaml#/components/schemas/FinishedDnsTestResult' @@ -757,6 +765,20 @@ components: $ref: 'schemas.yaml#/components/schemas/FailedTestStatus' rawOutput: $ref: 'schemas.yaml#/components/schemas/TestRawOutput' + OfflineTestResult: + type: object + title: OfflineTestResult + description: | + Represents an `offline` test where the requested probe is currently offline and most fields are not available. + Only possible when passing an `id` of a previous measurement to the `locations` field. + required: + - status + - rawOutput + properties: + status: + $ref: 'schemas.yaml#/components/schemas/OfflineTestStatus' + rawOutput: + $ref: 'schemas.yaml#/components/schemas/TestRawOutput' FinishedPingTestResult: title: FinishedPingTestResult allOf: @@ -1118,6 +1140,10 @@ components: allOf: - $ref: 'schemas.yaml#/components/schemas/BaseTestStatus' - const: failed + OfflineTestStatus: + allOf: + - $ref: 'schemas.yaml#/components/schemas/BaseTestStatus' + - const: offline TimingPacketRtt: type: number description: The round-trip time for this packet. diff --git a/public/v1/spec.yaml b/public/v1/spec.yaml index da8b58ae..83830ce3 100644 --- a/public/v1/spec.yaml +++ b/public/v1/spec.yaml @@ -47,6 +47,7 @@ paths: - Set the `inProgressUpdates` option to `true` if the application is running in interactive mode so that the user sees the results right away. - If the application is interactive by default but also implements a "CI" mode to be used in scripts, do not set the flag in the CI mode. + - To perform multiple measurements from exactly the same probes, create a single measurement first, then pass its `id` in the `locations` option for the other measurements. requestBody: content: application/json: @@ -61,6 +62,8 @@ paths: $ref: 'components/examples.yaml#/components/examples/createMeasurementPingLocationsMagic' pingCustom: $ref: 'components/examples.yaml#/components/examples/createMeasurementPingCustom' + pingLocationsMeasurementId: + $ref: 'components/examples.yaml#/components/examples/createMeasurementPingLocationsMeasurementId' responses: '202': $ref: 'components/responses.yaml#/components/responses/measurements202' diff --git a/src/lib/adopted-probes.ts b/src/lib/adopted-probes.ts index 0b0b9e26..993f52d3 100644 --- a/src/lib/adopted-probes.ts +++ b/src/lib/adopted-probes.ts @@ -200,10 +200,6 @@ export class AdoptedProbes { const adoptedValue = _.get(adoptedProbe, adoptedField) as string | number; const connectedValue = _.get(connectedProbe, connectedField) as string | number; - if (!adoptedValue && !connectedValue) { // undefined and null values are treated equal and don't require sync - return; - } - if (adoptedValue !== connectedValue) { updateObject[adoptedField] = connectedValue; } diff --git a/src/lib/geoip/client.ts b/src/lib/geoip/client.ts index c6aa89f9..dc8aac13 100644 --- a/src/lib/geoip/client.ts +++ b/src/lib/geoip/client.ts @@ -7,7 +7,6 @@ import type { ProbeLocation } from '../../probe/types.js'; import RedisCache from '../cache/redis-cache.js'; import { getRedisClient } from '../redis/client.js'; import { scopedLogger } from '../logger.js'; -import { getRegionByCountry } from '../location/location.js'; import { isAddrWhitelisted } from './whitelist.js'; import { ipinfoLookup } from './providers/ipinfo.js'; import { fastlyLookup } from './providers/fastly.js'; @@ -16,8 +15,8 @@ import { ipmapLookup } from './providers/ipmap.js'; import { type Ip2LocationBundledResponse, ip2LocationLookup } from './providers/ip2location.js'; import { isHostingOverrides } from './overrides.js'; -export type LocationInfo = Omit; type Provider = 'ipmap' | 'ip2location' | 'ipinfo' | 'maxmind' | 'fastly'; +export type LocationInfo = ProbeLocation & {isHosting: boolean | null}; export type LocationInfoWithProvider = LocationInfo & {provider: Provider}; export type NetworkInfo = { network: string; @@ -32,8 +31,8 @@ export const createGeoipClient = (): GeoipClient => new GeoipClient(new RedisCac export default class GeoipClient { constructor (private readonly cache: CacheInterface) {} - async lookup (addr: string): Promise { - let isHosting = undefined; + async lookup (addr: string): Promise { + let isHosting = null; const results = await Promise .allSettled([ this.lookupWithCache(`geoip:ipinfo:${addr}`, async () => ipinfoLookup(addr)), @@ -43,7 +42,7 @@ export default class GeoipClient { this.lookupWithCache(`geoip:fastly:${addr}`, async () => fastlyLookup(addr)), ]) .then(([ ipinfo, ip2location, maxmind, ipmap, fastly ]) => { - isHosting = ip2location.status === 'fulfilled' ? ip2location.value.isHosting : undefined; + isHosting = ip2location.status === 'fulfilled' ? ip2location.value.location.isHosting : null; const fulfilled: (LocationInfoWithProvider | null)[] = []; // Providers here are pushed in a desc prioritized order @@ -82,14 +81,12 @@ export default class GeoipClient { } } - const region = getRegionByCountry(match.country); - return { continent: match.continent, country: match.country, state: match.state, city: match.city, - region, + region: match.region, normalizedCity: match.normalizedCity, asn: Number(networkMatch.asn), latitude: Number(match.latitude), diff --git a/src/lib/geoip/fake-client.ts b/src/lib/geoip/fake-client.ts index 53502033..9f9a7cd2 100644 --- a/src/lib/geoip/fake-client.ts +++ b/src/lib/geoip/fake-client.ts @@ -1,10 +1,10 @@ -import type { ProbeLocation } from '../../probe/types.js'; +import type { LocationInfo } from './client.js'; -export const fakeLookup = (): ProbeLocation => { +export const fakeLookup = (): LocationInfo => { return { continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -13,5 +13,6 @@ export const fakeLookup = (): ProbeLocation => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: null, }; }; diff --git a/src/lib/geoip/providers/fastly.ts b/src/lib/geoip/providers/fastly.ts index 38ea34b2..42390ffe 100644 --- a/src/lib/geoip/providers/fastly.ts +++ b/src/lib/geoip/providers/fastly.ts @@ -1,4 +1,5 @@ import got from 'got'; +import { getRegionByCountry } from '../../location/location.js'; import { getCity } from '../city-approximation.js'; import type { LocationInfo } from '../client.js'; import { @@ -38,8 +39,9 @@ export const fastlyLookup = async (addr: string): Promise => { return { continent: data.continent_code, + region: getRegionByCountry(data.country_code), country: data.country_code, - state: data.country_code === 'US' ? data.region : undefined, + state: data.country_code === 'US' ? data.region : null, city: normalizeCityNamePublic(city), normalizedCity: normalizeCityName(city), asn: result.as.number, @@ -47,5 +49,6 @@ export const fastlyLookup = async (addr: string): Promise => { longitude: data.longitude, network: result.as.name, normalizedNetwork: normalizeNetworkName(result.as.name), + isHosting: null, }; }; diff --git a/src/lib/geoip/providers/ip2location.ts b/src/lib/geoip/providers/ip2location.ts index c8a3368d..d18f3fc8 100644 --- a/src/lib/geoip/providers/ip2location.ts +++ b/src/lib/geoip/providers/ip2location.ts @@ -2,6 +2,7 @@ import got from 'got'; import config from 'config'; import { getContinentByCountry, + getRegionByCountry, getStateIsoByName, } from '../../location/location.js'; import type { LocationInfo } from '../client.js'; @@ -30,7 +31,6 @@ type Ip2LocationResponse = { export type Ip2LocationBundledResponse = { location: LocationInfo, - isHosting: boolean | undefined, isProxy: boolean, }; @@ -50,7 +50,8 @@ export const ip2LocationLookup = async (addr: string): Promise => { return { continent: result.country ? getContinentByCountry(result.country) : '', - state: result.country === 'US' && result.region ? getStateIsoByName(result.region) : undefined, + region: result.country ? getRegionByCountry(result.country) : '', + state: result.country === 'US' && result.region ? getStateIsoByName(result.region) : null, country: result.country ?? '', city: normalizeCityNamePublic(city), normalizedCity: normalizeCityName(city), @@ -43,6 +44,6 @@ export const ipinfoLookup = async (addr: string): Promise => { longitude: Number(lon), network, normalizedNetwork: normalizeNetworkName(network), - isHosting: result.privacy?.hosting, + isHosting: result.privacy?.hosting ?? null, }; }; diff --git a/src/lib/geoip/providers/ipmap.ts b/src/lib/geoip/providers/ipmap.ts index efb2de90..515ebbb4 100644 --- a/src/lib/geoip/providers/ipmap.ts +++ b/src/lib/geoip/providers/ipmap.ts @@ -1,5 +1,5 @@ import got from 'got'; -import { getContinentByCountry } from '../../location/location.js'; +import { getContinentByCountry, getRegionByCountry } from '../../location/location.js'; import type { LocationInfo } from '../client.js'; import { normalizeCityName, @@ -24,7 +24,8 @@ export const ipmapLookup = async (addr: string): Promise => { return { continent: location.countryCodeAlpha2 ? getContinentByCountry(location.countryCodeAlpha2) : '', - state: location.countryCodeAlpha2 === 'US' ? location.stateAnsiCode : undefined, + region: location.countryCodeAlpha2 ? getRegionByCountry(location.countryCodeAlpha2) : '', + state: location.countryCodeAlpha2 === 'US' && location.stateAnsiCode ? location.stateAnsiCode : null, country: location.countryCodeAlpha2 ?? '', city: normalizeCityNamePublic(city), normalizedCity: normalizeCityName(city), @@ -33,5 +34,6 @@ export const ipmapLookup = async (addr: string): Promise => { longitude: Number(location.longitude) ?? 0, network: '', normalizedNetwork: '', + isHosting: null, }; }; diff --git a/src/lib/geoip/providers/maxmind.ts b/src/lib/geoip/providers/maxmind.ts index 09ffee21..47cf2119 100644 --- a/src/lib/geoip/providers/maxmind.ts +++ b/src/lib/geoip/providers/maxmind.ts @@ -9,6 +9,7 @@ import { normalizeNetworkName, } from '../utils.js'; import { getCity } from '../city-approximation.js'; +import { getRegionByCountry } from '../../location/location.js'; const client = new WebServiceClient(config.get('maxmind.accountId'), config.get('maxmind.licenseKey')); @@ -39,8 +40,9 @@ export const maxmindLookup = async (addr: string): Promise => { return { continent: data.continent?.code ?? '', + region: data.country?.isoCode ? getRegionByCountry(data.country?.isoCode) : '', country: data.country?.isoCode ?? '', - state: data.country?.isoCode === 'US' ? data.subdivisions?.map(s => s.isoCode)[0] ?? '' : undefined, + state: data.country?.isoCode === 'US' ? data.subdivisions?.map(s => s.isoCode)[0] ?? '' : null, city: normalizeCityNamePublic(city), normalizedCity: normalizeCityName(city), asn: data.traits?.autonomousSystemNumber ?? 0, @@ -48,5 +50,6 @@ export const maxmindLookup = async (addr: string): Promise => { longitude: data.location?.longitude ?? 0, network: data.traits?.isp ?? '', normalizedNetwork: normalizeNetworkName(data.traits?.isp ?? ''), + isHosting: null, }; }; diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts deleted file mode 100644 index 9451ff24..00000000 --- a/src/lib/http/middleware/ratelimit.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { Context, Next } from 'koa'; -import requestIp from 'request-ip'; -import type { RateLimiterRes } from 'rate-limiter-flexible'; -import createHttpError from 'http-errors'; -import rateLimiter, { defaultState } from '../../ratelimiter.js'; -import type { MeasurementRequest } from '../../../measurement/types.js'; - -const setResponseHeaders = (ctx: Context, response: RateLimiterRes) => { - ctx.set('X-RateLimit-Reset', `${Math.round(response.msBeforeNext / 1000)}`); - ctx.set('X-RateLimit-Limit', `${rateLimiter.points}`); - ctx.set('X-RateLimit-Remaining', `${response.remainingPoints}`); -}; - -export const rateLimitHandler = () => async (ctx: Context, next: Next) => { - const { isAdmin } = ctx; - const clientIp = requestIp.getClientIp(ctx.req) ?? ''; - const request = ctx.request.body as MeasurementRequest; - const limit = request.locations.some(l => l.limit) ? request.locations.reduce((sum, { limit = 1 }) => sum + limit, 0) : request.limit; - - if (isAdmin) { - return next(); - } - - const currentState = await rateLimiter.get(clientIp) ?? defaultState as RateLimiterRes; - - if (currentState.remainingPoints < limit) { - setResponseHeaders(ctx, currentState); - throw createHttpError(429, 'API rate limit exceeded.', { type: 'rate_limit_exceeded' }); - } - - await next(); - const response = ctx.response.body as object & { probesCount?: number }; - - if (!('probesCount' in response) || typeof response.probesCount !== 'number') { - throw new Error('Missing probesCount field in response object'); - } - - const newState = await rateLimiter.penalty(clientIp, response.probesCount); - setResponseHeaders(ctx, newState); -}; diff --git a/src/lib/ratelimiter.ts b/src/lib/ratelimiter.ts index f82871fd..560561cd 100644 --- a/src/lib/ratelimiter.ts +++ b/src/lib/ratelimiter.ts @@ -1,16 +1,13 @@ import config from 'config'; +import type { Context } from 'koa'; import { RateLimiterRedis } from 'rate-limiter-flexible'; +import requestIp from 'request-ip'; +import type { RateLimiterRes } from 'rate-limiter-flexible'; import { createRedisClient } from './redis/client.js'; +import createHttpError from 'http-errors'; const redisClient = await createRedisClient({ legacyMode: true }); -export const defaultState = { - remainingPoints: config.get('measurement.rateLimit'), - msBeforeNext: config.get('measurement.rateLimitReset') * 1000, - consumedPoints: 0, - isFirstInDuration: true, -}; - const rateLimiter = new RateLimiterRedis({ storeClient: redisClient, keyPrefix: 'rate', @@ -18,4 +15,25 @@ const rateLimiter = new RateLimiterRedis({ duration: config.get('measurement.rateLimitReset'), }); +const setRateLimitHeaders = (ctx: Context, result: RateLimiterRes) => { + ctx.set('X-RateLimit-Reset', `${Math.round(result.msBeforeNext / 1000)}`); + ctx.set('X-RateLimit-Limit', `${rateLimiter.points}`); + ctx.set('X-RateLimit-Remaining', `${result.remainingPoints}`); +}; + +export const rateLimit = async (ctx: Context, numberOfProbes: number) => { + if (ctx['isAdmin']) { + return; + } + + try { + const clientIp = requestIp.getClientIp(ctx.req) ?? ''; + const result = await rateLimiter.consume(clientIp, numberOfProbes); + setRateLimitHeaders(ctx, result); + } catch (error) { + setRateLimitHeaders(ctx, error as RateLimiterRes); + throw createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' }); + } +}; + export default rateLimiter; diff --git a/src/lib/redis/client.ts b/src/lib/redis/client.ts index 5b3c4305..57fc91d5 100644 --- a/src/lib/redis/client.ts +++ b/src/lib/redis/client.ts @@ -7,7 +7,7 @@ import { type RedisFunctions, } from 'redis'; import { scopedLogger } from '../logger.js'; -import { count, recordResult, type RedisScripts } from './scripts.js'; +import { count, recordResult, markFinished, type RedisScripts } from './scripts.js'; const logger = scopedLogger('redis-client'); @@ -23,7 +23,7 @@ export const createRedisClient = async (options?: RedisClientOptions): Promise; + transformArguments (key: string): string[]; transformReply (reply: number): number; } & { SHA1: string; @@ -19,9 +19,19 @@ export type RecordResultScript = { SHA1: string; }; +export type MarkFinishedScript = { + NUMBER_OF_KEYS: number; + SCRIPT: string; + transformArguments (measurementId: string): string[]; + transformReply (): null; +} & { + SHA1: string; +}; + export type RedisScripts = { count: CountScript; recordResult: RecordResultScript; + markFinished: MarkFinishedScript; }; export const count: CountScript = defineScript({ @@ -53,13 +63,14 @@ export const recordResult: RecordResultScript = defineScript({ local data = KEYS[3] local date = KEYS[4] local key = 'gp:measurement:'..measurementId + local awaitingKey = key..':probes_awaiting' - local probesAwaiting = redis.call('GET', key..':probes_awaiting') + local probesAwaiting = redis.call('GET', awaitingKey) if not probesAwaiting then return end - probesAwaiting = redis.call('DECR', key..':probes_awaiting') + probesAwaiting = redis.call('DECR', awaitingKey) redis.call('JSON.SET', key, '$.results['..testId..'].result', data) redis.call('JSON.SET', key, '$.updatedAt', date) @@ -67,10 +78,6 @@ export const recordResult: RecordResultScript = defineScript({ return end - redis.call('HDEL', 'gp:in-progress', measurementId) - redis.call('DEL', key..':probes_awaiting') - redis.call('JSON.SET', key, '$.status', '"finished"') - return redis.call('JSON.GET', key) `, transformArguments (measurementId, testId, data) { @@ -80,3 +87,22 @@ export const recordResult: RecordResultScript = defineScript({ return JSON.parse(reply) as MeasurementRecord | null; }, }); + +export const markFinished: MarkFinishedScript = defineScript({ + NUMBER_OF_KEYS: 1, + SCRIPT: ` + local measurementId = KEYS[1] + local key = 'gp:measurement:'..measurementId + local awaitingKey = key..':probes_awaiting' + + redis.call('HDEL', 'gp:in-progress', measurementId) + redis.call('DEL', awaitingKey) + redis.call('JSON.SET', key, '$.status', '"finished"') + `, + transformArguments (measurementId) { + return [ measurementId ]; + }, + transformReply () { + return null; + }, +}); diff --git a/src/measurement/route/create-measurement.ts b/src/measurement/route/create-measurement.ts index 6084108c..0569bd4b 100644 --- a/src/measurement/route/create-measurement.ts +++ b/src/measurement/route/create-measurement.ts @@ -2,18 +2,15 @@ import config from 'config'; import type { Context } from 'koa'; import type Router from '@koa/router'; import { getMeasurementRunner } from '../runner.js'; -import type { MeasurementRequest } from '../types.js'; import { bodyParser } from '../../lib/http/middleware/body-parser.js'; import { validate } from '../../lib/http/middleware/validate.js'; import { schema } from '../schema/global-schema.js'; -import { rateLimitHandler } from '../../lib/http/middleware/ratelimit.js'; const hostConfig = config.get('server.host'); const runner = getMeasurementRunner(); const handle = async (ctx: Context): Promise => { - const request = ctx.request.body as MeasurementRequest; - const { measurementId, probesCount } = await runner.run(request); + const { measurementId, probesCount } = await runner.run(ctx); ctx.status = 202; ctx.set('Location', `${hostConfig}/v1/measurements/${measurementId}`); @@ -25,5 +22,5 @@ const handle = async (ctx: Context): Promise => { }; export const registerCreateMeasurementRoute = (router: Router): void => { - router.post('/measurements', '/measurements', bodyParser(), validate(schema), rateLimitHandler(), handle); + router.post('/measurements', '/measurements', bodyParser(), validate(schema), handle); }; diff --git a/src/measurement/route/get-measurement.ts b/src/measurement/route/get-measurement.ts index 143a3e41..3d47ec01 100644 --- a/src/measurement/route/get-measurement.ts +++ b/src/measurement/route/get-measurement.ts @@ -12,7 +12,7 @@ const handle = async (ctx: ParameterizedContext { - const probes = await this.router.findMatchingProbes(request.locations, request.limit); + async run (ctx: Context): Promise<{measurementId: string; probesCount: number;}> { + const userRequest = ctx.request.body as UserRequest; + const { onlineProbesMap, allProbes, request } = await this.router.findMatchingProbes(userRequest); - if (probes.length === 0) { + if (allProbes.length === 0) { throw createHttpError(422, 'No suitable probes found.', { type: 'no_probes_found' }); } - const measurementId = await this.store.createMeasurement(request, probes); + await this.checkRateLimit(ctx, onlineProbesMap.size); + + const measurementId = await this.store.createMeasurement(request, onlineProbesMap, allProbes); + + if (onlineProbesMap.size) { + this.sendToProbes(measurementId, onlineProbesMap, request); + // If all selected probes are offline, immediately mark measurement as finished + } else { + await this.store.markFinished(measurementId); + } - this.sendToProbes(measurementId, probes, request); this.metrics.recordMeasurement(request.type); - return { measurementId, probesCount: probes.length }; + return { measurementId, probesCount: allProbes.length }; } async recordProgress (data: MeasurementProgressMessage): Promise { @@ -44,17 +49,17 @@ export class MeasurementRunner { } async recordResult (data: MeasurementResultMessage): Promise { - const record = await this.redis.recordResult(data.measurementId, data.testId, data.result); + const record = await this.store.storeMeasurementResult(data); if (record) { this.metrics.recordMeasurementTime(record.type, Date.now() - new Date(record.createdAt).getTime()); } } - private sendToProbes (measurementId: string, probes: Probe[], request: MeasurementRequest) { + private sendToProbes (measurementId: string, onlineProbesMap: Map, request: MeasurementRequest) { let inProgressProbes = 0; const maxInProgressProbes = config.get('measurement.maxInProgressProbes'); - probes.forEach((probe, index) => { + onlineProbesMap.forEach((probe, index) => { const inProgressUpdates = request.inProgressUpdates && inProgressProbes++ < maxInProgressProbes; this.io.of('probes').to(probe.client).emit('probe:measurement:request', { measurementId, @@ -76,7 +81,7 @@ let runner: MeasurementRunner; export const getMeasurementRunner = () => { if (!runner) { - runner = new MeasurementRunner(getWsServer(), getRedisClient(), getMeasurementStore(), getProbeRouter(), getMetricsAgent()); + runner = new MeasurementRunner(getWsServer(), getMeasurementStore(), getProbeRouter(), rateLimit, getMetricsAgent()); } return runner; diff --git a/src/measurement/schema/location-schema.ts b/src/measurement/schema/location-schema.ts index d009028e..11bf8967 100644 --- a/src/measurement/schema/location-schema.ts +++ b/src/measurement/schema/location-schema.ts @@ -12,22 +12,25 @@ const measurementConfig = config.get<{limits: {global: number; location: number} const normalizeValue = (value: string): string => anyAscii(value); -export const schema = Joi.array().items(Joi.object().keys({ - continent: Joi.string().valid(...Object.keys(continents)).insensitive() - .messages({ 'any.only': 'The continent must be a valid two-letter continent code' }), - region: Joi.string().valid(...regionNames).insensitive(), - country: Joi.string().valid(...Object.keys(countries)).insensitive() - .messages({ 'any.only': 'The country must be a valid two-letter ISO code' }), - state: Joi.string().valid(...Object.values(states)).insensitive() - .messages({ 'any.only': 'The US state must be a valid two-letter code, e.g. CA' }), - city: Joi.string().min(1).max(128).lowercase().custom(normalizeValue), - network: Joi.string().min(1).max(128).lowercase().custom(normalizeValue), - asn: Joi.number().integer().positive(), - magic: Joi.string().min(1).lowercase().custom(normalizeValue), - tags: Joi.array().items(Joi.string().min(1).max(128).lowercase().custom(normalizeValue)), - limit: Joi.number().min(1).max(measurementConfig.limits.location).when(Joi.ref('/limit'), { - is: Joi.exist(), - then: Joi.forbidden().messages({ 'any.unknown': 'limit per location is not allowed when a global limit is set' }), - otherwise: Joi.number().default(1), - }), -}).or('continent', 'region', 'country', 'state', 'city', 'network', 'asn', 'magic', 'tags')).default(GLOBAL_DEFAULTS.locations); +export const schema = Joi.alternatives().try( + Joi.string(), + Joi.array().items(Joi.object().keys({ + continent: Joi.string().valid(...Object.keys(continents)).insensitive() + .messages({ 'any.only': 'The continent must be a valid two-letter continent code' }), + region: Joi.string().valid(...regionNames).insensitive(), + country: Joi.string().valid(...Object.keys(countries)).insensitive() + .messages({ 'any.only': 'The country must be a valid two-letter ISO code' }), + state: Joi.string().valid(...Object.values(states)).insensitive() + .messages({ 'any.only': 'The US state must be a valid two-letter code, e.g. CA' }), + city: Joi.string().min(1).max(128).lowercase().custom(normalizeValue), + network: Joi.string().min(1).max(128).lowercase().custom(normalizeValue), + asn: Joi.number().integer().positive(), + magic: Joi.string().min(1).custom(normalizeValue), + tags: Joi.array().items(Joi.string().min(1).max(128).lowercase().custom(normalizeValue)), + limit: Joi.number().min(1).max(measurementConfig.limits.location).when(Joi.ref('/limit'), { + is: Joi.exist(), + then: Joi.forbidden().messages({ 'any.unknown': 'limit per location is not allowed when a global limit is set' }), + otherwise: Joi.number().default(1), + }), + }).or('continent', 'region', 'country', 'state', 'city', 'network', 'asn', 'magic', 'tags')), +).default(GLOBAL_DEFAULTS.locations); diff --git a/src/measurement/schema/utils.ts b/src/measurement/schema/utils.ts index 59ff09a1..42ef12f5 100644 --- a/src/measurement/schema/utils.ts +++ b/src/measurement/schema/utils.ts @@ -52,7 +52,13 @@ export const globalIpOptions: {version: string[]; cidr: PresenceMode} = { versio export const GLOBAL_DEFAULTS = { locations: [], - limit: (request: MeasurementRequest) => request.locations.length || 1, + limit: (request: MeasurementRequest) => { + if (typeof request.locations === 'string') { + return 1; + } + + return request.locations?.length || 1; + }, inProgressUpdates: false, }; diff --git a/src/measurement/store.ts b/src/measurement/store.ts index c854456f..58c0616d 100644 --- a/src/measurement/store.ts +++ b/src/measurement/store.ts @@ -1,16 +1,16 @@ import config from 'config'; import _ from 'lodash'; import cryptoRandomString from 'crypto-random-string'; -import type { Probe } from '../probe/types.js'; +import type { OfflineProbe, Probe } from '../probe/types.js'; import type { RedisClient } from '../lib/redis/client.js'; import { getRedisClient } from '../lib/redis/client.js'; import { scopedLogger } from '../lib/logger.js'; -import type { MeasurementRecord, MeasurementResult, MeasurementRequest, MeasurementProgressMessage } from './types.js'; +import type { MeasurementRecord, MeasurementResult, MeasurementRequest, MeasurementProgressMessage, RequestType, MeasurementResultMessage } from './types.js'; import { getDefaults } from './schema/utils.js'; const logger = scopedLogger('store'); -export const getMeasurementKey = (id: string, suffix: 'probes_awaiting' | undefined = undefined): string => { +export const getMeasurementKey = (id: string, suffix: string | undefined = undefined): string => { let key = `gp:measurement:${id}`; if (suffix) { @@ -27,7 +27,7 @@ const substractObjects = (obj1: Record, obj2: Record, value2 as Record); if (!_.isEmpty(difference)) { @@ -44,27 +44,37 @@ const substractObjects = (obj1: Record, obj2: Record { + async getMeasurementString (id: string): Promise { return this.redis.sendCommand([ 'JSON.GET', getMeasurementKey(id) ]); } - async createMeasurement (request: MeasurementRequest, probes: Probe[]): Promise { + async getMeasurement (id: string) { + return await this.redis.json.get(getMeasurementKey(id)) as MeasurementRecord | null; + } + + async getMeasurementIps (id: string): Promise { + const ips = await this.redis.json.get(getMeasurementKey(id, 'ips')) as string[] | null; + return ips || []; + } + + async createMeasurement (request: MeasurementRequest, onlineProbesMap: Map, allProbes: (Probe | OfflineProbe)[]): Promise { const id = cryptoRandomString({ length: 16, type: 'alphanumeric' }); const key = getMeasurementKey(id); - const results = this.probesToResults(probes, request.type); const probesAwaitingTtl = config.get('measurement.timeout') + 5; const startTime = new Date(); - const measurement: MeasurementRecord = { + const results = this.probesToResults(allProbes, request.type); + + const measurement: Partial = { id, type: request.type, status: 'in-progress', createdAt: startTime.toISOString(), updatedAt: startTime.toISOString(), target: request.target, - limit: request.limit, - probesCount: probes.length, - locations: request.locations, + ...(request.limit && { limit: request.limit }), + probesCount: allProbes.length, + ...(request.locations && { locations: request.locations }), measurementOptions: request.measurementOptions, results, }; @@ -72,9 +82,11 @@ export class MeasurementStore { await Promise.all([ this.redis.hSet('gp:in-progress', id, startTime.getTime()), - this.redis.set(getMeasurementKey(id, 'probes_awaiting'), probes.length, { EX: probesAwaitingTtl }), + this.redis.set(getMeasurementKey(id, 'probes_awaiting'), onlineProbesMap.size, { EX: probesAwaitingTtl }), this.redis.json.set(key, '$', measurementWithoutDefaults), this.redis.expire(key, config.get('measurement.resultTTL')), + this.redis.json.set(getMeasurementKey(id, 'ips'), '$', allProbes.map(probe => probe.ipAddress)), + this.redis.expire(getMeasurementKey(id, 'ips'), config.get('measurement.resultTTL')), ]); return id; @@ -97,6 +109,20 @@ export class MeasurementStore { ]); } + async storeMeasurementResult (data: MeasurementResultMessage) { + const record = await this.redis.recordResult(data.measurementId, data.testId, data.result); + + if (record) { + await this.markFinished(data.measurementId); + } + + return record; + } + + async markFinished (id: string) { + await this.redis.markFinished(id); + } + async markFinishedByTimeout (ids: string[]): Promise { if (ids.length === 0) { return; @@ -153,15 +179,18 @@ export class MeasurementStore { }, intervalTime); } - removeDefaults (measurement: MeasurementRecord, request: MeasurementRequest): Partial { + removeDefaults (measurement: Partial, request: MeasurementRequest): Partial { const defaults = getDefaults(request); + // Removes `"limit": 1` from locations. E.g. [{"country": "US", "limit": 1}] => [{"country": "US"}] - measurement.locations = measurement.locations.map(location => location.limit === 1 ? _.omit(location, 'limit') : location); + if (_.isArray(measurement.locations)) { + measurement.locations = measurement.locations.map(location => location.limit === 1 ? _.omit(location, 'limit') : location); + } return substractObjects(measurement, defaults) as Partial; } - probesToResults (probes: Probe[], type: string) { + probesToResults (probes: (Probe | OfflineProbe)[], type: RequestType) { const results = probes.map(probe => ({ probe: { continent: probe.location.continent, @@ -176,13 +205,20 @@ export class MeasurementStore { tags: probe.tags.map(({ value }) => value), resolvers: probe.resolvers, }, - result: this.getInitialResult(type), + result: this.getInitialResult(type, probe.status), } as MeasurementResult)); return results; } - getInitialResult (type: string) { + getInitialResult (type: RequestType, status: Probe['status'] | OfflineProbe['status']) { + if (status === 'offline') { + return { + status: 'offline', + rawOutput: 'This probe is currently offline. Please try again later.', + }; + } + if (type === 'http') { return { status: 'in-progress', diff --git a/src/measurement/types.ts b/src/measurement/types.ts index 854caa53..90fa3997 100644 --- a/src/measurement/types.ts +++ b/src/measurement/types.ts @@ -6,7 +6,7 @@ import type { Location } from '../lib/location/types.js'; type TestResult = { rawOutput: string; - status: 'in-progress' | 'finished' | 'failed'; + status: 'in-progress' | 'finished' | 'failed' | 'offline'; }; type PingTest = { @@ -18,7 +18,7 @@ type PingTiming = { ttl: number; }; -type PingResult = TestResult & { +export type PingResult = TestResult & { timings: PingTiming[]; stats: { min: number; @@ -170,14 +170,17 @@ export type LocationWithLimit = Location & {limit?: number}; * Measurement Objects */ -type MeasurementStatus = 'in-progress' | 'finished'; +export type UserRequest = Omit & { + locations: LocationWithLimit[] | string; + limit: number; +} export type MeasurementRequest = { type: 'ping' | 'traceroute' | 'dns' | 'http' | 'mtr'; target: string; measurementOptions: MeasurementOptions; - locations: LocationWithLimit[]; - limit: number; + locations: LocationWithLimit[] | undefined; + limit: number | undefined; inProgressUpdates: boolean; }; @@ -201,13 +204,13 @@ export type MeasurementResult = { export type MeasurementRecord = { id: string; type: MeasurementRequest['type']; - status: MeasurementStatus; + status: 'in-progress' | 'finished'; createdAt: string; updatedAt: string; target: string; - limit: number; + limit?: number; probesCount: number; - locations: LocationWithLimit[]; + locations?: LocationWithLimit[]; measurementOptions?: MeasurementOptions; results: MeasurementResult[]; }; diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 3fd20c64..1ee4be73 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -12,7 +12,7 @@ import { getRegionAliases, } from '../lib/location/location.js'; import { ProbeError } from '../lib/probe-error.js'; -import { createGeoipClient } from '../lib/geoip/client.js'; +import { createGeoipClient, LocationInfo } from '../lib/geoip/client.js'; import type GeoipClient from '../lib/geoip/client.js'; import getProbeIp from '../lib/get-probe-ip.js'; import { getRegion } from '../lib/ip-ranges.js'; @@ -118,7 +118,7 @@ export const getIndex = (location: ProbeLocation, tags: Tag[]) => { return index; }; -const getLocation = (ipInfo: ProbeLocation): ProbeLocation => ({ +const getLocation = (ipInfo: LocationInfo): ProbeLocation => ({ continent: ipInfo.continent, region: ipInfo.region, country: ipInfo.country, @@ -132,7 +132,7 @@ const getLocation = (ipInfo: ProbeLocation): ProbeLocation => ({ normalizedNetwork: ipInfo.normalizedNetwork, }); -const getTags = (clientIp: string, ipInfo: ProbeLocation) => { +const getTags = (clientIp: string, ipInfo: LocationInfo) => { const tags: Tag[] = []; const cloudRegion = getRegion(clientIp); diff --git a/src/probe/probes-location-filter.ts b/src/probe/probes-location-filter.ts new file mode 100644 index 00000000..fe6f3382 --- /dev/null +++ b/src/probe/probes-location-filter.ts @@ -0,0 +1,152 @@ +import config from 'config'; +import _ from 'lodash'; +import type { Location } from '../lib/location/types.js'; +import type { Probe, ProbeLocation } from './types.js'; + +/* + * [ + * [ public key, internal key] + * ] + * + * */ +const locationKeyMap = [ + [ 'network', 'normalizedNetwork' ], + [ 'city', 'normalizedCity' ], +]; + +export class ProbesLocationFilter { + static magicFilter (probes: Probe[], magicLocation: string) { + let filteredProbes = probes; + const keywords = magicLocation.split('+'); + + for (const keyword of keywords) { + const closestExactMatchPosition = probes.reduce((smallestExactMatchPosition, probe) => { + const exactMatchPosition = ProbesLocationFilter.getExactIndexPosition(probe, keyword); + + if (exactMatchPosition === -1) { + return smallestExactMatchPosition; + } + + return exactMatchPosition < smallestExactMatchPosition ? exactMatchPosition : smallestExactMatchPosition; + }, Number.POSITIVE_INFINITY); + const noExactMatches = closestExactMatchPosition === Number.POSITIVE_INFINITY; + + if (noExactMatches) { + filteredProbes = filteredProbes.filter(probe => ProbesLocationFilter.getIndexPosition(probe, keyword) !== -1); + } else { + filteredProbes = filteredProbes.filter(probe => ProbesLocationFilter.getExactIndexPosition(probe, keyword) === closestExactMatchPosition); + } + } + + return filteredProbes; + } + + static getExactIndexPosition (probe: Probe, value: string) { + return probe.index.findIndex(category => category.some(index => index === value.toLowerCase().replaceAll('-', ' ').trim())); + } + + static getIndexPosition (probe: Probe, value: string) { + return probe.index.findIndex(category => category.some(index => index.includes(value.toLowerCase().replaceAll('-', ' ').trim()))); + } + + static hasTag (probe: Probe, tag: string) { + return probe.tags.some(({ value }) => value.toLowerCase() === tag); + } + + public filterGloballyDistibuted (probes: Probe[], limit: number): Probe[] { + const distribution = this.getDistibutionConfig(); + return this.filterByLocationAndWeight(probes, distribution, limit); + } + + public filterByLocation (probes: Probe[], location: Location): Probe[] { + if (location.magic === 'world') { + return _.shuffle(this.filterGloballyDistibuted(probes, probes.length)); + } + + let filteredProbes = probes; + + Object.keys(location).forEach((key) => { + if (key === 'tags') { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + filteredProbes = probes.filter(probe => location.tags!.every(tag => ProbesLocationFilter.hasTag(probe, tag))); + } else if (key === 'magic') { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + filteredProbes = ProbesLocationFilter.magicFilter(filteredProbes, location.magic!); + } else { + const probeKey = locationKeyMap.find(m => m.includes(key))?.[1] ?? key; + filteredProbes = filteredProbes.filter(probe => location[key as keyof Location] === probe.location[probeKey as keyof ProbeLocation]); + } + }); + + const isMagicSorting = Object.keys(location).includes('magic'); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return isMagicSorting ? this.magicSort(filteredProbes, location.magic!) : _.shuffle(filteredProbes); + } + + public filterByLocationAndWeight (probes: Probe[], distribution: Map, limit: number): Probe[] { + const groupedByLocation = new Map(); + + for (const [ location ] of distribution) { + const foundProbes = this.filterByLocation(probes, location); + + if (foundProbes.length > 0) { + groupedByLocation.set(location, foundProbes); + } + } + + const pickedProbes = new Set(); + + while (groupedByLocation.size > 0 && pickedProbes.size < limit) { + const selectedCount = pickedProbes.size; + + for (const [ location, locationProbes ] of groupedByLocation) { + if (pickedProbes.size === limit) { + break; + } + + const locationWeight = distribution.get(location); + + if (!locationWeight) { + continue; + } + + let count = Math.ceil((limit - selectedCount) * locationWeight / 100); + const remainingSpace = limit - pickedProbes.size; + count = count > remainingSpace ? remainingSpace : count; + + for (const s of locationProbes.splice(0, count)) { + pickedProbes.add(s); + } + + if (locationProbes.length === 0) { + groupedByLocation.delete(location); + } + } + } + + return [ ...pickedProbes ]; + } + + private magicSort (probes: Probe[], magicString: string): Probe[] { + const getClosestIndexPosition = (probe: Probe) => { + const keywords = magicString.split('+'); + const closestIndexPosition = keywords.reduce((smallesIndex, keyword) => { + const indexPosition = ProbesLocationFilter.getIndexPosition(probe, keyword); + return indexPosition < smallesIndex ? indexPosition : smallesIndex; + }, Number.POSITIVE_INFINITY); + return closestIndexPosition; + }; + + const probesGroupedByIndexPosition = _.groupBy(probes, getClosestIndexPosition); + const groupsSortedByIndexPosition = Object.values(probesGroupedByIndexPosition); // Object.values sorts values by key + const groupsWithShuffledItems = groupsSortedByIndexPosition.map(group => _.shuffle(group)); + const resultProbes = groupsWithShuffledItems.flat(); + + return resultProbes; + } + + private getDistibutionConfig () { + return new Map(_.shuffle(Object.entries(config.get>('measurement.globalDistribution'))) + .map(([ value, weight ]) => [{ continent: value }, weight ])); + } +} diff --git a/src/probe/router.ts b/src/probe/router.ts index f4c0b7c2..7a8bf64b 100644 --- a/src/probe/router.ts +++ b/src/probe/router.ts @@ -1,64 +1,89 @@ import _ from 'lodash'; -import type { RemoteProbeSocket } from '../lib/ws/server.js'; import { fetchSockets } from '../lib/ws/fetch-sockets.js'; -import type { LocationWithLimit } from '../measurement/types.js'; +import type { LocationWithLimit, MeasurementRequest, MeasurementResult, UserRequest } from '../measurement/types.js'; import type { Location } from '../lib/location/types.js'; -import type { Probe } from './types.js'; -import { SocketsLocationFilter } from './sockets-location-filter.js'; +import type { OfflineProbe, Probe } from './types.js'; +import { ProbesLocationFilter } from './probes-location-filter.js'; +import { getMeasurementStore, MeasurementStore } from '../measurement/store.js'; +import { normalizeFromPublicName, normalizeNetworkName } from '../lib/geoip/utils.js'; export class ProbeRouter { + private readonly probesFilter = new ProbesLocationFilter(); + constructor ( private readonly fetchWsSockets: typeof fetchSockets, - private readonly socketsFilter = new SocketsLocationFilter(), + private readonly store: MeasurementStore, ) {} - public async findMatchingProbes ( - locations: LocationWithLimit[] = [], - globalLimit = 1, - ): Promise { - const sockets = await this.fetchSockets(); - let filtered: RemoteProbeSocket[] = []; + public async findMatchingProbes (userRequest: UserRequest): Promise<{ + onlineProbesMap: Map; + allProbes: (Probe | OfflineProbe)[]; + request: MeasurementRequest; + }> { + const locations = userRequest.locations ?? []; + const globalLimit = userRequest.limit ?? 1; + + const connectedProbes = await this.fetchProbes(); + + if (typeof locations === 'string') { + return this.findWithMeasurementId(connectedProbes, locations, userRequest); + } if (locations.some(l => l.limit)) { - filtered = this.findWithLocationLimit(sockets, locations); - } else if (locations.length > 0) { - filtered = this.findWithGlobalLimit(sockets, locations, globalLimit); - } else { - filtered = this.findGloballyDistributed(sockets, globalLimit); + const filtered = this.findWithLocationLimit(connectedProbes, locations); + return this.processFiltered(filtered, connectedProbes, locations, userRequest); + } + + if (locations.length > 0) { + const filtered = this.findWithGlobalLimit(connectedProbes, locations, globalLimit); + return this.processFiltered(filtered, connectedProbes, locations, userRequest); } - return filtered.map(s => s.data.probe); + const filtered = this.findGloballyDistributed(connectedProbes, globalLimit); + return this.processFiltered(filtered, connectedProbes, locations, userRequest); } - private async fetchSockets (): Promise { + private async processFiltered (filtered: Probe[], connectedProbes: Probe[], locations: LocationWithLimit[], request: UserRequest) { + if (filtered.length === 0 && locations.length === 1 && locations[0]?.magic) { + return this.findWithMeasurementId(connectedProbes, locations[0].magic, request); + } + + return { + allProbes: filtered, + onlineProbesMap: new Map(filtered.entries()), + request: request as MeasurementRequest, + }; + } + + private async fetchProbes (): Promise { const sockets = await this.fetchWsSockets(); - return sockets.filter(s => s.data.probe.status === 'ready'); + return sockets.filter(s => s.data.probe.status === 'ready').map(s => s.data.probe); } - private findGloballyDistributed (sockets: RemoteProbeSocket[], limit: number): RemoteProbeSocket[] { - return this.socketsFilter.filterGloballyDistibuted(sockets, limit); + private findGloballyDistributed (probes: Probe[], limit: number): Probe[] { + return this.probesFilter.filterGloballyDistibuted(probes, limit); } - private findWithGlobalLimit (sockets: RemoteProbeSocket[], locations: Location[], limit: number): RemoteProbeSocket[] { + private findWithGlobalLimit (probes: Probe[], locations: Location[], limit: number): Probe[] { const weight = Math.floor(100 / locations.length); const distribution = new Map(locations.map(l => [ l, weight ])); - return this.socketsFilter.filterByLocationAndWeight(sockets, distribution, limit); + return this.probesFilter.filterByLocationAndWeight(probes, distribution, limit); } - private findWithLocationLimit (sockets: RemoteProbeSocket[], locations: LocationWithLimit[]): RemoteProbeSocket[] { - const grouped = new Map(); + private findWithLocationLimit (probes: Probe[], locations: LocationWithLimit[]): Probe[] { + const grouped = new Map(); for (const location of locations) { const { limit, ...l } = location; - const found = this.socketsFilter.filterByLocation(sockets, l); + const found = this.probesFilter.filterByLocation(probes, l); if (found.length > 0) { grouped.set(location, found); } } - const picked = new Set(); + const picked = new Set(); for (const [ loc, soc ] of grouped) { for (const s of _.take(soc, loc.limit)) { @@ -68,6 +93,86 @@ export class ProbeRouter { return [ ...picked ]; } + + private async findWithMeasurementId (connectedProbes: Probe[], measurementId: string, userRequest: UserRequest): Promise<{ + onlineProbesMap: Map; + allProbes: (Probe | OfflineProbe)[]; + request: MeasurementRequest; + }> { + const ipToConnectedProbe = new Map(connectedProbes.map(probe => [ probe.ipAddress, probe ])); + const prevIps = await this.store.getMeasurementIps(measurementId); + const prevMeasurement = await this.store.getMeasurement(measurementId); + + const emptyResult = { onlineProbesMap: new Map(), allProbes: [], request: userRequest } as { + onlineProbesMap: Map; + allProbes: (Probe | OfflineProbe)[]; + request: MeasurementRequest; + }; + + if (!prevMeasurement || prevIps.length === 0) { + return emptyResult; + } + + const request: MeasurementRequest = { ...userRequest, limit: prevMeasurement.limit, locations: prevMeasurement.locations }; + const onlineProbesMap: Map = new Map(); + const allProbes: (Probe | OfflineProbe)[] = []; + + for (let i = 0; i < prevIps.length; i++) { + const ip = prevIps[i]!; + const connectedProbe = ipToConnectedProbe.get(ip); + + if (connectedProbe) { + onlineProbesMap.set(i, connectedProbe); + allProbes.push(connectedProbe); + } else { + const prevTest = prevMeasurement.results[i]; + + if (!prevTest) { + return emptyResult; + } + + const offlineProbe = this.testToOfflineProbe(prevTest, ip); + allProbes.push(offlineProbe); + } + } + + return { onlineProbesMap, allProbes, request }; + } + + private testToOfflineProbe = (test: MeasurementResult, ip: string): OfflineProbe => ({ + status: 'offline', + client: null, + version: null, + nodeVersion: null, + uuid: null, + isHardware: false, + hardwareDevice: null, + ipAddress: ip, + host: null, + location: { + continent: test.probe.continent, + region: test.probe.region, + country: test.probe.country, + city: test.probe.city, + normalizedCity: normalizeFromPublicName(test.probe.city), + asn: test.probe.asn, + latitude: test.probe.latitude, + longitude: test.probe.longitude, + state: test.probe.state, + network: test.probe.network, + normalizedNetwork: normalizeNetworkName(test.probe.network), + }, + index: [], + resolvers: test.probe.resolvers, + tags: test.probe.tags.map(tag => ({ value: tag, type: 'offline' })), + stats: { + cpu: { + count: 0, + load: [], + }, + jobs: { count: 0 }, + }, + } as OfflineProbe); } // Factory @@ -76,7 +181,7 @@ let router: ProbeRouter; export const getProbeRouter = () => { if (!router) { - router = new ProbeRouter(fetchSockets); + router = new ProbeRouter(fetchSockets, getMeasurementStore()); } return router; diff --git a/src/probe/sockets-location-filter.ts b/src/probe/sockets-location-filter.ts deleted file mode 100644 index 29322521..00000000 --- a/src/probe/sockets-location-filter.ts +++ /dev/null @@ -1,153 +0,0 @@ -import config from 'config'; -import _ from 'lodash'; -import type { Location } from '../lib/location/types.js'; -import type { RemoteProbeSocket } from '../lib/ws/server.js'; -import type { ProbeLocation } from './types.js'; - -/* - * [ - * [ public key, internal key] - * ] - * - * */ -const locationKeyMap = [ - [ 'network', 'normalizedNetwork' ], - [ 'city', 'normalizedCity' ], -]; - -export class SocketsLocationFilter { - static magicFilter (sockets: RemoteProbeSocket[], magicLocation: string) { - let filteredSockets = sockets; - const keywords = magicLocation.split('+'); - - for (const keyword of keywords) { - const closestExactMatchPosition = sockets.reduce((smallestExactMatchPosition, socket) => { - const exactMatchPosition = SocketsLocationFilter.getExactIndexPosition(socket, keyword); - - if (exactMatchPosition === -1) { - return smallestExactMatchPosition; - } - - return exactMatchPosition < smallestExactMatchPosition ? exactMatchPosition : smallestExactMatchPosition; - }, Number.POSITIVE_INFINITY); - const noExactMatches = closestExactMatchPosition === Number.POSITIVE_INFINITY; - - if (noExactMatches) { - filteredSockets = filteredSockets.filter(socket => SocketsLocationFilter.getIndexPosition(socket, keyword) !== -1); - } else { - filteredSockets = filteredSockets.filter(socket => SocketsLocationFilter.getExactIndexPosition(socket, keyword) === closestExactMatchPosition); - } - } - - return filteredSockets; - } - - static getExactIndexPosition (socket: RemoteProbeSocket, value: string) { - return socket.data.probe.index.findIndex(category => category.some(index => index === value.replaceAll('-', ' ').trim())); - } - - static getIndexPosition (socket: RemoteProbeSocket, value: string) { - return socket.data.probe.index.findIndex(category => category.some(index => index.includes(value.replaceAll('-', ' ').trim()))); - } - - static hasTag (socket: RemoteProbeSocket, tag: string) { - return socket.data.probe.tags.some(({ value }) => value.toLowerCase() === tag); - } - - public filterGloballyDistibuted (sockets: RemoteProbeSocket[], limit: number): RemoteProbeSocket[] { - const distribution = this.getDistibutionConfig(); - return this.filterByLocationAndWeight(sockets, distribution, limit); - } - - public filterByLocation (sockets: RemoteProbeSocket[], location: Location): RemoteProbeSocket[] { - if (location.magic === 'world') { - return _.shuffle(this.filterGloballyDistibuted(sockets, sockets.length)); - } - - let filteredSockets = sockets; - - Object.keys(location).forEach((key) => { - if (key === 'tags') { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - filteredSockets = filteredSockets.filter(socket => location.tags!.every(tag => SocketsLocationFilter.hasTag(socket, tag))); - } else if (key === 'magic') { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - filteredSockets = SocketsLocationFilter.magicFilter(filteredSockets, location.magic!); - } else { - const probeKey = locationKeyMap.find(m => m.includes(key))?.[1] ?? key; - filteredSockets = filteredSockets.filter(socket => location[key as keyof Location] === socket.data.probe.location[probeKey as keyof ProbeLocation]); - } - }); - - const isMagicSorting = Object.keys(location).includes('magic'); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return isMagicSorting ? this.magicSort(filteredSockets, location.magic!) : _.shuffle(filteredSockets); - } - - public filterByLocationAndWeight (sockets: RemoteProbeSocket[], distribution: Map, limit: number): RemoteProbeSocket[] { - const groupedByLocation = new Map(); - - for (const [ location ] of distribution) { - const foundSockets = this.filterByLocation(sockets, location); - - if (foundSockets.length > 0) { - groupedByLocation.set(location, foundSockets); - } - } - - const pickedSockets = new Set(); - - while (groupedByLocation.size > 0 && pickedSockets.size < limit) { - const selectedCount = pickedSockets.size; - - for (const [ location, locationSockets ] of groupedByLocation) { - if (pickedSockets.size === limit) { - break; - } - - const locationWeight = distribution.get(location); - - if (!locationWeight) { - continue; - } - - let count = Math.ceil((limit - selectedCount) * locationWeight / 100); - const remainingSpace = limit - pickedSockets.size; - count = count > remainingSpace ? remainingSpace : count; - - for (const s of locationSockets.splice(0, count)) { - pickedSockets.add(s); - } - - if (locationSockets.length === 0) { - groupedByLocation.delete(location); - } - } - } - - return [ ...pickedSockets ]; - } - - private magicSort (sockets: RemoteProbeSocket[], magicString: string): RemoteProbeSocket[] { - const getClosestIndexPosition = (socket: RemoteProbeSocket) => { - const keywords = magicString.split('+'); - const closestIndexPosition = keywords.reduce((smallesIndex, keyword) => { - const indexPosition = SocketsLocationFilter.getIndexPosition(socket, keyword); - return indexPosition < smallesIndex ? indexPosition : smallesIndex; - }, Number.POSITIVE_INFINITY); - return closestIndexPosition; - }; - - const socketsGroupedByIndexPosition = _.groupBy(sockets, getClosestIndexPosition); - const groupsSortedByIndexPosition = Object.values(socketsGroupedByIndexPosition); // Object.values sorts values by key - const groupsWithShuffledItems = groupsSortedByIndexPosition.map(group => _.shuffle(group)); - const resultSockets = groupsWithShuffledItems.flat(); - - return resultSockets; - } - - private getDistibutionConfig () { - return new Map(_.shuffle(Object.entries(config.get>('measurement.globalDistribution'))) - .map(([ value, weight ]) => [{ continent: value }, weight ])); - } -} diff --git a/src/probe/types.ts b/src/probe/types.ts index 0c5630ff..314d4cdb 100644 --- a/src/probe/types.ts +++ b/src/probe/types.ts @@ -7,10 +7,9 @@ export type ProbeLocation = { asn: number; latitude: number; longitude: number; - state?: string | undefined; + state: string | null; network: string; normalizedNetwork: string; - isHosting?: boolean | undefined; }; export type ProbeStats = { @@ -47,3 +46,31 @@ export type Probe = { tags: Tag[]; stats: ProbeStats; }; + +type Modify = Omit & Fields; + +export type OfflineProbe = Modify diff --git a/test/tests/integration/adoption-code/adoption-code.test.ts b/test/tests/integration/adoption-code.test.ts similarity index 95% rename from test/tests/integration/adoption-code/adoption-code.test.ts rename to test/tests/integration/adoption-code.test.ts index 902b655c..13ba3c17 100644 --- a/test/tests/integration/adoption-code/adoption-code.test.ts +++ b/test/tests/integration/adoption-code.test.ts @@ -3,8 +3,8 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; import type { Socket } from 'socket.io-client'; import request from 'supertest'; -import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; -import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../utils/server.js'; +import nockGeoIpProviders from '../../utils/nock-geo-ip.js'; let probe: Socket; const app = await getTestServer(); diff --git a/test/tests/integration/health/health.test.ts b/test/tests/integration/health.test.ts similarity index 96% rename from test/tests/integration/health/health.test.ts rename to test/tests/integration/health.test.ts index dcd9b517..f6e38b11 100644 --- a/test/tests/integration/health/health.test.ts +++ b/test/tests/integration/health.test.ts @@ -3,7 +3,7 @@ import process from 'node:process'; import { expect } from 'chai'; import * as sinon from 'sinon'; import request, { type SuperTest, type Test } from 'supertest'; -import { getTestServer } from '../../../utils/server.js'; +import { getTestServer } from '../../utils/server.js'; after(() => { process.removeAllListeners('SIGTERM'); diff --git a/test/tests/integration/measurement/create-measurement.test.ts b/test/tests/integration/measurement/create-measurement.test.ts index a777c845..9cfacae2 100644 --- a/test/tests/integration/measurement/create-measurement.test.ts +++ b/test/tests/integration/measurement/create-measurement.test.ts @@ -124,7 +124,33 @@ describe('Create measurement', () => { }); }); + it('should create measurement with a single probe by default', async () => { + let id; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + id = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.not.exist; + expect(response.body.locations).to.not.exist; + expect(response).to.matchApiSchema(); + }); + }); + it('should create measurement with global limit', async () => { + let id; await requestAgent.post('/v1/measurements') .send({ type: 'ping', @@ -138,6 +164,15 @@ describe('Create measurement', () => { expect(response.header.location).to.exist; expect(response.body.probesCount).to.equal(1); expect(response).to.matchApiSchema(); + id = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.equal(2); + expect(response.body.locations).to.deep.equal([{ country: 'US' }]); + expect(response).to.matchApiSchema(); }); }); @@ -370,6 +405,237 @@ describe('Create measurement', () => { }); }); + it('should create measurement with another measurement id location', async () => { + let id; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + }) + .expect(202) + .expect((response) => { + id = response.body.id; + }); + + let id2; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: id, + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + id2 = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id2}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.not.exist; + expect(response.body.locations).to.not.exist; + expect(response).to.matchApiSchema(); + }); + }); + + it('should create measurement with another measurement id location and copy its limit and locations', async () => { + let id; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + limit: 10, + locations: [{ + continent: 'NA', + }], + }) + .expect(202) + .expect((response) => { + id = response.body.id; + }); + + let id2; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: id, + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + id2 = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id2}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.equal(10); + expect(response.body.locations).to.deep.equal([{ continent: 'NA' }]); + expect(response).to.matchApiSchema(); + }); + }); + + it('should create measurement with measurement id created from measurement id', async () => { + let id1; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + }) + .expect(202) + .expect((response) => { + id1 = response.body.id; + }); + + let id2; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: id1, + }) + .expect(202) + .expect((response) => { + id2 = response.body.id; + }); + + + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: id2, + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + }); + }); + + it('should create measurement with another measurement id location passed in magic field', async () => { + let id; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + limit: 2, + locations: [{ country: 'US' }], + }) + .expect(202) + .expect((response) => { + id = response.body.id; + }); + + let id2; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: [{ magic: id }], + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + id2 = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id2}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.equal(2); + expect(response.body.locations).to.deep.equal([{ country: 'US' }]); + expect(response).to.matchApiSchema(); + }); + }); + + it('should respond with error if there is no requested measurement id', async () => { + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: 'nonExistingid', + }) + .expect(422) + .expect((response) => { + expect(response.body).to.deep.equal({ + error: { + message: 'No suitable probes found.', + type: 'no_probes_found', + }, + links: { + documentation: 'https://www.jsdelivr.com/docs/api.globalping.io#post-/v1/measurements', + }, + }); + + expect(response).to.matchApiSchema(); + }); + }); + + describe('offline probes', () => { + after(() => { + probe.emit('probe:status:update', 'ready'); + }); + + it('should create measurement with offline test result if requested probe is offline', async () => { + let id; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + limit: 2, + locations: [{ + continent: 'NA', + }], + }) + .expect(202) + .expect((response) => { + id = response.body.id; + }); + + probe.emit('probe:status:update', 'ping-test-failed'); + + let id2; + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: id, + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + id2 = response.body.id; + }); + + await requestAgent.get(`/v1/measurements/${id2}`) + .expect(200) + .expect((response) => { + expect(response.body.limit).to.equal(2); + expect(response.body.locations).to.deep.equal([{ continent: 'NA' }]); + expect(response.body.results[0].result.status).to.equal('offline'); + expect(response.body.results[0].result.rawOutput).to.equal('This probe is currently offline. Please try again later.'); + expect(response).to.matchApiSchema(); + }); + }); + }); + describe('adopted probes', () => { before(async () => { await client(ADOPTED_PROBES_TABLE).insert({ diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 785c1121..d25089e8 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -82,6 +82,7 @@ describe('Get Probes', () => { continent: 'SA', region: 'South America', country: 'AR', + state: null, city: 'Buenos Aires', asn: 61004, latitude: -34.6131, @@ -163,6 +164,7 @@ describe('Get Probes', () => { continent: 'SA', region: 'South America', country: 'AR', + state: null, city: 'Buenos Aires', asn: 61004, latitude: -34.6131, @@ -197,6 +199,7 @@ describe('Get Probes', () => { continent: 'SA', region: 'South America', country: 'AR', + state: null, city: 'Buenos Aires', asn: 61004, latitude: -34.6131, @@ -276,6 +279,7 @@ describe('Get Probes', () => { continent: 'SA', region: 'South America', country: 'AR', + state: null, city: 'Cordoba', latitude: -31.4135, longitude: -64.18105, diff --git a/test/tests/integration/middleware/ratelimit.test.ts b/test/tests/integration/ratelimit.test.ts similarity index 89% rename from test/tests/integration/middleware/ratelimit.test.ts rename to test/tests/integration/ratelimit.test.ts index 3e8ec9d8..4f26c571 100644 --- a/test/tests/integration/middleware/ratelimit.test.ts +++ b/test/tests/integration/ratelimit.test.ts @@ -4,8 +4,8 @@ import requestIp from 'request-ip'; import type { RateLimiterRedis } from 'rate-limiter-flexible'; import { expect } from 'chai'; import type { Socket } from 'socket.io-client'; -import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; -import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../utils/server.js'; +import nockGeoIpProviders from '../../utils/nock-geo-ip.js'; describe('rate limiter', () => { let app: Server; @@ -24,7 +24,7 @@ describe('rate limiter', () => { // Koa sees ipv6-ipv4 monster clientIpv6 = `::ffff:${clientIp ?? '127.0.0.1'}`; - const rateLimiter = await import('../../../../src/lib/ratelimiter.js'); + const rateLimiter = await import('../../../src/lib/ratelimiter.js'); rateLimiterInstance = rateLimiter.default; nockGeoIpProviders(); @@ -43,7 +43,7 @@ describe('rate limiter', () => { describe('headers', () => { it('should NOT include headers (GET)', async () => { - const response = await requestAgent.get('/v1/').send() as Response; + const response = await requestAgent.get('/v1/').send().expect(200) as Response; expect(response.headers['x-ratelimit-limit']).to.not.exist; expect(response.headers['x-ratelimit-remaining']).to.not.exist; @@ -51,7 +51,7 @@ describe('rate limiter', () => { }); it('should NOT include headers if body is not valid (POST)', async () => { - const response = await requestAgent.post('/v1/measurements').send() as Response; + const response = await requestAgent.post('/v1/measurements').send().expect(400) as Response; expect(response.headers['x-ratelimit-limit']).to.not.exist; expect(response.headers['x-ratelimit-remaining']).to.not.exist; @@ -62,7 +62,7 @@ describe('rate limiter', () => { const response = await requestAgent.post('/v1/measurements').send({ type: 'ping', target: 'jsdelivr.com', - }) as Response; + }).expect(202) as Response; expect(response.headers['x-ratelimit-limit']).to.exist; expect(response.headers['x-ratelimit-remaining']).to.exist; @@ -73,7 +73,7 @@ describe('rate limiter', () => { const response = await requestAgent.post('/v1/measurements').send({ type: 'ping', target: 'jsdelivr.com', - }) as Response; + }).expect(202) as Response; expect(response.headers['x-ratelimit-limit']).to.equal('100000'); expect(response.headers['x-ratelimit-remaining']).to.equal('99999'); @@ -82,7 +82,7 @@ describe('rate limiter', () => { const response2 = await requestAgent.post('/v1/measurements').send({ type: 'ping', target: 'jsdelivr.com', - }) as Response; + }).expect(202) as Response; expect(response2.headers['x-ratelimit-limit']).to.equal('100000'); expect(response2.headers['x-ratelimit-remaining']).to.equal('99998'); @@ -97,7 +97,7 @@ describe('rate limiter', () => { const response = await requestAgent.post('/v1/measurements').send({ type: 'ping', target: 'jsdelivr.com', - }) as Response; + }).expect(202) as Response; expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(99999); }); @@ -108,10 +108,9 @@ describe('rate limiter', () => { const response = await requestAgent.post('/v1/measurements').send({ type: 'ping', target: 'jsdelivr.com', - }) as Response; + }).expect(429) as Response; expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(0); - expect(response.statusCode).to.equal(429); }); }); }); diff --git a/test/tests/unit/adopted-probes.test.ts b/test/tests/unit/adopted-probes.test.ts index c731b3fb..569ade43 100644 --- a/test/tests/unit/adopted-probes.test.ts +++ b/test/tests/unit/adopted-probes.test.ts @@ -15,6 +15,7 @@ const defaultAdoptedProbe = { status: 'ready', version: '0.26.0', country: 'IE', + state: null, countryOfCustomCity: '', city: 'Dublin', latitude: 53.3331, @@ -33,6 +34,7 @@ const defaultConnectedProbe: Probe = { continent: 'EU', region: 'Northern Europe', country: 'IE', + state: null, city: 'Dublin', normalizedCity: 'dublin', asn: 16509, @@ -225,13 +227,14 @@ describe('AdoptedProbes', () => { continent: 'EU', region: 'Northern Europe', country: 'GB', + state: null, city: 'London', asn: 20473, latitude: 51.50853, longitude: -0.12574, network: 'The Constant Company, LLC', }, - }, + } as Probe, }, }]); @@ -269,6 +272,7 @@ describe('AdoptedProbes', () => { continent: 'EU', region: 'Northern Europe', country: 'GB', + state: null, city: 'London', asn: 20473, latitude: 51.50853, @@ -320,6 +324,7 @@ describe('AdoptedProbes', () => { continent: 'EU', region: 'Northern Europe', country: 'GB', + state: null, city: 'London', asn: 20473, latitude: 51.50853, @@ -393,6 +398,7 @@ describe('AdoptedProbes', () => { region: 'Northern Europe', country: 'IE', city: 'Dundalk', + state: null, normalizedCity: 'dundalk', asn: 16509, latitude: 54, diff --git a/test/tests/unit/geoip/client.test.ts b/test/tests/unit/geoip/client.test.ts index 34d1f46f..b6061edb 100644 --- a/test/tests/unit/geoip/client.test.ts +++ b/test/tests/unit/geoip/client.test.ts @@ -64,7 +64,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -89,7 +89,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -98,7 +98,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', - isHosting: undefined, + isHosting: null, }); }); @@ -119,7 +119,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -140,7 +140,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -152,7 +152,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -173,7 +173,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -182,7 +182,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', - isHosting: undefined, + isHosting: null, }); }); @@ -207,7 +207,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'AF', country: 'EG', - state: undefined, + state: null, city: 'El-Rashda', region: 'Northern Africa', normalizedCity: 'el-rashda', @@ -216,7 +216,7 @@ describe('geoip service', () => { longitude: 26.487, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -228,7 +228,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'EU', country: 'DE', - state: undefined, + state: null, city: 'Falkenstein', region: 'Western Europe', normalizedCity: 'falkenstein', @@ -237,7 +237,7 @@ describe('geoip service', () => { longitude: 12.371563, network: 'Hetzner Online GmbH', normalizedNetwork: 'hetzner online gmbh', - isHosting: undefined, + isHosting: null, }); }); @@ -249,7 +249,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'EU', country: 'DE', - state: undefined, + state: null, city: 'Zwickau', region: 'Western Europe', normalizedCity: 'zwickau', @@ -258,7 +258,7 @@ describe('geoip service', () => { longitude: 12.371563, network: 'Hetzner Online GmbH', normalizedNetwork: 'hetzner online gmbh', - isHosting: undefined, + isHosting: null, }); }); @@ -291,7 +291,7 @@ describe('geoip service', () => { expect(info).to.deep.equal({ continent: 'SA', country: 'AR', - state: undefined, + state: null, city: 'Buenos Aires', region: 'South America', normalizedCity: 'buenos aires', @@ -300,7 +300,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', - isHosting: undefined, + isHosting: null, }); }); @@ -342,7 +342,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -364,7 +364,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -385,7 +385,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -406,7 +406,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', - isHosting: undefined, + isHosting: null, }); }); @@ -446,7 +446,7 @@ describe('geoip service', () => { longitude: -77.039476, network: 'Verizon Business', normalizedNetwork: 'verizon business', - isHosting: undefined, + isHosting: null, }); }); diff --git a/test/tests/unit/measurement/runner.test.ts b/test/tests/unit/measurement/runner.test.ts index 9011e3a9..60954e77 100644 --- a/test/tests/unit/measurement/runner.test.ts +++ b/test/tests/unit/measurement/runner.test.ts @@ -1,54 +1,57 @@ +import type { Context } from 'koa'; import * as sinon from 'sinon'; import { Server } from 'socket.io'; import { expect } from 'chai'; import * as td from 'testdouble'; -import type { RedisClient } from '../../../../src/lib/redis/client.js'; import { MeasurementStore } from '../../../../src/measurement/store.js'; import { ProbeRouter } from '../../../../src/probe/router.js'; import { MetricsAgent } from '../../../../src/lib/metrics.js'; import type { Probe } from '../../../../src/probe/types.js'; import type { MeasurementRunner } from '../../../../src/measurement/runner.js'; import type { MeasurementRecord, MeasurementResultMessage } from '../../../../src/measurement/types.js'; +import createHttpError from 'http-errors'; const getProbe = (id: number) => ({ client: id } as unknown as Probe); +const req = { + headers: { + 'x-client-ip': '1.1.1.1', + }, +}; + describe('MeasurementRunner', () => { + const set = sinon.stub(); const emit = sinon.stub(); const to = sinon.stub(); const io = sinon.createStubInstance(Server); - const redis = { - recordResult: sinon.stub(), - } as sinon.SinonStubbedInstance; const store = sinon.createStubInstance(MeasurementStore); const router = sinon.createStubInstance(ProbeRouter); const metrics = sinon.createStubInstance(MetricsAgent); + const rateLimit = sinon.stub(); let runner: MeasurementRunner; let testId: number; before(async () => { td.replaceEsm('crypto-random-string', null, () => testId++); const { MeasurementRunner } = await import('../../../../src/measurement/runner.js'); - runner = new MeasurementRunner(io, redis, store, router, metrics); + runner = new MeasurementRunner(io, store, router, rateLimit, metrics); }); beforeEach(() => { - emit.reset(); - to.reset(); + sinon.resetHistory(); to.returns({ emit }); io.of.returns({ to } as any); - router.findMatchingProbes.reset(); - store.createMeasurement.reset(); store.createMeasurement.resolves('measurementid'); - metrics.recordMeasurement.reset(); - redis.recordResult.reset(); testId = 0; }); - it('should run measurement for the required amount of probes', async () => { - router.findMatchingProbes.resolves([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ]); + after(() => { + td.reset(); + }); - await runner.run({ - type: 'ping', + it('should run measurement for the required amount of probes', async () => { + const request = { + type: 'ping' as const, target: 'jsdelivr.com', measurementOptions: { packets: 3, @@ -56,14 +59,39 @@ describe('MeasurementRunner', () => { locations: [], limit: 10, inProgressUpdates: false, + }; + + router.findMatchingProbes.resolves({ + onlineProbesMap: new Map([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ].entries()), + allProbes: [ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ], + request, }); + await runner.run({ + set, + req, + request: { + body: request, + }, + } as unknown as Context); + expect(router.findMatchingProbes.callCount).to.equal(1); - expect(router.findMatchingProbes.args[0]).to.deep.equal([ [], 10 ]); + expect(router.findMatchingProbes.args[0]).to.deep.equal([ request ]); expect(store.createMeasurement.callCount).to.equal(1); - expect(store.createMeasurement.args[0]![1]).to.deep.equal([{ client: 0 }, { client: 1 }, { client: 2 }, { client: 3 }]); + expect(store.createMeasurement.args[0]).to.deep.equal([ + { + type: 'ping', + target: 'jsdelivr.com', + measurementOptions: { packets: 3 }, + locations: [], + limit: 10, + inProgressUpdates: false, + }, + new Map([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ].entries()), + [ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ], + ]); expect(to.callCount).to.equal(4); expect(emit.callCount).to.equal(4); @@ -124,10 +152,8 @@ describe('MeasurementRunner', () => { }); it('should send `inProgressUpdates: true` to the first N probes if requested', async () => { - router.findMatchingProbes.resolves([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ]); - - await runner.run({ - type: 'ping', + const request = { + type: 'ping'as const, target: 'jsdelivr.com', measurementOptions: { packets: 3, @@ -135,14 +161,39 @@ describe('MeasurementRunner', () => { locations: [], limit: 10, inProgressUpdates: true, + }; + + router.findMatchingProbes.resolves({ + onlineProbesMap: new Map([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ].entries()), + allProbes: [ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ], + request, }); + await runner.run({ + set, + req, + request: { + body: request, + }, + } as unknown as Context); + expect(router.findMatchingProbes.callCount).to.equal(1); - expect(router.findMatchingProbes.args[0]).to.deep.equal([ [], 10 ]); + expect(router.findMatchingProbes.args[0]).to.deep.equal([ request ]); expect(store.createMeasurement.callCount).to.equal(1); - expect(store.createMeasurement.args[0]![1]).to.deep.equal([{ client: 0 }, { client: 1 }, { client: 2 }, { client: 3 }]); + expect(store.createMeasurement.args[0]).to.deep.equal([ + { + type: 'ping', + target: 'jsdelivr.com', + measurementOptions: { packets: 3 }, + locations: [], + limit: 10, + inProgressUpdates: true, + }, + new Map([ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ].entries()), + [ getProbe(0), getProbe(1), getProbe(2), getProbe(3) ], + ]); expect(to.callCount).to.equal(4); expect(emit.callCount).to.equal(4); @@ -197,7 +248,7 @@ describe('MeasurementRunner', () => { it('should properly handle result events from probes', async () => { const sandbox = sinon.createSandbox({ useFakeTimers: { now: new Date('2023-05-24T09:56:55.000Z').getTime() } }); - redis.recordResult + store.storeMeasurementResult .onFirstCall().resolves(null) .onSecondCall().resolves({ type: 'ping', createdAt: '2023-05-24T09:56:30.000Z' } as MeasurementRecord) .onThirdCall().resolves(null); @@ -206,12 +257,103 @@ describe('MeasurementRunner', () => { await runner.recordResult({ measurementId: 'measurementid', testId: 'testid2', result: {} as MeasurementResultMessage['result'] }); await runner.recordResult({ measurementId: 'measurementid', testId: 'testid3', result: {} as MeasurementResultMessage['result'] }); - expect(redis.recordResult.callCount).to.equal(3); - expect(redis.recordResult.args[0]).to.deep.equal([ 'measurementid', 'testid1', {}]); - expect(redis.recordResult.args[1]).to.deep.equal([ 'measurementid', 'testid2', {}]); - expect(redis.recordResult.args[2]).to.deep.equal([ 'measurementid', 'testid3', {}]); + expect(store.storeMeasurementResult.callCount).to.equal(3); + expect(store.storeMeasurementResult.args[0]).to.deep.equal([{ measurementId: 'measurementid', testId: 'testid1', result: {} }]); + expect(store.storeMeasurementResult.args[1]).to.deep.equal([{ measurementId: 'measurementid', testId: 'testid2', result: {} }]); + expect(store.storeMeasurementResult.args[2]).to.deep.equal([{ measurementId: 'measurementid', testId: 'testid3', result: {} }]); expect(metrics.recordMeasurementTime.callCount).to.equal(1); expect(metrics.recordMeasurementTime.args[0]).to.deep.equal([ 'ping', 25000 ]); sandbox.restore(); }); + + it('should call ratelimiter with the number of online probes', async () => { + const request = { + type: 'ping' as const, + target: 'jsdelivr.com', + measurementOptions: { + packets: 3, + }, + locations: [], + limit: 10, + inProgressUpdates: false, + }; + + router.findMatchingProbes.resolves({ + onlineProbesMap: new Map([ getProbe(0) ].entries()), + allProbes: [ getProbe(0), getProbe(1) ], + request, + }); + + const ctx = { + set, + req, + request: { + body: request, + }, + } as unknown as Context; + + await runner.run(ctx); + + expect(rateLimit.callCount).to.equal(1); + expect(rateLimit.args[0]).to.deep.equal([ ctx, 1 ]); + }); + + it('should throw 422 error if no probes found', async () => { + const request = { + type: 'ping' as const, + target: 'jsdelivr.com', + measurementOptions: { + packets: 3, + }, + locations: [], + limit: 10, + inProgressUpdates: false, + }; + + router.findMatchingProbes.resolves({ + onlineProbesMap: new Map([].entries()), + allProbes: [], + request, + }); + + const err = await runner.run({ + set, + req, + request: { + body: request, + }, + } as unknown as Context).catch((err: unknown) => err); + expect(err).to.deep.equal(createHttpError(422, 'No suitable probes found.', { type: 'no_probes_found' })); + expect(store.markFinished.callCount).to.equal(0); + }); + + it('should immideately call store.markFinished if there are no online probes', async () => { + const request = { + type: 'ping' as const, + target: 'jsdelivr.com', + measurementOptions: { + packets: 3, + }, + locations: [], + limit: 10, + inProgressUpdates: false, + }; + + router.findMatchingProbes.resolves({ + onlineProbesMap: new Map([].entries()), + allProbes: [ getProbe(0) ], + request, + }); + + await runner.run({ + set, + req, + request: { + body: request, + }, + } as unknown as Context); + + expect(store.markFinished.callCount).to.equal(1); + expect(store.markFinished.args[0]).to.deep.equal([ 'measurementid' ]); + }); }); diff --git a/test/tests/unit/measurement/schema/schema.test.ts b/test/tests/unit/measurement/schema/schema.test.ts index 617f0979..4083e1d7 100644 --- a/test/tests/unit/measurement/schema/schema.test.ts +++ b/test/tests/unit/measurement/schema/schema.test.ts @@ -228,7 +228,7 @@ describe('command schema', async () => { const valid = locationSchema.validate(input); - expect(valid.value![0].magic).to.equal('petah tiqva'); + expect(valid.value![0].magic).to.equal('Petah Tiqva'); }); it('should correct region value (lowercase)', () => { diff --git a/test/tests/unit/measurement/store.test.ts b/test/tests/unit/measurement/store.test.ts index 9ba7bec6..014cfca3 100644 --- a/test/tests/unit/measurement/store.test.ts +++ b/test/tests/unit/measurement/store.test.ts @@ -2,9 +2,11 @@ import * as td from 'testdouble'; import { expect } from 'chai'; import * as sinon from 'sinon'; import type { MeasurementStore } from '../../../../src/measurement/store.js'; -import type { Probe } from '../../../../src/probe/types.js'; +import type { OfflineProbe, Probe } from '../../../../src/probe/types.js'; +import type { PingResult } from '../../../../src/measurement/types.js'; -const getProbe = (id: string) => ({ +const getProbe = (id: string, ip: string) => ({ + ipAddress: ip, location: { network: id, continent: 'continent', @@ -20,6 +22,11 @@ const getProbe = (id: string) => ({ resolvers: [], } as unknown as Probe); +const getOfflineProbe = (id: string, ip: string) => ({ + ...getProbe(id, ip), + status: 'offline', +} as unknown as OfflineProbe); + describe('measurement store', () => { let getMeasurementStore: () => MeasurementStore; let sandbox: sinon.SinonSandbox; @@ -35,6 +42,8 @@ describe('measurement store', () => { set: sinon.stub(), strAppend: sinon.stub(), }, + recordResult: sinon.stub(), + markFinished: sinon.stub(), }; before(async () => { @@ -46,14 +55,8 @@ describe('measurement store', () => { beforeEach(() => { sandbox = sinon.createSandbox({ useFakeTimers: { now: 1_678_000_000_000 } }); sandbox.stub(Math, 'random').returns(0.8); - redisMock.hScan.reset(); - redisMock.hDel.reset(); - redisMock.hSet.reset(); - redisMock.set.reset(); - redisMock.expire.reset(); - redisMock.json.mGet.reset(); - redisMock.json.set.reset(); - redisMock.json.strAppend.reset(); + redisMock.recordResult.reset(); + sinon.resetHistory(); }); afterEach(() => { @@ -116,6 +119,109 @@ describe('measurement store', () => { }]); }); + it('should store measurement probes in the same order as in arguments', async () => { + const store = getMeasurementStore(); + await store.createMeasurement( + { + type: 'ping', + measurementOptions: { packets: 3 }, + target: 'jsdelivr.com', + locations: [], + limit: 4, + inProgressUpdates: false, + }, + new Map([ getProbe('z', '1.1.1.1'), getProbe('10', '2.2.2.2'), getProbe('x', '3.3.3.3'), getProbe('0', '4.4.4.4') ].entries()), + [ getProbe('z', '1.1.1.1'), getProbe('10', '2.2.2.2'), getProbe('x', '3.3.3.3'), getProbe('0', '4.4.4.4') ], + ); + + expect(redisMock.hSet.callCount).to.equal(1); + expect(redisMock.hSet.args[0]).to.deep.equal([ 'gp:in-progress', 'measurementid', 1678000000000 ]); + expect(redisMock.set.callCount).to.equal(1); + expect(redisMock.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid:probes_awaiting', 4, { EX: 35 }]); + expect(redisMock.json.set.callCount).to.equal(2); + + expect(redisMock.json.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid', '$', { + id: 'measurementid', + type: 'ping', + status: 'in-progress', + createdAt: '2023-03-05T07:06:40.000Z', + updatedAt: '2023-03-05T07:06:40.000Z', + target: 'jsdelivr.com', + limit: 4, + probesCount: 4, + results: [{ + probe: { + continent: 'continent', + region: 'region', + country: 'country', + state: 'state', + city: 'city', + asn: 'asn', + longitude: 'longitude', + latitude: 'latitude', + network: 'z', + tags: [], + resolvers: [], + }, + result: { status: 'in-progress', rawOutput: '' }, + }, + { + probe: { + continent: 'continent', + region: 'region', + country: 'country', + state: 'state', + city: 'city', + asn: 'asn', + longitude: 'longitude', + latitude: 'latitude', + network: '10', + tags: [], + resolvers: [], + }, + result: { status: 'in-progress', rawOutput: '' }, + }, + { + probe: { + continent: 'continent', + region: 'region', + country: 'country', + state: 'state', + city: 'city', + asn: 'asn', + longitude: 'longitude', + latitude: 'latitude', + network: 'x', + tags: [], + resolvers: [], + }, + result: { status: 'in-progress', rawOutput: '' }, + }, + { + probe: { + continent: 'continent', + region: 'region', + country: 'country', + state: 'state', + city: 'city', + asn: 'asn', + longitude: 'longitude', + latitude: 'latitude', + network: '0', + tags: [], + resolvers: [], + }, + result: { status: 'in-progress', rawOutput: '' }, + }], + }]); + + expect(redisMock.expire.args[0]).to.deep.equal([ 'gp:measurement:measurementid', 604800 ]); + + expect(redisMock.json.set.args[1]).to.deep.equal([ 'gp:measurement:measurementid:ips', '$', [ '1.1.1.1', '2.2.2.2', '3.3.3.3', '4.4.4.4' ] ]); + + expect(redisMock.expire.args[1]).to.deep.equal([ 'gp:measurement:measurementid:ips', 604800 ]); + }); + it('should initialize measurement object with the proper default values', async () => { const store = getMeasurementStore(); await store.createMeasurement( @@ -127,7 +233,8 @@ describe('measurement store', () => { limit: 1, inProgressUpdates: false, }, - [ getProbe('id') ], + new Map([ [ 0, getProbe('id', '1.1.1.1') ] ]), + [ getProbe('id', '1.1.1.1') ], ); expect(redisMock.json.set.firstCall.args).to.deep.equal([ @@ -182,7 +289,8 @@ describe('measurement store', () => { limit: 1, inProgressUpdates: false, }, - [ getProbe('id') ], + new Map([ [ 0, getProbe('id', '1.1.1.1') ] ]), + [ getProbe('id', '1.1.1.1') ], ); expect(redisMock.json.set.firstCall.args).to.deep.equal([ @@ -223,7 +331,7 @@ describe('measurement store', () => { ]); }); - it('should store measurement probes in the same order as in arguments', async () => { + it('should initialize measurement object with the proper default in case of offline probes', async () => { const store = getMeasurementStore(); await store.createMeasurement( { @@ -231,95 +339,49 @@ describe('measurement store', () => { measurementOptions: { packets: 3 }, target: 'jsdelivr.com', locations: [], - limit: 4, + limit: 1, inProgressUpdates: false, }, - [ getProbe('z'), getProbe('10'), getProbe('x'), getProbe('0') ], + new Map(), + [ getOfflineProbe('id', '1.1.1.1') ], ); - expect(redisMock.hSet.callCount).to.equal(1); - expect(redisMock.hSet.args[0]).to.deep.equal([ 'gp:in-progress', 'measurementid', 1678000000000 ]); - expect(redisMock.set.callCount).to.equal(1); - expect(redisMock.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid:probes_awaiting', 4, { EX: 35 }]); - expect(redisMock.json.set.callCount).to.equal(1); - - expect(redisMock.json.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid', '$', { - id: 'measurementid', - type: 'ping', - status: 'in-progress', - createdAt: '2023-03-05T07:06:40.000Z', - updatedAt: '2023-03-05T07:06:40.000Z', - target: 'jsdelivr.com', - limit: 4, - probesCount: 4, - results: [{ - probe: { - continent: 'continent', - region: 'region', - country: 'country', - state: 'state', - city: 'city', - asn: 'asn', - longitude: 'longitude', - latitude: 'latitude', - network: 'z', - tags: [], - resolvers: [], - }, - result: { status: 'in-progress', rawOutput: '' }, - }, - { - probe: { - continent: 'continent', - region: 'region', - country: 'country', - state: 'state', - city: 'city', - asn: 'asn', - longitude: 'longitude', - latitude: 'latitude', - network: '10', - tags: [], - resolvers: [], - }, - result: { status: 'in-progress', rawOutput: '' }, - }, + expect(redisMock.json.set.firstCall.args).to.deep.equal([ + 'gp:measurement:measurementid', + '$', { - probe: { - continent: 'continent', - region: 'region', - country: 'country', - state: 'state', - city: 'city', - asn: 'asn', - longitude: 'longitude', - latitude: 'latitude', - network: 'x', - tags: [], - resolvers: [], - }, - result: { status: 'in-progress', rawOutput: '' }, + id: 'measurementid', + type: 'ping', + status: 'in-progress', + createdAt: '2023-03-05T07:06:40.000Z', + updatedAt: '2023-03-05T07:06:40.000Z', + target: 'jsdelivr.com', + probesCount: 1, + results: [ + { + probe: { + continent: 'continent', + region: 'region', + country: 'country', + state: 'state', + city: 'city', + asn: 'asn', + longitude: 'longitude', + latitude: 'latitude', + network: 'id', + tags: [], + resolvers: [], + }, + result: { + status: 'offline', + rawOutput: 'This probe is currently offline. Please try again later.', + }, + }, + ], }, - { - probe: { - continent: 'continent', - region: 'region', - country: 'country', - state: 'state', - city: 'city', - asn: 'asn', - longitude: 'longitude', - latitude: 'latitude', - network: '0', - tags: [], - resolvers: [], - }, - result: { status: 'in-progress', rawOutput: '' }, - }], - }]); + ]); - expect(redisMock.expire.callCount).to.equal(1); - expect(redisMock.expire.args[0]).to.deep.equal([ 'gp:measurement:measurementid', 604800 ]); + expect(redisMock.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid:probes_awaiting', 0, { EX: 35 }]); }); it('should store non-default fields of the measurement request', async () => { @@ -349,7 +411,8 @@ describe('measurement store', () => { limit: 2, inProgressUpdates: false, }, - [ getProbe('id') ], + new Map([ [ 0, getProbe('id', '1.1.1.1') ] ]), + [ getProbe('id', '1.1.1.1') ], ); expect(redisMock.json.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid', '$', { @@ -392,9 +455,6 @@ describe('measurement store', () => { }, }], }]); - - expect(redisMock.expire.callCount).to.equal(1); - expect(redisMock.expire.args[0]).to.deep.equal([ 'gp:measurement:measurementid', 604800 ]); }); it('shouldn\'t store fields of the measurement request which are equal to the default', async () => { @@ -416,7 +476,8 @@ describe('measurement store', () => { locations: [], inProgressUpdates: false, }, - [ getProbe('id') ], + new Map([ [ 0, getProbe('id', '1.1.1.1') ] ]), + [ getProbe('id', '1.1.1.1') ], ); expect(redisMock.json.set.args[0]).to.deep.equal([ 'gp:measurement:measurementid', '$', { @@ -449,9 +510,6 @@ describe('measurement store', () => { }, }], }]); - - expect(redisMock.expire.callCount).to.equal(1); - expect(redisMock.expire.args[0]).to.deep.equal([ 'gp:measurement:measurementid', 604800 ]); }); it('should store rawHeaders and rawBody fields for the http in-progress updates', async () => { @@ -509,4 +567,51 @@ describe('measurement store', () => { expect(redisMock.json.strAppend.callCount).to.equal(0); expect(redisMock.json.set.callCount).to.equal(2); }); + + it('should mark measurement as finished if storeMeasurementResult returned record', async () => { + redisMock.recordResult.resolves({}); + + const store = getMeasurementStore(); + await store.storeMeasurementResult({ + testId: 'testid', + measurementId: 'measurementid', + result: { + status: 'finished', + rawOutput: 'output', + } as PingResult, + }); + + expect(redisMock.recordResult.callCount).to.equal(1); + + expect(redisMock.recordResult.args[0]).to.deep.equal([ + 'measurementid', + 'testid', + { status: 'finished', rawOutput: 'output' }, + ]); + + expect(redisMock.markFinished.callCount).to.equal(1); + expect(redisMock.markFinished.args[0]).to.deep.equal([ 'measurementid' ]); + }); + + it('should not mark measurement as finished if storeMeasurementResult didn\'t return record', async () => { + const store = getMeasurementStore(); + await store.storeMeasurementResult({ + testId: 'testid', + measurementId: 'measurementid', + result: { + status: 'finished', + rawOutput: 'output', + } as PingResult, + }); + + expect(redisMock.recordResult.callCount).to.equal(1); + + expect(redisMock.recordResult.args[0]).to.deep.equal([ + 'measurementid', + 'testid', + { status: 'finished', rawOutput: 'output' }, + ]); + + expect(redisMock.markFinished.callCount).to.equal(0); + }); }); diff --git a/test/tests/unit/middleware/ratelimit.test.ts b/test/tests/unit/middleware/ratelimit.test.ts deleted file mode 100644 index ff81d70e..00000000 --- a/test/tests/unit/middleware/ratelimit.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -import * as sinon from 'sinon'; -import { expect } from 'chai'; -import rateLimiter from '../../../../src/lib/ratelimiter.js'; -import { rateLimitHandler } from '../../../../src/lib/http/middleware/ratelimit.js'; -import createHttpError from 'http-errors'; - -describe('rate limit middleware', () => { - const defaultCtx: any = { - set: sinon.stub(), - req: {}, - request: { - body: {}, - }, - response: {}, - }; - let ctx = { ...defaultCtx }; - - beforeEach(async () => { - defaultCtx.set.reset(); - ctx = { ...defaultCtx }; - await rateLimiter.delete(''); - }); - - it('should set rate limit headers based on the "probesCount" field value', async () => { - ctx.request.body = { - limit: 10, - locations: [], - }; - - const next: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 5, - }; - }; - - await rateLimitHandler()(ctx, next); - - expect(ctx.set.callCount).to.equal(3); - expect(ctx.set.firstCall.args[0]).to.equal('X-RateLimit-Reset'); - expect(ctx.set.secondCall.args).to.deep.equal([ 'X-RateLimit-Limit', '100000' ]); - expect(ctx.set.thirdCall.args).to.deep.equal([ 'X-RateLimit-Remaining', '99995' ]); - }); - - it('should throw an error if response body doesn\'t have "probesCount" field', async () => { - ctx.request.body = { - limit: 10, - locations: [], - }; - - const next: any = () => { - ctx.response.body = {}; - }; - - const err = await rateLimitHandler()(ctx, next).catch(err => err); - expect(err).to.deep.equal(new Error('Missing probesCount field in response object')); - }); - - it('should NOT set rate limit headers for admin', async () => { - ctx.request.body = { - limit: 10, - locations: [], - }; - - ctx.isAdmin = true; - - const next: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 10, - }; - }; - - await rateLimitHandler()(ctx, next); - expect(ctx.set.callCount).to.equal(0); - }); - - it('should validate request based on the "limit" field value', async () => { - ctx.request.body = { - limit: 60000, - locations: [], - }; - - const next: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 60000, - }; - }; - - await rateLimitHandler()(ctx, next); - expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); - - const err = await rateLimitHandler()(ctx, next).catch(err => err); // 60000 > 40000 so another request with the same body fails - expect(err).to.deep.equal(createHttpError(429, 'API rate limit exceeded.', { type: 'rate_limit_exceeded' })); - expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); - - ctx.request.body = { - limit: 40000, - locations: [], - }; - - const next2: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 40000, - }; - }; - - await rateLimitHandler()(ctx, next2); // 40000 === 40000 so request with the updated body works - expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); - }); - - it('should validate request based on the "location.limit" field value', async () => { - ctx.request.body = { - locations: [{ - continent: 'EU', - limit: 45000, - }, { - continent: 'NA', - limit: 45000, - }], - }; - - const next: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 90000, - }; - }; - - await rateLimitHandler()(ctx, next); - expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); - - const err = await rateLimitHandler()(ctx, next).catch(err => err); // only 10000 points remaining so another request with the same body fails - expect(err).to.deep.equal(createHttpError(429, 'API rate limit exceeded.', { type: 'rate_limit_exceeded' })); - expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); - - ctx.request.body = { - locations: [{ - continent: 'EU', - limit: 5000, - }, { - continent: 'NA', - limit: 5000, - }], - }; - - const next2: any = () => { - ctx.response.body = { - id: 'id', - probesCount: 10000, - }; - }; - - await rateLimitHandler()(ctx, next2); // request with 10000 probes will work fine - expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); - }); -}); diff --git a/test/tests/unit/probe/router.test.ts b/test/tests/unit/probe/router.test.ts index 52b351f2..c14b78b8 100644 --- a/test/tests/unit/probe/router.test.ts +++ b/test/tests/unit/probe/router.test.ts @@ -4,16 +4,18 @@ import { expect } from 'chai'; import * as td from 'testdouble'; import { ProbeRouter } from '../../../../src/probe/router.js'; +import { getRegionByCountry } from '../../../../src/lib/location/location.js'; import type { RemoteProbeSocket } from '../../../../src/lib/ws/server.js'; import type { DeepPartial } from '../../../types.js'; import type { Probe, ProbeLocation } from '../../../../src/probe/types.js'; import type { Location } from '../../../../src/lib/location/types.js'; -import { getRegionByCountry } from '../../../../src/lib/location/location.js'; +import type { MeasurementStore } from '../../../../src/measurement/store.js'; +import type { UserRequest } from '../../../../src/measurement/types.js'; const defaultLocation = { continent: '', country: 'PL', - state: undefined, + state: null, city: '', region: '', normalizedCity: '', @@ -25,12 +27,16 @@ const defaultLocation = { }; describe('probe router', () => { - const sandbox = sinon.createSandbox(); const fetchSocketsMock = sinon.stub(); const geoLookupMock = sinon.stub(); const getRegionMock = sinon.stub(); - const router = new ProbeRouter(fetchSocketsMock); + const store = { + getMeasurementIps: sinon.stub().resolves([]), + getMeasurement: sinon.stub(), + }; + const router = new ProbeRouter(fetchSocketsMock, store as unknown as MeasurementStore); let buildProbe: (socket: RemoteProbeSocket) => Promise; + let sandbox: sinon.SinonSandbox; const buildSocket = async ( id: string, @@ -57,6 +63,7 @@ describe('probe router', () => { }; before(async () => { + sandbox = sinon.createSandbox(); await td.replaceEsm('../../../../src/lib/geoip/client.ts', { createGeoipClient: () => ({ lookup: geoLookupMock }) }); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: getRegionMock }); buildProbe = (await import('../../../../src/probe/builder.js')).buildProbe as unknown as (socket: RemoteProbeSocket) => Promise; @@ -64,15 +71,12 @@ describe('probe router', () => { beforeEach(() => { sandbox.reset(); - }); - - afterEach(() => { - geoLookupMock.reset(); - fetchSocketsMock.reset(); - getRegionMock.reset(); + sinon.resetHistory(); + store.getMeasurementIps.resolves([]); }); after(() => { + sandbox.restore(); td.reset(); }); @@ -87,16 +91,21 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const userRequest = { locations: [ { country: 'UA', limit: 2 }, { country: 'PL', limit: 2 }, - ]); + ] } as UserRequest; + + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes(userRequest); + expect(request).to.equal(userRequest); + expect(onlineProbesMap.size).to.equal(4); expect(fetchSocketsMock.calledOnce).to.be.true; expect(fetchSocketsMock.firstCall.args).to.deep.equal([]); - expect(probes.length).to.equal(4); - expect(probes.filter(p => p.location.country === 'UA').length).to.equal(2); - expect(probes.filter(p => p.location.country === 'PL').length).to.equal(2); + expect(allProbes.length).to.equal(4); + expect(onlineProbesMap.size).to.equal(4); + expect(allProbes.filter(p => p.location.country === 'UA').length).to.equal(2); + expect(allProbes.filter(p => p.location.country === 'PL').length).to.equal(2); }); it('should return 1 probe if location limit is not set', async () => { @@ -110,17 +119,40 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { country: 'UA', limit: 2 }, { country: 'PL' }, - ]); + ] } as UserRequest); + + expect(fetchSocketsMock.calledOnce).to.be.true; + expect(fetchSocketsMock.firstCall.args).to.deep.equal([]); + + expect(onlineProbesMap.size).to.equal(3); + expect(allProbes.length).to.equal(3); + expect(allProbes.filter(p => p.location.country === 'UA').length).to.equal(2); + expect(allProbes.filter(p => p.location.country === 'PL').length).to.equal(1); + }); + + it('should return 1 probe if global limit is not set', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'UA' }), + await buildSocket('socket-2', { continent: 'EU', country: 'PL' }), + await buildSocket('socket-3', { continent: 'EU', country: 'PL' }), + await buildSocket('socket-4', { continent: 'NA', country: 'UA' }), + await buildSocket('socket-5', { continent: 'EU', country: 'PL' }), + ]; + + fetchSocketsMock.resolves(sockets as never); + const userRequest = { locations: [] } as unknown as UserRequest; + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes(userRequest); + + expect(request).to.equal(userRequest); expect(fetchSocketsMock.calledOnce).to.be.true; expect(fetchSocketsMock.firstCall.args).to.deep.equal([]); - expect(probes.length).to.equal(3); - expect(probes.filter(p => p.location.country === 'UA').length).to.equal(2); - expect(probes.filter(p => p.location.country === 'PL').length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes.length).to.equal(1); }); it('should shuffle result probes', async () => { @@ -138,15 +170,17 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes1 = await router.findMatchingProbes([ + const { allProbes: probes1, onlineProbesMap: online1 } = await router.findMatchingProbes({ locations: [ { continent: 'EU', limit: 10 }, - ]); - const probes2 = await router.findMatchingProbes([ + ] } as UserRequest); + const { allProbes: probes2, onlineProbesMap: online2 } = await router.findMatchingProbes({ locations: [ { continent: 'EU', limit: 10 }, - ]); + ] } as UserRequest); expect(fetchSocketsMock.calledTwice).to.be.true; + expect(online1.size).to.equal(10); expect(probes1.length).to.equal(10); + expect(online2.size).to.equal(10); expect(probes2.length).to.equal(10); const countries1 = probes1.map(probe => probe.location.country); const countries2 = probes2.map(probe => probe.location.country); @@ -165,17 +199,18 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { country: 'GB', limit: 2 }, { country: 'PL', limit: 2 }, - ]); + ] } as UserRequest); expect(fetchSocketsMock.calledOnce).to.be.true; expect(fetchSocketsMock.firstCall.args).to.deep.equal([]); - expect(probes.length).to.equal(2); - expect(probes.filter(p => p.location.country === 'GB').length).to.equal(1); - expect(probes.filter(p => p.location.country === 'PL').length).to.equal(1); + expect(onlineProbesMap.size).to.equal(2); + expect(allProbes.length).to.equal(2); + expect(allProbes.filter(p => p.location.country === 'GB').length).to.equal(1); + expect(allProbes.filter(p => p.location.country === 'PL').length).to.equal(1); }); }); @@ -186,10 +221,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([], 100); - const grouped = _.groupBy(probes, 'location.continent'); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [], limit: 100 } as unknown as UserRequest); + const grouped = _.groupBy(allProbes, 'location.continent'); - expect(probes.length).to.equal(100); + expect(onlineProbesMap.size).to.equal(100); + expect(allProbes.length).to.equal(100); expect(grouped['AF']?.length).to.equal(5); expect(grouped['AS']?.length).to.equal(15); expect(grouped['EU']?.length).to.equal(30); @@ -210,10 +246,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([], 100); - const grouped = _.groupBy(probes, 'location.continent'); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [], limit: 100 } as unknown as UserRequest); + const grouped = _.groupBy(allProbes, 'location.continent'); - expect(probes.length).to.equal(100); + expect(allProbes.length).to.equal(100); + expect(onlineProbesMap.size).to.equal(100); expect(grouped['AF']?.length).to.equal(13); expect(grouped['AS']?.length).to.equal(15); expect(grouped['EU']?.length).to.equal(20); @@ -231,8 +268,9 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ continent: 'AF' }, { continent: 'NA' }, { continent: 'SA' }], 7); - expect(probes.length).to.equal(7); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ continent: 'AF' }, { continent: 'NA' }, { continent: 'SA' }], limit: 7 } as UserRequest); + expect(allProbes.length).to.equal(7); + expect(onlineProbesMap.size).to.equal(7); }); it('should find when probes not enough', async () => { @@ -245,10 +283,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([], 100); - const grouped = _.groupBy(probes, 'location.continent'); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [], limit: 100 } as unknown as UserRequest); + const grouped = _.groupBy(allProbes, 'location.continent'); - expect(probes.length).to.equal(65); + expect(allProbes.length).to.equal(65); + expect(onlineProbesMap.size).to.equal(65); expect(grouped['AF']?.length).to.equal(15); expect(grouped['EU']?.length).to.equal(20); expect(grouped['OC']?.length).to.equal(10); @@ -270,8 +309,8 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes1 = await router.findMatchingProbes([], 100); - const probes2 = await router.findMatchingProbes([], 100); + const { allProbes: probes1 } = await router.findMatchingProbes({ locations: [], limit: 100 } as unknown as UserRequest); + const { allProbes: probes2 } = await router.findMatchingProbes({ locations: [], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledTwice).to.be.true; expect(probes1.length).to.equal(10); @@ -289,7 +328,16 @@ describe('probe router', () => { const cached = cache[location.country]; if (cached) { - return { ...cached }; + return { + ...cached, + data: { + ...cached.data, + probe: { + ...cached.data.probe, + }, + }, + id, + }; } const socket = await buildSocket(id, location); @@ -314,10 +362,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); - const grouped = _.groupBy(probes, 'location.country'); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); + const grouped = _.groupBy(allProbes, 'location.country'); - expect(probes.length).to.equal(100); + expect(allProbes.length).to.equal(100); + expect(onlineProbesMap.size).to.equal(100); expect(grouped['PL']?.length).to.equal(99); expect(grouped['UA']?.length).to.equal(1); }); @@ -336,10 +385,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); - const grouped = _.groupBy(probes, 'location.country'); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); + const grouped = _.groupBy(allProbes, 'location.country'); - expect(probes.length).to.equal(100); + expect(allProbes.length).to.equal(100); + expect(onlineProbesMap.size).to.equal(100); expect(grouped['PL']?.length).to.equal(34); expect(grouped['UA']?.length).to.equal(33); expect(grouped['NL']?.length).to.equal(33); @@ -360,16 +410,14 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes1 = await router.findMatchingProbes([ - { continent: 'EU' }, - ], 100); - const probes2 = await router.findMatchingProbes([ - { continent: 'EU' }, - ], 100); + const { allProbes: probes1, onlineProbesMap: online1 } = await router.findMatchingProbes({ locations: [{ continent: 'EU' }], limit: 100 } as unknown as UserRequest); + const { allProbes: probes2, onlineProbesMap: online2 } = await router.findMatchingProbes({ locations: [{ continent: 'EU' }], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledTwice).to.be.true; expect(probes1.length).to.equal(10); + expect(online1.size).to.equal(10); expect(probes2.length).to.equal(10); + expect(online2.size).to.equal(10); const countries1 = probes1.map(probe => probe.location.country); const countries2 = probes2.map(probe => probe.location.country); expect(countries1).to.not.deep.equal(countries2); @@ -399,10 +447,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('US'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('US'); }); it('should not find probe by continent alias if it is used not in magic field', async () => { @@ -411,9 +460,10 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ continent: 'NA' }], 100); - expect(probes.length).to.equal(1); - const probes2 = await router.findMatchingProbes([{ continent: 'North America' }], 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ continent: 'NA' }], limit: 100 } as unknown as UserRequest); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + const { allProbes: probes2 } = await router.findMatchingProbes({ locations: [{ continent: 'North America' }], limit: 100 } as unknown as UserRequest); expect(probes2.length).to.equal(0); }); @@ -424,9 +474,10 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ region: 'Northern Africa' }], 100); - expect(probes.length).to.equal(1); - const probes2 = await router.findMatchingProbes([{ region: 'North Africa' }], 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ region: 'Northern Africa' }], limit: 100 } as unknown as UserRequest); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + const { allProbes: probes2 } = await router.findMatchingProbes({ locations: [{ region: 'North Africa' }], limit: 100 } as unknown as UserRequest); expect(probes2.length).to.equal(0); }); }); @@ -436,7 +487,7 @@ describe('probe router', () => { continent: 'EU', region: getRegionByCountry('GB'), country: 'GB', - state: undefined, + state: null, city: 'London', normalizedCity: 'london', asn: 5089, @@ -450,10 +501,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ magic: 'europe' }], 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ magic: 'europe' }], limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); it('should return match (region alias)', async () => { @@ -463,10 +515,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ magic: 'north africa' }], 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ magic: 'north africa' }], limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.region).to.equal('Northern Africa'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.region).to.equal('Northern Africa'); }); it('should not return match (non-existing region alias)', async () => { @@ -476,9 +529,10 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([{ magic: 'south africa' }], 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [{ magic: 'south africa' }], limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(0); + expect(allProbes.length).to.equal(0); + expect(onlineProbesMap.size).to.equal(0); }); it('should return match (country alias)', async () => { @@ -492,10 +546,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); it('should return match (magic nested)', async () => { @@ -509,10 +564,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); it('should return result sorted by priority of magic fields in case of partial match', async () => { @@ -523,15 +579,16 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { magic: 'd' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledOnce).to.be.true; - expect(probes.length).to.equal(3); - expect(probes[0]!.location.country).to.equal('DE'); - expect(probes[1]!.location.country).to.equal('RS'); - expect(probes[2]!.location.country).to.equal('CZ'); + expect(allProbes.length).to.equal(3); + expect(onlineProbesMap.size).to.equal(3); + expect(allProbes[0]!.location.country).to.equal('DE'); + expect(allProbes[1]!.location.country).to.equal('RS'); + expect(allProbes[2]!.location.country).to.equal('CZ'); }); it('should ignore low-priority partial matches if there is an exact match', async () => { @@ -542,13 +599,14 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { magic: 'vn' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledOnce).to.be.true; - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('VN'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('VN'); }); it('should ignore high-priority partial matches if there is an exact match', async () => { @@ -559,13 +617,14 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { magic: 'wars' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledOnce).to.be.true; - expect(probes.length).to.equal(1); - expect(probes[0]!.location.normalizedCity).to.equal('poznan'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.normalizedCity).to.equal('poznan'); }); it('should ignore same-level partial matches if there is an exact match', async () => { @@ -575,13 +634,14 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes([ + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: [ { magic: 'york' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledOnce).to.be.true; - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); it('should shuffle result considering priority of magic fields', async () => { @@ -619,16 +679,18 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes1 = await router.findMatchingProbes([ + const { allProbes: probes1, onlineProbesMap: online1 } = await router.findMatchingProbes({ locations: [ { magic: 'd' }, - ], 100); - const probes2 = await router.findMatchingProbes([ + ], limit: 100 } as unknown as UserRequest); + const { allProbes: probes2, onlineProbesMap: online2 } = await router.findMatchingProbes({ locations: [ { magic: 'd' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledTwice).to.be.true; expect(probes1.length).to.equal(30); + expect(online1.size).to.equal(30); expect(probes2.length).to.equal(30); + expect(online2.size).to.equal(30); expect(probes1.slice(0, 9).every(probe => probe.location.country === 'DE')).to.be.true; expect(probes2.slice(0, 9).every(probe => probe.location.country === 'DE')).to.be.true; expect(probes1.slice(0, 9).map(probe => probe.location.normalizedCity)).to.not.deep.equal(probes2.slice(0, 9).map(probe => probe.location.normalizedCity)); @@ -661,16 +723,18 @@ describe('probe router', () => { ]; fetchSocketsMock.resolves(sockets as never); - const probes1 = await router.findMatchingProbes([ + const { allProbes: probes1, onlineProbesMap: online1 } = await router.findMatchingProbes({ locations: [ { magic: 'de' }, - ], 100); - const probes2 = await router.findMatchingProbes([ + ], limit: 100 } as unknown as UserRequest); + const { allProbes: probes2, onlineProbesMap: online2 } = await router.findMatchingProbes({ locations: [ { magic: 'de' }, - ], 100); + ], limit: 100 } as unknown as UserRequest); expect(fetchSocketsMock.calledTwice).to.be.true; expect(probes1.length).to.equal(10); + expect(online1.size).to.equal(10); expect(probes2.length).to.equal(10); + expect(online2.size).to.equal(10); expect(probes1.slice(0, 9).every(probe => probe.location.country === 'DE')).to.be.true; expect(probes2.slice(0, 9).every(probe => probe.location.country === 'DE')).to.be.true; expect(probes1.slice(0, 9).map(probe => probe.location.normalizedCity)).to.not.deep.equal(probes2.slice(0, 9).map(probe => probe.location.normalizedCity)); @@ -689,10 +753,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); } }); @@ -710,10 +775,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); } }); @@ -741,10 +807,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('US'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('US'); }); } }); @@ -763,10 +830,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); } }); @@ -777,7 +845,7 @@ describe('probe router', () => { continent: 'EU', region: getRegionByCountry('GB'), country: 'GB', - state: undefined, + state: null, city: 'london', asn: 5089, network: 'a-virgin media', @@ -795,10 +863,11 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); }); it('should return 0 matches for partial tag value', async () => { @@ -813,9 +882,9 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); - - expect(probes.length).to.equal(0); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); + expect(allProbes.length).to.equal(0); + expect(onlineProbesMap.size).to.equal(0); }); it('should return match for user tag', async () => { @@ -833,10 +902,194 @@ describe('probe router', () => { fetchSocketsMock.resolves(sockets as never); - const probes = await router.findMatchingProbes(locations, 100); + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations, limit: 100 } as unknown as UserRequest); + expect(allProbes.length).to.equal(1); + expect(onlineProbesMap.size).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('GB'); + }); + }); + + describe('route with measurement id string', async () => { + it('should find probes by prev measurement id', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([ '1.2.3.4' ]); + + store.getMeasurement.resolves({ + results: [{ + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }], + }); + + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes({ locations: 'measurementid' } as UserRequest); + + expect(request).to.deep.equal({ limit: undefined, locations: undefined }); + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(store.getMeasurement.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes[0]!.location.country).to.equal('PL'); + expect(allProbes[0]!.status).to.equal('ready'); + expect(onlineProbesMap.get(0)?.location.country).to.equal('PL'); + }); + + it('should find probes by prev measurement id in magic field', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([ '1.2.3.4' ]); + + store.getMeasurement.resolves({ + results: [{ + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }], + }); + + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes({ locations: [{ magic: 'measurementid' }] } as UserRequest); + + expect(request).to.deep.equal({ limit: undefined, locations: undefined }); + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(store.getMeasurement.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes[0]!.location.country).to.equal('PL'); + expect(allProbes[0]!.status).to.equal('ready'); + expect(onlineProbesMap.get(0)?.location.country).to.equal('PL'); + }); + + it('should not find probes without errors by prev measurement id in magic field if no such measurement found', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes({ locations: [{ magic: 'measurementid' }] } as UserRequest); + + expect(request).to.deep.equal({ locations: [{ magic: 'measurementid' }] }); + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(store.getMeasurement.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes.length).to.equal(0); + expect(onlineProbesMap.size).to.equal(0); + }); + + it('should return proper values for locations and limit in request object', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([ '1.2.3.4', '1.2.3.4' ]); + + store.getMeasurement.resolves({ + limit: 2, + locations: [{ + continent: 'EU', + }], + results: [{ + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }, { + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }], + }); + + const { request } = await router.findMatchingProbes({ locations: [{ magic: 'measurementid' }] } as UserRequest); + + expect(request).to.deep.equal({ locations: [{ continent: 'EU' }], limit: 2 }); + }); + + it('should replace non-connected probes with offline probe data', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([ '9.9.9.9' ]); + + store.getMeasurement.resolves({ + results: [{ + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }], + }); + + const { onlineProbesMap, allProbes, request } = await router.findMatchingProbes({ locations: 'measurementid' } as UserRequest); + + expect(request).to.deep.equal({ locations: undefined, limit: undefined }); + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(store.getMeasurement.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes.length).to.equal(1); + expect(allProbes[0]!.location.country).to.equal('PL'); + expect(allProbes[0]!.status).to.equal('offline'); + expect(onlineProbesMap.size).to.equal(0); + }); + + it('should return empty data if measurement ips wasn\'t found', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([]); + + store.getMeasurement.resolves({ + results: [{ + probe: { + continent: 'EU', + country: 'PL', + city: 'Warsaw', + network: 'Liberty Global B.V.', + tags: [], + }, + }], + }); + + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: 'measurementid' } as UserRequest); + + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes.length).to.equal(0); + expect(onlineProbesMap.size).to.equal(0); + }); + + it('should return empty data if measurement itself wasn\'t found', async () => { + const sockets: Array> = [ + await buildSocket('socket-1', { continent: 'EU', country: 'PL' }), + ]; + fetchSocketsMock.resolves(sockets as never); + store.getMeasurementIps.resolves([ '9.9.9.9' ]); + + store.getMeasurement.resolves(null); + + const { onlineProbesMap, allProbes } = await router.findMatchingProbes({ locations: 'measurementid' } as UserRequest); - expect(probes.length).to.equal(1); - expect(probes[0]!.location.country).to.equal('GB'); + expect(store.getMeasurementIps.args[0]).to.deep.equal([ 'measurementid' ]); + expect(store.getMeasurement.args[0]).to.deep.equal([ 'measurementid' ]); + expect(allProbes.length).to.equal(0); + expect(onlineProbesMap.size).to.equal(0); }); }); }); diff --git a/test/tests/unit/ws/fetch-sockets.test.ts b/test/tests/unit/ws/fetch-sockets.test.ts index 6da9ccca..433a0121 100644 --- a/test/tests/unit/ws/fetch-sockets.test.ts +++ b/test/tests/unit/ws/fetch-sockets.test.ts @@ -40,7 +40,7 @@ describe('fetchSockets', () => { continent: 'EU', region: 'Western Europe', country: 'FR', - state: undefined, + state: null, city: 'Paris', normalizedCity: 'paris', asn: 12876, @@ -97,7 +97,7 @@ describe('fetchSockets', () => { continent: 'EU', region: 'Western Europe', country: 'FR', - state: undefined, + state: null, city: 'Marseille', normalizedCity: 'marseille', asn: 12876, @@ -124,7 +124,7 @@ describe('fetchSockets', () => { continent: 'EU', region: 'Western Europe', country: 'FR', - state: undefined, + state: null, city: 'Marseille', normalizedCity: 'marseille', asn: 12876, diff --git a/test/utils/server.ts b/test/utils/server.ts index 801eae97..d79597cb 100644 --- a/test/utils/server.ts +++ b/test/utils/server.ts @@ -2,11 +2,14 @@ import _ from 'lodash'; import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import { io, type Socket } from 'socket.io-client'; +import { scopedLogger } from '../../src/lib/logger.js'; import { createServer } from '../../src/lib/server.js'; let app: Server; let url: string; +const logger = scopedLogger('test-server'); + export const getTestServer = async (): Promise => { if (!app) { app = await createServer(); @@ -37,6 +40,8 @@ export const addFakeProbe = async (events: object = {}, options: object = {}): P client.on(event, listener); } + client.on('connect_error', (error: Error) => logger.error(error)); + client.on('connect', () => { resolve(client); });