feat: allow reusing measurement IDs for selecting same group of probes (

#453)

* refactor: move suffix before the id

* feat: initially implement locations as id

* feat: make measurement cloning work

* fix: rate limits

* refactor: store empty location fields as nulls

* fix: get rid of undefined in probe location object

* feat: update offline probe fields values

* refactor: refactor

* feat: mark measurement as finished if all filtered probes are offline

* feat: small fixes

* test: fix part of the tests

* test: fix existing tests

* feat: user prev implementation of measurement key

* refactor: rename store methods

* test: add unit tests

* docs: update openapi docs

* feat: move isHosting to the location object

* fix: remove redundant commands from redis script

* docs: update descriptions

* feat: copy prev mes location and limit if measurement is created by id

* feat: add "offline" test status

* feat: allow to pass measurement id in the magic field

* feat: replace magic with id in locations field with actual location

* test: add offline test result int test

* docs: update descriptions

---------

Co-authored-by: Martin Kolárik <[email protected]>

public/v1/components/examples.yaml

@@ -75,6 +75,14 @@ components:
             "packets": 6
           }
         }
+    createMeasurementPingLocationsMeasurementId:
+      summary: 'ping: previous measurement id'
+      value:
+        {
+          "type": "ping",
+          "target": "cdn.jsdelivr.net",
+          "locations": "1wzMrzLBZfaPoT1c"
+        }
     createMeasurementResponse:
       value:
         {

public/v1/components/schemas.yaml

@@ -76,12 +76,18 @@ components:
           maximum: 200
           default: 1
     MeasurementLocations:
-      type: array
-      description: |
-        An array of locations from which to run the measurement.
-        Each object specifies a location using one or multiple keys.
-      items:
-        $ref: 'schemas.yaml#/components/schemas/MeasurementLocationOption'
+      oneOf:
+        - type: array
+          description: |
+            An array of locations from which to run the measurement. Each object specifies a location using one or multiple keys.
+          items:
+            $ref: 'schemas.yaml#/components/schemas/MeasurementLocationOption'
+        - type: string
+          description: |
+            `id` of a previous measurement - its probes are reused for the new measurement and returned in the same order.
+            Measurement type and options are not reused and need to be specified in the request.
+            Note that this option only works for the lifetime of the original measurement
+            and will result in a `422` response for expired or invalid `id` values.
     MeasurementOptions:
       anyOf:
         - $ref: 'schemas.yaml#/components/schemas/MeasurementPingOptions'
@@ -571,6 +577,7 @@ components:
         - continent
         - region
         - country
+        - state
         - city
         - asn
         - network
@@ -716,6 +723,7 @@ components:
       anyOf:
         - $ref: 'schemas.yaml#/components/schemas/InProgressTestResult'
         - $ref: 'schemas.yaml#/components/schemas/FailedTestResult'
+        - $ref: 'schemas.yaml#/components/schemas/OfflineTestResult'
         - $ref: 'schemas.yaml#/components/schemas/FinishedPingTestResult'
         - $ref: 'schemas.yaml#/components/schemas/FinishedTracerouteTestResult'
         - $ref: 'schemas.yaml#/components/schemas/FinishedDnsTestResult'
@@ -757,6 +765,20 @@ components:
           $ref: 'schemas.yaml#/components/schemas/FailedTestStatus'
         rawOutput:
           $ref: 'schemas.yaml#/components/schemas/TestRawOutput'
+    OfflineTestResult:
+      type: object
+      title: OfflineTestResult
+      description: |
+        Represents an `offline` test where the requested probe is currently offline and most fields are not available.
+        Only possible when passing an `id` of a previous measurement to the `locations` field.
+      required:
+        - status
+        - rawOutput
+      properties:
+        status:
+          $ref: 'schemas.yaml#/components/schemas/OfflineTestStatus'
+        rawOutput:
+          $ref: 'schemas.yaml#/components/schemas/TestRawOutput'
     FinishedPingTestResult:
       title: FinishedPingTestResult
       allOf:
@@ -1118,6 +1140,10 @@ components:
       allOf:
         - $ref: 'schemas.yaml#/components/schemas/BaseTestStatus'
         - const: failed
+    OfflineTestStatus:
+      allOf:
+        - $ref: 'schemas.yaml#/components/schemas/BaseTestStatus'
+        - const: offline
     TimingPacketRtt:
       type: number
       description: The round-trip time for this packet.

public/v1/spec.yaml

@@ -47,6 +47,7 @@ paths:
 
         - Set the `inProgressUpdates` option to `true` if the application is running in interactive mode so that the user sees the results right away.
           - If the application is interactive by default but also implements a "CI" mode to be used in scripts, do not set the flag in the CI mode.
+        - To perform multiple measurements from exactly the same probes, create a single measurement first, then pass its `id` in the `locations` option for the other measurements.
       requestBody:
         content:
           application/json:
@@ -61,6 +62,8 @@ paths:
                 $ref: 'components/examples.yaml#/components/examples/createMeasurementPingLocationsMagic'
               pingCustom:
                 $ref: 'components/examples.yaml#/components/examples/createMeasurementPingCustom'
+              pingLocationsMeasurementId:
+                $ref: 'components/examples.yaml#/components/examples/createMeasurementPingLocationsMeasurementId'
       responses:
         '202':
           $ref: 'components/responses.yaml#/components/responses/measurements202'

src/lib/adopted-probes.ts

@@ -200,10 +200,6 @@ export class AdoptedProbes {
 			const adoptedValue = _.get(adoptedProbe, adoptedField) as string | number;
 			const connectedValue = _.get(connectedProbe, connectedField) as string | number;
 
-			if (!adoptedValue && !connectedValue) { // undefined and null values are treated equal and don't require sync
-				return;
-			}
-
 			if (adoptedValue !== connectedValue) {
 				updateObject[adoptedField] = connectedValue;
 			}

src/lib/geoip/client.ts

@@ -7,7 +7,6 @@ import type { ProbeLocation } from '../../probe/types.js';
 import RedisCache from '../cache/redis-cache.js';
 import { getRedisClient } from '../redis/client.js';
 import { scopedLogger } from '../logger.js';
-import { getRegionByCountry } from '../location/location.js';
 import { isAddrWhitelisted } from './whitelist.js';
 import { ipinfoLookup } from './providers/ipinfo.js';
 import { fastlyLookup } from './providers/fastly.js';
@@ -16,8 +15,8 @@ import { ipmapLookup } from './providers/ipmap.js';
 import { type Ip2LocationBundledResponse, ip2LocationLookup } from './providers/ip2location.js';
 import { isHostingOverrides } from './overrides.js';
 
-export type LocationInfo = Omit<ProbeLocation, 'region'>;
 type Provider = 'ipmap' | 'ip2location' | 'ipinfo' | 'maxmind' | 'fastly';
+export type LocationInfo = ProbeLocation & {isHosting: boolean | null};
 export type LocationInfoWithProvider = LocationInfo & {provider: Provider};
 export type NetworkInfo = {
 	network: string;
@@ -32,8 +31,8 @@ export const createGeoipClient = (): GeoipClient => new GeoipClient(new RedisCac
 export default class GeoipClient {
 	constructor (private readonly cache: CacheInterface) {}
 
-	async lookup (addr: string): Promise<ProbeLocation> {
-		let isHosting = undefined;
+	async lookup (addr: string): Promise<LocationInfo> {
+		let isHosting = null;
 		const results = await Promise
 			.allSettled([
 				this.lookupWithCache<LocationInfo>(`geoip:ipinfo:${addr}`, async () => ipinfoLookup(addr)),
@@ -43,7 +42,7 @@ export default class GeoipClient {
 				this.lookupWithCache<LocationInfo>(`geoip:fastly:${addr}`, async () => fastlyLookup(addr)),
 			])
 			.then(([ ipinfo, ip2location, maxmind, ipmap, fastly ]) => {
-				isHosting = ip2location.status === 'fulfilled' ? ip2location.value.isHosting : undefined;
+				isHosting = ip2location.status === 'fulfilled' ? ip2location.value.location.isHosting : null;
 				const fulfilled: (LocationInfoWithProvider | null)[] = [];
 
 				// Providers here are pushed in a desc prioritized order
@@ -82,14 +81,12 @@ export default class GeoipClient {
 			}
 		}
 
-		const region = getRegionByCountry(match.country);
-
 		return {
 			continent: match.continent,
 			country: match.country,
 			state: match.state,
 			city: match.city,
-			region,
+			region: match.region,
 			normalizedCity: match.normalizedCity,
 			asn: Number(networkMatch.asn),
 			latitude: Number(match.latitude),

src/lib/geoip/fake-client.ts

@@ -1,10 +1,10 @@
-import type { ProbeLocation } from '../../probe/types.js';
+import type { LocationInfo } from './client.js';
 
-export const fakeLookup = (): ProbeLocation => {
+export const fakeLookup = (): LocationInfo => {
 	return {
 		continent: 'SA',
 		country: 'AR',
-		state: undefined,
+		state: null,
 		city: 'Buenos Aires',
 		region: 'South America',
 		normalizedCity: 'buenos aires',
@@ -13,5 +13,6 @@ export const fakeLookup = (): ProbeLocation => {
 		longitude: -58.3772,
 		network: 'InterBS S.R.L. (BAEHOST)',
 		normalizedNetwork: 'interbs s.r.l. (baehost)',
+		isHosting: null,
 	};
 };

src/lib/geoip/providers/fastly.ts

@@ -1,4 +1,5 @@
 import got from 'got';
+import { getRegionByCountry } from '../../location/location.js';
 import { getCity } from '../city-approximation.js';
 import type { LocationInfo } from '../client.js';
 import {
@@ -38,14 +39,16 @@ export const fastlyLookup = async (addr: string): Promise<LocationInfo> => {
 
 	return {
 		continent: data.continent_code,
+		region: getRegionByCountry(data.country_code),
 		country: data.country_code,
-		state: data.country_code === 'US' ? data.region : undefined,
+		state: data.country_code === 'US' ? data.region : null,
 		city: normalizeCityNamePublic(city),
 		normalizedCity: normalizeCityName(city),
 		asn: result.as.number,
 		latitude: data.latitude,
 		longitude: data.longitude,
 		network: result.as.name,
 		normalizedNetwork: normalizeNetworkName(result.as.name),
+		isHosting: null,
 	};
 };

src/lib/geoip/providers/ip2location.ts

@@ -2,6 +2,7 @@ import got from 'got';
 import config from 'config';
 import {
 	getContinentByCountry,
+	getRegionByCountry,
 	getStateIsoByName,
 } from '../../location/location.js';
 import type { LocationInfo } from '../client.js';
@@ -30,7 +31,6 @@ type Ip2LocationResponse = {
 
 export type Ip2LocationBundledResponse = {
 	location: LocationInfo,
-	isHosting: boolean | undefined,
 	isProxy: boolean,
 };
 
@@ -50,7 +50,8 @@ export const ip2LocationLookup = async (addr: string): Promise<Ip2LocationBundle
 
 	const location = {
 		continent: result.country_code ? getContinentByCountry(result.country_code) : '',
-		state: result.country_code === 'US' && result.region_name ? getStateIsoByName(result.region_name) : undefined,
+		region: result.country_code ? getRegionByCountry(result.country_code) : '',
+		state: result.country_code === 'US' && result.region_name ? getStateIsoByName(result.region_name) : null,
 		country: result.country_code ?? '',
 		city: normalizeCityNamePublic(city),
 		normalizedCity: normalizeCityName(city),
@@ -59,11 +60,11 @@ export const ip2LocationLookup = async (addr: string): Promise<Ip2LocationBundle
 		longitude: result.longitude ?? 0,
 		network: result.as ?? '',
 		normalizedNetwork: normalizeNetworkName(result.as ?? ''),
+		isHosting: result.usage_type ? HOSTING_USAGE_TYPES.includes(result.usage_type) : null,
 	};
 
 	return {
 		location,
-		isHosting: result.usage_type ? HOSTING_USAGE_TYPES.includes(result.usage_type) : undefined,
 		isProxy: result.is_proxy ?? false,
 	};
 };

src/lib/geoip/providers/ipinfo.ts

@@ -1,6 +1,6 @@
 import got from 'got';
 import config from 'config';
-import { getContinentByCountry, getStateIsoByName } from '../../location/location.js';
+import { getContinentByCountry, getRegionByCountry, getStateIsoByName } from '../../location/location.js';
 import type { LocationInfo } from '../client.js';
 import {
 	normalizeCityName,
@@ -34,7 +34,8 @@ export const ipinfoLookup = async (addr: string): Promise<LocationInfo> => {
 
 	return {
 		continent: result.country ? getContinentByCountry(result.country) : '',
-		state: result.country === 'US' && result.region ? getStateIsoByName(result.region) : undefined,
+		region: result.country ? getRegionByCountry(result.country) : '',
+		state: result.country === 'US' && result.region ? getStateIsoByName(result.region) : null,
 		country: result.country ?? '',
 		city: normalizeCityNamePublic(city),
 		normalizedCity: normalizeCityName(city),
@@ -43,6 +44,6 @@ export const ipinfoLookup = async (addr: string): Promise<LocationInfo> => {
 		longitude: Number(lon),
 		network,
 		normalizedNetwork: normalizeNetworkName(network),
-		isHosting: result.privacy?.hosting,
+		isHosting: result.privacy?.hosting ?? null,
 	};
 };

src/lib/geoip/providers/ipmap.ts

@@ -1,5 +1,5 @@
 import got from 'got';
-import { getContinentByCountry } from '../../location/location.js';
+import { getContinentByCountry, getRegionByCountry } from '../../location/location.js';
 import type { LocationInfo } from '../client.js';
 import {
 	normalizeCityName,
@@ -24,7 +24,8 @@ export const ipmapLookup = async (addr: string): Promise<LocationInfo> => {
 
 	return {
 		continent: location.countryCodeAlpha2 ? getContinentByCountry(location.countryCodeAlpha2) : '',
-		state: location.countryCodeAlpha2 === 'US' ? location.stateAnsiCode : undefined,
+		region: location.countryCodeAlpha2 ? getRegionByCountry(location.countryCodeAlpha2) : '',
+		state: location.countryCodeAlpha2 === 'US' && location.stateAnsiCode ? location.stateAnsiCode : null,
 		country: location.countryCodeAlpha2 ?? '',
 		city: normalizeCityNamePublic(city),
 		normalizedCity: normalizeCityName(city),
@@ -33,5 +34,6 @@ export const ipmapLookup = async (addr: string): Promise<LocationInfo> => {
 		longitude: Number(location.longitude) ?? 0,
 		network: '',
 		normalizedNetwork: '',
+		isHosting: null,
 	};
 };

src/lib/geoip/providers/maxmind.ts

@@ -9,6 +9,7 @@ import {
 	normalizeNetworkName,
 } from '../utils.js';
 import { getCity } from '../city-approximation.js';
+import { getRegionByCountry } from '../../location/location.js';
 
 const client = new WebServiceClient(config.get('maxmind.accountId'), config.get('maxmind.licenseKey'));
 
@@ -39,14 +40,16 @@ export const maxmindLookup = async (addr: string): Promise<LocationInfo> => {
 
 	return {
 		continent: data.continent?.code ?? '',
+		region: data.country?.isoCode ? getRegionByCountry(data.country?.isoCode) : '',
 		country: data.country?.isoCode ?? '',
-		state: data.country?.isoCode === 'US' ? data.subdivisions?.map(s => s.isoCode)[0] ?? '' : undefined,
+		state: data.country?.isoCode === 'US' ? data.subdivisions?.map(s => s.isoCode)[0] ?? '' : null,
 		city: normalizeCityNamePublic(city),
 		normalizedCity: normalizeCityName(city),
 		asn: data.traits?.autonomousSystemNumber ?? 0,
 		latitude: data.location?.latitude ?? 0,
 		longitude: data.location?.longitude ?? 0,
 		network: data.traits?.isp ?? '',
 		normalizedNetwork: normalizeNetworkName(data.traits?.isp ?? ''),
+		isHosting: null,
 	};
 };