From f8aac2e07536d604154b6f7a2ec9af7015d88400 Mon Sep 17 00:00:00 2001 From: lsabor Date: Thu, 23 Oct 2025 12:12:04 -0700 Subject: [PATCH 1/7] save work --- front_end/src/utils/forecasts/dataset.ts | 109 ++++++++++++++++++++--- 1 file changed, 96 insertions(+), 13 deletions(-) diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index 0895c5eb4c..c36474c1ff 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -15,6 +15,101 @@ import { nominalLocationToCdfLocation, } from "@/utils/math"; +function standardizeCdf( + cdf: number[], + lowerOpen: boolean, + upperOpen: boolean +): number[] { + if (cdf.length === 0) { + return []; + } + + const pmf: number[] = []; + pmf.push(cdf[0] ?? 0); + for (let i = 1; i < cdf.length; i++) { + pmf.push((cdf[i] ?? 0) - (cdf[i - 1] ?? 0)); + } + pmf.push(1 - (cdf[cdf.length - 1] ?? 1)); + // cap depends on cdf_size (0.2 if size is the default 201) + const cap = + ((0.2 - 1e-7) * (DefaultInboundOutcomeCount + 1)) / Math.max(cdf.length, 1); + const capPmf = (scale: number) => { + return pmf.map((value, i) => { + if (i == 0 || i == pmf.length - 1) { + return value; + } + return Math.min(cap, scale * value); + }); + }; + const cappedSum = (scale: number) => { + return capPmf(scale).reduce((acc, value) => acc + value, 0); + }; + + // find the appropriate scale search space + const inboundMass = (cdf[cdf.length - 1] ?? 1) - (cdf[0] ?? 0); + let lo = 1; + let hi = 1; + while (cappedSum(hi) < inboundMass) { + hi *= 1.1; + } + // home in on scale value that makes capped sum approx inboundMass + for (let i = 0; i < 100; i++) { + const scale = 0.5 * (lo + hi); + const s = cappedSum(scale); + if (s < inboundMass) { + lo = scale; + } else { + hi = scale; + } + if (hi - lo < 1e-12) { + break; + } + } + // apply scale and renormalize + const scaledPmf = capPmf(0.5 * (lo + hi)); + console.log(Math.max(...scaledPmf.slice(1, pmf.length - 1))); + const totalMass = scaledPmf.reduce((acc, value) => acc + value, 0); + const normalizedPmf = scaledPmf.map((value) => value / (totalMass ?? 1)); + // back to CDF space + const standardizedCdf: number[] = []; + let cumulative = 0; + for (let i = 0; i < normalizedPmf.length - 1; i++) { + cumulative += normalizedPmf[i] ?? 0; + standardizedCdf.push(cumulative); + } + + const scaleLowerTo = lowerOpen ? 0 : standardizedCdf[0] ?? 0; + const lastStandardizedIndex = standardizedCdf.length - 1; + const lastStandardizedCdf = + lastStandardizedIndex >= 0 ? standardizedCdf[lastStandardizedIndex] : 1; + const scaleUpperTo = upperOpen ? 1 : lastStandardizedCdf ?? 1; + const rescaledInboundMass = scaleUpperTo - scaleLowerTo; + + const applyMinimum = (F: number, location: number) => { + const rescaledInboundMassSafe = + rescaledInboundMass === 0 ? 1 : rescaledInboundMass; + const rescaledF = (F - scaleLowerTo) / rescaledInboundMassSafe; + if (lowerOpen && upperOpen) { + return 0.988 * rescaledF + 0.01 * location + 0.001; + } + if (lowerOpen) { + return 0.989 * rescaledF + 0.01 * location + 0.001; + } + if (upperOpen) { + return 0.989 * rescaledF + 0.01 * location; + } + return 0.99 * rescaledF + 0.01 * location; + }; + + const denominator = + standardizedCdf.length > 1 ? standardizedCdf.length - 1 : 1; + return standardizedCdf.map((F, index) => { + const location = denominator === 0 ? 0 : index / denominator; + const standardizedValue = applyMinimum(F, location); + return Math.round(standardizedValue * 1e10) / 1e10; + }); +} + /** * Get chart data from slider input */ @@ -51,19 +146,7 @@ export function getSliderNumericForecastDataset( }, Array(componentCdfs[0]!.length).fill(0)); cdf = cdf.map((F) => Number(F)); - // standardize cdf - const cdfOffset = - lowerOpen && upperOpen - ? (F: number, x: number) => 0.988 * F + 0.01 * x + 0.001 - : lowerOpen - ? (F: number, x: number) => 0.989 * F + 0.01 * x + 0.001 - : upperOpen - ? (F: number, x: number) => 0.989 * F + 0.01 * x - : (F: number, x: number) => 0.99 * F + 0.01 * x; - cdf = cdf.map( - (F, index) => - Math.round(cdfOffset(F, index / (cdf.length - 1)) * 1e10) / 1e10 - ); + cdf = standardizeCdf(cdf, lowerOpen, upperOpen, inboundOutcomeCount); return { cdf: cdf, From 5adbc22fc2715a8cd77c6d7780e0a5a174afa38a Mon Sep 17 00:00:00 2001 From: lsabor Date: Sun, 2 Nov 2025 10:13:09 -0800 Subject: [PATCH 2/7] frontend cap application fix, make more readable, swap order of operations, apply backend cap --- front_end/src/utils/forecasts/dataset.ts | 112 +++++++++++------------ questions/serializers/common.py | 8 +- 2 files changed, 54 insertions(+), 66 deletions(-) diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index c36474c1ff..cbf1e2540c 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -18,96 +18,86 @@ import { function standardizeCdf( cdf: number[], lowerOpen: boolean, - upperOpen: boolean + upperOpen: boolean, + inboundOutcomeCount: number ): number[] { if (cdf.length === 0) { return []; } - const pmf: number[] = []; + // apply lower bound + const scaleLowerTo = lowerOpen ? 0 : cdf[0] ?? 0; + const scaleUpperTo = upperOpen ? 1 : cdf[cdf.length - 1] ?? 1; + const rescaledInboundMass = scaleUpperTo - scaleLowerTo || 1; + const applyMinimum = (F: number, location: number) => { + const rescaledF = (F - scaleLowerTo) / rescaledInboundMass; + if (lowerOpen && upperOpen) { + return 0.988 * rescaledF + 0.01 * location + 0.001; + } + if (lowerOpen) { + return 0.989 * rescaledF + 0.01 * location + 0.001; + } + if (upperOpen) { + return 0.989 * rescaledF + 0.01 * location; + } + return 0.99 * rescaledF + 0.01 * location; + }; + cdf = cdf.map((F, index) => applyMinimum(F, index / cdf.length)); + + // apply upper bound + let pmf: number[] = []; pmf.push(cdf[0] ?? 0); for (let i = 1; i < cdf.length; i++) { pmf.push((cdf[i] ?? 0) - (cdf[i - 1] ?? 0)); } pmf.push(1 - (cdf[cdf.length - 1] ?? 1)); // cap depends on cdf_size (0.2 if size is the default 201) - const cap = - ((0.2 - 1e-7) * (DefaultInboundOutcomeCount + 1)) / Math.max(cdf.length, 1); - const capPmf = (scale: number) => { - return pmf.map((value, i) => { - if (i == 0 || i == pmf.length - 1) { - return value; - } - return Math.min(cap, scale * value); - }); - }; - const cappedSum = (scale: number) => { - return capPmf(scale).reduce((acc, value) => acc + value, 0); - }; + const cap = 0.2 * (DefaultInboundOutcomeCount / inboundOutcomeCount); + const capPmf = (scale: number) => + pmf.map((value, i) => + i == 0 || i == pmf.length - 1 ? value : Math.min(cap, scale * value) + ); + const cappedSum = (scale: number) => + capPmf(scale).reduce((acc, value) => acc + value, 0); // find the appropriate scale search space - const inboundMass = (cdf[cdf.length - 1] ?? 1) - (cdf[0] ?? 0); let lo = 1; let hi = 1; - while (cappedSum(hi) < inboundMass) { - hi *= 1.1; - } - // home in on scale value that makes capped sum approx inboundMass + let scale = 1; + while (cappedSum(hi) < 1) hi *= 1.2; + // hone in on scale value that makes capped sum 1 for (let i = 0; i < 100; i++) { - const scale = 0.5 * (lo + hi); + scale = 0.5 * (lo + hi); const s = cappedSum(scale); - if (s < inboundMass) { + if (s == 1) { + hi = scale; + break; + } else if (cappedSum(scale) < 1) { lo = scale; } else { hi = scale; } - if (hi - lo < 1e-12) { + if (hi - lo < 2e-5) { break; } } // apply scale and renormalize - const scaledPmf = capPmf(0.5 * (lo + hi)); - console.log(Math.max(...scaledPmf.slice(1, pmf.length - 1))); - const totalMass = scaledPmf.reduce((acc, value) => acc + value, 0); - const normalizedPmf = scaledPmf.map((value) => value / (totalMass ?? 1)); + pmf = capPmf(hi); + const inboundScaleFactor = + ((cdf[cdf.length - 1] ?? 1) - (cdf[0] ?? 0)) / + pmf.slice(1, pmf.length - 1).reduce((acc, value) => acc + value, 0); + pmf = pmf.map((value, i) => + i == 0 || i == pmf.length - 1 ? value : value * inboundScaleFactor + ); // back to CDF space - const standardizedCdf: number[] = []; + cdf = []; let cumulative = 0; - for (let i = 0; i < normalizedPmf.length - 1; i++) { - cumulative += normalizedPmf[i] ?? 0; - standardizedCdf.push(cumulative); + for (let i = 0; i < pmf.length - 1; i++) { + cumulative += pmf[i] ?? 0; + cdf.push(cumulative); } - const scaleLowerTo = lowerOpen ? 0 : standardizedCdf[0] ?? 0; - const lastStandardizedIndex = standardizedCdf.length - 1; - const lastStandardizedCdf = - lastStandardizedIndex >= 0 ? standardizedCdf[lastStandardizedIndex] : 1; - const scaleUpperTo = upperOpen ? 1 : lastStandardizedCdf ?? 1; - const rescaledInboundMass = scaleUpperTo - scaleLowerTo; - - const applyMinimum = (F: number, location: number) => { - const rescaledInboundMassSafe = - rescaledInboundMass === 0 ? 1 : rescaledInboundMass; - const rescaledF = (F - scaleLowerTo) / rescaledInboundMassSafe; - if (lowerOpen && upperOpen) { - return 0.988 * rescaledF + 0.01 * location + 0.001; - } - if (lowerOpen) { - return 0.989 * rescaledF + 0.01 * location + 0.001; - } - if (upperOpen) { - return 0.989 * rescaledF + 0.01 * location; - } - return 0.99 * rescaledF + 0.01 * location; - }; - - const denominator = - standardizedCdf.length > 1 ? standardizedCdf.length - 1 : 1; - return standardizedCdf.map((F, index) => { - const location = denominator === 0 ? 0 : index / denominator; - const standardizedValue = applyMinimum(F, location); - return Math.round(standardizedValue * 1e10) / 1e10; - }); + return cdf; } /** diff --git a/questions/serializers/common.py b/questions/serializers/common.py index 2bd9538d32..53fb54721c 100644 --- a/questions/serializers/common.py +++ b/questions/serializers/common.py @@ -489,11 +489,9 @@ def continuous_validation(self, continuous_cdf, question: Question): "continuous_cdf must be increasing by at least " f"{min_diff} at every step.\n" ) - # max diff for default CDF is derived empirically from slider positions - # TODO: make this lower and scale with inbound_outcome_count - max_diff = ( - 0.59 if len(continuous_cdf) == DEFAULT_INBOUND_OUTCOME_COUNT + 1 else 1 - ) + # Check if maximum difference between cdf points is acceptable + # (0.2 if inbound outcome count is the default 200) + max_diff = 0.2 * DEFAULT_INBOUND_OUTCOME_COUNT / inbound_outcome_count if not all(inbound_pmf <= max_diff): errors += ( "continuous_cdf must be increasing by no more than " From f8dc4fc44724fa8abde1a45691e2199b9d1cdeb7 Mon Sep 17 00:00:00 2001 From: lsabor Date: Sun, 2 Nov 2025 10:15:09 -0800 Subject: [PATCH 3/7] round to 10 decimal places --- front_end/src/utils/forecasts/dataset.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index cbf1e2540c..741bc49406 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -97,6 +97,8 @@ function standardizeCdf( cdf.push(cumulative); } + // finally round to 10 decimal places + cdf = cdf.map((value) => Math.round(value * 1e10) / 1e10); return cdf; } From 80206a73e455ff681fd39e791cef385706e75fb7 Mon Sep 17 00:00:00 2001 From: lsabor Date: Sun, 2 Nov 2025 10:31:20 -0800 Subject: [PATCH 4/7] return to simpler implementation of applying lower bound --- front_end/src/utils/forecasts/dataset.ts | 26 ++++++++---------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index 741bc49406..5a5802cf52 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -26,23 +26,15 @@ function standardizeCdf( } // apply lower bound - const scaleLowerTo = lowerOpen ? 0 : cdf[0] ?? 0; - const scaleUpperTo = upperOpen ? 1 : cdf[cdf.length - 1] ?? 1; - const rescaledInboundMass = scaleUpperTo - scaleLowerTo || 1; - const applyMinimum = (F: number, location: number) => { - const rescaledF = (F - scaleLowerTo) / rescaledInboundMass; - if (lowerOpen && upperOpen) { - return 0.988 * rescaledF + 0.01 * location + 0.001; - } - if (lowerOpen) { - return 0.989 * rescaledF + 0.01 * location + 0.001; - } - if (upperOpen) { - return 0.989 * rescaledF + 0.01 * location; - } - return 0.99 * rescaledF + 0.01 * location; - }; - cdf = cdf.map((F, index) => applyMinimum(F, index / cdf.length)); + const cdfOffset = + lowerOpen && upperOpen + ? (F: number, x: number) => 0.988 * F + 0.01 * x + 0.001 + : lowerOpen + ? (F: number, x: number) => 0.989 * F + 0.01 * x + 0.001 + : upperOpen + ? (F: number, x: number) => 0.989 * F + 0.01 * x + : (F: number, x: number) => 0.99 * F + 0.01 * x; + cdf = cdf.map((F, index) => cdfOffset(F, index / (cdf.length - 1))); // apply upper bound let pmf: number[] = []; From e62321f06311476679c53f3886739810efaccdc1 Mon Sep 17 00:00:00 2001 From: lsabor Date: Sun, 2 Nov 2025 10:34:48 -0800 Subject: [PATCH 5/7] use the scale variable --- front_end/src/utils/forecasts/dataset.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index 5a5802cf52..e50ca66a36 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -62,7 +62,6 @@ function standardizeCdf( scale = 0.5 * (lo + hi); const s = cappedSum(scale); if (s == 1) { - hi = scale; break; } else if (cappedSum(scale) < 1) { lo = scale; @@ -74,7 +73,7 @@ function standardizeCdf( } } // apply scale and renormalize - pmf = capPmf(hi); + pmf = capPmf(scale); const inboundScaleFactor = ((cdf[cdf.length - 1] ?? 1) - (cdf[0] ?? 0)) / pmf.slice(1, pmf.length - 1).reduce((acc, value) => acc + value, 0); From 251164b9629976642b26c02d2d69bb3008646407 Mon Sep 17 00:00:00 2001 From: lsabor Date: Sun, 2 Nov 2025 15:12:04 -0800 Subject: [PATCH 6/7] update api docs and minor rework of frontend logic --- docs/openapi.yml | 66 +++++++++++++++++++----- front_end/src/utils/forecasts/dataset.ts | 8 ++- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/docs/openapi.yml b/docs/openapi.yml index c4fdb37a86..dcb012fc5f 100644 --- a/docs/openapi.yml +++ b/docs/openapi.yml @@ -183,12 +183,13 @@ info: that the Metaculus api will accept. But wait! Just because we have a cdf that represents out beliefs, it doesn't mean Metaculus will accept it. We'll have to make sure it obeys a few rules, lest it be rejected as invalid. - 1. The cdf must be strictly increasing by at least 0.00005 per step. This is because Metaculus evaluates continuous forecasts - by their PDF (technically a PMF) dervied as the set of differences between consecutive CDF points, and 0.00005 is the - minimum value allowed to avoid scores getting too arbitrarily negative. + 1. The cdf must be strictly increasing by at least 0.00005 per step (1% / inbound_outcome_count). This is because Metaculus + evaluates continuous forecasts by their PDF (technically a PMF) dervied as the set of differences between consecutive + CDF points, and 0.00005 is the minimum value allowed to avoid scores getting too arbitrarily negative. Note that if the + inbound_outcome_count is less than the normal 200, this threshold will be larger. - 2. The cdf must not increase by more than 0.59 at any step, as this is the maximum value attainable via the sliders in the UI. - This threshold may be lowered in the future. + 2. The cdf must not increase by more than 0.2 at any step if the question has the normal inbound_outcome_count of 200. Otherwise, + that threshold is scaled as such: 0.2 * (200 / inbound_outcome_count). This prevents too extreme spikes in the distribution. 3. The cdf must obey bounds. If a boundary is open, at least 0.1% of probability mass must be assigned outside of it; if it is closed, no probability mass may be outside of it. @@ -197,19 +198,27 @@ info: so those with an abundance of precision in their forecasts way want to skip it. The cdfs (and thus their derived pdfs) you see on the website have been standardized in this way. ```python - def standardize_cdf(cdf: list[float], question_data: dict) -> list[float]: + def standardize_cdf(cdf, question_data: dict): """ Takes a cdf and returns a standardized version of it - assigns no mass outside of closed bounds (scales accordingly) - assigns at least a minimum amount of mass outside of open bounds - increasing by at least the minimum amount (0.01 / 200 = 0.0005) + - caps the maximum growth to 0.2 - TODO: add smoothing over cdfs that spike too heavily (exceed a change of 0.59) + Note, thresholds change with different `inbound_outcome_count`s """ lower_open = question_data["open_lower_bound"] upper_open = question_data["open_upper_bound"] + inbound_outcome_count = question_data["inbound_outcome_count"] + default_inbound_outcome_count = 200 + cdf = np.asarray(cdf, dtype=float) + if not cdf.size: + return [] + + # apply lower bound & enforce boundary values scale_lower_to = 0 if lower_open else cdf[0] scale_upper_to = 1.0 if upper_open else cdf[-1] rescaled_inbound_mass = scale_upper_to - scale_lower_to @@ -227,13 +236,44 @@ info: return 0.989 * rescaled_F + 0.01 * location return 0.99 * rescaled_F + 0.01 * location - standardized_cdf = [] - for i, F in enumerate(cdf): - standardized_F = standardize(F, i / (len(cdf) - 1)) - # round to avoid floating point errors - standardized_cdf.append(round(standardized_F, 10)) + for i, value in enumerate(cdf): + cdf[i] = standardize(value, i / (len(cdf) - 1)) + + # apply upper bound + # operate in PMF space + pmf = np.diff(cdf, prepend=0, append=1) + # cap depends on inboundOutcomeCount (0.2 if it is the default 200) + cap = 0.2 * (default_inbound_outcome_count / inbound_outcome_count) + + def cap_pmf(scale: float) -> np.ndarray: + return np.concatenate([pmf[:1], np.minimum(cap, scale * pmf[1:-1]), pmf[-1:]]) + + def capped_sum(scale: float) -> float: + return float(cap_pmf(scale).sum()) + + # find the appropriate scale search space + lo = hi = scale = 1.0 + while capped_sum(hi) < 1.0: + hi *= 1.2 + # hone in on scale value that makes capped sum 1 + for _ in range(100): + scale = 0.5 * (lo + hi) + s = capped_sum(scale) + if s < 1.0: + lo = scale + else: + hi = scale + if s == 1.0 or (hi - lo) < 2e-5: + break + # apply scale and renormalize + pmf = cap_pmf(scale) + pmf[1:-1] *= (cdf[-1] - cdf[0]) / pmf[1:-1].sum() + # back to CDF space + cdf = np.cumsum(pmf)[:-1] - return standardized_cdf + # round to minimize floating point errors + cdf = np.round(cdf, 10) + return cdf.tolist() ``` With this tiny guide, you can be well along your way to submitting continuous forecasts to Metaculus. diff --git a/front_end/src/utils/forecasts/dataset.ts b/front_end/src/utils/forecasts/dataset.ts index e50ca66a36..7e8c9d4e9b 100644 --- a/front_end/src/utils/forecasts/dataset.ts +++ b/front_end/src/utils/forecasts/dataset.ts @@ -43,7 +43,7 @@ function standardizeCdf( pmf.push((cdf[i] ?? 0) - (cdf[i - 1] ?? 0)); } pmf.push(1 - (cdf[cdf.length - 1] ?? 1)); - // cap depends on cdf_size (0.2 if size is the default 201) + // cap depends on inboundOutcomeCount (0.2 if it is the default 200) const cap = 0.2 * (DefaultInboundOutcomeCount / inboundOutcomeCount); const capPmf = (scale: number) => pmf.map((value, i) => @@ -61,14 +61,12 @@ function standardizeCdf( for (let i = 0; i < 100; i++) { scale = 0.5 * (lo + hi); const s = cappedSum(scale); - if (s == 1) { - break; - } else if (cappedSum(scale) < 1) { + if (s < 1) { lo = scale; } else { hi = scale; } - if (hi - lo < 2e-5) { + if (s === 1 || hi - lo < 2e-5) { break; } } From 4c1f0383bddece54a194a9af6cead74d28048bba Mon Sep 17 00:00:00 2001 From: lsabor Date: Wed, 5 Nov 2025 14:33:55 -0800 Subject: [PATCH 7/7] change value back to 0.59 --- questions/serializers/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/questions/serializers/common.py b/questions/serializers/common.py index 53fb54721c..137f5b381c 100644 --- a/questions/serializers/common.py +++ b/questions/serializers/common.py @@ -490,8 +490,9 @@ def continuous_validation(self, continuous_cdf, question: Question): f"{min_diff} at every step.\n" ) # Check if maximum difference between cdf points is acceptable - # (0.2 if inbound outcome count is the default 200) - max_diff = 0.2 * DEFAULT_INBOUND_OUTCOME_COUNT / inbound_outcome_count + # (0.59 if inbound outcome count is the default 200) + # TODO: switch this value to 0.2 after coordinating + max_diff = 0.59 * DEFAULT_INBOUND_OUTCOME_COUNT / inbound_outcome_count if not all(inbound_pmf <= max_diff): errors += ( "continuous_cdf must be increasing by no more than "