|
| 1 | +/** |
| 2 | + * Ox Alpha is a Web/Cloud-only row served at a list price of zero by an |
| 3 | + * anonymous host, and every unusual thing about it is unusual on purpose. These |
| 4 | + * tests pin the three decisions that a later reader would otherwise correct |
| 5 | + * back to convention: it is metered by no pool, it warns without claiming a |
| 6 | + * training grant, and it names its own reasoning effort rather than taking the |
| 7 | + * provider's. |
| 8 | + */ |
| 9 | +import { describe, expect, it } from 'bun:test' |
| 10 | + |
| 11 | +import { |
| 12 | + FREEBUFF_MODELS, |
| 13 | + FREEBUFF_SERVICE_ONLY_MODEL_IDS, |
| 14 | + FREEBUFF_WEB_GEO_EXEMPT_MODEL_IDS, |
| 15 | + FREEBUFF_WEB_LIMITED_MODEL_IDS, |
| 16 | + isFreebuffServiceOnlyModelId, |
| 17 | + isFreebuffSessionModelAllowedForAccessTier, |
| 18 | + isFreebuffWebModelAllowedForLimitedTier, |
| 19 | + LIMITED_FREEBUFF_MODEL_IDS, |
| 20 | + FREEBUFF_MODEL_CONTEXT_WINDOWS, |
| 21 | + FREEBUFF_OX_ALPHA_MAX_PRICE, |
| 22 | + FREEBUFF_OX_ALPHA_MODEL_ID, |
| 23 | + FREEBUFF_MIMO_V25_MODEL_ID, |
| 24 | + FREEBUFF_STANDARD_MODEL_IDS, |
| 25 | + FREEBUFF_TRACED_MODEL_IDS, |
| 26 | + FREEBUFF_WEB_ALL_MODELS, |
| 27 | + FREEBUFF_WEB_GOD_ONLY_MODEL_IDS, |
| 28 | + FREEBUFF_WEB_MODELS, |
| 29 | + FREEBUFF_WEB_PREMIUM_MODEL_IDS, |
| 30 | + getFreebuffModelReasoningEffort, |
| 31 | + isFreebuffOxAlphaModelId, |
| 32 | + isFreebuffWebModelId, |
| 33 | + SUPPORTED_FREEBUFF_MODELS, |
| 34 | +} from '../constants/freebuff-models' |
| 35 | +import { |
| 36 | + FREEBUFF_ROOT_AGENT_IDS, |
| 37 | + FREEBUFF_ROOT_AGENT_ID_BY_MODEL, |
| 38 | + FREEBUFF_WEB_BASE3_AGENT_ID_BY_MODEL, |
| 39 | + FREEBUFF_CLI_BASE3_AGENT_ID_BY_MODEL, |
| 40 | + FREE_MODE_AGENT_MODELS, |
| 41 | +} from '../constants/free-agents' |
| 42 | + |
| 43 | +const OX = FREEBUFF_OX_ALPHA_MODEL_ID |
| 44 | +const row = FREEBUFF_WEB_MODELS.find((m) => m.id === OX) |
| 45 | + |
| 46 | +describe('Ox Alpha is a Web/Cloud row', () => { |
| 47 | + it('is offered on the browser surfaces and to every user', () => { |
| 48 | + expect(row).toBeDefined() |
| 49 | + expect(isFreebuffWebModelId(OX)).toBe(true) |
| 50 | + // Not a god-only test row — this one is meant to carry real traffic. |
| 51 | + expect(FREEBUFF_WEB_GOD_ONLY_MODEL_IDS as readonly string[]).not.toContain( |
| 52 | + OX, |
| 53 | + ) |
| 54 | + }) |
| 55 | + |
| 56 | + it('stays off the CLI and Desktop', () => { |
| 57 | + // The absence from these two lists IS the surface gate: FREEBUFF_MODELS is |
| 58 | + // the CLI/Desktop catalog and SUPPORTED_FREEBUFF_MODELS is what |
| 59 | + // isFreebuffSessionModelId admits there. Adding it to either ships the |
| 60 | + // model to surfaces whose capacity we have not agreed to spend yet. |
| 61 | + expect(FREEBUFF_MODELS.map((m) => m.id)).not.toContain(OX) |
| 62 | + expect(SUPPORTED_FREEBUFF_MODELS.map((m) => m.id)).not.toContain(OX) |
| 63 | + expect(FREEBUFF_CLI_BASE3_AGENT_ID_BY_MODEL[OX]).toBeUndefined() |
| 64 | + expect(FREEBUFF_WEB_ALL_MODELS.map((m) => m.id)).toContain(OX) |
| 65 | + }) |
| 66 | + |
| 67 | + it('matches dated builds of itself, so a variant cannot escape the fence', () => { |
| 68 | + expect(isFreebuffOxAlphaModelId(OX)).toBe(true) |
| 69 | + expect(isFreebuffOxAlphaModelId('stealth/ox-alpha-20260820')).toBe(true) |
| 70 | + expect(isFreebuffOxAlphaModelId('stealth/ox-beta')).toBe(false) |
| 71 | + expect(isFreebuffOxAlphaModelId(null)).toBe(false) |
| 72 | + }) |
| 73 | +}) |
| 74 | + |
| 75 | +describe('Ox Alpha is metered by no pool, and that rests on the price fence', () => { |
| 76 | + it('is standard rather than premium', () => { |
| 77 | + expect(row?.premium).toBe(false) |
| 78 | + expect(FREEBUFF_WEB_PREMIUM_MODEL_IDS as readonly string[]).not.toContain( |
| 79 | + OX, |
| 80 | + ) |
| 81 | + expect(FREEBUFF_STANDARD_MODEL_IDS).toContain(OX) |
| 82 | + }) |
| 83 | + |
| 84 | + it('fences the price at exactly zero', () => { |
| 85 | + // The two numbers are the whole justification for the assertion above. A |
| 86 | + // ceiling above zero admits a repriced endpoint, and this row has no pool |
| 87 | + // to bound what that would cost — so raising either of these without also |
| 88 | + // giving the model a quota is the mistake this test exists to stop. |
| 89 | + expect(FREEBUFF_OX_ALPHA_MAX_PRICE.prompt).toBe(0) |
| 90 | + expect(FREEBUFF_OX_ALPHA_MAX_PRICE.completion).toBe(0) |
| 91 | + }) |
| 92 | +}) |
| 93 | + |
| 94 | +describe('Ox Alpha discloses retention without claiming a training grant', () => { |
| 95 | + it('warns, while staying dataUse: service', () => { |
| 96 | + // The one row in the catalog where a warning does NOT imply |
| 97 | + // `dataUse: 'training'`. OpenRouter's stealth terms say the host retains |
| 98 | + // prompts and completions and does not train on them, so the warning is |
| 99 | + // owed and the training label is not. `dataUse` is machine-readable and |
| 100 | + // drives trace storage, which is why it may not be used as a stand-in for |
| 101 | + // "something to disclose". |
| 102 | + expect(row?.warning).toBeDefined() |
| 103 | + expect(row?.dataUse).toBe('service') |
| 104 | + expect(FREEBUFF_TRACED_MODEL_IDS as readonly string[]).not.toContain(OX) |
| 105 | + }) |
| 106 | +}) |
| 107 | + |
| 108 | +describe('Ox Alpha names its own reasoning effort', () => { |
| 109 | + it('sends high, not the provider default of max', () => { |
| 110 | + // Reasoning is MANDATORY on this model and the provider's default rung is |
| 111 | + // `max` — measured at ~4x the completion tokens and ~4x the latency of |
| 112 | + // `high`, with samples truncated mid-answer. An unset effort here is not a |
| 113 | + // neutral choice; it is that rung. |
| 114 | + expect(getFreebuffModelReasoningEffort(OX)).toBe('high') |
| 115 | + expect(row?.efforts).toEqual(['low', 'high', 'max']) |
| 116 | + expect(row?.defaultEffort).toBe('high') |
| 117 | + }) |
| 118 | + |
| 119 | + it('carries a context window rather than falling back to the default', () => { |
| 120 | + // Absent, base-chat budgets a million-token model 131,072 and summarizes a |
| 121 | + // thread ~8x early, throwing away the prompt cache each time. |
| 122 | + expect(FREEBUFF_MODEL_CONTEXT_WINDOWS[OX]).toBe(1_000_000) |
| 123 | + }) |
| 124 | +}) |
| 125 | + |
| 126 | +describe('Ox Alpha is registered everywhere a root must be', () => { |
| 127 | + it('has a base2 and a base3 root, both allowlisted and both pinned', () => { |
| 128 | + expect(FREEBUFF_ROOT_AGENT_ID_BY_MODEL[OX]).toBe('base2-free-ox-alpha') |
| 129 | + expect(FREEBUFF_WEB_BASE3_AGENT_ID_BY_MODEL[OX]).toBe('base3-free-ox-alpha') |
| 130 | + for (const id of ['base2-free-ox-alpha', 'base3-free-ox-alpha']) { |
| 131 | + // Missing from this list, the root's own requests 403 with |
| 132 | + // free_mode_invalid_agent_hierarchy. |
| 133 | + expect(FREEBUFF_ROOT_AGENT_IDS as readonly string[]).toContain(id) |
| 134 | + // Pinned to exactly one model: a root that allows more than one is a |
| 135 | + // door onto everything else it allows. |
| 136 | + expect([...(FREE_MODE_AGENT_MODELS[id] ?? [])]).toEqual([OX]) |
| 137 | + } |
| 138 | + }) |
| 139 | +}) |
| 140 | + |
| 141 | +describe('Ox Alpha reaches limited-tier regions', () => { |
| 142 | + it('is offered and admitted at limited tier', () => { |
| 143 | + // Both halves, because they are enforced in different places and only one |
| 144 | + // of them is visible to a user: the picker list decides whether the row is |
| 145 | + // drawn, and the session gate decides whether their first send works. A row |
| 146 | + // that passes one and fails the other is the exact shape of the GLM/Desktop |
| 147 | + // bug freebuff-offer-invariants.ts was written for. |
| 148 | + expect(isFreebuffWebModelAllowedForLimitedTier(OX)).toBe(true) |
| 149 | + expect(isFreebuffSessionModelAllowedForAccessTier(OX, 'limited')).toBe(true) |
| 150 | + expect(FREEBUFF_WEB_LIMITED_MODEL_IDS).toContain(OX) |
| 151 | + expect(FREEBUFF_WEB_GEO_EXEMPT_MODEL_IDS as readonly string[]).toContain(OX) |
| 152 | + }) |
| 153 | + |
| 154 | + it('reaches limited regions WITHOUT joining the CLI/Desktop limited catalog', () => { |
| 155 | + // The split is the whole trick. LIMITED_FREEBUFF_MODEL_IDS maps over |
| 156 | + // SUPPORTED_FREEBUFF_MODELS and feeds the CLI picker, the home FAQ and the |
| 157 | + // README, so adding a Web/Cloud-only row there would advertise it on |
| 158 | + // surfaces that cannot run it — and `LIMITED_FREEBUFF_MODELS` would carry an |
| 159 | + // `undefined!` where the row should be. |
| 160 | + expect(LIMITED_FREEBUFF_MODEL_IDS as readonly string[]).not.toContain(OX) |
| 161 | + }) |
| 162 | + |
| 163 | + it('still costs the limited pool nothing extra', () => { |
| 164 | + // The limited tier is metered by REGION, not by model: every limited |
| 165 | + // session draws on the same pool whichever row it picks. This widens what |
| 166 | + // those users may choose, never how much they get — so no quota list |
| 167 | + // mentions the model, and none should. |
| 168 | + expect(FREEBUFF_WEB_PREMIUM_MODEL_IDS as readonly string[]).not.toContain( |
| 169 | + OX, |
| 170 | + ) |
| 171 | + }) |
| 172 | +}) |
| 173 | + |
| 174 | +describe('Ox Alpha is unreachable from anything but our own runner', () => { |
| 175 | + it('is service-account-only', () => { |
| 176 | + // The one gate here that an attacker cannot restate. Everything else that |
| 177 | + // keeps this model off a surface is a client-side fact — absent from the |
| 178 | + // CLI catalog means no shipped build renders it, which stops our users and |
| 179 | + // nobody else. A hand-written caller names the agent id and model directly |
| 180 | + // and passes every allowlist, because that pair IS valid; what it cannot |
| 181 | + // produce is the runner's API key. |
| 182 | + expect(isFreebuffServiceOnlyModelId(OX)).toBe(true) |
| 183 | + expect(FREEBUFF_SERVICE_ONLY_MODEL_IDS as readonly string[]).toContain(OX) |
| 184 | + }) |
| 185 | + |
| 186 | + it('covers dated builds and nothing else', () => { |
| 187 | + expect(isFreebuffServiceOnlyModelId('stealth/ox-alpha-20260820')).toBe(true) |
| 188 | + expect(isFreebuffServiceOnlyModelId(FREEBUFF_MIMO_V25_MODEL_ID)).toBe(false) |
| 189 | + expect(isFreebuffServiceOnlyModelId(null)).toBe(false) |
| 190 | + }) |
| 191 | + |
| 192 | + it('is the only model on that list, and the reason is the missing pool', () => { |
| 193 | + // Muse Spark and Kimi K3 Eco are browser-only too, and would pass this gate |
| 194 | + // in normal operation — but both are premium-pooled, so a third party |
| 195 | + // reaching one spends a quota that runs out. Ox Alpha is metered by |
| 196 | + // nothing, which is what makes it worth a gate of its own. If that ever |
| 197 | + // stops being the distinction, add them here; do not delete this test. |
| 198 | + expect([...FREEBUFF_SERVICE_ONLY_MODEL_IDS]).toEqual([OX]) |
| 199 | + expect(FREEBUFF_STANDARD_MODEL_IDS).toContain(OX) |
| 200 | + }) |
| 201 | +}) |
| 202 | + |
| 203 | +describe('the data tag', () => { |
| 204 | + it('renders wherever the row does, on both tiers', () => { |
| 205 | + // `warning` is what the picker draws the compact "Data" label from, and the |
| 206 | + // row is reachable from the full-access standard group AND the limited |
| 207 | + // tier's list — so a warning that only existed on one of those would be a |
| 208 | + // disclosure a limited-region user never sees. |
| 209 | + expect(row?.warning).toBe('Anonymous provider retains prompts') |
| 210 | + expect(FREEBUFF_WEB_MODELS.find((m) => m.id === OX)?.warning).toBeDefined() |
| 211 | + expect( |
| 212 | + FREEBUFF_WEB_LIMITED_MODEL_IDS.includes(OX) && row?.warning !== undefined, |
| 213 | + ).toBe(true) |
| 214 | + }) |
| 215 | +}) |
0 commit comments