authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-25 00:58:13-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-25 00:58:13-07:00
loge3e7cc3fc2e02966efe4efc45216dadcd046bfbb
tree54cf656522a8a77a294015fb80ca33fe3caecec0
parent86054b6df2c5e38b6e27b980eae2716bfe1da202
signature Signed by PGP key B5690EEEBB952194

fix(usage): price Claude fast-mode requests at the fast rate (#13599)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

8 files changed, 129 insertions(+), 38 deletions(-)

apps/server/src/usage/usageAggregation.test.ts+2
......@@ -12,6 +12,7 @@ const rates: RateTable = new Map([
1212 outputCostPerToken: 5e-5,
1313 cacheReadCostPerToken: 1e-6,
1414 cacheCreationCostPerToken: 1.25e-5,
15 fastMultiplier: 1,
1516 },
1617 ],
1718]);
......@@ -31,6 +32,7 @@ function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
3132 reasoningTokens: 0,
3233 },
3334 reportedCostUsd: null,
35 fast: false,
3436 dedupeKey: null,
3537 ...overrides,
3638 };
apps/server/src/usage/usageAggregation.ts+2-9
......@@ -161,20 +161,13 @@ export class UsageAggregator {
161161 this.#buckets.set(key, bucket);
162162 }
163163
164 const priced = priceUsage(
165 this.#options.rates,
166 record.model,
167 record.totals,
168 record.reportedCostUsd,
169 this.#options.priceOverrides,
170 );
164 const priced = priceUsage(this.#options.rates, record, this.#options.priceOverrides);
171165
172166 bucket.totals = addTotals(bucket.totals, record.totals);
173167 bucket.costUsd += priced.costUsd;
174168 bucket.cacheSavingsUsd += cacheSavingsUsd(
175169 this.#options.rates,
176 record.model,
177 record.totals,
170 record,
178171 this.#options.priceOverrides,
179172 );
180173 bucket.records += 1;
apps/server/src/usage/usagePricing.test.ts+35-7
......@@ -22,6 +22,12 @@ describe("usage pricing", () => {
2222 outputTokens: 1_000_000,
2323 reasoningTokens: 500_000,
2424 };
25 const record = (model: string, reportedCostUsd: number | null = null, fast = false) => ({
26 model,
27 totals,
28 reportedCostUsd,
29 fast,
30 });
2531
2632 it("uses custom token rates ahead of public and provider-reported costs", () => {
2733 const table = parseRateTable({ "example-model": rate(1) });
......@@ -35,12 +41,12 @@ describe("usage pricing", () => {
3541 });
3642
3743 for (const reportedCostUsd of [null, 99]) {
38 expect(priceUsage(table, "example-model", totals, reportedCostUsd, overrides)).toEqual({
44 expect(priceUsage(table, record("example-model", reportedCostUsd), overrides)).toEqual({
3945 costUsd: 13.5,
4046 costSource: "modelPriced",
4147 });
4248 }
43 expect(cacheSavingsUsd(table, "example-model", totals, overrides)).toBe(1.5);
49 expect(cacheSavingsUsd(table, record("example-model"), overrides)).toBe(1.5);
4450 });
4551
4652 it("prices unknown models offline and uses input prices for omitted cache rates", () => {
......@@ -49,11 +55,11 @@ describe("usage pricing", () => {
4955 "example-model": { inputCostPerMillionTokens: 2, outputCostPerMillionTokens: 8 },
5056 });
5157
52 expect(priceUsage(table, "example-model", totals, null, overrides)).toEqual({
58 expect(priceUsage(table, record("example-model"), overrides)).toEqual({
5359 costUsd: 14,
5460 costSource: "modelPriced",
5561 });
56 expect(cacheSavingsUsd(table, "example-model", totals, overrides)).toBe(0);
62 expect(cacheSavingsUsd(table, record("example-model"), overrides)).toBe(0);
5763 });
5864
5965 it("preserves explicit zero rates and matches only the exact trimmed model ID", () => {
......@@ -64,7 +70,7 @@ describe("usage pricing", () => {
6470 outputCostPerMillionTokens: 0,
6571 },
6672 });
67 expect(priceUsage(table, " vendor/example-model[1m] ", totals, 99, overrides)).toEqual({
73 expect(priceUsage(table, record(" vendor/example-model[1m] ", 99), overrides)).toEqual({
6874 costUsd: 0,
6975 costSource: "modelPriced",
7076 });
......@@ -74,14 +80,36 @@ describe("usage pricing", () => {
7480 "vendor/Example-model[1m]",
7581 "other/example-model[1m]",
7682 ]) {
77 expect(priceUsage(table, model, totals, null, overrides).costSource).toBe("unpriced");
78 expect(priceUsage(table, model, totals, 99, overrides)).toEqual({
83 expect(priceUsage(table, record(model), overrides).costSource).toBe("unpriced");
84 expect(priceUsage(table, record(model, 99), overrides)).toEqual({
7985 costUsd: 99,
8086 costSource: "providerReported",
8187 });
8288 }
8389 });
8490
91 it("prices fast-mode requests at the model's published fast multiple", () => {
92 const table = parseRateTable({
93 "claude-opus-5-5": { ...rate(4e-6, 2e-7), provider_specific_entry: { fast: 2, us: 1.1 } },
94 "claude-fable-5-1": { ...rate(1e-5, 2.5e-7), provider_specific_entry: { us: 1.1 } },
95 });
96 const overrides = createOverrideRateTable({
97 "claude-opus-5-5": { inputCostPerMillionTokens: 4, outputCostPerMillionTokens: 20 },
98 });
99 const cost = (model: string, fast: boolean, custom?: typeof overrides) =>
100 priceUsage(table, record(model, null, fast), custom).costUsd;
101
102 expect(cost("claude-opus-5-5", true)).toBeCloseTo(2 * cost("claude-opus-5-5", false));
103 expect(cacheSavingsUsd(table, record("claude-opus-5-5", null, true))).toBeCloseTo(
104 2 * cacheSavingsUsd(table, record("claude-opus-5-5")),
105 );
106 // No published fast tier, and custom prices, both stay at the standard rate.
107 expect(cost("claude-fable-5-1", true)).toBe(cost("claude-fable-5-1", false));
108 expect(cost("claude-opus-5-5", true, overrides)).toBe(
109 cost("claude-opus-5-5", false, overrides),
110 );
111 });
112
85113 it("keeps the canonical Fable rate separate from DeepInfra in either order", () => {
86114 const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const;
87115 const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const;
apps/server/src/usage/usagePricing.ts+43-17
......@@ -7,11 +7,9 @@
77 *
88 * @module usagePricing
99 */
10import type {
11 UsageCostSource,
12 UsageModelPriceOverride,
13 UsageTokenTotals,
14} from "@t3tools/contracts";
10import type { UsageCostSource, UsageModelPriceOverride } from "@t3tools/contracts";
11
12import type { UsageRecord } from "./usageTranscripts.ts";
1513
1614/**
1715 * The subset of a LiteLLM entry we price against. All values are USD per token.
......@@ -26,11 +24,19 @@ export interface ModelRate {
2624 readonly outputCostPerToken: number;
2725 readonly cacheReadCostPerToken: number;
2826 readonly cacheCreationCostPerToken: number;
27 /**
28 * Multiple of the rates above billed for a fast-mode request, from LiteLLM's
29 * `provider_specific_entry.fast`. `1` when the model publishes no fast tier.
30 */
31 readonly fastMultiplier: number;
2932}
3033
3134export type RateTable = ReadonlyMap<string, ModelRate>;
3235
33/** Custom IDs keep their case, provider prefix, and variant suffix. */
36/**
37 * Custom IDs keep their case, provider prefix, and variant suffix. Custom rates
38 * apply as entered, fast-mode requests included.
39 */
3440export function createOverrideRateTable(
3541 overrides: Readonly<Record<string, UsageModelPriceOverride>>,
3642): RateTable {
......@@ -44,6 +50,7 @@ export function createOverrideRateTable(
4450 (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
4551 cacheCreationCostPerToken:
4652 (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
53 fastMultiplier: 1,
4754 },
4855 ]),
4956 );
......@@ -55,12 +62,21 @@ interface LiteLlmEntry {
5562 readonly output_cost_per_token?: unknown;
5663 readonly cache_read_input_token_cost?: unknown;
5764 readonly cache_creation_input_token_cost?: unknown;
65 readonly provider_specific_entry?: unknown;
5866}
5967
6068function finiteNumber(value: unknown): number | null {
6169 return typeof value === "number" && Number.isFinite(value) ? value : null;
6270}
6371
72/** Reads `provider_specific_entry.fast`, e.g. `2` for Claude Opus 5.5. */
73function fastMultiplier(entry: LiteLlmEntry): number {
74 const specific = entry.provider_specific_entry;
75 if (typeof specific !== "object" || specific === null) return 1;
76 const fast = finiteNumber((specific as Record<string, unknown>)["fast"]);
77 return fast !== null && fast > 0 ? fast : 1;
78}
79
6480/**
6581 * Projects the LiteLLM document into a rate table.
6682 *
......@@ -92,6 +108,7 @@ export function parseRateTable(document: unknown): RateTable {
92108 // input rather than as free.
93109 cacheReadCostPerToken: finiteNumber(entry.cache_read_input_token_cost) ?? input,
94110 cacheCreationCostPerToken: finiteNumber(entry.cache_creation_input_token_cost) ?? input,
111 fastMultiplier: fastMultiplier(entry),
95112 });
96113 }
97114
......@@ -119,7 +136,8 @@ function sameRate(a: ModelRate, b: ModelRate): boolean {
119136 a.inputCostPerToken === b.inputCostPerToken &&
120137 a.outputCostPerToken === b.outputCostPerToken &&
121138 a.cacheReadCostPerToken === b.cacheReadCostPerToken &&
122 a.cacheCreationCostPerToken === b.cacheCreationCostPerToken
139 a.cacheCreationCostPerToken === b.cacheCreationCostPerToken &&
140 a.fastMultiplier === b.fastMultiplier
123141 );
124142}
125143
......@@ -165,24 +183,26 @@ export function lookupRate(table: RateTable, model: string): ModelRate | null {
165183 return table.get(key) ?? null;
166184}
167185
186/** The parts of a transcript record that decide its price. */
187export type PricedRecord = Pick<UsageRecord, "model" | "totals" | "fast" | "reportedCostUsd">;
188
168189export interface PricedUsage {
169190 readonly costUsd: number;
170191 readonly costSource: UsageCostSource;
171192}
172193
173194/**
174 * Prices a bucket's tokens.
195 * Prices one record's tokens.
175196 *
176197 * `reasoningTokens` is intentionally not charged separately: it is already
177198 * counted inside `outputTokens`.
178199 */
179200export function priceUsage(
180201 table: RateTable,
181 model: string,
182 totals: UsageTokenTotals,
183 reportedCostUsd: number | null,
202 record: PricedRecord,
184203 overrides?: RateTable,
185204): PricedUsage {
205 const { model, totals, reportedCostUsd } = record;
186206 const override = overrides?.get(model.trim());
187207 if (override === undefined && reportedCostUsd !== null && Number.isFinite(reportedCostUsd)) {
188208 return { costUsd: reportedCostUsd, costSource: "providerReported" };
......@@ -191,13 +211,16 @@ export function priceUsage(
191211 const rate = override ?? lookupRate(table, model);
192212 if (rate === null) return { costUsd: 0, costSource: "unpriced" };
193213
194 const costUsd =
214 const standardCostUsd =
195215 totals.uncachedInputTokens * rate.inputCostPerToken +
196216 totals.cachedInputTokens * rate.cacheReadCostPerToken +
197217 totals.cacheCreationTokens * rate.cacheCreationCostPerToken +
198218 totals.outputTokens * rate.outputCostPerToken;
199219
200 return { costUsd, costSource: "modelPriced" };
220 return {
221 costUsd: standardCostUsd * (record.fast ? rate.fastMultiplier : 1),
222 costSource: "modelPriced",
223 };
201224}
202225
203226/**
......@@ -206,11 +229,14 @@ export function priceUsage(
206229 */
207230export function cacheSavingsUsd(
208231 table: RateTable,
209 model: string,
210 totals: UsageTokenTotals,
232 record: PricedRecord,
211233 overrides?: RateTable,
212234): number {
213 const rate = overrides?.get(model.trim()) ?? lookupRate(table, model);
235 const rate = overrides?.get(record.model.trim()) ?? lookupRate(table, record.model);
214236 if (rate === null) return 0;
215 return totals.cachedInputTokens * (rate.inputCostPerToken - rate.cacheReadCostPerToken);
237 return (
238 record.totals.cachedInputTokens *
239 (rate.inputCostPerToken - rate.cacheReadCostPerToken) *
240 (record.fast ? rate.fastMultiplier : 1)
241 );
216242}
apps/server/src/usage/usageScanCache.test.ts+18-2
......@@ -24,6 +24,7 @@ function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
2424 reasoningTokens: 0,
2525 },
2626 reportedCostUsd: null,
27 fast: false,
2728 dedupeKey: "msg_1:",
2829 ...overrides,
2930 };
......@@ -57,7 +58,11 @@ function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][])
5758describe("scan cache round trip", () => {
5859 it("restores records unchanged", () => {
5960 const original = cacheWith([
60 ["/a.jsonl", 100, [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5" })]],
61 [
62 "/a.jsonl",
63 100,
64 [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", fast: true })],
65 ],
6166 ["/b.jsonl", 200, [record({ sessionId: "session-b", reportedCostUsd: 1.5 })]],
6267 ]);
6368 original.set("/grok.jsonl", {
......@@ -123,9 +128,20 @@ describe("scan cache round trip", () => {
123128 expect(decodeScanCache(JSON.parse(JSON.stringify(poisoned))).has("/a.jsonl")).toBe(false);
124129 });
125130
131 it("drops an entry whose fast flag is not 0 or 1", () => {
132 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record({ fast: true })]]]));
133 const row = encoded.files["/a.jsonl"]!.r[0]!;
134 const poisoned = {
135 ...encoded,
136 files: { "/a.jsonl": { ...encoded.files["/a.jsonl"]!, r: [[...row.slice(0, 10), true]] } },
137 };
138
139 expect(decodeScanCache(JSON.parse(JSON.stringify(poisoned))).has("/a.jsonl")).toBe(false);
140 });
141
126142 it("rejects a document from the previous cache version", () => {
127143 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
128 const previous = { ...encoded, version: 2 };
144 const previous = { ...encoded, version: 3 };
129145
130146 expect(decodeScanCache(JSON.parse(JSON.stringify(previous))).size).toBe(0);
131147 });
apps/server/src/usage/usageScanCache.ts+9-3
......@@ -23,7 +23,8 @@ import type { CodexScanState, UsageRecord } from "./usageTranscripts.ts";
2323// entries would keep serving double-counted records forever.
2424// v3: entries carry the parse position and reducer state so a grown file
2525// re-parses only its appended bytes instead of starting over.
26const USAGE_SCAN_CACHE_VERSION = 3 as const;
26// v4: records carry Claude fast mode, which v3 rows never captured.
27const USAGE_SCAN_CACHE_VERSION = 4 as const;
2728
2829export interface CachedFile {
2930 readonly size: number;
......@@ -58,6 +59,7 @@ type SerializedRecord = readonly [
5859 reasoningTokens: number,
5960 dedupeKey: string | null,
6061 reportedCostUsd: number | null,
62 fast: 0 | 1,
6163];
6264
6365interface SerializedFile {
......@@ -109,6 +111,7 @@ export function encodeScanCache(cache: ScanCache): SerializedCache {
109111 record.totals.reasoningTokens,
110112 record.dedupeKey,
111113 record.reportedCostUsd,
114 record.fast ? 1 : 0,
112115 ];
113116
114117 const files: Record<string, SerializedFile> = {};
......@@ -165,7 +168,7 @@ export function decodeScanCache(document: unknown): ScanCache {
165168 ): UsageRecord[] | null => {
166169 const records: UsageRecord[] = [];
167170 for (const row of rows) {
168 if (!isRecordArray(row) || row.length < 10) return null;
171 if (!isRecordArray(row) || row.length < 11) return null;
169172 const [
170173 timestampMs,
171174 modelIndex,
......@@ -177,6 +180,7 @@ export function decodeScanCache(document: unknown): ScanCache {
177180 reasoning,
178181 dedupeKey,
179182 reportedCostUsd,
183 fast,
180184 ] = row as SerializedRecord;
181185
182186 const model = typeof modelIndex === "number" ? models[modelIndex] : undefined;
......@@ -188,7 +192,8 @@ export function decodeScanCache(document: unknown): ScanCache {
188192 !Number.isFinite(cached) ||
189193 !Number.isFinite(cacheCreation) ||
190194 !Number.isFinite(output) ||
191 !Number.isFinite(reasoning)
195 !Number.isFinite(reasoning) ||
196 (fast !== 0 && fast !== 1)
192197 ) {
193198 return null;
194199 }
......@@ -206,6 +211,7 @@ export function decodeScanCache(document: unknown): ScanCache {
206211 reasoningTokens: reasoning,
207212 },
208213 reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null,
214 fast: fast === 1,
209215 dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null,
210216 });
211217 }
apps/server/src/usage/usageTranscripts.test.ts+11
......@@ -15,6 +15,7 @@ function claudeLine(overrides: {
1515 contentType: string;
1616 model?: string;
1717 outputTokens?: number;
18 speed?: string;
1819}): string {
1920 return JSON.stringify({
2021 type: "assistant",
......@@ -31,6 +32,7 @@ function claudeLine(overrides: {
3132 cache_creation_input_tokens: 66818,
3233 cache_read_input_tokens: 1000,
3334 output_tokens: overrides.outputTokens ?? 286,
35 ...(overrides.speed === undefined ? {} : { speed: overrides.speed }),
3436 },
3537 },
3638 });
......@@ -51,6 +53,15 @@ describe("parseClaudeLine", () => {
5153 reasoningTokens: 0,
5254 });
5355 expect(record?.dedupeKey).toBe("msg_1:");
56 expect(record?.fast).toBe(false);
57 });
58
59 it("marks fast-mode requests", () => {
60 const line = (speed: string) =>
61 parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text", speed }));
62
63 expect(line("fast")?.fast).toBe(true);
64 expect(line("standard")?.fast).toBe(false);
5465 });
5566
5667 it("gives every content block of one message the same dedupe key", () => {
apps/server/src/usage/usageTranscripts.ts+9
......@@ -15,6 +15,11 @@ export interface UsageRecord {
1515 readonly sessionId: string;
1616 readonly totals: UsageTokenTotals;
1717 readonly reportedCostUsd: number | null;
18 /**
19 * Whether the request ran in fast mode, which bills at a model-specific
20 * multiple of the standard rate. Only Claude Code records this.
21 */
22 readonly fast: boolean;
1823 /**
1924 * Key for cross-file de-duplication, or `null` when the record is inherently
2025 * unique and needs no dedup.
......@@ -145,6 +150,7 @@ export function parseClaudeLine(line: string): UsageRecord | null {
145150 reasoningTokens: 0,
146151 },
147152 reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null,
153 fast: usageRecord["speed"] === "fast",
148154 dedupeKey,
149155 };
150156}
......@@ -304,6 +310,7 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord
304310 totals,
305311 // Codex does not report cost in the rollout.
306312 reportedCostUsd: null,
313 fast: false,
307314 // Events surviving the fork-copy suppression above are unique to this
308315 // rollout, so they need no global dedup.
309316 dedupeKey: null,
......@@ -433,6 +440,7 @@ export function parseGrokLine(line: string): readonly UsageRecord[] {
433440 sessionId,
434441 totals: grokTotalsToUsage(topLevel),
435442 reportedCostUsd: grokCostTicksToUsd(topLevel.costUsdTicks),
443 fast: false,
436444 // No prompt id means we cannot tell two same-second updates apart.
437445 dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:grok`,
438446 },
......@@ -479,6 +487,7 @@ export function parseGrokLine(line: string): readonly UsageRecord[] {
479487 sessionId,
480488 totals,
481489 reportedCostUsd,
490 fast: false,
482491 dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:${entry.model}`,
483492 });
484493 }