authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-18 11:23:34-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-18 11:23:34-07:00
log8ebb6112377a110afb956ac01fb44fee68f5709c
tree59c32a528bf81f640bb370bc75936221f832841c
parent6d68677ff90915c17de06d78bb498d41a1adfe6a
signature Signed by PGP key B5690EEEBB952194

fix(ci): accept V2 transfer reports without cross-scenario comparisons (#12492)

2 files changed, 20 insertions(+), 2 deletions(-)

.github/scripts/thread-transfer-report.cjs+2-2
...@@ -61,7 +61,7 @@ function validateResult(value) {...@@ -61,7 +61,7 @@ function validateResult(value) {
61 }61 }
6262
63 assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");63 assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
64 if (value.scenario.id !== "thread-transfer-v1") {64 if (!["thread-transfer-v1", "thread-transfer-v2"].includes(value.scenario.id)) {
65 throw new Error("result.scenario.id is not supported");65 throw new Error("result.scenario.id is not supported");
66 }66 }
67 for (const key of SCENARIO_KEYS.slice(1)) {67 for (const key of SCENARIO_KEYS.slice(1)) {
...@@ -157,7 +157,7 @@ function renderComment(input) {...@@ -157,7 +157,7 @@ function renderComment(input) {
157 `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,157 `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
158 );158 );
159159
160 if (baseline && baseline.providers[provider].ceiling[metric.key] !== ceiling) {160 if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
161 ceilingChanges.push(161 ceilingChanges.push(
162 `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,162 `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
163 );163 );
.github/scripts/thread-transfer-report.test.cjs+18
...@@ -290,3 +290,21 @@ test("preserves a successful result when a same-SHA rerun has no artifact", asyn...@@ -290,3 +290,21 @@ test("preserves a successful result when a same-SHA rerun has no artifact", asyn
290 assert.equal(published, true);290 assert.equal(published, true);
291 assert.equal(updatedComment, false);291 assert.equal(updatedComment, false);
292});292});
293
294test("accepts V2 without comparing it to the V1 scenario", () => {
295 const current = result();
296 current.scenario.id = "thread-transfer-v2";
297 current.providers.codex.ceiling = { ...current.providers.codex.ceiling, totalWireBytes: 3000000 };
298 assert.equal(validateResult(current), current);
299 const comment = renderComment({
300 current,
301 baseline: result(),
302 currentRun: { sha: "bbbbbbbb", conclusion: "success", url: "https://example.com/current" },
303 baselineRun: { sha: "aaaaaaaa", matchesBase: true, url: "https://example.com/baseline" },
304 });
305 assert.match(comment, /fixture changed/);
306 assert.doesNotMatch(comment, /This PR changes transfer ceilings/);
307 assert.doesNotMatch(comment, /[+-]\d+\.\d+%/);
308 current.scenario.id = "unrecognized";
309 assert.throws(() => validateResult(current), /not supported/);
310});