1const fs = require("node:fs");
2const path = require("node:path");
3
4const ARTIFACT_NAME = "thread-transfer-results";
5const RESULT_FILE = "thread-transfer-result.json";
6const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
7const PROVIDERS = ["codex", "claudeAgent"];
8const OBSERVED_KEYS = [
9 "totalWireBytes",
10 "threadSnapshotWireBytes",
11 "threadSnapshotDecodedBytes",
12 "measuredTurnWebSocketWireBytes",
13 "measuredTurnWebSocketDecodedBytes",
14 "measuredTurnWebSocketMessages",
15];
16const CEILING_KEYS = [
17 "totalWireBytes",
18 "threadSnapshotWireBytes",
19 "measuredTurnWebSocketWireBytes",
20 "measuredTurnWebSocketDecodedBytes",
21 "measuredTurnWebSocketMessages",
22];
23const SCENARIO_KEYS = [
24 "id",
25 "historyTurns",
26 "historyCommandToolsPerTurn",
27 "historyMcpResultBytes",
28 "measuredCommandTools",
29 "measuredMcpResultBytes",
30];
31
32function resultShaMarker(sha) {
33 return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
34}
35
36function assertObject(value, label) {
37 if (value === null || typeof value !== "object" || Array.isArray(value)) {
38 throw new Error(`${label} must be an object`);
39 }
40}
41
42function assertExactKeys(value, expected, label) {
43 assertObject(value, label);
44 const actual = Object.keys(value).sort();
45 const wanted = [...expected].sort();
46 if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
47 throw new Error(`${label} has unexpected fields`);
48 }
49}
50
51function assertMetric(value, label) {
52 if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) {
53 throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`);
54 }
55}
56
57function validateResult(value) {
58 assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result");
59 if (value.schemaVersion !== 1) {
60 throw new Error("result.schemaVersion must be 1");
61 }
62
63 assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
64 if (!["thread-transfer-v1", "thread-transfer-v2"].includes(value.scenario.id)) {
65 throw new Error("result.scenario.id is not supported");
66 }
67 for (const key of SCENARIO_KEYS.slice(1)) {
68 assertMetric(value.scenario[key], `result.scenario.${key}`);
69 }
70
71 assertExactKeys(value.providers, PROVIDERS, "result.providers");
72 for (const provider of PROVIDERS) {
73 const entry = value.providers[provider];
74 assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`);
75 assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`);
76 assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`);
77 for (const key of OBSERVED_KEYS) {
78 assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`);
79 }
80 for (const key of CEILING_KEYS) {
81 assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`);
82 }
83 }
84
85 return value;
86}
87
88function readResult(directory) {
89 if (!directory) return undefined;
90 const file = path.join(directory, RESULT_FILE);
91 if (!fs.existsSync(file)) return undefined;
92 const stat = fs.lstatSync(file);
93 if (!stat.isFile() || stat.size > 64 * 1_024) {
94 throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
95 }
96 return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
97}
98
99function formatBytes(bytes) {
100 if (bytes < 1_024) return `${bytes} B`;
101 if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`;
102 return `${(bytes / 1_024).toFixed(1)} KiB`;
103}
104
105function formatValue(value, kind) {
106 return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
107}
108
109function formatImpact(current, baseline, kind) {
110 if (baseline === undefined) return "—";
111 const delta = current - baseline;
112 const prefix = delta > 0 ? "+" : delta < 0 ? "−" : "";
113 const magnitude = formatValue(Math.abs(delta), kind);
114 const percent =
115 baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`;
116 return `${prefix}${magnitude}${percent}`;
117}
118
119function sameScenario(left, right) {
120 return SCENARIO_KEYS.every((key) => left[key] === right[key]);
121}
122
123const METRICS = [
124 { key: "totalWireBytes", label: "Total thread wire", kind: "bytes" },
125 { key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" },
126 {
127 key: "measuredTurnWebSocketWireBytes",
128 label: "Live turn WebSocket wire",
129 kind: "bytes",
130 },
131 {
132 key: "measuredTurnWebSocketDecodedBytes",
133 label: "Live turn WebSocket decoded",
134 kind: "bytes",
135 },
136 { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
137];
138
139function renderComment(input) {
140 const current = input.current;
141 const baseline = input.baseline;
142 const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario);
143 const rows = [];
144 const ceilingChanges = [];
145 let failed = false;
146
147 for (const provider of PROVIDERS) {
148 for (const metric of METRICS) {
149 const observed = current.providers[provider].observed[metric.key];
150 const ceiling = current.providers[provider].ceiling[metric.key];
151 const baselineObserved = comparable
152 ? baseline.providers[provider].observed[metric.key]
153 : undefined;
154 const pass = observed <= ceiling;
155 failed ||= !pass;
156 rows.push(
157 `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
158 );
159
160 if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
161 ceilingChanges.push(
162 `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
163 );
164 }
165 }
166 }
167
168 const baselineLink = input.baselineRun
169 ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
170 : "unavailable";
171 const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`;
172 const notices = [];
173 if (!baseline) {
174 notices.push(
175 "> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.",
176 );
177 } else if (!comparable) {
178 notices.push(
179 "> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.",
180 );
181 } else if (!input.baselineRun.matchesBase) {
182 notices.push(
183 "> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.",
184 );
185 }
186 if (ceilingChanges.length > 0) {
187 notices.push(
188 `> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`,
189 );
190 }
191
192 return [
193 COMMENT_MARKER,
194 resultShaMarker(input.currentRun.sha),
195 "## Thread transfer impact",
196 "",
197 failed
198 ? "❌ One or more thread transfer ceilings were exceeded."
199 : "✅ Thread transfer remains within every enforced ceiling.",
200 ...(notices.length > 0 ? ["", ...notices] : []),
201 "",
202 "| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |",
203 "| --- | --- | ---: | ---: | ---: | ---: | --- |",
204 ...rows,
205 "",
206 `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
207 "",
208 "<details>",
209 "<summary>Scenario and decoded snapshot size</summary>",
210 "",
211 `${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`,
212 "",
213 ...PROVIDERS.map(
214 (provider) =>
215 `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
216 ),
217 "",
218 "</details>",
219 "",
220 "_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._",
221 ].join("\n");
222}
223
224async function artifactsForRun(github, owner, repo, runId) {
225 return github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
226 owner,
227 repo,
228 run_id: runId,
229 per_page: 100,
230 });
231}
232
233function findResultArtifact(artifacts) {
234 return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired);
235}
236
237async function resolve({ github, context, core }) {
238 const source = context.payload.workflow_run;
239 const { owner, repo } = context.repo;
240 if (source.event !== "pull_request") {
241 core.setOutput("publish", "false");
242 return;
243 }
244
245 let pullNumber = source.pull_requests?.[0]?.number;
246 if (!pullNumber) {
247 const associated = await github.paginate(
248 github.rest.repos.listPullRequestsAssociatedWithCommit,
249 { owner, repo, commit_sha: source.head_sha, per_page: 100 },
250 );
251 const matchingPulls = associated.filter(
252 (pull) =>
253 pull.state === "open" &&
254 pull.head.sha === source.head_sha &&
255 pull.head.ref === source.head_branch,
256 );
257 if (matchingPulls.length !== 1) {
258 core.info(
259 `Expected one open pull request for ${source.head_repository?.full_name ?? "unknown repository"}:${source.head_branch ?? "unknown branch"} at ${source.head_sha}; found ${matchingPulls.length}.`,
260 );
261 core.setOutput("publish", "false");
262 return;
263 }
264 pullNumber = matchingPulls[0].number;
265 }
266 if (!pullNumber) {
267 core.info("No open pull request is associated with the completed CI run.");
268 core.setOutput("publish", "false");
269 return;
270 }
271
272 const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber });
273 if (pull.head.sha !== source.head_sha) {
274 core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`);
275 core.setOutput("publish", "false");
276 return;
277 }
278
279 const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id);
280 const sourceArtifact = findResultArtifact(sourceArtifacts);
281 const workflowRuns = await github.paginate(github.rest.actions.listWorkflowRuns, {
282 owner,
283 repo,
284 workflow_id: source.workflow_id,
285 branch: pull.base.ref,
286 event: "push",
287 status: "success",
288 per_page: 100,
289 });
290 const orderedRuns = [
291 ...workflowRuns.filter((run) => run.head_sha === pull.base.sha),
292 ...workflowRuns.filter((run) => run.head_sha !== pull.base.sha),
293 ].slice(0, 20);
294
295 let baselineRun;
296 for (const run of orderedRuns) {
297 const artifacts = await artifactsForRun(github, owner, repo, run.id);
298 if (findResultArtifact(artifacts)) {
299 baselineRun = run;
300 break;
301 }
302 }
303
304 core.setOutput("publish", "true");
305 core.setOutput("pull_number", String(pullNumber));
306 core.setOutput("pr_artifact", sourceArtifact ? "true" : "false");
307 core.setOutput("pr_run_id", String(source.id));
308 core.setOutput("pr_sha", source.head_sha);
309 core.setOutput("pr_conclusion", source.conclusion ?? "unknown");
310 core.setOutput("baseline_artifact", baselineRun ? "true" : "false");
311 core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : "");
312 core.setOutput("baseline_sha", baselineRun?.head_sha ?? "");
313 core.setOutput(
314 "baseline_matches_base",
315 baselineRun?.head_sha === pull.base.sha ? "true" : "false",
316 );
317}
318
319async function upsertComment(github, context, pullNumber, body, options = {}) {
320 const { owner, repo } = context.repo;
321 const comments = await github.paginate(github.rest.issues.listComments, {
322 owner,
323 repo,
324 issue_number: pullNumber,
325 per_page: 100,
326 });
327 const existing = comments.find(
328 (comment) =>
329 comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER),
330 );
331 if (
332 options.preserveResultSha &&
333 existing?.body?.includes(resultShaMarker(options.preserveResultSha))
334 ) {
335 return;
336 }
337 if (existing) {
338 await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
339 } else {
340 await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body });
341 }
342}
343
344async function upsertCommentForCurrentHead(
345 github,
346 context,
347 core,
348 pullNumber,
349 expectedSha,
350 body,
351 options,
352) {
353 const { owner, repo } = context.repo;
354 const { data: pull } = await github.rest.pulls.get({
355 owner,
356 repo,
357 pull_number: pullNumber,
358 });
359 if (pull.head.sha !== expectedSha) {
360 core.info(`Skipping stale CI result ${expectedSha}; PR head is ${pull.head.sha}.`);
361 return false;
362 }
363
364 await upsertComment(github, context, pullNumber, body, options);
365 return true;
366}
367
368async function publish({ github, context, core }) {
369 const pullNumber = Number(process.env.PR_NUMBER);
370 if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {
371 throw new Error("PR_NUMBER is invalid");
372 }
373
374 const current = readResult(process.env.PR_RESULT_DIR);
375 const currentRun = {
376 sha: process.env.PR_SHA,
377 conclusion: process.env.PR_CONCLUSION,
378 url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`,
379 };
380 if (!current) {
381 await upsertCommentForCurrentHead(
382 github,
383 context,
384 core,
385 pullNumber,
386 currentRun.sha,
387 [
388 COMMENT_MARKER,
389 "## Thread transfer impact",
390 "",
391 `⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`,
392 "",
393 "_This comment will update automatically after the next completed run._",
394 ].join("\n"),
395 { preserveResultSha: currentRun.sha },
396 );
397 return;
398 }
399
400 const baseline = readResult(process.env.BASELINE_RESULT_DIR);
401 const baselineRun = baseline
402 ? {
403 sha: process.env.BASELINE_SHA,
404 matchesBase: process.env.BASELINE_MATCHES_BASE === "true",
405 url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
406 }
407 : undefined;
408 const body = renderComment({ current, baseline, currentRun, baselineRun });
409 const published = await upsertCommentForCurrentHead(
410 github,
411 context,
412 core,
413 pullNumber,
414 currentRun.sha,
415 body,
416 );
417 if (published) {
418 core.info(`Updated thread transfer report on PR #${pullNumber}.`);
419 }
420}
421
422module.exports = {
423 publish,
424 readResult,
425 renderComment,
426 resolve,
427 upsertCommentForCurrentHead,
428 validateResult,
429};