authorgravatar for saiansh2525@gmail.comYash Singh <saiansh2525@gmail.com> 2026-09-06 20:28:38-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-06 20:28:38-05:00
log95f9b14f873c7b119f0ffb6dbd194c7063293aa2
treec8ff705d47efb60ed43f61c7e7339fc2fd0be882
parent8c9a49afb5a4c9b28c1ef3540248a1cfa87cdad5
signature Signed by PGP key B5690EEEBB952194

fix(server): import transcripts with oversized tool records (#10430)

5 files changed, 338 insertions(+), 44 deletions(-)

apps/server/package.json+2
...@@ -31,6 +31,8 @@...@@ -31,6 +31,8 @@
31 "effect": "catalog:",31 "effect": "catalog:",
32 "msgpackr-extract": "3.0.4",32 "msgpackr-extract": "3.0.4",
33 "node-pty": "^1.1.0",33 "node-pty": "^1.1.0",
34 "stream-chain": "^4.2.5",
35 "stream-json": "3.6.0",
34 "yaml": "catalog:",36 "yaml": "catalog:",
35 "yauzl": "^3.4.0"37 "yauzl": "^3.4.0"
36 },38 },
apps/server/src/project/AgentSessionJson.ts created+155
...@@ -0,0 +1,155 @@
1import * as SchemaAST from "effect/SchemaAST";
2import { isMany, none, type Many } from "stream-chain/defs.js";
3import { Assembler } from "stream-json/core/assembler.js";
4import { filter } from "stream-json/core/filters/filter.js";
5import * as StreamJson from "stream-json/core/parser.js";
6import type { ParserOptions, Token } from "stream-json/core/parser.js";
7
8type JsonPath = ReadonlyArray<string | number | null>;
9
10/** Select schema fields before assembling their values, without a second field list. */
11export function createTranscriptJsonSelector(schema: { readonly ast: SchemaAST.AST }) {
12 const ast = SchemaAST.toEncoded(schema.ast);
13 const includes = (node: SchemaAST.AST, path: JsonPath, index: number): boolean => {
14 if (index === path.length) return true;
15 switch (node._tag) {
16 case "Objects":
17 // Records have dynamic keys. Keep their values for the decoder to validate.
18 if (node.indexSignatures.length > 0) return true;
19 return node.propertySignatures.some(
20 (property) =>
21 String(property.name) === path[index] && includes(property.type, path, index + 1),
22 );
23 case "Arrays": {
24 const key = path[index];
25 if (typeof key !== "number") return true;
26 const element = node.elements[key];
27 if (element) return includes(element, path, index + 1);
28 return node.rest.length === 0 || node.rest.some((item) => includes(item, path, index + 1));
29 }
30 case "Union":
31 return node.types.some((type) => includes(type, path, index));
32 case "Suspend":
33 return includes(node.thunk(), path, index);
34 case "Unknown":
35 case "Any":
36 case "ObjectKeyword":
37 case "Declaration":
38 // Unstructured/custom schemas must reach the decoder intact. The
39 // shared budget still bounds their allocations.
40 return true;
41 default:
42 return false;
43 }
44 };
45 return (path: JsonPath) => includes(ast, path, 0);
46}
47
48export class TranscriptJsonLimitError extends Error {}
49
50/**
51 * Project a single JSONL record without materializing unselected string values.
52 * The caller supplies a shared allocation budget for the entire transcript.
53 * Budget exhaustion rejects the transcript, never a message within it.
54 */
55export function createTranscriptJsonReader(
56 reserve: (bytes: number) => void,
57 selectPath: (path: JsonPath) => boolean,
58) {
59 // The synchronous tokenizer is exported at runtime in 3.6.0, but omitted
60 // from its bundled types. Unlike parser(), it does not wrap tokens in an
61 // async generator; the file reader already supplies backpressure and UTF-8.
62 const { jsonParser } = StreamJson as typeof StreamJson & {
63 jsonParser: (
64 options: ParserOptions,
65 ) => (input: string | typeof none) => Many<Token> | typeof none;
66 };
67 const tokenize = jsonParser({ packValues: false });
68 const select = filter({ filter: selectPath, streamKeys: false }) as (
69 input: Token | typeof none,
70 ) => Token | Many<Token> | typeof none;
71 const assembler = new Assembler();
72 let key: string | null = null;
73 let value = "";
74 let depth = 0;
75 let complete = false;
76 let malformed = false;
77
78 const assemble = (token: Token) => {
79 reserve(
80 64 + ("value" in token && typeof token.value === "string" ? token.value.length * 2 : 0),
81 );
82 switch (token.name) {
83 case "startString":
84 case "startNumber":
85 value = "";
86 break;
87 case "stringChunk":
88 case "numberChunk":
89 value += token.value;
90 break;
91 case "endString":
92 assembler.consume({ name: "stringValue", value });
93 value = "";
94 break;
95 case "endNumber":
96 assembler.consume({ name: "numberValue", value });
97 value = "";
98 break;
99 default:
100 assembler.consume(token);
101 }
102 };
103 const selectToken = (token: Token | typeof none) => {
104 const selected = select(token);
105 if (selected === none) return;
106 if (isMany(selected)) {
107 for (const item of selected.values) assemble(item);
108 } else {
109 assemble(selected);
110 }
111 };
112 const consume = (input: string | typeof none) => {
113 if (malformed) return;
114 try {
115 const tokens = tokenize(input);
116 if (tokens === none) return;
117 for (const token of tokens.values) {
118 if (token.name === "startObject" || token.name === "startArray") {
119 if (++depth > 128)
120 throw new TranscriptJsonLimitError("Transcript JSON nesting exceeds 128 levels");
121 } else if (token.name === "endObject" || token.name === "endArray") {
122 if (--depth === 0) complete = true;
123 }
124 // Charge keys before assembling them, including unknown names. Reject
125 // the transcript on exhaustion instead of silently shortening a key.
126 if (token.name === "startKey") {
127 key = "";
128 } else if (token.name === "stringChunk" && key !== null) {
129 reserve(token.value.length * 2);
130 key += token.value;
131 } else if (token.name === "endKey") {
132 selectToken({ name: "keyValue", value: key ?? "" });
133 key = null;
134 } else {
135 selectToken(token);
136 }
137 }
138 } catch (cause) {
139 if (cause instanceof Error && cause.message.startsWith("Parser ")) {
140 malformed = true;
141 } else {
142 throw cause;
143 }
144 }
145 };
146 return {
147 write: (chunk: string) => consume(chunk),
148 finish: (): unknown => {
149 consume(none);
150 if (malformed || !complete) return undefined;
151 selectToken(none);
152 return assembler.done ? assembler.current : undefined;
153 },
154 };
155}
apps/server/src/project/AgentSessionScanner.test.ts+18-9
...@@ -1522,7 +1522,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1522,7 +1522,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1522 }),1522 }),
1523 );1523 );
15241524
1525 it.effect("shares a 64 MiB full-read budget across providers without hiding projects", () =>1525 it.effect("streams large transcripts across providers without hiding projects", () =>
1526 Effect.gen(function* () {1526 Effect.gen(function* () {
1527 const path = yield* Path.Path;1527 const path = yield* Path.Path;
1528 const fileSystem = yield* FileSystem.FileSystem;1528 const fileSystem = yield* FileSystem.FileSystem;
...@@ -1619,9 +1619,9 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1619,9 +1619,9 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1619 "Importable",1619 "Importable",
1620 "Importable",1620 "Importable",
1621 "Importable",1621 "Importable",
1622 "Skipped",1622 "Importable",
1623 ]);1623 ]);
1624 expect(fullReadBytes).toBe(64 * 1024 * 1024);1624 expect(fullReadBytes).toBe(80 * 1024 * 1024);
1625 }),1625 }),
1626 );1626 );
16271627
...@@ -1835,7 +1835,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1835,7 +1835,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1835 }),1835 }),
1836 );1836 );
18371837
1838 it.effect("reports an eligible transcript over 16 MiB as skipped", () =>1838 it.effect("imports visible history from a transcript with an oversized tool record", () =>
1839 Effect.gen(function* () {1839 Effect.gen(function* () {
1840 const path = yield* Path.Path;1840 const path = yield* Path.Path;
1841 const nowMs = Date.parse("2026-08-24T12:00:00.000Z");1841 const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
...@@ -1843,7 +1843,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1843,7 +1843,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1843 const claudeHomePath = yield* makeTempDir("t3code-claude-home-");1843 const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
1844 const codexHomePath = yield* makeTempDir("t3code-codex-home-");1844 const codexHomePath = yield* makeTempDir("t3code-codex-home-");
1845 const workspace = yield* makeTempDir("t3code-workspace-");1845 const workspace = yield* makeTempDir("t3code-workspace-");
1846 const transcript = [1846 const transcript = `${[
1847 encodeTranscriptRecord({1847 encodeTranscriptRecord({
1848 type: "session_meta",1848 type: "session_meta",
1849 payload: { id: "large-session", cwd: workspace },1849 payload: { id: "large-session", cwd: workspace },
...@@ -1852,9 +1852,10 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1852,9 +1852,10 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1852 type: "event_msg",1852 type: "event_msg",
1853 payload: { type: "user_message", message: "Import this large session" },1853 payload: { type: "user_message", message: "Import this large session" },
1854 }),1854 }),
1855 ]1855 ].join("\n")}\n${encodeTranscriptRecord({ type: "tool_result", data: "" }).padEnd(
1856 .join("\n")1856 16 * 1024 * 1024 + 1,
1857 .padEnd(16 * 1024 * 1024 + 1, " ");1857 " ",
1858 )}`;
1858 yield* writeTranscript({1859 yield* writeTranscript({
1859 filePath: path.join(codexHomePath, "sessions", "2026", "08", "24", "rollout-large.jsonl"),1860 filePath: path.join(codexHomePath, "sessions", "2026", "08", "24", "rollout-large.jsonl"),
1860 contents: transcript,1861 contents: transcript,
...@@ -1867,7 +1868,15 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {...@@ -1867,7 +1868,15 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1867 workspaceRoot: workspace,1868 workspaceRoot: workspace,
1868 });1869 });
18691870
1870 expect(outcomes).toEqual([{ _tag: "Skipped" }]);1871 expect(outcomes).toMatchObject([
1872 {
1873 _tag: "Importable",
1874 thread: {
1875 providerSessionId: "large-session",
1876 messages: [{ role: "user", text: "Import this large session" }],
1877 },
1878 },
1879 ]);
1871 }),1880 }),
1872 );1881 );
18731882
apps/server/src/project/AgentSessionScanner.ts+144-35
...@@ -35,6 +35,7 @@ import * as Layer from "effect/Layer";...@@ -35,6 +35,7 @@ import * as Layer from "effect/Layer";
35import * as Option from "effect/Option";35import * as Option from "effect/Option";
36import * as Path from "effect/Path";36import * as Path from "effect/Path";
37import * as Schema from "effect/Schema";37import * as Schema from "effect/Schema";
38import * as Semaphore from "effect/Semaphore";
38import * as Stream from "effect/Stream";39import * as Stream from "effect/Stream";
3940
40import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess";41import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess";
...@@ -45,6 +46,11 @@ import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSn...@@ -45,6 +46,11 @@ import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSn
45import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts";46import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts";
46import { expandHomePath } from "../pathExpansion.ts";47import { expandHomePath } from "../pathExpansion.ts";
47import * as ServerSettings from "../serverSettings.ts";48import * as ServerSettings from "../serverSettings.ts";
49import {
50 createTranscriptJsonReader,
51 createTranscriptJsonSelector,
52 TranscriptJsonLimitError,
53} from "./AgentSessionJson.ts";
4854
49/** Chunk size for full transcript reads. */55/** Chunk size for full transcript reads. */
50const TRANSCRIPT_PREFIX_BYTES = 32 * 1024;56const TRANSCRIPT_PREFIX_BYTES = 32 * 1024;
...@@ -71,9 +77,15 @@ const MAX_METADATA_OPERATIONS_PER_SOURCE = MAX_TRANSCRIPTS_PER_SOURCE * 4;...@@ -71,9 +77,15 @@ const MAX_METADATA_OPERATIONS_PER_SOURCE = MAX_TRANSCRIPTS_PER_SOURCE * 4;
71const MAX_METADATA_RECORDS_PER_SOURCE = 100_000;77const MAX_METADATA_RECORDS_PER_SOURCE = 100_000;
72const MAX_METADATA_RECORDS_PER_TRANSCRIPT = 1_000;78const MAX_METADATA_RECORDS_PER_TRANSCRIPT = 1_000;
73const RECENT_THREAD_WINDOW_MS = 30 * 24 * 60 * 60 * 1000;79const RECENT_THREAD_WINDOW_MS = 30 * 24 * 60 * 60 * 1000;
74const MAX_IMPORTED_TRANSCRIPT_BYTES = 16 * 1024 * 1024;80/**
81 * Large tool results (especially screenshots) can make an otherwise ordinary
82 * Codex transcript several GiB. Streaming field selection avoids allocating
83 * those payloads. Raw I/O and selected history have separate budgets.
84 */
85const MAX_IMPORTED_TRANSCRIPT_BYTES = 4 * 1024 * 1024 * 1024;
75const MAX_IMPORTED_MESSAGES = 200;86const MAX_IMPORTED_MESSAGES = 200;
76const MAX_IMPORT_BYTES = 64 * 1024 * 1024;87const MAX_IMPORT_HISTORY_BYTES = 32 * 1024 * 1024;
88const MAX_IMPORT_BYTES = 4 * 1024 * 1024 * 1024;
77const MAX_IMPORT_TRANSCRIPTS = 100;89const MAX_IMPORT_TRANSCRIPTS = 100;
78const MAX_IMPORT_RECORDS = 100_000;90const MAX_IMPORT_RECORDS = 100_000;
7991
...@@ -95,6 +107,7 @@ const CodexTurnMetadata = Schema.Struct({...@@ -95,6 +107,7 @@ const CodexTurnMetadata = Schema.Struct({
95const TranscriptRecord = Schema.Struct({107const TranscriptRecord = Schema.Struct({
96 type: Schema.optional(Schema.String),108 type: Schema.optional(Schema.String),
97 timestamp: Schema.optional(Schema.String),109 timestamp: Schema.optional(Schema.String),
110 cwd: Schema.optional(Schema.String),
98 sessionId: Schema.optional(Schema.String),111 sessionId: Schema.optional(Schema.String),
99 aiTitle: Schema.optional(Schema.String),112 aiTitle: Schema.optional(Schema.String),
100 isSidechain: Schema.optional(Schema.Boolean),113 isSidechain: Schema.optional(Schema.Boolean),
...@@ -109,6 +122,7 @@ const TranscriptRecord = Schema.Struct({...@@ -109,6 +122,7 @@ const TranscriptRecord = Schema.Struct({
109 role: Schema.optional(Schema.String),122 role: Schema.optional(Schema.String),
110 message: Schema.optional(Schema.String),123 message: Schema.optional(Schema.String),
111 model: Schema.optional(Schema.String),124 model: Schema.optional(Schema.String),
125 cwd: Schema.optional(Schema.String),
112 content: Schema.optional(Schema.Array(TranscriptContentBlock)),126 content: Schema.optional(Schema.Array(TranscriptContentBlock)),
113 internal_chat_message_metadata_passthrough: Schema.optional(Schema.Unknown),127 internal_chat_message_metadata_passthrough: Schema.optional(Schema.Unknown),
114 }),128 }),
...@@ -118,8 +132,19 @@ const TranscriptRecord = Schema.Struct({...@@ -118,8 +132,19 @@ const TranscriptRecord = Schema.Struct({
118const decodeClaudeSettings = Schema.decodeUnknownOption(ClaudeSettings);132const decodeClaudeSettings = Schema.decodeUnknownOption(ClaudeSettings);
119const decodeCodexSettings = Schema.decodeUnknownOption(CodexSettings);133const decodeCodexSettings = Schema.decodeUnknownOption(CodexSettings);
120const decodeTranscriptRecord = Schema.decodeUnknownOption(Schema.fromJsonString(TranscriptRecord));134const decodeTranscriptRecord = Schema.decodeUnknownOption(Schema.fromJsonString(TranscriptRecord));
135const decodeTranscriptValue = Schema.decodeUnknownOption(TranscriptRecord);
136const selectTranscriptPath = createTranscriptJsonSelector(TranscriptRecord);
121const decodeCodexTurnMetadata = Schema.decodeUnknownOption(CodexTurnMetadata);137const decodeCodexTurnMetadata = Schema.decodeUnknownOption(CodexTurnMetadata);
122138
139type DecodedTranscriptRecord = typeof TranscriptRecord.Type;
140
141interface AgentSessionTranscriptMetadata {
142 readonly source: AgentSessionSource;
143 readonly providerInstanceId: ProviderInstanceId;
144 readonly fallbackSessionId: string;
145 readonly lastActiveAtMs: number;
146}
147
123export interface AgentSessionThreadMessage {148export interface AgentSessionThreadMessage {
124 readonly role: "user" | "assistant";149 readonly role: "user" | "assistant";
125 readonly text: string;150 readonly text: string;
...@@ -254,16 +279,20 @@ function codexTurnId(metadata: unknown): string | null {...@@ -254,16 +279,20 @@ function codexTurnId(metadata: unknown): string | null {
254279
255/** Keep visible user and assistant text while ignoring tools, reasoning, and malformed records. */280/** Keep visible user and assistant text while ignoring tools, reasoning, and malformed records. */
256export function parseAgentSessionTranscript(281export function parseAgentSessionTranscript(
257 input: {282 input: AgentSessionTranscriptMetadata & {
258 readonly contents: string;283 readonly contents: string;
259 readonly source: AgentSessionSource;
260 readonly providerInstanceId: ProviderInstanceId;
261 readonly fallbackSessionId: string;
262 readonly lastActiveAtMs: number;
263 },284 },
264 lines = splitTranscriptRecords(input.contents, MAX_IMPORT_RECORDS + 1),285 lines = splitTranscriptRecords(input.contents, MAX_IMPORT_RECORDS + 1),
265): AgentSessionThread | null {286): AgentSessionThread | null {
266 if (lines.length > MAX_IMPORT_RECORDS) return null;287 if (lines.length > MAX_IMPORT_RECORDS) return null;
288 const records = lines.flatMap((line) => Option.toArray(decodeTranscriptRecord(line)));
289 return parseAgentSessionRecords(input, records);
290}
291
292function parseAgentSessionRecords(
293 input: AgentSessionTranscriptMetadata,
294 records: ReadonlyArray<DecodedTranscriptRecord>,
295): AgentSessionThread | null {
267 const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs));296 const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs));
268 // Claude filenames are session IDs. Codex rollout filenames include extra297 // Claude filenames are session IDs. Codex rollout filenames include extra
269 // timestamp text, so only transcript metadata can provide a resumable ID.298 // timestamp text, so only transcript metadata can provide a resumable ID.
...@@ -275,13 +304,6 @@ export function parseAgentSessionTranscript(...@@ -275,13 +304,6 @@ export function parseAgentSessionTranscript(
275 let firstUserMessage:304 let firstUserMessage:
276 | (AgentSessionThreadMessage & { readonly codexResponseUser: boolean })305 | (AgentSessionThreadMessage & { readonly codexResponseUser: boolean })
277 | undefined;306 | undefined;
278 function* decodedRecords() {
279 for (const line of lines) {
280 const decoded = decodeTranscriptRecord(line);
281 if (Option.isSome(decoded)) yield decoded.value;
282 }
283 }
284
285 // A Codex response item can include generated setup text beside the real307 // A Codex response item can include generated setup text beside the real
286 // prompt. Suppress response-user records only when the shared turn ID and a308 // prompt. Suppress response-user records only when the shared turn ID and a
287 // verbatim event copy prove which prompt the user submitted.309 // verbatim event copy prove which prompt the user submitted.
...@@ -308,7 +330,7 @@ export function parseAgentSessionTranscript(...@@ -308,7 +330,7 @@ export function parseAgentSessionTranscript(
308 };330 };
309 if (input.source === "codex") {331 if (input.source === "codex") {
310 let recordIndex = -1;332 let recordIndex = -1;
311 for (const record of decodedRecords()) {333 for (const record of records) {
312 recordIndex += 1;334 recordIndex += 1;
313 if (335 if (
314 record.type === "response_item" &&336 record.type === "response_item" &&
...@@ -365,7 +387,7 @@ export function parseAgentSessionTranscript(...@@ -365,7 +387,7 @@ export function parseAgentSessionTranscript(
365 };387 };
366388
367 let recordIndex = -1;389 let recordIndex = -1;
368 for (const record of decodedRecords()) {390 for (const record of records) {
369 recordIndex += 1;391 recordIndex += 1;
370 if (input.source === "claudeAgent") {392 if (input.source === "claudeAgent") {
371 if (393 if (
...@@ -478,6 +500,35 @@ export function parseAgentSessionTranscript(...@@ -478,6 +500,35 @@ export function parseAgentSessionTranscript(
478 };500 };
479}501}
480502
503function extractDecodedCwd(record: DecodedTranscriptRecord): string | null {
504 const cwd = record.cwd?.trim() || record.payload?.cwd?.trim();
505 return cwd && cwd.length > 0 ? cwd : null;
506}
507
508function shouldRetainDecodedRecord(
509 source: AgentSessionSource,
510 record: DecodedTranscriptRecord,
511): boolean {
512 if (extractDecodedCwd(record) !== null) return true;
513 if (source === "claudeAgent") {
514 return (
515 record.type === "user" ||
516 record.type === "assistant" ||
517 record.sessionId !== undefined ||
518 record.aiTitle !== undefined ||
519 record.message?.model !== undefined
520 );
521 }
522 return (
523 record.type === "session_meta" ||
524 record.type === "turn_context" ||
525 (record.type === "event_msg" && record.payload?.type === "user_message") ||
526 (record.type === "response_item" &&
527 record.payload?.type === "message" &&
528 (record.payload.role === "user" || record.payload.role === "assistant"))
529 );
530}
531
481/**532/**
482 * T3 Code runs its own agent sessions inside disposable worktrees. Their533 * T3 Code runs its own agent sessions inside disposable worktrees. Their
483 * transcripts look exactly like user sessions, but re-importing the app's own534 * transcripts look exactly like user sessions, but re-importing the app's own
...@@ -560,6 +611,9 @@ function sameTranscriptIdentity(...@@ -560,6 +611,9 @@ function sameTranscriptIdentity(
560611
561export const make = Effect.gen(function* () {612export const make = Effect.gen(function* () {
562 const fileSystem = yield* FileSystem.FileSystem;613 const fileSystem = yield* FileSystem.FileSystem;
614 // Different project imports can arrive concurrently from multiple clients.
615 // Only one transcript may hold its selected-history budget at a time.
616 const importReadLock = yield* Semaphore.make(1);
563 const path = yield* Path.Path;617 const path = yield* Path.Path;
564 const serverConfig = yield* ServerConfig.ServerConfig;618 const serverConfig = yield* ServerConfig.ServerConfig;
565 const serverSettings = yield* ServerSettings.ServerSettingsService;619 const serverSettings = yield* ServerSettings.ServerSettingsService;
...@@ -692,10 +746,16 @@ export const make = Effect.gen(function* () {...@@ -692,10 +746,16 @@ export const make = Effect.gen(function* () {
692 ).pipe(Effect.orElseSucceed(() => null));746 ).pipe(Effect.orElseSucceed(() => null));
693 });747 });
694748
695 /** Check the open file before and after reading, without reading past its reserved byte budget. */749 /**
750 * Project history fields while reading, before allocating whole JSON records.
751 * Check the file identity on both sides of the read. A selected-history budget
752 * failure rejects the entire transcript before any imported messages persist.
753 */
696 const readTranscript = Effect.fn("AgentSessionScanner.readTranscript")(function* (754 const readTranscript = Effect.fn("AgentSessionScanner.readTranscript")(function* (
697 filePath: string,755 filePath: string,
698 expected: ReturnType<typeof transcriptIdentity>,756 expected: ReturnType<typeof transcriptIdentity>,
757 recordLimit: number,
758 source: AgentSessionSource,
699 ) {759 ) {
700 if (expected.size > MAX_IMPORTED_TRANSCRIPT_BYTES) return null;760 if (expected.size > MAX_IMPORTED_TRANSCRIPT_BYTES) return null;
701761
...@@ -706,9 +766,38 @@ export const make = Effect.gen(function* () {...@@ -706,9 +766,38 @@ export const make = Effect.gen(function* () {
706 if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) {766 if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) {
707 return null;767 return null;
708 }768 }
709 const decoder = new TextDecoder();769 const records: Array<DecodedTranscriptRecord> = [];
710 let contents = "";770 let historyBytes = 0;
771 let recordBytes = 0;
772 let recordCount = 0;
711 let bytesRead = 0;773 let bytesRead = 0;
774 const reserve = (bytes: number) => {
775 recordBytes += bytes;
776 if (historyBytes + recordBytes > MAX_IMPORT_HISTORY_BYTES) {
777 throw new TranscriptJsonLimitError(
778 "Transcript selected history exceeds the 32 MiB memory budget",
779 );
780 }
781 };
782 let reader = createTranscriptJsonReader(reserve, selectTranscriptPath);
783 let decoder = new TextDecoder();
784 let recordStarted = false;
785
786 const finishRecord = () => {
787 reader.write(decoder.decode());
788 recordCount += 1;
789 if (recordCount > recordLimit) return false;
790 const decoded = decodeTranscriptValue(reader.finish());
791 if (Option.isSome(decoded) && shouldRetainDecodedRecord(source, decoded.value)) {
792 records.push(decoded.value);
793 historyBytes += recordBytes;
794 }
795 recordBytes = 0;
796 reader = createTranscriptJsonReader(reserve, selectTranscriptPath);
797 decoder = new TextDecoder();
798 recordStarted = false;
799 return true;
800 };
712801
713 while (bytesRead < expected.size) {802 while (bytesRead < expected.size) {
714 const next = yield* file.readAlloc(803 const next = yield* file.readAlloc(
...@@ -719,16 +808,36 @@ export const make = Effect.gen(function* () {...@@ -719,16 +808,36 @@ export const make = Effect.gen(function* () {
719 }808 }
720809
721 bytesRead += next.value.byteLength;810 bytesRead += next.value.byteLength;
722 contents += decoder.decode(next.value, { stream: true });811 const withinBudget = yield* Effect.try(() => {
812 let start = 0;
813 while (start < next.value.byteLength) {
814 const newline = next.value.indexOf(10, start);
815 const end = newline === -1 ? next.value.byteLength : newline;
816 recordStarted = true;
817 reader.write(decoder.decode(next.value.subarray(start, end), { stream: true }));
818 if (newline === -1) break;
819 if (!finishRecord()) return false;
820 start = newline + 1;
821 }
822 return true;
823 });
824 if (!withinBudget) return null;
723 }825 }
724826
827 if (recordStarted && !(yield* Effect.try(finishRecord))) return null;
725 return sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))828 return sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))
726 ? contents + decoder.decode()829 ? { records, recordCount }
727 : null;830 : null;
728 }),831 }),
729 ),832 ),
730 ),833 ),
731 ).pipe(Effect.orElseSucceed(() => null));834 ).pipe(
835 Effect.catch((cause) =>
836 Effect.logWarning("Could not read imported transcript", { filePath, cause }).pipe(
837 Effect.as(null),
838 ),
839 ),
840 );
732 });841 });
733842
734 /**843 /**
...@@ -1238,20 +1347,21 @@ export const make = Effect.gen(function* () {...@@ -1238,20 +1347,21 @@ export const make = Effect.gen(function* () {
1238 // Reserve the whole file even if its read or parse fails.1347 // Reserve the whole file even if its read or parse fails.
1239 transcriptsRemaining -= 1;1348 transcriptsRemaining -= 1;
1240 bytesRemaining -= identity.size;1349 bytesRemaining -= identity.size;
1241 const contents = yield* readTranscript(transcript.filePath, identity);1350 const snapshot = yield* readTranscript(
1242 if (contents === null) {1351 transcript.filePath,
1243 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });1352 identity,
1244 }1353 recordsRemaining,
1245 const lines = splitTranscriptRecords(contents, recordsRemaining + 1);1354 candidate.source,
1246 if (lines.length > recordsRemaining) {1355 );
1356 if (snapshot === null) {
1247 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });1357 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
1248 }1358 }
1249 recordsRemaining -= lines.length;1359 recordsRemaining -= snapshot.recordCount;
12501360
1251 // A stable replacement file can belong to a different project than the cached candidate.1361 // A stable replacement file can belong to a different project than the cached candidate.
1252 let snapshotCwd: string | null = null;1362 let snapshotCwd: string | null = null;
1253 for (const line of lines) {1363 for (const record of snapshot.records) {
1254 snapshotCwd = extractCwd(line);1364 snapshotCwd = extractDecodedCwd(record);
1255 if (snapshotCwd !== null) break;1365 if (snapshotCwd !== null) break;
1256 }1366 }
1257 if (snapshotCwd === null) {1367 if (snapshotCwd === null) {
...@@ -1265,15 +1375,14 @@ export const make = Effect.gen(function* () {...@@ -1265,15 +1375,14 @@ export const make = Effect.gen(function* () {
1265 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });1375 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
1266 }1376 }
12671377
1268 const parsedThread = parseAgentSessionTranscript(1378 const parsedThread = parseAgentSessionRecords(
1269 {1379 {
1270 contents,
1271 source: candidate.source,1380 source: candidate.source,
1272 providerInstanceId: candidate.providerInstanceId,1381 providerInstanceId: candidate.providerInstanceId,
1273 fallbackSessionId: path.basename(transcript.filePath, ".jsonl"),1382 fallbackSessionId: path.basename(transcript.filePath, ".jsonl"),
1274 lastActiveAtMs: transcript.mtimeMs,1383 lastActiveAtMs: transcript.mtimeMs,
1275 },1384 },
1276 lines,1385 snapshot.records,
1277 );1386 );
1278 if (parsedThread === null) {1387 if (parsedThread === null) {
1279 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });1388 return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
...@@ -1295,7 +1404,7 @@ export const make = Effect.gen(function* () {...@@ -1295,7 +1404,7 @@ export const make = Effect.gen(function* () {
1295 thread: parsedThread,1404 thread: parsedThread,
1296 source,1405 source,
1297 });1406 });
1298 }),1407 }).pipe(importReadLock.withPermits(1)),
1299 ),1408 ),
1300 Stream.map(Option.toArray),1409 Stream.map(Option.toArray),
1301 Stream.flattenIterable,1410 Stream.flattenIterable,
pnpm-lock.yaml+19
...@@ -502,6 +502,12 @@ importers:...@@ -502,6 +502,12 @@ importers:
502 node-pty:502 node-pty:
503 specifier: ^1.1.0503 specifier: ^1.1.0
504 version: 1.1.0504 version: 1.1.0
505 stream-chain:
506 specifier: ^4.2.5
507 version: 4.2.5
508 stream-json:
509 specifier: 3.6.0
510 version: 3.6.0
505 yaml:511 yaml:
506 specifier: ^2.9.0512 specifier: ^2.9.0
507 version: 2.9.0513 version: 2.9.0
...@@ -9848,6 +9854,13 @@ packages:...@@ -9848,6 +9854,13 @@ packages:
9848 resolution: {integrity: sha512-uyQK/mx5QjHun80FLJTfaWE7JtwfRMKBLkMne6udYOmvH0CawotVa7TfgYHzAnpphn4+TweIx1QKMnRIbipmUg==}9854 resolution: {integrity: sha512-uyQK/mx5QjHun80FLJTfaWE7JtwfRMKBLkMne6udYOmvH0CawotVa7TfgYHzAnpphn4+TweIx1QKMnRIbipmUg==}
9849 engines: {node: '>= 0.10.0'}9855 engines: {node: '>= 0.10.0'}
98509856
9857 stream-chain@4.2.5:
9858 resolution: {integrity: sha512-Wtyq3bNE3ggLR0v2vftqvuhltym3WbZAkZpfIrkr5F/6vpeUmWmwTgXa16zD87gpahwJ/Qulq3zVfUlgIc0J2A==}
9859 engines: {node: '>=22'}
9860
9861 stream-json@3.6.0:
9862 resolution: {integrity: sha512-NiJdqxKyau579z/E8vfqcjWfSDWxW/AT99javFXdPXF147Z5za85LRXSHEmSX9TKOakB7gaIccfD0fOIctb7KQ==}
9863
9851 strict-event-emitter@0.5.1:9864 strict-event-emitter@0.5.1:
9852 resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==}9865 resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==}
98539866
...@@ -20965,6 +20978,12 @@ snapshots:...@@ -20965,6 +20978,12 @@ snapshots:
2096520978
20966 stream-buffers@2.2.0: {}20979 stream-buffers@2.2.0: {}
2096720980
20981 stream-chain@4.2.5: {}
20982
20983 stream-json@3.6.0:
20984 dependencies:
20985 stream-chain: 4.2.5
20986
20968 strict-event-emitter@0.5.1:20987 strict-event-emitter@0.5.1:
20969 optional: true20988 optional: true
2097020989