| ... | ... | @@ -168,6 +168,7 @@ function makeHarness(config?: { |
| 168 | 168 | readonly claudeConfig?: Partial<ClaudeSettings>; |
| 169 | 169 | readonly instanceId?: ProviderInstanceId; |
| 170 | 170 | readonly scopedLimitNames?: ClaudeAdapterLiveOptions["scopedLimitNames"]; |
| 171 | readonly environment?: ClaudeAdapterLiveOptions["environment"]; |
| 171 | 172 | }) { |
| 172 | 173 | const query = new FakeClaudeQuery(); |
| 173 | 174 | let createInput: |
| ... | ... | @@ -178,6 +179,7 @@ function makeHarness(config?: { |
| 178 | 179 | | undefined; |
| 179 | 180 | |
| 180 | 181 | const adapterOptions: ClaudeAdapterLiveOptions = { |
| 182 | ...(config?.environment ? { environment: config.environment } : {}), |
| 181 | 183 | ...(config?.instanceId ? { instanceId: config.instanceId } : {}), |
| 182 | 184 | ...(config?.scopedLimitNames ? { scopedLimitNames: config.scopedLimitNames } : {}), |
| 183 | 185 | modelCatalog: Effect.succeed(SYNTHETIC_CLAUDE_MODEL_CATALOG), |
| ... | ... | @@ -2199,6 +2201,411 @@ describe("ClaudeAdapterLive", () => { |
| 2199 | 2201 | ); |
| 2200 | 2202 | }); |
| 2201 | 2203 | |
| 2204 | const AUTH_FAILURE_ASSISTANT = { |
| 2205 | type: "assistant", |
| 2206 | session_id: "sdk-session-auth", |
| 2207 | uuid: "assistant-auth", |
| 2208 | parent_tool_use_id: null, |
| 2209 | error: "authentication_failed", |
| 2210 | is_api_error_message: true, |
| 2211 | message: { |
| 2212 | id: "assistant-message-auth", |
| 2213 | model: "<synthetic>", |
| 2214 | content: [{ type: "text", text: "Not logged in \u00b7 Please run /login" }], |
| 2215 | }, |
| 2216 | } as unknown as SDKMessage; |
| 2217 | |
| 2218 | const completedTurn = (runtimeEvents: ReadonlyArray<ProviderRuntimeEvent>) => { |
| 2219 | const event = runtimeEvents[runtimeEvents.length - 1]; |
| 2220 | assert.equal(event?.type, "turn.completed"); |
| 2221 | assert(event?.type === "turn.completed"); |
| 2222 | return event.payload; |
| 2223 | }; |
| 2224 | |
| 2225 | it.effect.each([ |
| 2226 | { |
| 2227 | name: "an api_error terminal reason", |
| 2228 | result: { subtype: "success", is_error: false, terminal_reason: "api_error", errors: [] }, |
| 2229 | state: "failed", |
| 2230 | errorMessage: /claude auth login/, |
| 2231 | }, |
| 2232 | { |
| 2233 | name: "an is_error success with no terminal reason", |
| 2234 | result: { subtype: "success", is_error: true, errors: [] }, |
| 2235 | state: "failed", |
| 2236 | errorMessage: /claude auth login/, |
| 2237 | }, |
| 2238 | // Every other outcome names its own cause, and the latch must not speak over it. |
| 2239 | { |
| 2240 | name: "a terminal reason of its own", |
| 2241 | result: { |
| 2242 | subtype: "success", |
| 2243 | is_error: false, |
| 2244 | terminal_reason: "prompt_too_long", |
| 2245 | errors: [], |
| 2246 | }, |
| 2247 | state: "failed", |
| 2248 | errorMessage: /prompt exceeds the model's context window/, |
| 2249 | }, |
| 2250 | { |
| 2251 | name: "a listed tool failure", |
| 2252 | result: { |
| 2253 | subtype: "error_during_execution", |
| 2254 | is_error: true, |
| 2255 | errors: ["Tool execution failed: EACCES"], |
| 2256 | }, |
| 2257 | state: "failed", |
| 2258 | errorMessage: /EACCES/, |
| 2259 | }, |
| 2260 | { |
| 2261 | name: "a user interrupt", |
| 2262 | result: { |
| 2263 | subtype: "error_during_execution", |
| 2264 | is_error: true, |
| 2265 | terminal_reason: "aborted_tools", |
| 2266 | errors: [], |
| 2267 | }, |
| 2268 | state: "interrupted", |
| 2269 | errorMessage: undefined, |
| 2270 | }, |
| 2271 | { |
| 2272 | name: "a cancellation", |
| 2273 | result: { subtype: "error_during_execution", is_error: true, errors: ["cancelled"] }, |
| 2274 | state: "cancelled", |
| 2275 | errorMessage: /cancelled/, |
| 2276 | }, |
| 2277 | ])( |
| 2278 | "reports the real cause when an expired login is followed by $name", |
| 2279 | ({ result, state, errorMessage }) => { |
| 2280 | const harness = makeHarness(); |
| 2281 | return Effect.gen(function* () { |
| 2282 | const adapter = yield* ClaudeAdapter; |
| 2283 | |
| 2284 | const runtimeEventsFiber = yield* adapter.streamEvents.pipe( |
| 2285 | Stream.takeUntil((event) => event.type === "turn.completed"), |
| 2286 | Stream.runCollect, |
| 2287 | Effect.forkChild, |
| 2288 | ); |
| 2289 | |
| 2290 | const session = yield* adapter.startSession({ |
| 2291 | threadId: THREAD_ID, |
| 2292 | provider: ProviderDriverKind.make("claudeAgent"), |
| 2293 | runtimeMode: "full-access", |
| 2294 | }); |
| 2295 | yield* adapter.sendTurn({ threadId: session.threadId, input: "hello", attachments: [] }); |
| 2296 | |
| 2297 | harness.query.emit(AUTH_FAILURE_ASSISTANT); |
| 2298 | harness.query.emit({ |
| 2299 | type: "result", |
| 2300 | ...result, |
| 2301 | session_id: "sdk-session-auth", |
| 2302 | uuid: "result-auth", |
| 2303 | } as unknown as SDKMessage); |
| 2304 | |
| 2305 | const payload = completedTurn(Array.from(yield* Fiber.join(runtimeEventsFiber))); |
| 2306 | assert.equal(payload.state, state); |
| 2307 | if (errorMessage === undefined) { |
| 2308 | assert.equal(payload.errorMessage, undefined); |
| 2309 | } else { |
| 2310 | assert.match(payload.errorMessage ?? "", errorMessage); |
| 2311 | } |
| 2312 | }).pipe( |
| 2313 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 2314 | Effect.provide(harness.layer), |
| 2315 | ); |
| 2316 | }, |
| 2317 | ); |
| 2318 | |
| 2319 | it.effect("fails a usage-limited turn with the limit it parked on", () => { |
| 2320 | const harness = makeHarness(); |
| 2321 | return Effect.gen(function* () { |
| 2322 | const adapter = yield* ClaudeAdapter; |
| 2323 | |
| 2324 | const runtimeEventsFiber = yield* adapter.streamEvents.pipe( |
| 2325 | Stream.takeUntil((event) => event.type === "turn.completed"), |
| 2326 | Stream.runCollect, |
| 2327 | Effect.forkChild, |
| 2328 | ); |
| 2329 | |
| 2330 | const session = yield* adapter.startSession({ |
| 2331 | threadId: THREAD_ID, |
| 2332 | provider: ProviderDriverKind.make("claudeAgent"), |
| 2333 | runtimeMode: "full-access", |
| 2334 | }); |
| 2335 | yield* adapter.sendTurn({ threadId: session.threadId, input: "hello", attachments: [] }); |
| 2336 | |
| 2337 | const nowMs = yield* Clock.currentTimeMillis; |
| 2338 | harness.query.emit({ |
| 2339 | type: "rate_limit_event", |
| 2340 | rate_limit_info: { |
| 2341 | status: "rejected", |
| 2342 | rateLimitType: "five_hour", |
| 2343 | resetsAt: Math.floor(nowMs / 1000) + 2 * 60 * 60, |
| 2344 | }, |
| 2345 | session_id: "sdk-session-limit", |
| 2346 | uuid: "rate-limit-rejected", |
| 2347 | } as unknown as SDKMessage); |
| 2348 | harness.query.emit({ |
| 2349 | type: "result", |
| 2350 | subtype: "success", |
| 2351 | is_error: false, |
| 2352 | terminal_reason: "api_error", |
| 2353 | errors: [], |
| 2354 | session_id: "sdk-session-limit", |
| 2355 | uuid: "result-limit", |
| 2356 | } as unknown as SDKMessage); |
| 2357 | |
| 2358 | const payload = completedTurn(Array.from(yield* Fiber.join(runtimeEventsFiber))); |
| 2359 | assert.equal(payload.state, "failed"); |
| 2360 | assert.equal( |
| 2361 | payload.errorMessage, |
| 2362 | "Claude usage limit reached. Send the message again once the limit resets.", |
| 2363 | ); |
| 2364 | }).pipe( |
| 2365 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 2366 | Effect.provide(harness.layer), |
| 2367 | ); |
| 2368 | }); |
| 2369 | |
| 2370 | it.effect.each([ |
| 2371 | { |
| 2372 | name: "listed error with api_error", |
| 2373 | evidence: "auth", |
| 2374 | result: { |
| 2375 | subtype: "error_during_execution", |
| 2376 | is_error: true, |
| 2377 | terminal_reason: "api_error", |
| 2378 | errors: ["Tool execution failed: EACCES"], |
| 2379 | }, |
| 2380 | expected: /EACCES/, |
| 2381 | expectedState: "failed", |
| 2382 | }, |
| 2383 | { |
| 2384 | name: "overload with api_error", |
| 2385 | evidence: "auth", |
| 2386 | result: { |
| 2387 | subtype: "success", |
| 2388 | is_error: true, |
| 2389 | terminal_reason: "api_error", |
| 2390 | api_error_status: 529, |
| 2391 | errors: [], |
| 2392 | }, |
| 2393 | expected: /overloaded \(529\)/, |
| 2394 | expectedState: "failed", |
| 2395 | }, |
| 2396 | { |
| 2397 | name: "listed error on an is_error success", |
| 2398 | evidence: "auth", |
| 2399 | result: { |
| 2400 | subtype: "success", |
| 2401 | is_error: true, |
| 2402 | errors: ["Tool execution failed: EACCES"], |
| 2403 | }, |
| 2404 | expected: /EACCES/, |
| 2405 | expectedState: "failed", |
| 2406 | }, |
| 2407 | { |
| 2408 | name: "recovered same window", |
| 2409 | evidence: "recovered", |
| 2410 | result: { subtype: "success", is_error: false, terminal_reason: "api_error", errors: [] }, |
| 2411 | expected: /repeated API errors/, |
| 2412 | expectedState: "failed", |
| 2413 | }, |
| 2414 | { |
| 2415 | name: "nested assistant does not poison parent", |
| 2416 | evidence: "nested-auth", |
| 2417 | result: { subtype: "success", is_error: false, terminal_reason: "api_error", errors: [] }, |
| 2418 | expected: /repeated API errors/, |
| 2419 | expectedState: "failed", |
| 2420 | }, |
| 2421 | ...[ |
| 2422 | "recovered-missing-reset", |
| 2423 | "recovered-next-reset", |
| 2424 | "recovered-warning", |
| 2425 | "two-windows-recovered", |
| 2426 | ].map((evidence) => ({ |
| 2427 | name: evidence, |
| 2428 | evidence, |
| 2429 | result: { subtype: "success", is_error: false, terminal_reason: "api_error", errors: [] }, |
| 2430 | expected: /repeated API errors/, |
| 2431 | expectedState: "failed", |
| 2432 | })), |
| 2433 | ...["two-windows-one-recovered", "rejected-again"].map((evidence) => ({ |
| 2434 | name: evidence, |
| 2435 | evidence, |
| 2436 | result: { subtype: "success", is_error: false, terminal_reason: "api_error", errors: [] }, |
| 2437 | expected: /usage limit reached/, |
| 2438 | expectedState: "failed", |
| 2439 | })), |
| 2440 | { |
| 2441 | name: "successful turn stays successful", |
| 2442 | evidence: "recovered", |
| 2443 | result: { subtype: "success", is_error: false, errors: [] }, |
| 2444 | expected: undefined, |
| 2445 | expectedState: "completed", |
| 2446 | }, |
| 2447 | ])( |
| 2448 | "preserves terminal failure evidence after $name", |
| 2449 | ({ evidence, result, expected, expectedState }) => { |
| 2450 | const harness = makeHarness(); |
| 2451 | return Effect.gen(function* () { |
| 2452 | const adapter = yield* ClaudeAdapter; |
| 2453 | const eventsFiber = yield* adapter.streamEvents.pipe( |
| 2454 | Stream.takeUntil((event) => event.type === "turn.completed"), |
| 2455 | Stream.runCollect, |
| 2456 | Effect.forkChild, |
| 2457 | ); |
| 2458 | const session = yield* adapter.startSession({ |
| 2459 | threadId: THREAD_ID, |
| 2460 | provider: ProviderDriverKind.make("claudeAgent"), |
| 2461 | runtimeMode: "full-access", |
| 2462 | }); |
| 2463 | yield* adapter.sendTurn({ |
| 2464 | threadId: session.threadId, |
| 2465 | input: "synthetic hello", |
| 2466 | attachments: [], |
| 2467 | }); |
| 2468 | if (evidence === "auth" || evidence === "nested-auth") { |
| 2469 | harness.query.emit({ |
| 2470 | type: "assistant", |
| 2471 | session_id: "sdk-audit", |
| 2472 | uuid: "audit-auth", |
| 2473 | parent_tool_use_id: evidence === "nested-auth" ? "synthetic-parent-tool" : null, |
| 2474 | error: "authentication_failed", |
| 2475 | is_api_error_message: true, |
| 2476 | message: { |
| 2477 | id: "audit-message", |
| 2478 | model: "synthetic-audit-model", |
| 2479 | content: [{ type: "text", text: "Not logged in. Please run /login" }], |
| 2480 | }, |
| 2481 | } as unknown as SDKMessage); |
| 2482 | } else { |
| 2483 | const nowMs = yield* Clock.currentTimeMillis; |
| 2484 | const resetsAt = Math.floor(nowMs / 1000) + 7200; |
| 2485 | harness.query.emit({ |
| 2486 | type: "rate_limit_event", |
| 2487 | rate_limit_info: { status: "rejected", rateLimitType: "five_hour", resetsAt }, |
| 2488 | session_id: "sdk-audit", |
| 2489 | uuid: "audit-limit-rejected", |
| 2490 | } as unknown as SDKMessage); |
| 2491 | if (evidence.startsWith("two-windows")) { |
| 2492 | harness.query.emit({ |
| 2493 | type: "rate_limit_event", |
| 2494 | rate_limit_info: { status: "rejected", rateLimitType: "seven_day", resetsAt }, |
| 2495 | session_id: "sdk-audit", |
| 2496 | uuid: "audit-weekly-rejected", |
| 2497 | } as unknown as SDKMessage); |
| 2498 | } |
| 2499 | harness.query.emit({ |
| 2500 | type: "rate_limit_event", |
| 2501 | rate_limit_info: { |
| 2502 | status: evidence === "recovered-warning" ? "allowed_warning" : "allowed", |
| 2503 | rateLimitType: "five_hour", |
| 2504 | ...(evidence === "recovered-missing-reset" |
| 2505 | ? {} |
| 2506 | : { |
| 2507 | resetsAt: evidence === "recovered-next-reset" ? resetsAt + 18000 : resetsAt, |
| 2508 | }), |
| 2509 | }, |
| 2510 | session_id: "sdk-audit", |
| 2511 | uuid: "audit-limit-allowed", |
| 2512 | } as unknown as SDKMessage); |
| 2513 | if (evidence === "two-windows-recovered" || evidence === "rejected-again") { |
| 2514 | harness.query.emit({ |
| 2515 | type: "rate_limit_event", |
| 2516 | rate_limit_info: { |
| 2517 | status: evidence === "rejected-again" ? "rejected" : "allowed", |
| 2518 | rateLimitType: evidence === "rejected-again" ? "five_hour" : "seven_day", |
| 2519 | resetsAt, |
| 2520 | }, |
| 2521 | session_id: "sdk-audit", |
| 2522 | uuid: "audit-final-quota-update", |
| 2523 | } as unknown as SDKMessage); |
| 2524 | } |
| 2525 | } |
| 2526 | harness.query.emit({ |
| 2527 | type: "result", |
| 2528 | ...result, |
| 2529 | session_id: "sdk-audit", |
| 2530 | uuid: "audit-result", |
| 2531 | } as unknown as SDKMessage); |
| 2532 | const events = Array.from(yield* Fiber.join(eventsFiber)); |
| 2533 | const complete = events.at(-1); |
| 2534 | assert(complete?.type === "turn.completed"); |
| 2535 | assert.equal(complete.payload.state, expectedState); |
| 2536 | if (expected) assert.match(complete.payload.errorMessage ?? "", expected); |
| 2537 | else assert.equal(complete.payload.errorMessage, undefined); |
| 2538 | if (evidence === "rejected-again") |
| 2539 | assert.equal(events.filter((event) => event.type === "runtime.warning").length, 1); |
| 2540 | }).pipe( |
| 2541 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 2542 | Effect.provide(harness.layer), |
| 2543 | ); |
| 2544 | }, |
| 2545 | ); |
| 2546 | |
| 2547 | it.effect.each([ |
| 2548 | { homePath: "./synthetic config's $literal", inherited: undefined }, |
| 2549 | { homePath: "", inherited: ".synthetic config's $literal" }, |
| 2550 | { homePath: "", inherited: " /synthetic/path with edge spaces " }, |
| 2551 | ])( |
| 2552 | "reports the same Claude config and cwd used by the spawned query ($homePath, $inherited)", |
| 2553 | ({ homePath, inherited }) => { |
| 2554 | const harness = makeHarness({ |
| 2555 | claudeConfig: { homePath }, |
| 2556 | environment: { ...process.env, CLAUDE_CONFIG_DIR: inherited }, |
| 2557 | }); |
| 2558 | return Effect.gen(function* () { |
| 2559 | const adapter = yield* ClaudeAdapter; |
| 2560 | const eventsFiber = yield* adapter.streamEvents.pipe( |
| 2561 | Stream.takeUntil((event) => event.type === "turn.completed"), |
| 2562 | Stream.runCollect, |
| 2563 | Effect.forkChild, |
| 2564 | ); |
| 2565 | const cwd = NodePath.resolve("/tmp/synthetic-audit-project"); |
| 2566 | const session = yield* adapter.startSession({ |
| 2567 | threadId: THREAD_ID, |
| 2568 | provider: ProviderDriverKind.make("claudeAgent"), |
| 2569 | runtimeMode: "full-access", |
| 2570 | cwd, |
| 2571 | }); |
| 2572 | yield* adapter.sendTurn({ |
| 2573 | threadId: session.threadId, |
| 2574 | input: "synthetic", |
| 2575 | attachments: [], |
| 2576 | }); |
| 2577 | harness.query.emit(AUTH_FAILURE_ASSISTANT); |
| 2578 | harness.query.emit({ |
| 2579 | type: "result", |
| 2580 | subtype: "success", |
| 2581 | is_error: false, |
| 2582 | terminal_reason: "api_error", |
| 2583 | errors: [], |
| 2584 | session_id: "sdk-session-auth", |
| 2585 | uuid: "result-auth", |
| 2586 | } as unknown as SDKMessage); |
| 2587 | const events = Array.from(yield* Fiber.join(eventsFiber)); |
| 2588 | const completed = events.at(-1); |
| 2589 | assert(completed?.type === "turn.completed"); |
| 2590 | const actualQuery = harness.getLastCreateQueryInput(); |
| 2591 | assert(actualQuery !== undefined); |
| 2592 | const expectedConfigDir = homePath ? NodePath.resolve(homePath) : inherited; |
| 2593 | assert.equal(actualQuery.options.env?.CLAUDE_CONFIG_DIR, expectedConfigDir); |
| 2594 | assert.equal(actualQuery.options.cwd, cwd); |
| 2595 | assert( |
| 2596 | completed.payload.errorMessage?.includes( |
| 2597 | `CLAUDE_CONFIG_DIR set to ${encodeUnknownJsonString(expectedConfigDir)}`, |
| 2598 | ), |
| 2599 | ); |
| 2600 | assert(completed.payload.errorMessage?.includes(`from ${encodeUnknownJsonString(cwd)}`)); |
| 2601 | assert(!completed.payload.errorMessage?.includes("CLAUDE_CONFIG_DIR=")); |
| 2602 | }).pipe( |
| 2603 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 2604 | Effect.provide(harness.layer), |
| 2605 | ); |
| 2606 | }, |
| 2607 | ); |
| 2608 | |
| 2202 | 2609 | it.effect("fails a turn for every dead-turn terminal_reason", () => { |
| 2203 | 2610 | const reasons = [ |
| 2204 | 2611 | "blocking_limit", |