| author |  Theo Browne <me@t3.gg> 2026-10-03 01:43:08-07:00 |
| committer |  GitHub <noreply@github.com> 2026-10-03 01:43:08-07:00 |
| log | 6108ef3d3d0ef2a598088a1bb64980580e935830 |
| tree | d18b977a832634312ba99f1cbce51671133195e0 |
| parent | fed41fa88bb27cb4325cb208d571393850bc63c2 |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): runs no longer get stuck (#15048)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>9 files changed, 959 insertions(+), 115 deletions(-)
apps/server/src/orchestration-v2/EffectOutbox.ts+21-3
| ... | @@ -278,9 +278,16 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La | ... | @@ -278,9 +278,16 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La |
| 278 | available, | 278 | available, |
| 279 | Array.from({ length: Math.min(64, Math.max(0, Math.floor(count))) }, () => undefined), | 279 | Array.from({ length: Math.min(64, Math.max(0, Math.floor(count))) }, () => undefined), |
| 280 | ).pipe(Effect.asVoid); | 280 | ).pipe(Effect.asVoid); |
| 281 | // Each thread runs its effects one at a time, in enqueue (rowid) order. An earlier | ||
| 282 | // effect waiting out a retry backoff still blocks later ones, so a turn | ||
| 283 | // cannot start while a failed rollback is about to restore files. A claim | ||
| 284 | // that skips restart continuations is not blocked by them either. | ||
| 281 | // Title generation is correlated metadata work, so it has its own | 285 | // Title generation is correlated metadata work, so it has its own |
| 282 | // per-thread lane and cannot delay provider lifecycle effects. | 286 | // per-thread lane and cannot delay provider lifecycle effects. |
| 283 | const claimableCandidatePredicate = (availableBefore?: string) => | 287 | const claimableCandidatePredicate = ( |
| 288 | availableBefore?: string, | ||
| 289 | excludeRestartContinuations = false, | ||
| 290 | ) => | ||
| 284 | sql` | 291 | sql` |
| 285 | ${ | 292 | ${ |
| 286 | availableBefore === undefined | 293 | availableBefore === undefined |
| ... | @@ -292,7 +299,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La | ... | @@ -292,7 +299,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La |
| 292 | SELECT 1 | 299 | SELECT 1 |
| 293 | FROM orchestration_v2_effect_outbox AS active | 300 | FROM orchestration_v2_effect_outbox AS active |
| 294 | WHERE active.thread_id = candidate.thread_id | 301 | WHERE active.thread_id = candidate.thread_id |
| 295 | AND active.status = 'running' | 302 | AND ( |
| 303 | active.status = 'running' | ||
| 304 | OR ( | ||
| 305 | active.status = 'pending' | ||
| 306 | AND active.rowid < candidate.rowid | ||
| 307 | AND ${ | ||
| 308 | excludeRestartContinuations | ||
| 309 | ? sql`active.effect_type != 'provider-runtime.continue'` | ||
| 310 | : sql`1 = 1` | ||
| 311 | } | ||
| 312 | ) | ||
| 313 | ) | ||
| 296 | AND ( | 314 | AND ( |
| 297 | ( | 315 | ( |
| 298 | candidate.effect_type = 'thread-title.generate' | 316 | candidate.effect_type = 'thread-title.generate' |
| ... | @@ -492,7 +510,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La | ... | @@ -492,7 +510,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La |
| 492 | WHERE effect_id = ( | 510 | WHERE effect_id = ( |
| 493 | SELECT candidate.effect_id | 511 | SELECT candidate.effect_id |
| 494 | FROM orchestration_v2_effect_outbox AS candidate | 512 | FROM orchestration_v2_effect_outbox AS candidate |
| 495 | WHERE ${claimableCandidatePredicate(nowIso)} | 513 | WHERE ${claimableCandidatePredicate(nowIso, excludeRestartContinuations)} |
| 496 | AND ${excludeRestartContinuations ? sql`candidate.effect_type != 'provider-runtime.continue'` : sql`1 = 1`} | 514 | AND ${excludeRestartContinuations ? sql`candidate.effect_type != 'provider-runtime.continue'` : sql`1 = 1`} |
| 497 | ORDER BY candidate.available_at ASC, candidate.created_at ASC, candidate.effect_id ASC | 515 | ORDER BY candidate.available_at ASC, candidate.created_at ASC, candidate.effect_id ASC |
| 498 | LIMIT 1 | 516 | LIMIT 1 |
apps/server/src/orchestration-v2/EventSink.ts+59-32
| ... | @@ -17,6 +17,7 @@ import * as Effect from "effect/Effect"; | ... | @@ -17,6 +17,7 @@ import * as Effect from "effect/Effect"; |
| 17 | import * as Layer from "effect/Layer"; | 17 | import * as Layer from "effect/Layer"; |
| 18 | import * as Option from "effect/Option"; | 18 | import * as Option from "effect/Option"; |
| 19 | import * as PubSub from "effect/PubSub"; | 19 | import * as PubSub from "effect/PubSub"; |
| 20 | import * as Semaphore from "effect/Semaphore"; | ||
| 20 | import * as Schema from "effect/Schema"; | 21 | import * as Schema from "effect/Schema"; |
| 21 | import * as Stream from "effect/Stream"; | 22 | import * as Stream from "effect/Stream"; |
| 22 | import * as SqlClient from "effect/unstable/sql/SqlClient"; | 23 | import * as SqlClient from "effect/unstable/sql/SqlClient"; |
| ... | @@ -224,6 +225,40 @@ const baseLayer: Layer.Layer< | ... | @@ -224,6 +225,40 @@ const baseLayer: Layer.Layer< |
| 224 | ); | 225 | ); |
| 225 | } | 226 | } |
| 226 | }); | 227 | }); |
| 228 | const publishStoredEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) => | ||
| 229 | eventStore.publishCommitted(events).pipe(Effect.andThen(publishLiveEvents(events))); | ||
| 230 | |||
| 231 | // Transactions commit one at a time, but each writer publishes after its | ||
| 232 | // commit. If a writer is descheduled in between, a later commit reaches | ||
| 233 | // subscribers first, and clients drop any event at or below the newest | ||
| 234 | // sequence they have applied. So a writer takes this lane as the last step | ||
| 235 | // of its transaction and holds it until it has published. Publishing never | ||
| 236 | // waits, so a writer that holds the transaction while it waits for the | ||
| 237 | // lane is not blocked for long. | ||
| 238 | const publishLane = yield* Semaphore.make(1); | ||
| 239 | const commitThenPublish = <A, E, R>( | ||
| 240 | transaction: Effect.Effect<A, E, R>, | ||
| 241 | publish: (committed: A) => Effect.Effect<void>, | ||
| 242 | ) => | ||
| 243 | Effect.suspend(() => { | ||
| 244 | let holdsLane = false; | ||
| 245 | const takeLane = publishLane.take(1).pipe( | ||
| 246 | Effect.andThen( | ||
| 247 | Effect.sync(() => { | ||
| 248 | holdsLane = true; | ||
| 249 | }), | ||
| 250 | ), | ||
| 251 | Effect.uninterruptible, | ||
| 252 | ); | ||
| 253 | return sql | ||
| 254 | .withTransaction(Effect.tap(transaction, () => takeLane)) | ||
| 255 | .pipe( | ||
| 256 | Effect.tap(publish), | ||
| 257 | Effect.ensuring( | ||
| 258 | Effect.suspend(() => (holdsLane ? publishLane.release(1) : Effect.void)), | ||
| 259 | ), | ||
| 260 | ); | ||
| 261 | }); | ||
| 227 | 262 | ||
| 228 | // A user can answer after terminal normalization reads the pending request. | 263 | // A user can answer after terminal normalization reads the pending request. |
| 229 | // Recheck inside the write transaction so stale cleanup cannot erase answers. | 264 | // Recheck inside the write transaction so stale cleanup cannot erase answers. |
| ... | @@ -332,7 +367,7 @@ const baseLayer: Layer.Layer< | ... | @@ -332,7 +367,7 @@ const baseLayer: Layer.Layer< |
| 332 | "orchestration_v2.thread_id": input.events[0]?.threadId ?? null, | 367 | "orchestration_v2.thread_id": input.events[0]?.threadId ?? null, |
| 333 | }); | 368 | }); |
| 334 | 369 | ||
| 335 | const storedEvents = yield* sql.withTransaction( | 370 | return yield* commitThenPublish( |
| 336 | Effect.gen(function* () { | 371 | Effect.gen(function* () { |
| 337 | const normalized = yield* normalizeEvents( | 372 | const normalized = yield* normalizeEvents( |
| 338 | input.guardPendingUserInputCancellations === true | 373 | input.guardPendingUserInputCancellations === true |
| ... | @@ -347,13 +382,14 @@ const baseLayer: Layer.Layer< | ... | @@ -347,13 +382,14 @@ const baseLayer: Layer.Layer< |
| 347 | yield* effectOutbox.enqueue(input.effects); | 382 | yield* effectOutbox.enqueue(input.effects); |
| 348 | return committed; | 383 | return committed; |
| 349 | }), | 384 | }), |
| 385 | (storedEvents) => | ||
| 386 | Effect.gen(function* () { | ||
| 387 | if (input.effects.length > 0) { | ||
| 388 | yield* effectOutbox.notifyAvailable(input.effects.length); | ||
| 389 | } | ||
| 390 | yield* publishStoredEvents(storedEvents); | ||
| 391 | }), | ||
| 350 | ); | 392 | ); |
| 351 | if (input.effects.length > 0) { | ||
| 352 | yield* effectOutbox.notifyAvailable(input.effects.length); | ||
| 353 | } | ||
| 354 | yield* eventStore.publishCommitted(storedEvents); | ||
| 355 | yield* publishLiveEvents(storedEvents); | ||
| 356 | return storedEvents; | ||
| 357 | }); | 393 | }); |
| 358 | 394 | ||
| 359 | const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")( | 395 | const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")( |
| ... | @@ -365,7 +401,7 @@ const baseLayer: Layer.Layer< | ... | @@ -365,7 +401,7 @@ const baseLayer: Layer.Layer< |
| 365 | "orchestration_v2.thread_id": input.threadId, | 401 | "orchestration_v2.thread_id": input.threadId, |
| 366 | }); | 402 | }); |
| 367 | 403 | ||
| 368 | const result = yield* sql.withTransaction( | 404 | return yield* commitThenPublish( |
| 369 | Effect.gen(function* () { | 405 | Effect.gen(function* () { |
| 370 | const rows = yield* sql<{ | 406 | const rows = yield* sql<{ |
| 371 | readonly status: string; | 407 | readonly status: string; |
| ... | @@ -403,12 +439,8 @@ const baseLayer: Layer.Layer< | ... | @@ -403,12 +439,8 @@ const baseLayer: Layer.Layer< |
| 403 | yield* applyStoredEvents(storedEvents); | 439 | yield* applyStoredEvents(storedEvents); |
| 404 | return { committed: true as const, storedEvents }; | 440 | return { committed: true as const, storedEvents }; |
| 405 | }), | 441 | }), |
| 442 | (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void), | ||
| 406 | ); | 443 | ); |
| 407 | if (result.committed) { | ||
| 408 | yield* eventStore.publishCommitted(result.storedEvents); | ||
| 409 | yield* publishLiveEvents(result.storedEvents); | ||
| 410 | } | ||
| 411 | return result; | ||
| 412 | }, | 444 | }, |
| 413 | ); | 445 | ); |
| 414 | 446 | ||
| ... | @@ -424,7 +456,7 @@ const baseLayer: Layer.Layer< | ... | @@ -424,7 +456,7 @@ const baseLayer: Layer.Layer< |
| 424 | "orchestration_v2.expected_last_run_ordinal": input.expectedLastRunOrdinal, | 456 | "orchestration_v2.expected_last_run_ordinal": input.expectedLastRunOrdinal, |
| 425 | }); | 457 | }); |
| 426 | 458 | ||
| 427 | const result = yield* sql.withTransaction( | 459 | return yield* commitThenPublish( |
| 428 | Effect.gen(function* () { | 460 | Effect.gen(function* () { |
| 429 | const rows = yield* sql<{ | 461 | const rows = yield* sql<{ |
| 430 | readonly active_attempt_id: string | null; | 462 | readonly active_attempt_id: string | null; |
| ... | @@ -464,12 +496,8 @@ const baseLayer: Layer.Layer< | ... | @@ -464,12 +496,8 @@ const baseLayer: Layer.Layer< |
| 464 | yield* applyStoredEvents(storedEvents); | 496 | yield* applyStoredEvents(storedEvents); |
| 465 | return { committed: true as const, storedEvents }; | 497 | return { committed: true as const, storedEvents }; |
| 466 | }), | 498 | }), |
| 499 | (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void), | ||
| 467 | ); | 500 | ); |
| 468 | if (result.committed) { | ||
| 469 | yield* eventStore.publishCommitted(result.storedEvents); | ||
| 470 | yield* publishLiveEvents(result.storedEvents); | ||
| 471 | } | ||
| 472 | return result; | ||
| 473 | }); | 501 | }); |
| 474 | 502 | ||
| 475 | const existingCommandResult = (commandId: CommandId) => | 503 | const existingCommandResult = (commandId: CommandId) => |
| ... | @@ -490,7 +518,7 @@ const baseLayer: Layer.Layer< | ... | @@ -490,7 +518,7 @@ const baseLayer: Layer.Layer< |
| 490 | const commitCommandEffect = Effect.fn("orchestrationV2.EventSink.commitCommand")(function* ( | 518 | const commitCommandEffect = Effect.fn("orchestrationV2.EventSink.commitCommand")(function* ( |
| 491 | input: Parameters<EventSinkV2Shape["commitCommand"]>[0], | 519 | input: Parameters<EventSinkV2Shape["commitCommand"]>[0], |
| 492 | ) { | 520 | ) { |
| 493 | const result = yield* sql.withTransaction( | 521 | const result = yield* commitThenPublish( |
| 494 | Effect.gen(function* () { | 522 | Effect.gen(function* () { |
| 495 | const reserved = yield* commandReceipts.insertIfAbsent({ | 523 | const reserved = yield* commandReceipts.insertIfAbsent({ |
| 496 | commandId: input.commandId, | 524 | commandId: input.commandId, |
| ... | @@ -538,15 +566,15 @@ const baseLayer: Layer.Layer< | ... | @@ -538,15 +566,15 @@ const baseLayer: Layer.Layer< |
| 538 | }); | 566 | }); |
| 539 | return { receipt, storedEvents, committed: true as const, cancelledEffectIds }; | 567 | return { receipt, storedEvents, committed: true as const, cancelledEffectIds }; |
| 540 | }), | 568 | }), |
| 569 | (result) => | ||
| 570 | Effect.gen(function* () { | ||
| 571 | yield* effectOutbox.signalCancellations(result.cancelledEffectIds); | ||
| 572 | if (result.committed && input.effects.length > 0) { | ||
| 573 | yield* effectOutbox.notifyAvailable(input.effects.length); | ||
| 574 | } | ||
| 575 | if (result.committed) yield* publishStoredEvents(result.storedEvents); | ||
| 576 | }), | ||
| 541 | ); | 577 | ); |
| 542 | yield* effectOutbox.signalCancellations(result.cancelledEffectIds); | ||
| 543 | if (result.committed && input.effects.length > 0) { | ||
| 544 | yield* effectOutbox.notifyAvailable(input.effects.length); | ||
| 545 | } | ||
| 546 | if (result.committed) { | ||
| 547 | yield* eventStore.publishCommitted(result.storedEvents); | ||
| 548 | yield* publishLiveEvents(result.storedEvents); | ||
| 549 | } | ||
| 550 | return { | 578 | return { |
| 551 | receipt: result.receipt, | 579 | receipt: result.receipt, |
| 552 | storedEvents: result.storedEvents, | 580 | storedEvents: result.storedEvents, |
| ... | @@ -593,7 +621,7 @@ const baseLayer: Layer.Layer< | ... | @@ -593,7 +621,7 @@ const baseLayer: Layer.Layer< |
| 593 | 621 | ||
| 594 | const commitProjectCommandEffect = Effect.fn("orchestrationV2.EventSink.commitProjectCommand")( | 622 | const commitProjectCommandEffect = Effect.fn("orchestrationV2.EventSink.commitProjectCommand")( |
| 595 | function* (input: Parameters<EventSinkV2Shape["commitProjectCommand"]>[0]) { | 623 | function* (input: Parameters<EventSinkV2Shape["commitProjectCommand"]>[0]) { |
| 596 | const result = yield* sql.withTransaction( | 624 | const result = yield* commitThenPublish( |
| 597 | Effect.gen(function* () { | 625 | Effect.gen(function* () { |
| 598 | const reserved: CommandReceiptStore.ProjectCommandReceiptV2 = { | 626 | const reserved: CommandReceiptStore.ProjectCommandReceiptV2 = { |
| 599 | commandId: input.commandId, | 627 | commandId: input.commandId, |
| ... | @@ -613,10 +641,9 @@ const baseLayer: Layer.Layer< | ... | @@ -613,10 +641,9 @@ const baseLayer: Layer.Layer< |
| 613 | yield* commandReceipts.upsert(receipt); | 641 | yield* commandReceipts.upsert(receipt); |
| 614 | return { receipt, event }; | 642 | return { receipt, event }; |
| 615 | }), | 643 | }), |
| 644 | (result) => | ||
| 645 | result.event === undefined ? Effect.void : eventStore.publishCommitted([result.event]), | ||
| 616 | ); | 646 | ); |
| 617 | if (result.event !== undefined) { | ||
| 618 | yield* eventStore.publishCommitted([result.event]); | ||
| 619 | } | ||
| 620 | return { receipt: result.receipt, committed: result.event !== undefined }; | 647 | return { receipt: result.receipt, committed: result.event !== undefined }; |
| 621 | }, | 648 | }, |
| 622 | ); | 649 | ); |
apps/server/src/orchestration-v2/FoundationPersistence.test.ts+159
| ... | @@ -51,7 +51,9 @@ import * as EventStore from "./EventStore.ts"; | ... | @@ -51,7 +51,9 @@ import * as EventStore from "./EventStore.ts"; |
| 51 | import * as IdAllocator from "./IdAllocator.ts"; | 51 | import * as IdAllocator from "./IdAllocator.ts"; |
| 52 | import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; | 52 | import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; |
| 53 | import * as ProjectionStore from "./ProjectionStore.ts"; | 53 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 54 | import * as ProjectStore from "./ProjectStore.ts"; | ||
| 54 | import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; | 55 | import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; |
| 56 | import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; | ||
| 55 | 57 | ||
| 56 | const isLiveStreamBufferError = Schema.is(LiveStreamBufferError); | 58 | const isLiveStreamBufferError = Schema.is(LiveStreamBufferError); |
| 57 | const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); | 59 | const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); |
| ... | @@ -2303,6 +2305,76 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { | ... | @@ -2303,6 +2305,76 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { |
| 2303 | }).pipe(Effect.provide(Layer.fresh(effectOutboxProvided))), | 2305 | }).pipe(Effect.provide(Layer.fresh(effectOutboxProvided))), |
| 2304 | ); | 2306 | ); |
| 2305 | 2307 | ||
| 2308 | it.effect("keeps later thread effects behind an earlier effect waiting to retry", () => | ||
| 2309 | Effect.gen(function* () { | ||
| 2310 | const outbox = yield* EffectOutbox.EffectOutboxV2; | ||
| 2311 | const workerId = "retry-order-worker"; | ||
| 2312 | const commandId = CommandId.make("command:foundation-retry-order"); | ||
| 2313 | const threadId = ThreadId.make("thread:foundation-retry-order"); | ||
| 2314 | yield* outbox.enqueue([ | ||
| 2315 | { | ||
| 2316 | id: "effect:foundation-retry-order:z-rollback", | ||
| 2317 | commandId, | ||
| 2318 | threadId, | ||
| 2319 | request: { | ||
| 2320 | type: "provider-thread.rollback", | ||
| 2321 | providerThreadId: ProviderThreadId.make("provider-thread:foundation-retry-order"), | ||
| 2322 | checkpointId: CheckpointId.make("checkpoint:foundation-retry-order"), | ||
| 2323 | scopeId: CheckpointScopeId.make("scope:foundation-retry-order"), | ||
| 2324 | }, | ||
| 2325 | }, | ||
| 2326 | ]); | ||
| 2327 | const rollback = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); | ||
| 2328 | assert.isTrue(Option.isSome(rollback)); | ||
| 2329 | if (Option.isNone(rollback)) return; | ||
| 2330 | yield* outbox.retry({ | ||
| 2331 | effectId: rollback.value.id, | ||
| 2332 | workerId, | ||
| 2333 | error: "rollback failed once", | ||
| 2334 | delayMs: 60_000, | ||
| 2335 | }); | ||
| 2336 | |||
| 2337 | // A turn the user starts during the rollback's backoff must not run first, | ||
| 2338 | // even when its timestamp ties and its id sorts first. | ||
| 2339 | yield* outbox.enqueue([ | ||
| 2340 | { | ||
| 2341 | id: "effect:foundation-retry-order:a-start", | ||
| 2342 | commandId: CommandId.make("command:foundation-retry-order:start"), | ||
| 2343 | threadId, | ||
| 2344 | request: { type: "provider-turn.start", runId: RunId.make("run:foundation-retry-order") }, | ||
| 2345 | }, | ||
| 2346 | { | ||
| 2347 | id: "effect:foundation-retry-order:b-title", | ||
| 2348 | commandId: CommandId.make("command:foundation-retry-order:title"), | ||
| 2349 | threadId, | ||
| 2350 | request: { type: "thread-title.generate", kind: { type: "regenerate" } }, | ||
| 2351 | }, | ||
| 2352 | ]); | ||
| 2353 | const title = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); | ||
| 2354 | assert.equal(Option.getOrUndefined(title)?.id, "effect:foundation-retry-order:b-title"); | ||
| 2355 | const blocked = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); | ||
| 2356 | assert.isTrue(Option.isNone(blocked)); | ||
| 2357 | const nextClaimable = yield* outbox.nextClaimableAt; | ||
| 2358 | assert.isTrue(Option.isSome(nextClaimable)); | ||
| 2359 | if (Option.isSome(nextClaimable)) { | ||
| 2360 | assert.equal( | ||
| 2361 | DateTime.formatIso(nextClaimable.value), | ||
| 2362 | (yield* outbox.get(rollback.value.id)).pipe(Option.getOrThrow).availableAt, | ||
| 2363 | ); | ||
| 2364 | } | ||
| 2365 | |||
| 2366 | yield* outbox.cancelUnsettled({ | ||
| 2367 | threadId, | ||
| 2368 | effectTypes: ["provider-thread.rollback"], | ||
| 2369 | reason: "Test cleanup.", | ||
| 2370 | }); | ||
| 2371 | const unblocked = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); | ||
| 2372 | assert.equal(Option.getOrUndefined(unblocked)?.id, "effect:foundation-retry-order:a-start"); | ||
| 2373 | yield* outbox.succeed({ effectId: "effect:foundation-retry-order:a-start", workerId }); | ||
| 2374 | yield* outbox.succeed({ effectId: "effect:foundation-retry-order:b-title", workerId }); | ||
| 2375 | }).pipe(Effect.provide(Layer.fresh(effectOutboxProvided))), | ||
| 2376 | ); | ||
| 2377 | |||
| 2306 | it.effect("executes a retry at its durable deadline instead of the liveness interval", () => | 2378 | it.effect("executes a retry at its durable deadline instead of the liveness interval", () => |
| 2307 | Effect.gen(function* () { | 2379 | Effect.gen(function* () { |
| 2308 | const outbox = yield* EffectOutbox.EffectOutboxV2; | 2380 | const outbox = yield* EffectOutbox.EffectOutboxV2; |
| ... | @@ -3222,3 +3294,90 @@ it.live("keeps claiming new work after repeated idle periods", () => | ... | @@ -3222,3 +3294,90 @@ it.live("keeps claiming new work after repeated idle periods", () => |
| 3222 | }).pipe(Effect.provide(workerLayer), Effect.scoped); | 3294 | }).pipe(Effect.provide(workerLayer), Effect.scoped); |
| 3223 | }).pipe(Effect.provide(TestLayer)), | 3295 | }).pipe(Effect.provide(TestLayer)), |
| 3224 | ); | 3296 | ); |
| 3297 | |||
| 3298 | it.effect("publishes live events in commit order across concurrent writers", () => | ||
| 3299 | Effect.gen(function* () { | ||
| 3300 | const firstCommitted = yield* Deferred.make<void>(); | ||
| 3301 | const releaseFirst = yield* Deferred.make<void>(); | ||
| 3302 | // The first writer's post-commit wakeup stands in for any scheduler yield | ||
| 3303 | // between its commit and its publish. | ||
| 3304 | const pausingOutbox = Layer.effect( | ||
| 3305 | EffectOutbox.EffectOutboxV2, | ||
| 3306 | Effect.gen(function* () { | ||
| 3307 | const delegate = yield* EffectOutbox.EffectOutboxV2; | ||
| 3308 | return EffectOutbox.EffectOutboxV2.of({ | ||
| 3309 | ...delegate, | ||
| 3310 | notifyAvailable: (count) => | ||
| 3311 | Deferred.succeed(firstCommitted, undefined).pipe( | ||
| 3312 | Effect.andThen(Deferred.await(releaseFirst)), | ||
| 3313 | Effect.andThen(delegate.notifyAvailable(count)), | ||
| 3314 | ), | ||
| 3315 | }); | ||
| 3316 | }), | ||
| 3317 | ).pipe(Layer.provide(effectOutboxProvided)); | ||
| 3318 | const eventSinkLayer = EventSink.layerFromStores.pipe( | ||
| 3319 | Layer.provide( | ||
| 3320 | Layer.mergeAll( | ||
| 3321 | storesProvided, | ||
| 3322 | pausingOutbox, | ||
| 3323 | commandReceiptStoreProvided, | ||
| 3324 | ProjectStore.layer.pipe(Layer.provide(databaseLayer)), | ||
| 3325 | TurnItemPositionStore.layer.pipe(Layer.provide(databaseLayer)), | ||
| 3326 | ), | ||
| 3327 | ), | ||
| 3328 | ); | ||
| 3329 | |||
| 3330 | yield* Effect.gen(function* () { | ||
| 3331 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 3332 | const now = yield* DateTime.now; | ||
| 3333 | const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now); | ||
| 3334 | const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now); | ||
| 3335 | const published = yield* eventSink | ||
| 3336 | .stream({ afterSequence: yield* eventSink.latestSequence() }) | ||
| 3337 | .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true })); | ||
| 3338 | |||
| 3339 | const firstWrite = yield* eventSink | ||
| 3340 | .writeWithEffects({ | ||
| 3341 | events: [ | ||
| 3342 | threadCreatedEvent({ id: "event:foundation-publish-order:first", thread: first, now }), | ||
| 3343 | ], | ||
| 3344 | effects: [ | ||
| 3345 | { | ||
| 3346 | id: "effect:foundation-publish-order:first", | ||
| 3347 | commandId: CommandId.make("command:foundation-publish-order:first"), | ||
| 3348 | threadId: first.id, | ||
| 3349 | request: { type: "terminal.cleanup" }, | ||
| 3350 | }, | ||
| 3351 | ], | ||
| 3352 | }) | ||
| 3353 | .pipe(Effect.forkScoped); | ||
| 3354 | yield* Deferred.await(firstCommitted); | ||
| 3355 | // The second writer commits after the first. It may run as far as it can | ||
| 3356 | // before the first writer resumes. | ||
| 3357 | const secondWrite = yield* eventSink | ||
| 3358 | .write({ | ||
| 3359 | events: [ | ||
| 3360 | threadCreatedEvent({ | ||
| 3361 | id: "event:foundation-publish-order:second", | ||
| 3362 | thread: second, | ||
| 3363 | now, | ||
| 3364 | }), | ||
| 3365 | ], | ||
| 3366 | }) | ||
| 3367 | .pipe( | ||
| 3368 | Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY), | ||
| 3369 | Effect.forkScoped, | ||
| 3370 | ); | ||
| 3371 | yield* Effect.yieldNow; | ||
| 3372 | yield* Deferred.succeed(releaseFirst, undefined); | ||
| 3373 | yield* Fiber.join(firstWrite); | ||
| 3374 | yield* Fiber.join(secondWrite); | ||
| 3375 | |||
| 3376 | const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence); | ||
| 3377 | assert.deepEqual( | ||
| 3378 | sequences, | ||
| 3379 | [...sequences].sort((left, right) => left - right), | ||
| 3380 | ); | ||
| 3381 | }).pipe(Effect.provide(eventSinkLayer)); | ||
| 3382 | }).pipe(Effect.provide(databaseLayer)), | ||
| 3383 | ); |
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+347-3
| ... | @@ -20,6 +20,7 @@ import * as Cause from "effect/Cause"; | ... | @@ -20,6 +20,7 @@ import * as Cause from "effect/Cause"; |
| 20 | import * as DateTime from "effect/DateTime"; | 20 | import * as DateTime from "effect/DateTime"; |
| 21 | import * as Deferred from "effect/Deferred"; | 21 | import * as Deferred from "effect/Deferred"; |
| 22 | import * as Effect from "effect/Effect"; | 22 | import * as Effect from "effect/Effect"; |
| 23 | import * as Exit from "effect/Exit"; | ||
| 23 | import * as Fiber from "effect/Fiber"; | 24 | import * as Fiber from "effect/Fiber"; |
| 24 | import * as FileSystem from "effect/FileSystem"; | 25 | import * as FileSystem from "effect/FileSystem"; |
| 25 | import * as Layer from "effect/Layer"; | 26 | import * as Layer from "effect/Layer"; |
| ... | @@ -80,6 +81,52 @@ const FailingReleaseEventSinkLayer = Layer.effect( | ... | @@ -80,6 +81,52 @@ const FailingReleaseEventSinkLayer = Layer.effect( |
| 80 | }), | 81 | }), |
| 81 | ).pipe(Layer.provide(TestEventSinkLayer)); | 82 | ).pipe(Layer.provide(TestEventSinkLayer)); |
| 82 | 83 | ||
| 84 | interface FlakyReleaseWrites { | ||
| 85 | /** Which release writes fail right now. */ | ||
| 86 | readonly failing: Ref.Ref<"none" | "session" | "session-and-requests">; | ||
| 87 | /** Receives one item per failed write. */ | ||
| 88 | readonly failures: Queue.Queue<void>; | ||
| 89 | /** Holds runtime request writes: completes `paused`, then waits for `resume`. */ | ||
| 90 | readonly pauseRequestWrites?: { | ||
| 91 | readonly paused: Deferred.Deferred<void>; | ||
| 92 | readonly resume: Deferred.Deferred<void>; | ||
| 93 | }; | ||
| 94 | } | ||
| 95 | |||
| 96 | // Fails release writes with a defect, the way a failed SQL commit surfaces. | ||
| 97 | const makeFlakyReleaseEventSinkLayer = (flaky: FlakyReleaseWrites) => | ||
| 98 | Layer.effect( | ||
| 99 | EventSink.EventSinkV2, | ||
| 100 | Effect.gen(function* () { | ||
| 101 | const delegate = yield* EventSink.EventSinkV2; | ||
| 102 | return EventSink.EventSinkV2.of({ | ||
| 103 | ...delegate, | ||
| 104 | write: (input) => | ||
| 105 | Effect.gen(function* () { | ||
| 106 | const failing = yield* Ref.get(flaky.failing); | ||
| 107 | const fails = input.events.some( | ||
| 108 | (event) => | ||
| 109 | (failing !== "none" && | ||
| 110 | event.type === "provider-session.updated" && | ||
| 111 | (event.payload.status === "stopped" || event.payload.status === "error")) || | ||
| 112 | (failing === "session-and-requests" && event.type === "runtime-request.updated"), | ||
| 113 | ); | ||
| 114 | const pause = flaky.pauseRequestWrites; | ||
| 115 | if ( | ||
| 116 | pause !== undefined && | ||
| 117 | input.events.some((event) => event.type === "runtime-request.updated") | ||
| 118 | ) { | ||
| 119 | yield* Deferred.succeed(pause.paused, undefined); | ||
| 120 | yield* Deferred.await(pause.resume); | ||
| 121 | } | ||
| 122 | if (!fails) return yield* delegate.write(input); | ||
| 123 | yield* Queue.offer(flaky.failures, undefined); | ||
| 124 | return yield* Effect.die(new Error("simulated commit failure")); | ||
| 125 | }), | ||
| 126 | }); | ||
| 127 | }), | ||
| 128 | ).pipe(Layer.provide(TestEventSinkLayer)); | ||
| 129 | |||
| 83 | const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2; | 130 | const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2; |
| 84 | const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = { | 131 | const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = { |
| 85 | ...CodexCapabilities, | 132 | ...CodexCapabilities, |
| ... | @@ -359,15 +406,19 @@ function makeTestLayer(input: { | ... | @@ -359,15 +406,19 @@ function makeTestLayer(input: { |
| 359 | readonly initialProviderItemIdentityVersion?: 2; | 406 | readonly initialProviderItemIdentityVersion?: 2; |
| 360 | }) => Effect.Effect<void>; | 407 | }) => Effect.Effect<void>; |
| 361 | readonly failReleaseEventWrites?: boolean; | 408 | readonly failReleaseEventWrites?: boolean; |
| 409 | readonly flakyReleaseWrites?: FlakyReleaseWrites; | ||
| 362 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; | 410 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; |
| 363 | readonly hangSessionScopeClose?: boolean; | 411 | readonly hangSessionScopeClose?: boolean; |
| 364 | readonly beforeUnload?: Effect.Effect<void>; | 412 | readonly beforeUnload?: Effect.Effect<void>; |
| 365 | readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>; | 413 | readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>; |
| 366 | readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>; | 414 | readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>; |
| 367 | }) { | 415 | }) { |
| 368 | const configuredEventSinkLayer = input.failReleaseEventWrites | 416 | const configuredEventSinkLayer = |
| 369 | ? FailingReleaseEventSinkLayer | 417 | input.flakyReleaseWrites !== undefined |
| 370 | : TestEventSinkLayer; | 418 | ? makeFlakyReleaseEventSinkLayer(input.flakyReleaseWrites) |
| 419 | : input.failReleaseEventWrites | ||
| 420 | ? FailingReleaseEventSinkLayer | ||
| 421 | : TestEventSinkLayer; | ||
| 371 | const registryLayer = ProviderAdapterRegistry.makeSingleLayer( | 422 | const registryLayer = ProviderAdapterRegistry.makeSingleLayer( |
| 372 | makeProviderAdapter(input.state, { | 423 | makeProviderAdapter(input.state, { |
| 373 | failEventStream: input.failEventStream ?? false, | 424 | failEventStream: input.failEventStream ?? false, |
| ... | @@ -2299,6 +2350,299 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r | ... | @@ -2299,6 +2350,299 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r |
| 2299 | yield* effect.pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 1000 }))); | 2350 | yield* effect.pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 1000 }))); |
| 2300 | }), | 2351 | }), |
| 2301 | ); | 2352 | ); |
| 2353 | it.effect("ProviderSessionManagerV2 retries release records that failed to persist", () => | ||
| 2354 | Effect.gen(function* () { | ||
| 2355 | const state = yield* Ref.make(emptyState); | ||
| 2356 | const flaky: FlakyReleaseWrites = { | ||
| 2357 | failing: yield* Ref.make<"none" | "session" | "session-and-requests">("session"), | ||
| 2358 | failures: yield* Queue.unbounded<void>(), | ||
| 2359 | }; | ||
| 2360 | const effect = Effect.gen(function* () { | ||
| 2361 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2362 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2363 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2364 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 2365 | const now = yield* DateTime.now; | ||
| 2366 | const threadId = ThreadId.make("thread-provider-session-manager-release-retry"); | ||
| 2367 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 2368 | providerInstanceId: modelSelection.instanceId, | ||
| 2369 | threadId, | ||
| 2370 | }); | ||
| 2371 | const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now }); | ||
| 2372 | yield* eventSink.write({ | ||
| 2373 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 2374 | }); | ||
| 2375 | const pendingRequest = yield* makePendingRuntimeRequestEvents({ | ||
| 2376 | idAllocator, | ||
| 2377 | threadId, | ||
| 2378 | providerSessionId, | ||
| 2379 | providerThread, | ||
| 2380 | now, | ||
| 2381 | }); | ||
| 2382 | yield* eventSink.write({ events: pendingRequest.events }); | ||
| 2383 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 2384 | |||
| 2385 | assert.isTrue(Exit.isFailure(yield* Effect.exit(manager.close(providerSessionId)))); | ||
| 2386 | yield* Queue.take(flaky.failures); | ||
| 2387 | // The failed session write does not keep the approval answerable. | ||
| 2388 | const afterClose = yield* projectionStore.getThreadProjection(threadId); | ||
| 2389 | assert.equal(afterClose.runtimeRequests.at(-1)?.responseCapability.type, "not_resumable"); | ||
| 2390 | assert.equal(afterClose.providerSessions.at(-1)?.status, "ready"); | ||
| 2391 | |||
| 2392 | // The first retry fails as well, and the retries continue. | ||
| 2393 | yield* TestClock.adjust("1 second"); | ||
| 2394 | yield* Queue.take(flaky.failures); | ||
| 2395 | yield* Ref.set(flaky.failing, "none"); | ||
| 2396 | const stopped = yield* eventSink | ||
| 2397 | .stream({ | ||
| 2398 | threadId, | ||
| 2399 | afterSequence: yield* eventSink.latestSequence({ threadId }), | ||
| 2400 | eventType: "provider-session.updated", | ||
| 2401 | }) | ||
| 2402 | .pipe(Stream.runHead, Effect.forkScoped); | ||
| 2403 | yield* TestClock.adjust("1 second"); | ||
| 2404 | yield* Fiber.join(stopped); | ||
| 2405 | |||
| 2406 | const afterRetry = yield* projectionStore.getThreadProjection(threadId); | ||
| 2407 | assert.equal(afterRetry.providerSessions.at(-1)?.status, "stopped"); | ||
| 2408 | }); | ||
| 2409 | |||
| 2410 | yield* effect.pipe( | ||
| 2411 | Effect.provide(makeTestLayer({ state, idleTimeoutMs: 60_000, flakyReleaseWrites: flaky })), | ||
| 2412 | ); | ||
| 2413 | }), | ||
| 2414 | ); | ||
| 2415 | |||
| 2416 | it.effect("ProviderSessionManagerV2 release retries leave a replacement session alone", () => | ||
| 2417 | Effect.gen(function* () { | ||
| 2418 | const state = yield* Ref.make(emptyState); | ||
| 2419 | const flaky: FlakyReleaseWrites = { | ||
| 2420 | failing: yield* Ref.make<"none" | "session" | "session-and-requests">("none"), | ||
| 2421 | failures: yield* Queue.unbounded<void>(), | ||
| 2422 | }; | ||
| 2423 | const effect = Effect.gen(function* () { | ||
| 2424 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2425 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2426 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2427 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 2428 | const threadId = ThreadId.make("thread-provider-session-manager-release-replacement"); | ||
| 2429 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 2430 | providerInstanceId: modelSelection.instanceId, | ||
| 2431 | threadId, | ||
| 2432 | }); | ||
| 2433 | const writePendingRequest = Effect.gen(function* () { | ||
| 2434 | const now = yield* DateTime.now; | ||
| 2435 | const request = yield* makePendingRuntimeRequestEvents({ | ||
| 2436 | idAllocator, | ||
| 2437 | threadId, | ||
| 2438 | providerSessionId, | ||
| 2439 | providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), | ||
| 2440 | now, | ||
| 2441 | }); | ||
| 2442 | yield* eventSink.write({ events: request.events }); | ||
| 2443 | return request.requestId; | ||
| 2444 | }); | ||
| 2445 | yield* eventSink.write({ | ||
| 2446 | events: [ | ||
| 2447 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), | ||
| 2448 | ], | ||
| 2449 | }); | ||
| 2450 | const oldRequestId = yield* writePendingRequest; | ||
| 2451 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 2452 | yield* Ref.set(flaky.failing, "session-and-requests"); | ||
| 2453 | assert.isTrue(Exit.isFailure(yield* Effect.exit(manager.close(providerSessionId)))); | ||
| 2454 | yield* Queue.take(flaky.failures); | ||
| 2455 | yield* Queue.take(flaky.failures); | ||
| 2456 | yield* Ref.set(flaky.failing, "none"); | ||
| 2457 | |||
| 2458 | // A replacement opens with the same id before the retry runs. | ||
| 2459 | yield* TestClock.adjust("500 millis"); | ||
| 2460 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 2461 | const newRequestId = yield* writePendingRequest; | ||
| 2462 | const replacementStatus = (yield* projectionStore.getThreadProjection( | ||
| 2463 | threadId, | ||
| 2464 | )).providerSessions.at(-1)?.status; | ||
| 2465 | const settled = yield* eventSink | ||
| 2466 | .stream({ | ||
| 2467 | threadId, | ||
| 2468 | afterSequence: yield* eventSink.latestSequence({ threadId }), | ||
| 2469 | eventType: "runtime-request.updated", | ||
| 2470 | }) | ||
| 2471 | .pipe(Stream.runHead, Effect.forkScoped); | ||
| 2472 | yield* TestClock.adjust("500 millis"); | ||
| 2473 | yield* Fiber.join(settled); | ||
| 2474 | |||
| 2475 | const projection = yield* projectionStore.getThreadProjection(threadId); | ||
| 2476 | const request = (id: typeof oldRequestId) => | ||
| 2477 | projection.runtimeRequests.find((candidate) => candidate.id === id); | ||
| 2478 | assert.equal(request(oldRequestId)?.responseCapability.type, "not_resumable"); | ||
| 2479 | assert.equal(request(newRequestId)?.responseCapability.type, "live"); | ||
| 2480 | assert.equal(projection.providerSessions.at(-1)?.status, replacementStatus); | ||
| 2481 | }); | ||
| 2482 | |||
| 2483 | yield* effect.pipe( | ||
| 2484 | Effect.provide(makeTestLayer({ state, idleTimeoutMs: 60_000, flakyReleaseWrites: flaky })), | ||
| 2485 | ); | ||
| 2486 | }), | ||
| 2487 | ); | ||
| 2488 | |||
| 2489 | it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () => | ||
| 2490 | Effect.gen(function* () { | ||
| 2491 | const state = yield* Ref.make(emptyState); | ||
| 2492 | const flaky: FlakyReleaseWrites = { | ||
| 2493 | failing: yield* Ref.make<"none" | "session" | "session-and-requests">("none"), | ||
| 2494 | failures: yield* Queue.unbounded<void>(), | ||
| 2495 | }; | ||
| 2496 | const effect = Effect.gen(function* () { | ||
| 2497 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2498 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2499 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2500 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 2501 | const now = yield* DateTime.now; | ||
| 2502 | const firstThreadId = ThreadId.make("thread-provider-session-manager-release-each-a"); | ||
| 2503 | const secondThreadId = ThreadId.make("thread-provider-session-manager-release-each-b"); | ||
| 2504 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 2505 | providerInstanceId: modelSelection.instanceId, | ||
| 2506 | threadId: firstThreadId, | ||
| 2507 | }); | ||
| 2508 | yield* eventSink.write({ | ||
| 2509 | events: [ | ||
| 2510 | yield* makeThreadCreatedEvent({ idAllocator, threadId: firstThreadId, now }), | ||
| 2511 | yield* makeThreadCreatedEvent({ idAllocator, threadId: secondThreadId, now }), | ||
| 2512 | ], | ||
| 2513 | }); | ||
| 2514 | const secondThreadRequest = yield* makePendingRuntimeRequestEvents({ | ||
| 2515 | idAllocator, | ||
| 2516 | threadId: secondThreadId, | ||
| 2517 | providerSessionId, | ||
| 2518 | providerThread: makeProviderThread({ | ||
| 2519 | idAllocator, | ||
| 2520 | threadId: secondThreadId, | ||
| 2521 | providerSessionId, | ||
| 2522 | now, | ||
| 2523 | }), | ||
| 2524 | now, | ||
| 2525 | }); | ||
| 2526 | yield* eventSink.write({ events: secondThreadRequest.events }); | ||
| 2527 | const failRelease = (failedWrites: number) => | ||
| 2528 | Effect.gen(function* () { | ||
| 2529 | yield* Ref.set(flaky.failing, "session-and-requests"); | ||
| 2530 | assert.isTrue(Exit.isFailure(yield* Effect.exit(manager.close(providerSessionId)))); | ||
| 2531 | yield* Effect.repeat(Queue.take(flaky.failures), { times: failedWrites - 1 }); | ||
| 2532 | yield* Ref.set(flaky.failing, "none"); | ||
| 2533 | }); | ||
| 2534 | |||
| 2535 | // The first session serves both threads. Its replacement serves one. | ||
| 2536 | yield* manager.open({ | ||
| 2537 | threadId: firstThreadId, | ||
| 2538 | providerSessionId, | ||
| 2539 | modelSelection, | ||
| 2540 | runtimePolicy, | ||
| 2541 | }); | ||
| 2542 | yield* manager.open({ | ||
| 2543 | threadId: secondThreadId, | ||
| 2544 | providerSessionId, | ||
| 2545 | modelSelection, | ||
| 2546 | runtimePolicy, | ||
| 2547 | }); | ||
| 2548 | // Both the session write and the second thread's request write fail. | ||
| 2549 | yield* failRelease(2); | ||
| 2550 | yield* TestClock.adjust("500 millis"); | ||
| 2551 | yield* manager.open({ | ||
| 2552 | threadId: firstThreadId, | ||
| 2553 | providerSessionId, | ||
| 2554 | modelSelection, | ||
| 2555 | runtimePolicy, | ||
| 2556 | }); | ||
| 2557 | // Only the session write fails: the first thread has no requests. | ||
| 2558 | yield* failRelease(1); | ||
| 2559 | |||
| 2560 | const settled = yield* eventSink | ||
| 2561 | .stream({ | ||
| 2562 | threadId: secondThreadId, | ||
| 2563 | afterSequence: yield* eventSink.latestSequence({ threadId: secondThreadId }), | ||
| 2564 | eventType: "runtime-request.updated", | ||
| 2565 | }) | ||
| 2566 | .pipe(Stream.runHead, Effect.forkScoped); | ||
| 2567 | yield* TestClock.adjust("500 millis"); | ||
| 2568 | yield* Fiber.join(settled); | ||
| 2569 | |||
| 2570 | const projection = yield* projectionStore.getThreadProjection(secondThreadId); | ||
| 2571 | assert.equal(projection.runtimeRequests.at(-1)?.responseCapability.type, "not_resumable"); | ||
| 2572 | }); | ||
| 2573 | |||
| 2574 | yield* effect.pipe( | ||
| 2575 | Effect.provide(makeTestLayer({ state, idleTimeoutMs: 60_000, flakyReleaseWrites: flaky })), | ||
| 2576 | ); | ||
| 2577 | }), | ||
| 2578 | ); | ||
| 2579 | |||
| 2580 | it.effect("ProviderSessionManagerV2 settles a request the event pump persists during release", () => | ||
| 2581 | Effect.gen(function* () { | ||
| 2582 | const state = yield* Ref.make(emptyState); | ||
| 2583 | const flaky: FlakyReleaseWrites = { | ||
| 2584 | failing: yield* Ref.make<"none" | "session" | "session-and-requests">("none"), | ||
| 2585 | failures: yield* Queue.unbounded<void>(), | ||
| 2586 | pauseRequestWrites: { | ||
| 2587 | paused: yield* Deferred.make<void>(), | ||
| 2588 | resume: yield* Deferred.make<void>(), | ||
| 2589 | }, | ||
| 2590 | }; | ||
| 2591 | const pause = flaky.pauseRequestWrites!; | ||
| 2592 | const effect = Effect.gen(function* () { | ||
| 2593 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2594 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2595 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2596 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 2597 | const now = yield* DateTime.now; | ||
| 2598 | const threadId = ThreadId.make("thread-provider-session-manager-release-pump"); | ||
| 2599 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 2600 | providerInstanceId: modelSelection.instanceId, | ||
| 2601 | threadId, | ||
| 2602 | }); | ||
| 2603 | yield* eventSink.write({ | ||
| 2604 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 2605 | }); | ||
| 2606 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 2607 | // The runtime creates the request a moment after the release starts. | ||
| 2608 | const createdAt = DateTime.add(now, { seconds: 1 }); | ||
| 2609 | const pendingRequest = yield* makePendingRuntimeRequestEvents({ | ||
| 2610 | idAllocator, | ||
| 2611 | threadId, | ||
| 2612 | providerSessionId, | ||
| 2613 | providerThread: makeProviderThread({ | ||
| 2614 | idAllocator, | ||
| 2615 | threadId, | ||
| 2616 | providerSessionId, | ||
| 2617 | now: createdAt, | ||
| 2618 | }), | ||
| 2619 | now: createdAt, | ||
| 2620 | }); | ||
| 2621 | const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); | ||
| 2622 | assert.isDefined(adapterEvents); | ||
| 2623 | yield* Queue.offerAll(adapterEvents!, pendingRequest.providerEvents); | ||
| 2624 | // The event pump holds the request permit while it persists the request. | ||
| 2625 | yield* Deferred.await(pause.paused); | ||
| 2626 | const closed = yield* manager | ||
| 2627 | .close(providerSessionId) | ||
| 2628 | .pipe(Effect.forkScoped({ startImmediately: true })); | ||
| 2629 | yield* TestClock.adjust("1 second"); | ||
| 2630 | yield* Deferred.succeed(pause.resume, undefined); | ||
| 2631 | yield* Fiber.join(closed); | ||
| 2632 | |||
| 2633 | const projection = yield* projectionStore.getThreadProjection(threadId); | ||
| 2634 | const request = projection.runtimeRequests.find( | ||
| 2635 | (candidate) => candidate.id === pendingRequest.requestId, | ||
| 2636 | ); | ||
| 2637 | assert.equal(request?.responseCapability.type, "not_resumable"); | ||
| 2638 | }); | ||
| 2639 | |||
| 2640 | yield* effect.pipe( | ||
| 2641 | Effect.provide(makeTestLayer({ state, idleTimeoutMs: 60_000, flakyReleaseWrites: flaky })), | ||
| 2642 | ); | ||
| 2643 | }), | ||
| 2644 | ); | ||
| 2645 | |||
| 2302 | it.effect("ProviderSessionManagerV2 terminalizes a pending input transcript item on release", () => | 2646 | it.effect("ProviderSessionManagerV2 terminalizes a pending input transcript item on release", () => |
| 2303 | Effect.gen(function* () { | 2647 | Effect.gen(function* () { |
| 2304 | const state = yield* Ref.make(emptyState); | 2648 | const state = yield* Ref.make(emptyState); |
apps/server/src/orchestration-v2/ProviderSessionManager.ts+144-28
| ... | @@ -17,11 +17,13 @@ import * as Duration from "effect/Duration"; | ... | @@ -17,11 +17,13 @@ import * as Duration from "effect/Duration"; |
| 17 | import * as Effect from "effect/Effect"; | 17 | import * as Effect from "effect/Effect"; |
| 18 | import * as Exit from "effect/Exit"; | 18 | import * as Exit from "effect/Exit"; |
| 19 | import * as Fiber from "effect/Fiber"; | 19 | import * as Fiber from "effect/Fiber"; |
| 20 | import * as FiberSet from "effect/FiberSet"; | ||
| 20 | import * as FileSystem from "effect/FileSystem"; | 21 | import * as FileSystem from "effect/FileSystem"; |
| 21 | import * as Layer from "effect/Layer"; | 22 | import * as Layer from "effect/Layer"; |
| 22 | import * as Option from "effect/Option"; | 23 | import * as Option from "effect/Option"; |
| 23 | import * as Queue from "effect/Queue"; | 24 | import * as Queue from "effect/Queue"; |
| 24 | import * as Ref from "effect/Ref"; | 25 | import * as Ref from "effect/Ref"; |
| 26 | import * as Schedule from "effect/Schedule"; | ||
| 25 | import * as Schema from "effect/Schema"; | 27 | import * as Schema from "effect/Schema"; |
| 26 | import * as Scope from "effect/Scope"; | 28 | import * as Scope from "effect/Scope"; |
| 27 | import * as Semaphore from "effect/Semaphore"; | 29 | import * as Semaphore from "effect/Semaphore"; |
| ... | @@ -361,6 +363,9 @@ export const layerWithOptions = ( | ... | @@ -361,6 +363,9 @@ export const layerWithOptions = ( |
| 361 | ); | 363 | ); |
| 362 | const layerScope = yield* Effect.scope; | 364 | const layerScope = yield* Effect.scope; |
| 363 | const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>()); | 365 | const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>()); |
| 366 | // One retry per released entry, so a later release with the same id | ||
| 367 | // cannot drop cleanup for threads only the earlier session served. | ||
| 368 | const releaseRecordRetries = yield* FiberSet.make(); | ||
| 364 | const nextSubscriberId = yield* Ref.make(0); | 369 | const nextSubscriberId = yield* Ref.make(0); |
| 365 | const sessionOpen = yield* makeKeyedSerialExecutor<ProviderSessionId>(); | 370 | const sessionOpen = yield* makeKeyedSerialExecutor<ProviderSessionId>(); |
| 366 | // Orders a thread's attach against a detach unloading it on the same session. | 371 | // Orders a thread's attach against a detach unloading it on the same session. |
| ... | @@ -608,6 +613,8 @@ export const layerWithOptions = ( | ... | @@ -608,6 +613,8 @@ export const layerWithOptions = ( |
| 608 | const writeReleasedRuntimeRequestEvents = (input: { | 613 | const writeReleasedRuntimeRequestEvents = (input: { |
| 609 | readonly entry: LiveSessionEntry; | 614 | readonly entry: LiveSessionEntry; |
| 610 | readonly reason: ProviderSessionReleaseReason; | 615 | readonly reason: ProviderSessionReleaseReason; |
| 616 | /** Requests created later belong to a replacement session with the same id. */ | ||
| 617 | readonly releasedAt: DateTime.Utc; | ||
| 611 | }) => | 618 | }) => |
| 612 | Effect.gen(function* () { | 619 | Effect.gen(function* () { |
| 613 | const providerSessionId = input.entry.runtime.providerSessionId; | 620 | const providerSessionId = input.entry.runtime.providerSessionId; |
| ... | @@ -629,7 +636,8 @@ export const layerWithOptions = ( | ... | @@ -629,7 +636,8 @@ export const layerWithOptions = ( |
| 629 | (request) => | 636 | (request) => |
| 630 | request.status === "pending" && | 637 | request.status === "pending" && |
| 631 | request.responseCapability.type === "live" && | 638 | request.responseCapability.type === "live" && |
| 632 | request.responseCapability.providerSessionId === providerSessionId, | 639 | request.responseCapability.providerSessionId === providerSessionId && |
| 640 | DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt), | ||
| 633 | ); | 641 | ); |
| 634 | 642 | ||
| 635 | for (const request of releasedRequests) { | 643 | for (const request of releasedRequests) { |
| ... | @@ -708,6 +716,125 @@ export const layerWithOptions = ( | ... | @@ -708,6 +716,125 @@ export const layerWithOptions = ( |
| 708 | } | 716 | } |
| 709 | }); | 717 | }); |
| 710 | 718 | ||
| 719 | // Records a released session as stopped and resolves the live runtime | ||
| 720 | // requests it left. Each write runs even if the other fails. Once a | ||
| 721 | // replacement session opens with the same id, it owns the session status, | ||
| 722 | // so only the requests are settled. | ||
| 723 | const writeReleaseRecords = (input: { | ||
| 724 | readonly entry: LiveSessionEntry; | ||
| 725 | readonly reason: ProviderSessionReleaseReason; | ||
| 726 | readonly detail?: string; | ||
| 727 | readonly releasedAt: DateTime.Utc; | ||
| 728 | readonly replaced: boolean; | ||
| 729 | }) => | ||
| 730 | Effect.all( | ||
| 731 | [ | ||
| 732 | input.replaced | ||
| 733 | ? Effect.succeed(Exit.void) | ||
| 734 | : Effect.exit(writeReleasedSessionEvents(input)), | ||
| 735 | Effect.exit( | ||
| 736 | writeReleasedRuntimeRequestEvents(input).pipe( | ||
| 737 | input.entry.requestEventPermit.withPermits(1), | ||
| 738 | ), | ||
| 739 | ), | ||
| 740 | ], | ||
| 741 | { concurrency: 1 }, | ||
| 742 | ).pipe(Effect.flatMap(Exit.asVoidAll)); | ||
| 743 | |||
| 744 | // The session already left the live map, so a later release finds | ||
| 745 | // nothing to do. Without a retry the UI would keep a ready session and | ||
| 746 | // answerable approvals until a server restart. Each attempt holds the | ||
| 747 | // session's open lock, so it sees a replacement that opened meanwhile. | ||
| 748 | const retryReleaseRecords = ( | ||
| 749 | input: Omit<Parameters<typeof writeReleaseRecords>[0], "replaced">, | ||
| 750 | ) => { | ||
| 751 | const providerSessionId = input.entry.runtime.providerSessionId; | ||
| 752 | const attempt = Effect.gen(function* () { | ||
| 753 | const exit = yield* Effect.exit( | ||
| 754 | sessionOpen.withLock( | ||
| 755 | providerSessionId, | ||
| 756 | Effect.gen(function* () { | ||
| 757 | const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId)); | ||
| 758 | yield* writeReleaseRecords({ ...input, replaced }); | ||
| 759 | }), | ||
| 760 | ), | ||
| 761 | ); | ||
| 762 | if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; | ||
| 763 | yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", { | ||
| 764 | providerSessionId, | ||
| 765 | cause: exit.cause, | ||
| 766 | }); | ||
| 767 | // A failed SQL commit is a defect, so every failure but interruption | ||
| 768 | // is retried. | ||
| 769 | return yield* Effect.fail(exit.cause); | ||
| 770 | }); | ||
| 771 | return attempt.pipe( | ||
| 772 | Effect.retry({ | ||
| 773 | schedule: Schedule.exponential("1 second").pipe( | ||
| 774 | Schedule.modifyDelay(({ duration }) => | ||
| 775 | Effect.succeed(Duration.min(duration, Duration.seconds(30))), | ||
| 776 | ), | ||
| 777 | ), | ||
| 778 | }), | ||
| 779 | Effect.delay("1 second"), | ||
| 780 | FiberSet.run(releaseRecordRetries), | ||
| 781 | ); | ||
| 782 | }; | ||
| 783 | |||
| 784 | const logReleaseFailure = | ||
| 785 | (providerSessionId: ProviderSessionId) => | ||
| 786 | <E, R>(release: Effect.Effect<void, E, R>) => | ||
| 787 | release.pipe( | ||
| 788 | Effect.catchCause((cause) => | ||
| 789 | Effect.logWarning("orchestration-v2.provider-session-release-failed", { | ||
| 790 | providerSessionId, | ||
| 791 | cause, | ||
| 792 | }), | ||
| 793 | ), | ||
| 794 | ); | ||
| 795 | |||
| 796 | // Removes the live entry and reads the request cleanup cutoff while | ||
| 797 | // holding the entry's request permit. A request the event pump is | ||
| 798 | // persisting for this runtime lands before the cutoff, and once the | ||
| 799 | // entry is gone the pump persists no more for it. A replacement's | ||
| 800 | // requests come after its own open. | ||
| 801 | const removeLiveEntry = (input: { | ||
| 802 | readonly providerSessionId: ProviderSessionId; | ||
| 803 | readonly onlyIfIdleGeneration?: number; | ||
| 804 | }): Effect.Effect<readonly [Option.Option<LiveSessionEntry>, DateTime.Utc]> => | ||
| 805 | Effect.gen(function* () { | ||
| 806 | const key = sessionKey(input.providerSessionId); | ||
| 807 | const candidate = (yield* Ref.get(sessions)).get(key); | ||
| 808 | if (candidate === undefined) { | ||
| 809 | return [Option.none<LiveSessionEntry>(), yield* DateTime.now] as const; | ||
| 810 | } | ||
| 811 | const removed = yield* Effect.zip( | ||
| 812 | Ref.modify(sessions, (current) => { | ||
| 813 | const existing = current.get(key); | ||
| 814 | if (existing !== candidate) { | ||
| 815 | return [existing === undefined ? "gone" : "changed", current] as const; | ||
| 816 | } | ||
| 817 | if ( | ||
| 818 | input.onlyIfIdleGeneration !== undefined && | ||
| 819 | (existing.busyCount > 0 || existing.idleGeneration !== input.onlyIfIdleGeneration) | ||
| 820 | ) { | ||
| 821 | return ["kept", current] as const; | ||
| 822 | } | ||
| 823 | const updated = new Map(current); | ||
| 824 | updated.delete(key); | ||
| 825 | return ["removed", updated] as const; | ||
| 826 | }), | ||
| 827 | DateTime.now, | ||
| 828 | ).pipe(candidate.requestEventPermit.withPermits(1)); | ||
| 829 | const [outcome, releasedAt] = removed; | ||
| 830 | // Another entry took this id while the permit was held; release it instead. | ||
| 831 | if (outcome === "changed") return yield* removeLiveEntry(input); | ||
| 832 | return [ | ||
| 833 | outcome === "removed" ? Option.some(candidate) : Option.none<LiveSessionEntry>(), | ||
| 834 | releasedAt, | ||
| 835 | ] as const; | ||
| 836 | }); | ||
| 837 | |||
| 711 | const releaseEntry = (input: { | 838 | const releaseEntry = (input: { |
| 712 | readonly providerSessionId: ProviderSessionId; | 839 | readonly providerSessionId: ProviderSessionId; |
| 713 | readonly reason: ProviderSessionReleaseReason; | 840 | readonly reason: ProviderSessionReleaseReason; |
| ... | @@ -717,23 +844,8 @@ export const layerWithOptions = ( | ... | @@ -717,23 +844,8 @@ export const layerWithOptions = ( |
| 717 | readonly gracefulSubscribers?: boolean; | 844 | readonly gracefulSubscribers?: boolean; |
| 718 | }) => | 845 | }) => |
| 719 | Effect.acquireUseRelease( | 846 | Effect.acquireUseRelease( |
| 720 | Ref.modify(sessions, (current) => { | 847 | removeLiveEntry(input), |
| 721 | const key = sessionKey(input.providerSessionId); | 848 | ([entry, releasedAt]) => |
| 722 | const existing = current.get(key); | ||
| 723 | if (existing === undefined) { | ||
| 724 | return [Option.none<LiveSessionEntry>(), current] as const; | ||
| 725 | } | ||
| 726 | if ( | ||
| 727 | input.onlyIfIdleGeneration !== undefined && | ||
| 728 | (existing.busyCount > 0 || existing.idleGeneration !== input.onlyIfIdleGeneration) | ||
| 729 | ) { | ||
| 730 | return [Option.none<LiveSessionEntry>(), current] as const; | ||
| 731 | } | ||
| 732 | const updated = new Map(current); | ||
| 733 | updated.delete(key); | ||
| 734 | return [Option.some(existing), updated] as const; | ||
| 735 | }), | ||
| 736 | (entry) => | ||
| 737 | Option.match(entry, { | 849 | Option.match(entry, { |
| 738 | onNone: () => Effect.void, | 850 | onNone: () => Effect.void, |
| 739 | onSome: (entry) => | 851 | onSome: (entry) => |
| ... | @@ -794,21 +906,25 @@ export const layerWithOptions = ( | ... | @@ -794,21 +906,25 @@ export const layerWithOptions = ( |
| 794 | Effect.forkDetach, | 906 | Effect.forkDetach, |
| 795 | ); | 907 | ); |
| 796 | } | 908 | } |
| 797 | yield* writeReleasedSessionEvents({ | 909 | const records = { |
| 798 | entry, | 910 | entry, |
| 799 | reason: input.reason, | 911 | reason: input.reason, |
| 800 | ...(input.detail === undefined ? {} : { detail: input.detail }), | 912 | ...(input.detail === undefined ? {} : { detail: input.detail }), |
| 801 | }); | 913 | releasedAt, |
| 802 | yield* writeReleasedRuntimeRequestEvents({ | 914 | }; |
| 803 | entry, | 915 | const recorded = yield* Effect.exit( |
| 804 | reason: input.reason, | 916 | writeReleaseRecords({ ...records, replaced: false }), |
| 805 | }).pipe(entry.requestEventPermit.withPermits(1)); | 917 | ); |
| 918 | if (Exit.isFailure(recorded)) { | ||
| 919 | yield* retryReleaseRecords(records); | ||
| 920 | return yield* recorded; | ||
| 921 | } | ||
| 806 | if (Option.isSome(closeExit) && Exit.isFailure(closeExit.value)) { | 922 | if (Option.isSome(closeExit) && Exit.isFailure(closeExit.value)) { |
| 807 | return yield* Effect.failCause(closeExit.value.cause); | 923 | return yield* Effect.failCause(closeExit.value.cause); |
| 808 | } | 924 | } |
| 809 | }), | 925 | }), |
| 810 | }), | 926 | }), |
| 811 | (entry) => | 927 | ([entry]) => |
| 812 | Option.match(entry, { | 928 | Option.match(entry, { |
| 813 | onNone: () => Effect.void, | 929 | onNone: () => Effect.void, |
| 814 | onSome: (entry) => | 930 | onSome: (entry) => |
| ... | @@ -1495,7 +1611,7 @@ export const layerWithOptions = ( | ... | @@ -1495,7 +1611,7 @@ export const layerWithOptions = ( |
| 1495 | providerSessionId: entry.runtime.providerSessionId, | 1611 | providerSessionId: entry.runtime.providerSessionId, |
| 1496 | reason: "manual_shutdown", | 1612 | reason: "manual_shutdown", |
| 1497 | gracefulSubscribers: true, | 1613 | gracefulSubscribers: true, |
| 1498 | }).pipe(Effect.ignore); | 1614 | }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); |
| 1499 | return; | 1615 | return; |
| 1500 | } | 1616 | } |
| 1501 | const cause = Exit.isFailure(exit) | 1617 | const cause = Exit.isFailure(exit) |
| ... | @@ -1516,7 +1632,7 @@ export const layerWithOptions = ( | ... | @@ -1516,7 +1632,7 @@ export const layerWithOptions = ( |
| 1516 | providerSessionId: entry.runtime.providerSessionId, | 1632 | providerSessionId: entry.runtime.providerSessionId, |
| 1517 | reason: "runtime_error", | 1633 | reason: "runtime_error", |
| 1518 | detail: Cause.pretty(cause), | 1634 | detail: Cause.pretty(cause), |
| 1519 | }).pipe(Effect.ignore); | 1635 | }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); |
| 1520 | }), | 1636 | }), |
| 1521 | ), | 1637 | ), |
| 1522 | Effect.forkIn(layerScope), | 1638 | Effect.forkIn(layerScope), |
| ... | @@ -1705,7 +1821,7 @@ export const layerWithOptions = ( | ... | @@ -1705,7 +1821,7 @@ export const layerWithOptions = ( |
| 1705 | providerSessionId: input.providerSessionId, | 1821 | providerSessionId: input.providerSessionId, |
| 1706 | reason: "runtime_error", | 1822 | reason: "runtime_error", |
| 1707 | detail: "Failed to persist the provider-session attachment.", | 1823 | detail: "Failed to persist the provider-session attachment.", |
| 1708 | }).pipe(Effect.ignore), | 1824 | }).pipe(logReleaseFailure(input.providerSessionId)), |
| 1709 | ), | 1825 | ), |
| 1710 | ); | 1826 | ); |
| 1711 | yield* startEventPump(entry); | 1827 | yield* startEventPump(entry); |
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts+56-3
| ... | @@ -30,6 +30,7 @@ import * as ProviderAuthService from "../provider/Services/ProviderAuthService.t | ... | @@ -30,6 +30,7 @@ import * as ProviderAuthService from "../provider/Services/ProviderAuthService.t |
| 30 | import * as ContextHandoffService from "./ContextHandoffService.ts"; | 30 | import * as ContextHandoffService from "./ContextHandoffService.ts"; |
| 31 | import * as EventSink from "./EventSink.ts"; | 31 | import * as EventSink from "./EventSink.ts"; |
| 32 | import * as IdAllocator from "./IdAllocator.ts"; | 32 | import * as IdAllocator from "./IdAllocator.ts"; |
| 33 | import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; | ||
| 33 | import * as ProjectionStore from "./ProjectionStore.ts"; | 34 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 34 | import { ProviderAdapterEventStreamError } from "./ProviderAdapter.ts"; | 35 | import { ProviderAdapterEventStreamError } from "./ProviderAdapter.ts"; |
| 35 | import * as ProviderSessionManager from "./ProviderSessionManager.ts"; | 36 | import * as ProviderSessionManager from "./ProviderSessionManager.ts"; |
| ... | @@ -168,6 +169,8 @@ function makeLocalCommandHarness(input: { | ... | @@ -168,6 +169,8 @@ function makeLocalCommandHarness(input: { |
| 168 | readonly interruptOpen?: boolean; | 169 | readonly interruptOpen?: boolean; |
| 169 | readonly interruptRunBeforeOpenFailure?: boolean; | 170 | readonly interruptRunBeforeOpenFailure?: boolean; |
| 170 | readonly writeFailure?: unknown; | 171 | readonly writeFailure?: unknown; |
| 172 | /** Loads the thread and starts the run, then fails every later state read. */ | ||
| 173 | readonly failReadsAfterRunning?: boolean; | ||
| 171 | }) { | 174 | }) { |
| 172 | const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z"); | 175 | const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z"); |
| 173 | const threadId = ThreadId.make("thread-native-account-command"); | 176 | const threadId = ThreadId.make("thread-native-account-command"); |
| ... | @@ -411,9 +414,40 @@ function makeLocalCommandHarness(input: { | ... | @@ -411,9 +414,40 @@ function makeLocalCommandHarness(input: { |
| 411 | ), | 414 | ), |
| 412 | ), | 415 | ), |
| 413 | ) | 416 | ) |
| 414 | : Effect.die("A local command must not open a native session."), | 417 | : input.failReadsAfterRunning === true |
| 418 | ? Effect.succeed({ | ||
| 419 | driver: providerThread.driver, | ||
| 420 | providerSession: { | ||
| 421 | id: providerSessionId, | ||
| 422 | driver: providerThread.driver, | ||
| 423 | providerInstanceId: newInstanceId, | ||
| 424 | status: "ready", | ||
| 425 | cwd: "/tmp/native-account-command", | ||
| 426 | model: null, | ||
| 427 | capabilities: CodexProviderCapabilitiesV2, | ||
| 428 | createdAt: now, | ||
| 429 | updatedAt: now, | ||
| 430 | lastError: null, | ||
| 431 | }, | ||
| 432 | ensureThread: () => Effect.succeed(providerThread), | ||
| 433 | } as never) | ||
| 434 | : Effect.die("A local command must not open a native session."), | ||
| 435 | ); | ||
| 436 | const startRootRun = vi.fn< | ||
| 437 | (input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void> | ||
| 438 | >(() => | ||
| 439 | input.failReadsAfterRunning === true | ||
| 440 | ? Effect.void | ||
| 441 | : Effect.die("A local command must not start a native turn."), | ||
| 442 | ); | ||
| 443 | const failReadIfRunning = Effect.suspend(() => | ||
| 444 | input.failReadsAfterRunning === true && | ||
| 445 | projection.runs.find((candidate) => candidate.id === runId)?.status === "running" | ||
| 446 | ? Effect.fail( | ||
| 447 | new ProjectionStore.ProjectionStoreReadError({ threadId, cause: "database unavailable" }), | ||
| 448 | ) | ||
| 449 | : Effect.void, | ||
| 415 | ); | 450 | ); |
| 416 | const startRootRun = vi.fn(() => Effect.die("A local command must not start a native turn.")); | ||
| 417 | const tryHandlePromptCommand = vi.fn(() => | 451 | const tryHandlePromptCommand = vi.fn(() => |
| 418 | input.logoutFailure === undefined | 452 | input.logoutFailure === undefined |
| 419 | ? Effect.succeed(true) | 453 | ? Effect.succeed(true) |
| ... | @@ -471,7 +505,7 @@ function makeLocalCommandHarness(input: { | ... | @@ -471,7 +505,7 @@ function makeLocalCommandHarness(input: { |
| 471 | ), | 505 | ), |
| 472 | }), | 506 | }), |
| 473 | getRuntimeRecoveryProjection: () => | 507 | getRuntimeRecoveryProjection: () => |
| 474 | Effect.succeed({ | 508 | Effect.as(failReadIfRunning, { |
| 475 | ...projection, | 509 | ...projection, |
| 476 | hasConversation: projection.messages.some( | 510 | hasConversation: projection.messages.some( |
| 477 | (m) => | 511 | (m) => |
| ... | @@ -703,6 +737,25 @@ effectIt.effect( | ... | @@ -703,6 +737,25 @@ effectIt.effect( |
| 703 | }), | 737 | }), |
| 704 | ); | 738 | ); |
| 705 | 739 | ||
| 740 | effectIt.effect("does not mistake a failed state read for a superseded run", () => | ||
| 741 | Effect.gen(function* () { | ||
| 742 | const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true }); | ||
| 743 | |||
| 744 | yield* harness.start; | ||
| 745 | |||
| 746 | expect(harness.projection().runs.at(-1)?.status).toBe("running"); | ||
| 747 | const controls = harness.startRootRun.mock.calls[0]?.[0]; | ||
| 748 | expect(controls).toBeDefined(); | ||
| 749 | if (controls === undefined) return; | ||
| 750 | // "false" would skip the provider turn or the terminal write and leave the | ||
| 751 | // run active. A read failure must reach the caller instead. | ||
| 752 | const startCheck = yield* Effect.flip(controls.shouldStartProviderTurn!()); | ||
| 753 | const finalizeCheck = yield* Effect.flip(controls.shouldFinalizeRun!()); | ||
| 754 | expect(startCheck._tag).toBe("ProjectionStoreReadError"); | ||
| 755 | expect(finalizeCheck._tag).toBe("ProjectionStoreReadError"); | ||
| 756 | }), | ||
| 757 | ); | ||
| 758 | |||
| 706 | effectIt.effect("does not overwrite a run interrupted while its thread loads", () => | 759 | effectIt.effect("does not overwrite a run interrupted while its thread loads", () => |
| 707 | Effect.gen(function* () { | 760 | Effect.gen(function* () { |
| 708 | const harness = makeLocalCommandHarness({ | 761 | const harness = makeLocalCommandHarness({ |
apps/server/src/orchestration-v2/ProviderTurnStartService.ts+2-2
| ... | @@ -124,13 +124,14 @@ export const layer: Layer.Layer< | ... | @@ -124,13 +124,14 @@ export const layer: Layer.Layer< |
| 124 | }) => { | 124 | }) => { |
| 125 | // Guards and background routing need live execution state, not a fresh | 125 | // Guards and background routing need live execution state, not a fresh |
| 126 | // allocation of every completed message and tool output in the thread. | 126 | // allocation of every completed message and tool output in the thread. |
| 127 | // `false` means the run moved on or is gone. A failed read is an error, | ||
| 128 | // so the caller fails the start or the run instead of skipping it. | ||
| 127 | const isCurrentAttemptInStatus = (expectedStatus: OrchestrationV2Run["status"]) => | 129 | const isCurrentAttemptInStatus = (expectedStatus: OrchestrationV2Run["status"]) => |
| 128 | projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( | 130 | projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( |
| 129 | Effect.map((current) => { | 131 | Effect.map((current) => { |
| 130 | const run = current.runs.find((candidate) => candidate.id === input.runId); | 132 | const run = current.runs.find((candidate) => candidate.id === input.runId); |
| 131 | return run?.activeAttemptId === input.attemptId && run.status === expectedStatus; | 133 | return run?.activeAttemptId === input.attemptId && run.status === expectedStatus; |
| 132 | }), | 134 | }), |
| 133 | Effect.catchCause(() => Effect.succeed(false)), | ||
| 134 | ); | 135 | ); |
| 135 | return { | 136 | return { |
| 136 | isCurrentAttemptInStatus, | 137 | isCurrentAttemptInStatus, |
| ... | @@ -157,7 +158,6 @@ export const layer: Layer.Layer< | ... | @@ -157,7 +158,6 @@ export const layer: Layer.Layer< |
| 157 | (run.status === "starting" || run.status === "running") | 158 | (run.status === "starting" || run.status === "running") |
| 158 | ); | 159 | ); |
| 159 | }), | 160 | }), |
| 160 | Effect.catchCause(() => Effect.succeed(false)), | ||
| 161 | ), | 161 | ), |
| 162 | hasUnpairedRunInterruptRequest: () => | 162 | hasUnpairedRunInterruptRequest: () => |
| 163 | projectionStore | 163 | projectionStore |
apps/server/src/orchestration-v2/RunExecutionService.test.ts+144-17
| ... | @@ -50,6 +50,7 @@ import { | ... | @@ -50,6 +50,7 @@ import { |
| 50 | type ProviderAdapterV2Event, | 50 | type ProviderAdapterV2Event, |
| 51 | type ProviderAdapterV2SessionRuntime, | 51 | type ProviderAdapterV2SessionRuntime, |
| 52 | } from "./ProviderAdapter.ts"; | 52 | } from "./ProviderAdapter.ts"; |
| 53 | import * as ProjectionStore from "./ProjectionStore.ts"; | ||
| 53 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; | 54 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; |
| 54 | import * as RunExecutionService from "./RunExecutionService.ts"; | 55 | import * as RunExecutionService from "./RunExecutionService.ts"; |
| 55 | import * as RunFinalizationService from "./RunFinalizationService.ts"; | 56 | import * as RunFinalizationService from "./RunFinalizationService.ts"; |
| ... | @@ -590,6 +591,113 @@ it.effect("rechecks run ownership immediately before calling the provider", () = | ... | @@ -590,6 +591,113 @@ it.effect("rechecks run ownership immediately before calling the provider", () = |
| 590 | }).pipe(Effect.provide(RunExecutionTestLayer)), | 591 | }).pipe(Effect.provide(RunExecutionTestLayer)), |
| 591 | ); | 592 | ); |
| 592 | 593 | ||
| 594 | it.effect("fails the run when its ownership check cannot be read before calling the provider", () => | ||
| 595 | Effect.gen(function* () { | ||
| 596 | const guardCalls = yield* Ref.make(0); | ||
| 597 | const providerStarts = yield* Ref.make(0); | ||
| 598 | const writes = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]); | ||
| 599 | const threadId = ThreadId.make("thread:run-execution-start-guard-read"); | ||
| 600 | const runId = RunId.make("run:run-execution-start-guard-read"); | ||
| 601 | const attemptId = RunAttemptId.make("attempt:run-execution-start-guard-read"); | ||
| 602 | const providerInstanceId = ProviderInstanceId.make("codex"); | ||
| 603 | const testLayer = RunExecutionService.layer.pipe( | ||
| 604 | Layer.provide( | ||
| 605 | Layer.mergeAll( | ||
| 606 | Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), | ||
| 607 | Layer.mock(EventSink.EventSinkV2)({ | ||
| 608 | writeIfRunCurrent: (input) => | ||
| 609 | Ref.update(writes, (current) => [...current, ...input.events]).pipe( | ||
| 610 | Effect.as({ committed: true, storedEvents: [] }), | ||
| 611 | ), | ||
| 612 | }), | ||
| 613 | IdAllocator.layer, | ||
| 614 | Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ | ||
| 615 | ingestNormalized: () => Effect.succeed([]), | ||
| 616 | }), | ||
| 617 | ServerSettings.layerTest(), | ||
| 618 | ), | ||
| 619 | ), | ||
| 620 | ); | ||
| 621 | |||
| 622 | yield* Effect.gen(function* () { | ||
| 623 | const runExecution = yield* RunExecutionService.RunExecutionServiceV2; | ||
| 624 | yield* runExecution.startRootRun({ | ||
| 625 | commandId: CommandId.make("command:run-execution-start-guard-read"), | ||
| 626 | appThread: { id: threadId } as OrchestrationV2AppThread, | ||
| 627 | providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"), | ||
| 628 | session: { | ||
| 629 | events: Stream.never, | ||
| 630 | startTurn: () => Ref.update(providerStarts, (count) => count + 1), | ||
| 631 | } as unknown as ProviderAdapterV2SessionRuntime, | ||
| 632 | run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run, | ||
| 633 | rootNode: { | ||
| 634 | id: NodeId.make("node:run-execution-start-guard-read"), | ||
| 635 | } as OrchestrationV2ExecutionNode, | ||
| 636 | checkpointScope: { | ||
| 637 | id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard-read"), | ||
| 638 | } as OrchestrationV2CheckpointScope, | ||
| 639 | providerThread: { | ||
| 640 | id: ProviderThreadId.make("provider-thread:run-execution-start-guard-read"), | ||
| 641 | driver, | ||
| 642 | } as OrchestrationV2ProviderThread, | ||
| 643 | attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt, | ||
| 644 | attemptId, | ||
| 645 | providerTurnOrdinal: 1, | ||
| 646 | // The preparation check passes; the check right before the provider | ||
| 647 | // call cannot read the run. | ||
| 648 | shouldStartProviderTurn: () => | ||
| 649 | Ref.getAndUpdate(guardCalls, (calls) => calls + 1).pipe( | ||
| 650 | Effect.flatMap((calls) => | ||
| 651 | calls === 0 | ||
| 652 | ? Effect.succeed(true) | ||
| 653 | : Effect.fail( | ||
| 654 | new ProjectionStore.ProjectionStoreReadError({ | ||
| 655 | threadId, | ||
| 656 | cause: "database unavailable", | ||
| 657 | }), | ||
| 658 | ), | ||
| 659 | ), | ||
| 660 | ), | ||
| 661 | // The failure is settled by the guarded write, not by another read. | ||
| 662 | shouldFinalizeRun: () => | ||
| 663 | Effect.fail( | ||
| 664 | new ProjectionStore.ProjectionStoreReadError({ | ||
| 665 | threadId, | ||
| 666 | cause: "database unavailable", | ||
| 667 | }), | ||
| 668 | ), | ||
| 669 | message: { | ||
| 670 | messageId: MessageId.make("message:run-execution-start-guard-read"), | ||
| 671 | text: "Start while the store is down.", | ||
| 672 | attachments: [], | ||
| 673 | createdBy: "user", | ||
| 674 | creationSource: "web", | ||
| 675 | }, | ||
| 676 | modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, | ||
| 677 | runtimePolicy: { | ||
| 678 | runtimeMode: "full-access", | ||
| 679 | interactionMode: "default", | ||
| 680 | cwd: process.cwd(), | ||
| 681 | approvalPolicy: "never", | ||
| 682 | sandboxPolicy: { | ||
| 683 | type: "readOnly", | ||
| 684 | access: { type: "fullAccess" }, | ||
| 685 | networkAccess: false, | ||
| 686 | }, | ||
| 687 | }, | ||
| 688 | }); | ||
| 689 | }).pipe(Effect.provide(testLayer)); | ||
| 690 | |||
| 691 | assert.equal(yield* Ref.get(guardCalls), 2); | ||
| 692 | assert.equal(yield* Ref.get(providerStarts), 0); | ||
| 693 | const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated"); | ||
| 694 | assert.equal( | ||
| 695 | runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined, | ||
| 696 | "failed", | ||
| 697 | ); | ||
| 698 | }), | ||
| 699 | ); | ||
| 700 | |||
| 593 | it.effect( | 701 | it.effect( |
| 594 | "dispatches only attachment-free compact commands through the native compaction path", | 702 | "dispatches only attachment-free compact commands through the native compaction path", |
| 595 | () => | 703 | () => |
| ... | @@ -3165,6 +3273,25 @@ it.effect.each(["completed", "interrupted", "cancelled", "failed"] as const)( | ... | @@ -3165,6 +3273,25 @@ it.effect.each(["completed", "interrupted", "cancelled", "failed"] as const)( |
| 3165 | }), | 3273 | }), |
| 3166 | ); | 3274 | ); |
| 3167 | 3275 | ||
| 3276 | it.effect("records a finished run as failed when its ownership check cannot be read", () => | ||
| 3277 | Effect.gen(function* () { | ||
| 3278 | const { observed } = yield* captureRootRunTermination({ | ||
| 3279 | key: "finalize-guard-read-failure", | ||
| 3280 | shouldFinalizeRun: () => | ||
| 3281 | Effect.fail( | ||
| 3282 | new ProjectionStore.ProjectionStoreReadError({ | ||
| 3283 | threadId: ThreadId.make("thread:finalize-guard-read-failure"), | ||
| 3284 | cause: "database unavailable", | ||
| 3285 | }), | ||
| 3286 | ), | ||
| 3287 | events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")), | ||
| 3288 | }); | ||
| 3289 | // The fallback settles through the guarded write instead of the same | ||
| 3290 | // failing read, so the run does not stay running. | ||
| 3291 | assert.include(observed, "run:failed"); | ||
| 3292 | }), | ||
| 3293 | ); | ||
| 3294 | |||
| 3168 | it.effect("does not refresh pull requests for auxiliary or stale provider terminals", () => | 3295 | it.effect("does not refresh pull requests for auxiliary or stale provider terminals", () => |
| 3169 | Effect.gen(function* () { | 3296 | Effect.gen(function* () { |
| 3170 | const { observed } = yield* captureRootRunTermination({ | 3297 | const { observed } = yield* captureRootRunTermination({ |
| ... | @@ -3254,7 +3381,7 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = | ... | @@ -3254,7 +3381,7 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = |
| 3254 | 3381 | ||
| 3255 | function captureRootRunTermination(input: { | 3382 | function captureRootRunTermination(input: { |
| 3256 | readonly key: string; | 3383 | readonly key: string; |
| 3257 | readonly shouldFinalizeRun: () => Effect.Effect<boolean, never>; | 3384 | readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>; |
| 3258 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; | 3385 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; |
| 3259 | readonly seedOpenSubagent?: boolean; | 3386 | readonly seedOpenSubagent?: boolean; |
| 3260 | readonly events?: ( | 3387 | readonly events?: ( |
| ... | @@ -3278,6 +3405,17 @@ function captureRootRunTermination(input: { | ... | @@ -3278,6 +3405,17 @@ function captureRootRunTermination(input: { |
| 3278 | const ingestionDone = yield* Deferred.make<void>(); | 3405 | const ingestionDone = yield* Deferred.make<void>(); |
| 3279 | const captureTurnItem = (payload: OrchestrationV2TurnItem) => | 3406 | const captureTurnItem = (payload: OrchestrationV2TurnItem) => |
| 3280 | Ref.update(writtenItems, (current) => [...current, payload]); | 3407 | Ref.update(writtenItems, (current) => [...current, payload]); |
| 3408 | const captureFinalEvents = (events: ReadonlyArray<OrchestrationV2DomainEvent>) => | ||
| 3409 | Effect.gen(function* () { | ||
| 3410 | for (const event of events) { | ||
| 3411 | if (event.type === "turn-item.updated") { | ||
| 3412 | yield* captureTurnItem(event.payload); | ||
| 3413 | } | ||
| 3414 | if (event.type === "run.updated") { | ||
| 3415 | yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]); | ||
| 3416 | } | ||
| 3417 | } | ||
| 3418 | }); | ||
| 3281 | const testLayer = RunExecutionService.layer.pipe( | 3419 | const testLayer = RunExecutionService.layer.pipe( |
| 3282 | Layer.provide( | 3420 | Layer.provide( |
| 3283 | Layer.mergeAll( | 3421 | Layer.mergeAll( |
| ... | @@ -3292,22 +3430,11 @@ function captureRootRunTermination(input: { | ... | @@ -3292,22 +3430,11 @@ function captureRootRunTermination(input: { |
| 3292 | } | 3430 | } |
| 3293 | return []; | 3431 | return []; |
| 3294 | }), | 3432 | }), |
| 3295 | writeWithEffects: (payload) => | 3433 | writeWithEffects: (payload) => captureFinalEvents(payload.events).pipe(Effect.as([])), |
| 3296 | Effect.gen(function* () { | 3434 | writeIfRunCurrent: (payload) => |
| 3297 | for (const event of payload.events) { | 3435 | captureFinalEvents(payload.events).pipe( |
| 3298 | if (event.type === "turn-item.updated") { | 3436 | Effect.as({ committed: true, storedEvents: [] }), |
| 3299 | yield* captureTurnItem(event.payload); | 3437 | ), |
| 3300 | } | ||
| 3301 | if (event.type === "run.updated") { | ||
| 3302 | yield* Ref.update(observed, (current) => [ | ||
| 3303 | ...current, | ||
| 3304 | `run:${event.payload.status}`, | ||
| 3305 | ]); | ||
| 3306 | } | ||
| 3307 | } | ||
| 3308 | return []; | ||
| 3309 | }), | ||
| 3310 | writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), | ||
| 3311 | }), | 3438 | }), |
| 3312 | IdAllocator.layer, | 3439 | IdAllocator.layer, |
| 3313 | Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ | 3440 | Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ |
apps/server/src/orchestration-v2/RunExecutionService.ts+27-27
| ... | @@ -28,6 +28,7 @@ import * as Context from "effect/Context"; | ... | @@ -28,6 +28,7 @@ import * as Context from "effect/Context"; |
| 28 | import * as Cause from "effect/Cause"; | 28 | import * as Cause from "effect/Cause"; |
| 29 | import * as DateTime from "effect/DateTime"; | 29 | import * as DateTime from "effect/DateTime"; |
| 30 | import * as Effect from "effect/Effect"; | 30 | import * as Effect from "effect/Effect"; |
| 31 | import * as Exit from "effect/Exit"; | ||
| 31 | import * as Fiber from "effect/Fiber"; | 32 | import * as Fiber from "effect/Fiber"; |
| 32 | import * as Layer from "effect/Layer"; | 33 | import * as Layer from "effect/Layer"; |
| 33 | import * as Ref from "effect/Ref"; | 34 | import * as Ref from "effect/Ref"; |
| ... | @@ -47,6 +48,7 @@ import type { | ... | @@ -47,6 +48,7 @@ import type { |
| 47 | } from "./ProviderAdapter.ts"; | 48 | } from "./ProviderAdapter.ts"; |
| 48 | import { ProviderAdapterTurnStartError } from "./ProviderAdapter.ts"; | 49 | import { ProviderAdapterTurnStartError } from "./ProviderAdapter.ts"; |
| 49 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; | 50 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; |
| 51 | import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; | ||
| 50 | import { makeProviderFailure, makeProviderFailureTurnItem } from "./ProviderFailure.ts"; | 52 | import { makeProviderFailure, makeProviderFailureTurnItem } from "./ProviderFailure.ts"; |
| 51 | import * as RunFinalizationService from "./RunFinalizationService.ts"; | 53 | import * as RunFinalizationService from "./RunFinalizationService.ts"; |
| 52 | 54 | ||
| ... | @@ -511,8 +513,8 @@ export interface RunExecutionServiceV2StartRootRunInput { | ... | @@ -511,8 +513,8 @@ export interface RunExecutionServiceV2StartRootRunInput { |
| 511 | >; | 513 | >; |
| 512 | readonly relatedThreadIds?: ReadonlyArray<ThreadId>; | 514 | readonly relatedThreadIds?: ReadonlyArray<ThreadId>; |
| 513 | readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>; | 515 | readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>; |
| 514 | readonly shouldStartProviderTurn?: () => Effect.Effect<boolean, never>; | 516 | readonly shouldStartProviderTurn?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 515 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, never>; | 517 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 516 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; | 518 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; |
| 517 | readonly message: ProviderAdapterV2TurnMessage; | 519 | readonly message: ProviderAdapterV2TurnMessage; |
| 518 | readonly modelSelection: ModelSelection; | 520 | readonly modelSelection: ModelSelection; |
| ... | @@ -557,7 +559,7 @@ export const layer: Layer.Layer< | ... | @@ -557,7 +559,7 @@ export const layer: Layer.Layer< |
| 557 | readonly checkpointScope: OrchestrationV2CheckpointScope; | 559 | readonly checkpointScope: OrchestrationV2CheckpointScope; |
| 558 | readonly providerThread: OrchestrationV2ProviderThread; | 560 | readonly providerThread: OrchestrationV2ProviderThread; |
| 559 | readonly attempt: OrchestrationV2RunAttempt; | 561 | readonly attempt: OrchestrationV2RunAttempt; |
| 560 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, never>; | 562 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 561 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; | 563 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; |
| 562 | readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; | 564 | readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; |
| 563 | readonly terminal: ProviderTerminalEvent; | 565 | readonly terminal: ProviderTerminalEvent; |
| ... | @@ -1286,15 +1288,12 @@ export const layer: Layer.Layer< | ... | @@ -1286,15 +1288,12 @@ export const layer: Layer.Layer< |
| 1286 | checkpointScope: input.checkpointScope, | 1288 | checkpointScope: input.checkpointScope, |
| 1287 | providerThread, | 1289 | providerThread, |
| 1288 | attempt: input.attempt, | 1290 | attempt: input.attempt, |
| 1289 | ...(input.shouldFinalizeRun === undefined | 1291 | // The failure may be the ownership |
| 1290 | ? {} | 1292 | // read itself, so check in the write. |
| 1291 | : { shouldFinalizeRun: input.shouldFinalizeRun }), | 1293 | writeIfRunCurrent: { |
| 1292 | ...(input.hasUnpairedRunInterruptRequest === undefined | 1294 | activeAttemptId: input.attempt.id, |
| 1293 | ? {} | 1295 | expectedStatus: "running", |
| 1294 | : { | 1296 | }, |
| 1295 | hasUnpairedRunInterruptRequest: | ||
| 1296 | input.hasUnpairedRunInterruptRequest, | ||
| 1297 | }), | ||
| 1298 | openRunOwnedSubagents: openSubagents, | 1297 | openRunOwnedSubagents: openSubagents, |
| 1299 | terminal: makeFailedTerminalEvent( | 1298 | terminal: makeFailedTerminalEvent( |
| 1300 | makeProviderFailure({ | 1299 | makeProviderFailure({ |
| ... | @@ -1328,10 +1327,13 @@ export const layer: Layer.Layer< | ... | @@ -1328,10 +1327,13 @@ export const layer: Layer.Layer< |
| 1328 | Effect.forkDetach, | 1327 | Effect.forkDetach, |
| 1329 | ); | 1328 | ); |
| 1330 | 1329 | ||
| 1331 | if ( | 1330 | // A failed read fails the start below, so the run is recorded as |
| 1332 | input.shouldStartProviderTurn !== undefined && | 1331 | // failed instead of staying active with no provider turn. |
| 1333 | !(yield* input.shouldStartProviderTurn()) | 1332 | const shouldStart = |
| 1334 | ) { | 1333 | input.shouldStartProviderTurn === undefined |
| 1334 | ? Exit.succeed(true) | ||
| 1335 | : yield* Effect.exit(input.shouldStartProviderTurn()); | ||
| 1336 | if (Exit.isSuccess(shouldStart) && !shouldStart.value) { | ||
| 1335 | yield* Fiber.interrupt(providerEventFiber); | 1337 | yield* Fiber.interrupt(providerEventFiber); |
| 1336 | return; | 1338 | return; |
| 1337 | } | 1339 | } |
| ... | @@ -1373,7 +1375,7 @@ export const layer: Layer.Layer< | ... | @@ -1373,7 +1375,7 @@ export const layer: Layer.Layer< |
| 1373 | }), | 1375 | }), |
| 1374 | )) | 1376 | )) |
| 1375 | : input.session.startTurn(turnInput); | 1377 | : input.session.startTurn(turnInput); |
| 1376 | yield* startTurn.pipe( | 1378 | yield* Effect.andThen(shouldStart, startTurn).pipe( |
| 1377 | Effect.catchCause((cause) => | 1379 | Effect.catchCause((cause) => |
| 1378 | Effect.logError("orchestration V2 provider turn start failed", { | 1380 | Effect.logError("orchestration V2 provider turn start failed", { |
| 1379 | runId: input.run.id, | 1381 | runId: input.run.id, |
| ... | @@ -1392,20 +1394,18 @@ export const layer: Layer.Layer< | ... | @@ -1392,20 +1394,18 @@ export const layer: Layer.Layer< |
| 1392 | checkpointScope: input.checkpointScope, | 1394 | checkpointScope: input.checkpointScope, |
| 1393 | providerThread, | 1395 | providerThread, |
| 1394 | attempt: input.attempt, | 1396 | attempt: input.attempt, |
| 1395 | ...(input.shouldFinalizeRun === undefined | 1397 | // Checked in the write transaction, not by another |
| 1396 | ? {} | 1398 | // read that can fail like the one before the start. |
| 1397 | : { shouldFinalizeRun: input.shouldFinalizeRun }), | 1399 | writeIfRunCurrent: { |
| 1398 | ...(input.hasUnpairedRunInterruptRequest === undefined | 1400 | activeAttemptId: input.attempt.id, |
| 1399 | ? {} | 1401 | expectedStatus: "running", |
| 1400 | : { | 1402 | }, |
| 1401 | hasUnpairedRunInterruptRequest: | ||
| 1402 | input.hasUnpairedRunInterruptRequest, | ||
| 1403 | }), | ||
| 1404 | openRunOwnedSubagents: openSubagents, | 1403 | openRunOwnedSubagents: openSubagents, |
| 1405 | terminal: makeFailedTerminalEvent( | 1404 | terminal: makeFailedTerminalEvent( |
| 1406 | makeProviderFailure({ | 1405 | makeProviderFailure({ |
| 1407 | cause: Cause.squash(cause), | 1406 | cause: Cause.squash(cause), |
| 1408 | class: "provider_error", | 1407 | // A failed ownership read is not the provider's fault. |
| 1408 | class: Exit.isFailure(shouldStart) ? "unknown" : "provider_error", | ||
| 1409 | }), | 1409 | }), |
| 1410 | latestItemOrdinal + 1, | 1410 | latestItemOrdinal + 1, |
| 1411 | ), | 1411 | ), |