authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-08 21:43:53-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-08 18:43:53-07:00
loga6c9b41f902fba2a4137806c09e829935e91baac
tree3dbb440a6b9f7015c4b2ff8dd1ad32132b59a065
parent70c423a5e48ab34d8b1a033e798ab378b48dde5d
signature Signed by PGP key B5690EEEBB952194

feat(server): agents can now open the images you paste into chat (#5757)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

4 files changed, 113 insertions(+), 7 deletions(-)

apps/server/integration/providerService.integration.test.ts+2
...@@ -23,6 +23,7 @@ import {...@@ -23,6 +23,7 @@ import {
23 ProviderService,23 ProviderService,
24 type ProviderServiceShape,24 type ProviderServiceShape,
25} from "../src/provider/Services/ProviderService.ts";25} from "../src/provider/Services/ProviderService.ts";
26import * as ServerConfig from "../src/config.ts";
26import { ServerSettingsService } from "../src/serverSettings.ts";27import { ServerSettingsService } from "../src/serverSettings.ts";
27import { AnalyticsService } from "../src/telemetry/Services/AnalyticsService.ts";28import { AnalyticsService } from "../src/telemetry/Services/AnalyticsService.ts";
28import { SqlitePersistenceMemory } from "../src/persistence/Layers/Sqlite.ts";29import { SqlitePersistenceMemory } from "../src/persistence/Layers/Sqlite.ts";
...@@ -93,6 +94,7 @@ const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<Ana...@@ -93,6 +94,7 @@ const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<Ana
93 const shared = Layer.mergeAll(94 const shared = Layer.mergeAll(
94 directoryLayer,95 directoryLayer,
95 Layer.succeed(ProviderAdapterRegistry, registry),96 Layer.succeed(ProviderAdapterRegistry, registry),
97 ServerConfig.layerTest(cwd, cwd).pipe(Layer.provide(NodeServices.layer)),
96 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),98 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
97 options?.analytics ?? AnalyticsService.layerTest,99 options?.analytics ?? AnalyticsService.layerTest,
98 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),100 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
apps/server/src/provider/Layers/ClaudeAdapter.ts+10-2
...@@ -4090,6 +4090,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (...@@ -4090,6 +4090,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
4090 ...(ultracode ? { ultracode: true } : {}),4090 ...(ultracode ? { ultracode: true } : {}),
4091 };4091 };
4092 const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId);4092 const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId);
4093 // The attachments dir grant lets the agent Read/copy pasted images at
4094 // the paths ProviderService injects into the turn text, without an
4095 // approval prompt. It is a leaf directory holding only attachment
4096 // files; siblings like secrets/ and state.sqlite stay ungranted.
4097 const additionalDirectories = [
4098 ...(input.cwd ? [input.cwd] : []),
4099 serverConfig.attachmentsDir,
4100 ];
4093 const queryOptions: ClaudeQueryOptions = {4101 const queryOptions: ClaudeQueryOptions = {
4094 ...(input.cwd ? { cwd: input.cwd } : {}),4102 ...(input.cwd ? { cwd: input.cwd } : {}),
4095 ...(apiModelId ? { model: apiModelId } : {}),4103 ...(apiModelId ? { model: apiModelId } : {}),
...@@ -4113,7 +4121,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (...@@ -4113,7 +4121,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
4113 includePartialMessages: true,4121 includePartialMessages: true,
4114 canUseTool,4122 canUseTool,
4115 env: claudeEnvironment,4123 env: claudeEnvironment,
4116 ...(input.cwd ? { additionalDirectories: [input.cwd] } : {}),4124 additionalDirectories,
4117 ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}),4125 ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}),
4118 ...(mcpSession4126 ...(mcpSession
4119 ? {4127 ? {
...@@ -4148,7 +4156,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (...@@ -4148,7 +4156,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
4148 "claude.query.resume": existingResumeSessionId ?? "",4156 "claude.query.resume": existingResumeSessionId ?? "",
4149 "claude.query.session_id": newSessionId ?? "",4157 "claude.query.session_id": newSessionId ?? "",
4150 "claude.query.include_partial_messages": true,4158 "claude.query.include_partial_messages": true,
4151 "claude.query.additional_directories": input.cwd ? [input.cwd] : [],4159 "claude.query.additional_directories": additionalDirectories,
4152 "claude.query.setting_sources": [...CLAUDE_SETTING_SOURCES],4160 "claude.query.setting_sources": [...CLAUDE_SETTING_SOURCES],
4153 "claude.query.settings_json": encodeJsonStringForDiagnostics(settings) ?? "",4161 "claude.query.settings_json": encodeJsonStringForDiagnostics(settings) ?? "",
4154 "claude.query.extra_args_json": encodeJsonStringForDiagnostics(extraArgs) ?? "",4162 "claude.query.extra_args_json": encodeJsonStringForDiagnostics(extraArgs) ?? "",
apps/server/src/provider/Layers/ProviderService.test.ts+65
...@@ -55,11 +55,15 @@ import {...@@ -55,11 +55,15 @@ import {
55 makeSqlitePersistenceLive,55 makeSqlitePersistenceLive,
56 SqlitePersistenceMemory,56 SqlitePersistenceMemory,
57} from "../../persistence/Layers/Sqlite.ts";57} from "../../persistence/Layers/Sqlite.ts";
58import * as ServerConfig from "../../config.ts";
58import * as ServerSettings from "../../serverSettings.ts";59import * as ServerSettings from "../../serverSettings.ts";
59import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";60import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";
60import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts";61import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts";
6162
62const defaultServerSettingsLayer = ServerSettings.ServerSettingsService.layerTest();63const defaultServerSettingsLayer = ServerSettings.ServerSettingsService.layerTest();
64const serverConfigTestLayer = ServerConfig.layerTest(process.cwd(), process.cwd()).pipe(
65 Layer.provide(NodeServices.layer),
66);
6367
64const asRequestId = (value: string): ApprovalRequestId => ApprovalRequestId.make(value);68const asRequestId = (value: string): ApprovalRequestId => ApprovalRequestId.make(value);
65const asEventId = (value: string): EventId => EventId.make(value);69const asEventId = (value: string): EventId => EventId.make(value);
...@@ -292,6 +296,7 @@ function makeProviderServiceLayer() {...@@ -292,6 +296,7 @@ function makeProviderServiceLayer() {
292 Layer.provide(providerAdapterLayer),296 Layer.provide(providerAdapterLayer),
293 Layer.provide(directoryLayer),297 Layer.provide(directoryLayer),
294 Layer.provide(defaultServerSettingsLayer),298 Layer.provide(defaultServerSettingsLayer),
299 Layer.provide(serverConfigTestLayer),
295 Layer.provideMerge(AnalyticsService.layerTest),300 Layer.provideMerge(AnalyticsService.layerTest),
296 Layer.provide(301 Layer.provide(
297 Layer.succeed(302 Layer.succeed(
...@@ -343,6 +348,7 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () =>...@@ -343,6 +348,7 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () =>
343 Layer.provide(providerAdapterLayer),348 Layer.provide(providerAdapterLayer),
344 Layer.provide(directoryLayer),349 Layer.provide(directoryLayer),
345 Layer.provide(defaultServerSettingsLayer),350 Layer.provide(defaultServerSettingsLayer),
351 Layer.provide(serverConfigTestLayer),
346 Layer.provideMerge(AnalyticsService.layerTest),352 Layer.provideMerge(AnalyticsService.layerTest),
347 Layer.provide(353 Layer.provide(
348 Layer.succeed(354 Layer.succeed(
...@@ -402,6 +408,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled providers", ()...@@ -402,6 +408,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled providers", ()
402 Layer.provide(providerAdapterLayer),408 Layer.provide(providerAdapterLayer),
403 Layer.provide(directoryLayer),409 Layer.provide(directoryLayer),
404 Layer.provide(defaultServerSettingsLayer),410 Layer.provide(defaultServerSettingsLayer),
411 Layer.provide(serverConfigTestLayer),
405 Layer.provide(AnalyticsService.layerTest),412 Layer.provide(AnalyticsService.layerTest),
406 Layer.provide(413 Layer.provide(
407 Layer.succeed(414 Layer.succeed(
...@@ -486,6 +493,7 @@ it.effect(...@@ -486,6 +493,7 @@ it.effect(
486 Layer.provide(providerAdapterLayer),493 Layer.provide(providerAdapterLayer),
487 Layer.provide(directoryLayer),494 Layer.provide(directoryLayer),
488 Layer.provide(serverSettingsLayer),495 Layer.provide(serverSettingsLayer),
496 Layer.provide(serverConfigTestLayer),
489 Layer.provide(AnalyticsService.layerTest),497 Layer.provide(AnalyticsService.layerTest),
490 Layer.provide(498 Layer.provide(
491 Layer.succeed(499 Layer.succeed(
...@@ -556,6 +564,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled custom instance...@@ -556,6 +564,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled custom instance
556 Layer.provide(providerAdapterLayer),564 Layer.provide(providerAdapterLayer),
557 Layer.provide(directoryLayer),565 Layer.provide(directoryLayer),
558 Layer.provide(defaultServerSettingsLayer),566 Layer.provide(defaultServerSettingsLayer),
567 Layer.provide(serverConfigTestLayer),
559 Layer.provide(AnalyticsService.layerTest),568 Layer.provide(AnalyticsService.layerTest),
560 Layer.provide(569 Layer.provide(
561 Layer.succeed(570 Layer.succeed(
...@@ -611,6 +620,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se...@@ -611,6 +620,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se
611 Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),620 Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),
612 Layer.provide(directoryLayer),621 Layer.provide(directoryLayer),
613 Layer.provide(defaultServerSettingsLayer),622 Layer.provide(defaultServerSettingsLayer),
623 Layer.provide(serverConfigTestLayer),
614 Layer.provide(AnalyticsService.layerTest),624 Layer.provide(AnalyticsService.layerTest),
615 Layer.provide(625 Layer.provide(
616 Layer.succeed(626 Layer.succeed(
...@@ -671,6 +681,7 @@ it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", (...@@ -671,6 +681,7 @@ it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", (
671 Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),681 Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),
672 Layer.provide(directoryLayer),682 Layer.provide(directoryLayer),
673 Layer.provide(defaultServerSettingsLayer),683 Layer.provide(defaultServerSettingsLayer),
684 Layer.provide(serverConfigTestLayer),
674 Layer.provide(AnalyticsService.layerTest),685 Layer.provide(AnalyticsService.layerTest),
675 Layer.provide(686 Layer.provide(
676 Layer.succeed(687 Layer.succeed(
...@@ -737,6 +748,7 @@ it.effect(...@@ -737,6 +748,7 @@ it.effect(
737 ),748 ),
738 Layer.provide(firstDirectoryLayer),749 Layer.provide(firstDirectoryLayer),
739 Layer.provide(defaultServerSettingsLayer),750 Layer.provide(defaultServerSettingsLayer),
751 Layer.provide(serverConfigTestLayer),
740 Layer.provide(AnalyticsService.layerTest),752 Layer.provide(AnalyticsService.layerTest),
741 Layer.provide(753 Layer.provide(
742 Layer.succeed(754 Layer.succeed(
...@@ -796,6 +808,7 @@ it.effect(...@@ -796,6 +808,7 @@ it.effect(
796 ),808 ),
797 Layer.provide(secondDirectoryLayer),809 Layer.provide(secondDirectoryLayer),
798 Layer.provide(defaultServerSettingsLayer),810 Layer.provide(defaultServerSettingsLayer),
811 Layer.provide(serverConfigTestLayer),
799 Layer.provide(AnalyticsService.layerTest),812 Layer.provide(AnalyticsService.layerTest),
800 Layer.provide(813 Layer.provide(
801 Layer.succeed(814 Layer.succeed(
...@@ -927,6 +940,54 @@ routing.layer("ProviderServiceLive routing", (it) => {...@@ -927,6 +940,54 @@ routing.layer("ProviderServiceLive routing", (it) => {
927 }),940 }),
928 );941 );
929942
943 it.effect("appends attachment file paths to the turn input text", () =>
944 Effect.gen(function* () {
945 const provider = yield* ProviderService.ProviderService;
946
947 const session = yield* provider.startSession(asThreadId("thread-attach"), {
948 provider: ProviderDriverKind.make("codex"),
949 providerInstanceId: codexInstanceId,
950 threadId: asThreadId("thread-attach"),
951 cwd: "/tmp/project",
952 runtimeMode: "full-access",
953 });
954
955 const attachment = {
956 type: "image" as const,
957 id: "thread-attach-12345678-1234-1234-1234-123456789abc",
958 name: "screenshot.png",
959 mimeType: "image/png",
960 sizeBytes: 123,
961 };
962
963 routing.codex.sendTurn.mockClear();
964 yield* provider.sendTurn({
965 threadId: session.threadId,
966 input: "use this screenshot",
967 attachments: [attachment],
968 });
969
970 const turnInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;
971 assert.equal(typeof turnInput.input, "string");
972 const turnText = turnInput.input ?? "";
973 assert.equal(turnText.startsWith("use this screenshot"), true);
974 assert.include(turnText, '[Attached image "screenshot.png" is saved at: ');
975 assert.equal(turnText.endsWith(`${attachment.id}.png]`), true);
976
977 // An attachment-only turn stays valid and the injected line becomes the
978 // whole input text, so the agent still learns the path.
979 routing.codex.sendTurn.mockClear();
980 yield* provider.sendTurn({
981 threadId: session.threadId,
982 attachments: [attachment],
983 });
984 const imageOnlyInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;
985 assert.equal(imageOnlyInput.input?.startsWith('[Attached image "screenshot.png"'), true);
986
987 yield* provider.stopSession({ threadId: session.threadId });
988 }),
989 );
990
930 it.effect("recovers stale persisted sessions for rollback by resuming thread identity", () =>991 it.effect("recovers stale persisted sessions for rollback by resuming thread identity", () =>
931 Effect.gen(function* () {992 Effect.gen(function* () {
932 const provider = yield* ProviderService.ProviderService;993 const provider = yield* ProviderService.ProviderService;
...@@ -1307,6 +1368,7 @@ routing.layer("ProviderServiceLive routing", (it) => {...@@ -1307,6 +1368,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
1307 ),1368 ),
1308 Layer.provide(firstDirectoryLayer),1369 Layer.provide(firstDirectoryLayer),
1309 Layer.provide(defaultServerSettingsLayer),1370 Layer.provide(defaultServerSettingsLayer),
1371 Layer.provide(serverConfigTestLayer),
1310 Layer.provide(AnalyticsService.layerTest),1372 Layer.provide(AnalyticsService.layerTest),
1311 Layer.provide(1373 Layer.provide(
1312 Layer.succeed(1374 Layer.succeed(
...@@ -1345,6 +1407,7 @@ routing.layer("ProviderServiceLive routing", (it) => {...@@ -1345,6 +1407,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
1345 ),1407 ),
1346 Layer.provide(secondDirectoryLayer),1408 Layer.provide(secondDirectoryLayer),
1347 Layer.provide(defaultServerSettingsLayer),1409 Layer.provide(defaultServerSettingsLayer),
1410 Layer.provide(serverConfigTestLayer),
1348 Layer.provide(AnalyticsService.layerTest),1411 Layer.provide(AnalyticsService.layerTest),
1349 Layer.provide(1412 Layer.provide(
1350 Layer.succeed(1413 Layer.succeed(
...@@ -1413,6 +1476,7 @@ routing.layer("ProviderServiceLive routing", (it) => {...@@ -1413,6 +1476,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
1413 ),1476 ),
1414 Layer.provide(firstDirectoryLayer),1477 Layer.provide(firstDirectoryLayer),
1415 Layer.provide(defaultServerSettingsLayer),1478 Layer.provide(defaultServerSettingsLayer),
1479 Layer.provide(serverConfigTestLayer),
1416 Layer.provide(AnalyticsService.layerTest),1480 Layer.provide(AnalyticsService.layerTest),
1417 Layer.provide(1481 Layer.provide(
1418 Layer.succeed(1482 Layer.succeed(
...@@ -1446,6 +1510,7 @@ routing.layer("ProviderServiceLive routing", (it) => {...@@ -1446,6 +1510,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
1446 ),1510 ),
1447 Layer.provide(secondDirectoryLayer),1511 Layer.provide(secondDirectoryLayer),
1448 Layer.provide(defaultServerSettingsLayer),1512 Layer.provide(defaultServerSettingsLayer),
1513 Layer.provide(serverConfigTestLayer),
1449 Layer.provide(AnalyticsService.layerTest),1514 Layer.provide(AnalyticsService.layerTest),
1450 Layer.provide(1515 Layer.provide(
1451 Layer.succeed(1516 Layer.succeed(
apps/server/src/provider/Layers/ProviderService.ts+36-5
...@@ -35,6 +35,8 @@ import * as Schema from "effect/Schema";...@@ -35,6 +35,8 @@ import * as Schema from "effect/Schema";
35import * as SchemaIssue from "effect/SchemaIssue";35import * as SchemaIssue from "effect/SchemaIssue";
36import * as Stream from "effect/Stream";36import * as Stream from "effect/Stream";
3737
38import { resolveAttachmentPath } from "../../attachmentStore.ts";
39import * as ServerConfig from "../../config.ts";
38import {40import {
39 increment,41 increment,
40 providerMetricAttributes,42 providerMetricAttributes,
...@@ -203,6 +205,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -203,6 +205,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
203 options?: ProviderServiceLiveOptions,205 options?: ProviderServiceLiveOptions,
204) {206) {
205 const analytics = yield* Effect.service(AnalyticsService.AnalyticsService);207 const analytics = yield* Effect.service(AnalyticsService.AnalyticsService);
208 const serverConfig = yield* ServerConfig.ServerConfig;
206 const eventLoggers = yield* ProviderEventLoggers.ProviderEventLoggers;209 const eventLoggers = yield* ProviderEventLoggers.ProviderEventLoggers;
207 // Options-provided logger wins (test overrides); otherwise we take whatever210 // Options-provided logger wins (test overrides); otherwise we take whatever
208 // the `ProviderEventLoggers` tag exposes — `undefined` means "no canonical211 // the `ProviderEventLoggers` tag exposes — `undefined` means "no canonical
...@@ -665,16 +668,44 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -665,16 +668,44 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
665 payload: rawInput,668 payload: rawInput,
666 });669 });
667670
668 const input = {671 const attachments = parsed.attachments ?? [];
669 ...parsed,672 if (!parsed.input && attachments.length === 0) {
670 attachments: parsed.attachments ?? [],
671 };
672 if (!input.input && input.attachments.length === 0) {
673 return yield* toValidationError(673 return yield* toValidationError(
674 "ProviderService.sendTurn",674 "ProviderService.sendTurn",
675 "Either input text or at least one attachment is required",675 "Either input text or at least one attachment is required",
676 );676 );
677 }677 }
678
679 // Adapters inline attachment pixels into the model prompt, but the model's
680 // tools cannot dereference pixels. Appending the on-disk path is what lets
681 // a turn like "include this screenshot in the PR" copy the actual file.
682 // This runs after schema decode, so the appended lines are exempt from the
683 // PROVIDER_SEND_TURN_MAX_INPUT_CHARS check; attachment count is capped, so
684 // the overhead is bounded. Unresolvable ids are skipped here and surface
685 // as adapter errors when the file is read for inlining.
686 const attachmentPathLines = attachments.flatMap((attachment) => {
687 const attachmentPath = resolveAttachmentPath({
688 attachmentsDir: serverConfig.attachmentsDir,
689 attachment,
690 });
691 return attachmentPath === null
692 ? []
693 : [`[Attached ${attachment.type} "${attachment.name}" is saved at: ${attachmentPath}]`];
694 });
695 const inputTextWithAttachmentPaths =
696 attachmentPathLines.length === 0
697 ? parsed.input
698 : [parsed.input, attachmentPathLines.join("\n")]
699 .filter((part): part is string => typeof part === "string" && part.length > 0)
700 .join("\n\n");
701
702 const input = {
703 ...parsed,
704 ...(inputTextWithAttachmentPaths !== undefined
705 ? { input: inputTextWithAttachmentPaths }
706 : {}),
707 attachments,
708 };
678 yield* Effect.annotateCurrentSpan({709 yield* Effect.annotateCurrentSpan({
679 "provider.operation": "send-turn",710 "provider.operation": "send-turn",
680 "provider.thread_id": input.threadId,711 "provider.thread_id": input.threadId,