authorgravatar for 159104116+whoisaldo@users.noreply.github.comAli Younes <159104116+whoisaldo@users.noreply.github.com> 2026-10-11 03:40:36-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-11 02:40:36-05:00
log8d199b4f7b918079161d5f43e0146c099301be08
tree35fc1bf71253d38407d478bc551265d25693ea1b
parent20254307084f1101ced00dfadeddc7804a7de45d
signature Signed by PGP key B5690EEEBB952194

fix(mobile): image-only messages ask the agent to respond, like desktop (#16480)

Co-authored-by: Yash Singh <saiansh2525@gmail.com>

7 files changed, 59 insertions(+), 12 deletions(-)

apps/mobile/src/lib/composerContext.test.ts+15
...@@ -8,6 +8,7 @@ import {...@@ -8,6 +8,7 @@ import {
8} from "@t3tools/contracts";8} from "@t3tools/contracts";
9import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens";9import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens";
10import {10import {
11 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
11 collectComposerContextReferences,12 collectComposerContextReferences,
12 formatComposerContextReference,13 formatComposerContextReference,
13 projectComposerContextForProvider,14 projectComposerContextForProvider,
...@@ -207,6 +208,20 @@ describe("mobile composer context", () => {...@@ -207,6 +208,20 @@ describe("mobile composer context", () => {
207});208});
208209
209describe("host context compatibility", () => {210describe("host context compatibility", () => {
211 it("sends an image-only prompt as the attachment-only prompt and keeps the image", () => {
212 const context: OrchestrationMessageContext = { version: 1, records: [image] };
213 const reference = formatComposerContextReference(image);
214 expect(serializeComposerMessageForServer(reference, context, true)).toEqual({
215 text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
216 context,
217 });
218 const withProse = `${reference} what is this?`;
219 expect(serializeComposerMessageForServer(withProse, context, true)).toEqual({
220 text: withProse,
221 context,
222 });
223 });
224
210 it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => {225 it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => {
211 const pr = pullRequestComposerContext(226 const pr = pullRequestComposerContext(
212 {227 {
apps/mobile/src/lib/composerContext.ts+11-2
...@@ -13,8 +13,10 @@ import {...@@ -13,8 +13,10 @@ import {
13} from "@t3tools/contracts";13} from "@t3tools/contracts";
14import * as Schema from "effect/Schema";14import * as Schema from "effect/Schema";
15import {15import {
16 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
16 collectComposerContextReferences,17 collectComposerContextReferences,
17 formatComposerContextReference,18 formatComposerContextReference,
19 isImageReferencesOnly,
18 replaceComposerContextReferences,20 replaceComposerContextReferences,
19 sanitizeComposerContextLabel,21 sanitizeComposerContextLabel,
20} from "@t3tools/shared/composerContextReferences";22} from "@t3tools/shared/composerContextReferences";
...@@ -229,12 +231,19 @@ export function reidentifyComposerContext(...@@ -229,12 +231,19 @@ export function reidentifyComposerContext(
229 };231 };
230}232}
231233
232/** Keep queued records canonical; choose the wire format against the host at dispatch time. */234/**
235 * Keep queued records canonical; choose the wire format against the host at dispatch time.
236 * An image-only prompt goes out as the attachment-only prompt web sends, so the agent is still
237 * asked to respond. The images stay in the message's attachments and context records.
238 */
233export function serializeComposerMessageForServer(239export function serializeComposerMessageForServer(
234 text: string,240 composerText: string,
235 context: OrchestrationMessageContext | undefined,241 context: OrchestrationMessageContext | undefined,
236 supportsInlineMessageContext: boolean,242 supportsInlineMessageContext: boolean,
237): { text: string; context?: OrchestrationMessageContext } {243): { text: string; context?: OrchestrationMessageContext } {
244 const text = isImageReferencesOnly(composerText)
245 ? ATTACHMENT_ONLY_BOOTSTRAP_PROMPT
246 : composerText;
238 return supportsInlineMessageContext247 return supportsInlineMessageContext
239 ? { text, ...(context ? { context } : {}) }248 ? { text, ...(context ? { context } : {}) }
240 : { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) };249 : { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) };
apps/web/src/components/ChatView.tsx+2-4
...@@ -607,10 +607,8 @@ import {...@@ -607,10 +607,8 @@ import {
607 supportsServerUpdateThreadContinuation,607 supportsServerUpdateThreadContinuation,
608} from "../versionSkew";608} from "../versionSkew";
609import { useAssetUrls } from "../assets/assetUrls";609import { useAssetUrls } from "../assets/assetUrls";
610import {610import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
611 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,611import { recallableComposerPrompt } from "./chat/composerPromptHistory";
612 recallableComposerPrompt,
613} from "./chat/composerPromptHistory";
614import { observeResize } from "~/lib/observeResize";612import { observeResize } from "~/lib/observeResize";
615613
616const EMPTY_PROVIDERS: ServerProvider[] = [];614const EMPTY_PROVIDERS: ServerProvider[] = [];
apps/web/src/components/chat/composerPromptHistory.test.ts+1-1
...@@ -1,8 +1,8 @@...@@ -1,8 +1,8 @@
1import { describe, expect, it } from "vite-plus/test";1import { describe, expect, it } from "vite-plus/test";
22
3import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
3import { buildPlanImplementationPrompt } from "../../proposedPlan";4import { buildPlanImplementationPrompt } from "../../proposedPlan";
4import {5import {
5 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
6 buildComposerPromptHistoryEntries,6 buildComposerPromptHistoryEntries,
7 recallableComposerPrompt,7 recallableComposerPrompt,
8 stepComposerPromptHistory,8 stepComposerPromptHistory,
apps/web/src/components/chat/composerPromptHistory.ts+4-5
...@@ -1,4 +1,7 @@...@@ -1,4 +1,7 @@
1import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";1import {
2 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
3 collectComposerContextReferences,
4} from "@t3tools/shared/composerContextReferences";
2import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan";5import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan";
36
4/**7/**
...@@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /<review_comment\b[^>]*>[\s\S]*?<\/review_c...@@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /<review_comment\b[^>]*>[\s\S]*?<\/review_c
15const TRAILING_LEGACY_CONTEXT =18const TRAILING_LEGACY_CONTEXT =
16 /\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/;19 /\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/;
1720
18/** Text sent in place of an empty prompt when a message is attachments only. */
19export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
20 "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
21
22export interface ComposerPromptHistoryMessage {21export interface ComposerPromptHistoryMessage {
23 readonly id: string;22 readonly id: string;
24 readonly role: string;23 readonly role: string;
packages/shared/src/composerContextReferences.test.ts+12
...@@ -6,6 +6,7 @@ import {...@@ -6,6 +6,7 @@ import {
6 formatComposerContextHref,6 formatComposerContextHref,
7 formatComposerContextProviderMarker,7 formatComposerContextProviderMarker,
8 formatComposerContextReference,8 formatComposerContextReference,
9 isImageReferencesOnly,
9 parseComposerContextHref,10 parseComposerContextHref,
10 projectComposerContextForProvider,11 projectComposerContextForProvider,
11 replaceComposerContextReferences,12 replaceComposerContextReferences,
...@@ -86,6 +87,17 @@ describe("labels and reference links", () => {...@@ -86,6 +87,17 @@ describe("labels and reference links", () => {
86 expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]);87 expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]);
87 });88 });
8889
90 it("recognizes a prompt made only of image references", () => {
91 const image = "![a.png](t3-context://v1/image/ctx_1)";
92 const file = "[notes.md](t3-context://v1/file/ctx_3)";
93 expect(isImageReferencesOnly(image)).toBe(true);
94 expect(isImageReferencesOnly(` ${image}\n![b.png](t3-context://v1/image/ctx_2) `)).toBe(true);
95 expect(isImageReferencesOnly(`${image} what is this?`)).toBe(false);
96 expect(isImageReferencesOnly(file)).toBe(false);
97 expect(isImageReferencesOnly(`${image} ${file}`)).toBe(false);
98 expect(isImageReferencesOnly(" ")).toBe(false);
99 });
100
89 it("replaces occurrences in place", () => {101 it("replaces occurrences in place", () => {
90 const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c";102 const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c";
91 expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe(103 expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe(
packages/shared/src/composerContextReferences.ts+14
...@@ -92,6 +92,20 @@ export function collectComposerContextReferences(...@@ -92,6 +92,20 @@ export function collectComposerContextReferences(
92 return occurrences;92 return occurrences;
93}93}
9494
95/** Text sent in place of an empty prompt when a message is attachments only. */
96export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
97 "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
98
99/** Whether `text` is image references and nothing else, which is how an image-only prompt reads. */
100export function isImageReferencesOnly(text: string): boolean {
101 const references = collectComposerContextReferences(text);
102 return (
103 references.length > 0 &&
104 references.every((reference) => reference.kind === "image") &&
105 replaceComposerContextReferences(text, () => "").trim() === ""
106 );
107}
108
95export function replaceComposerContextReferences(109export function replaceComposerContextReferences(
96 text: string,110 text: string,
97 replace: (occurrence: ComposerContextReferenceOccurrence) => string,111 replace: (occurrence: ComposerContextReferenceOccurrence) => string,