authorgravatar for yashafaid@gmail.comCestercian <yashafaid@gmail.com> 2026-09-20 03:38:53+05:30
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-19 15:08:53-07:00
logdcf8942304cfef382af55c0068cb2f04c86757fc
treeb920978d181fdc59df0368f473ade848081f2efe
parentf9e8f578fa0fb1ea25e458334a28bbb4fe22302d
signature Signed by PGP key B5690EEEBB952194

fix(web): keep desktop annotation screenshots under CSP (#12636)

4 files changed, 44 insertions(+), 89 deletions(-)

apps/web/src/components/preview/PreviewView.test.tsx+2-10
......@@ -27,7 +27,6 @@ const mocks = vi.hoisted(() => ({
2727 openPictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
2828 closePictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
2929 pickElement: vi.fn(),
30 capturePreviewAnnotationScreenshot: vi.fn(),
3130 addPreviewAnnotation: vi.fn(),
3231 addImage: vi.fn(),
3332 toggleAnnotation: null as (() => void) | null,
......@@ -90,10 +89,6 @@ vi.mock("~/composerDraftStore", () => ({
9089 }),
9190}));
9291
93vi.mock("~/lib/previewAnnotation", () => ({
94 capturePreviewAnnotationScreenshot: mocks.capturePreviewAnnotationScreenshot,
95}));
96
9792vi.mock("~/localApi", () => ({
9893 ensureLocalApi: vi.fn(),
9994}));
......@@ -342,8 +337,6 @@ describe("PreviewView navigation", () => {
342337 mocks.openPictureInPicture.mockClear();
343338 mocks.closePictureInPicture.mockClear();
344339 mocks.pickElement.mockReset();
345 mocks.capturePreviewAnnotationScreenshot.mockReset();
346 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "none" });
347340 mocks.addPreviewAnnotation.mockClear();
348341 vi.mocked(toastManager.add).mockClear();
349342 mocks.addImage.mockClear();
......@@ -585,7 +578,7 @@ describe("PreviewView navigation", () => {
585578 expect(toastManager.add).toHaveBeenCalledTimes(1);
586579 });
587580
588 it("still sends when the picked element's crop cannot be captured", async () => {
581 it("still sends annotation text when the picked element's crop is malformed", async () => {
589582 const annotation = {
590583 id: "annotation-2",
591584 pageUrl: "https://example.com/dashboard",
......@@ -596,7 +589,7 @@ describe("PreviewView navigation", () => {
596589 strokes: [],
597590 styleChanges: [],
598591 screenshot: {
599 dataUrl: "data:image/png;base64,c2NyZWVuc2hvdA==",
592 dataUrl: "data:image/png;base64,%%%",
600593 width: 10,
601594 height: 10,
602595 cropRect: { x: 0, y: 0, width: 10, height: 10 },
......@@ -605,7 +598,6 @@ describe("PreviewView navigation", () => {
605598 };
606599 const onSendAnnotation = vi.fn();
607600 mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });
608 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "failed" });
609601
610602 renderToStaticMarkup(
611603 <PreviewView
apps/web/src/components/preview/PreviewView.tsx+1-1
......@@ -603,7 +603,7 @@ export function PreviewView({
603603 // instead of holding the composer for an attachment that never lands.
604604 // The stored copy drops the screenshot on failure, otherwise the prompt
605605 // would tell the agent a crop is attached when none was sent.
606 const capture = await capturePreviewAnnotationScreenshot(picked);
606 const capture = capturePreviewAnnotationScreenshot(picked);
607607 // Main reports a crop that failed or timed out on its side; the local
608608 // conversion can fail too. Either way the user should hear about it.
609609 const cropDropped = screenshotFailed || capture.status === "failed";
apps/web/src/lib/previewAnnotation.test.ts+28-46
......@@ -9,66 +9,48 @@ const annotation: PreviewAnnotationPayload = {
99 pageTitle: "Example",
1010 comment: "Make these cards feel related.",
1111 elements: [],
12 regions: [{ id: "region_1", rect: { x: 10, y: 20, width: 100, height: 80 } }],
13 strokes: [
14 {
15 id: "stroke_1",
16 color: "#7c3aed",
17 width: 4,
18 points: [
19 { x: 10, y: 10 },
20 { x: 20, y: 20 },
21 ],
22 bounds: { x: 6, y: 6, width: 18, height: 18 },
23 },
24 ],
25 styleChanges: [
26 {
27 targetId: "element_1",
28 selector: ".card",
29 property: "border-radius",
30 previousValue: "4px",
31 value: "16px",
32 },
33 ],
12 regions: [],
13 strokes: [],
14 styleChanges: [],
3415 screenshot: {
35 dataUrl: "data:image/png;base64,AA==",
36 width: 100,
37 height: 80,
38 cropRect: { x: 10, y: 20, width: 100, height: 80 },
16 dataUrl:
17 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=",
18 width: 1,
19 height: 1,
20 cropRect: { x: 0, y: 0, width: 1, height: 1 },
3921 },
4022 createdAt: "2026-06-11T00:00:00.000Z",
4123};
4224
4325describe("preview annotation capture", () => {
4426 afterEach(() => {
45 vi.useRealTimers();
4627 vi.unstubAllGlobals();
4728 });
4829
49 it("returns the crop when the fetch resolves", async () => {
50 vi.stubGlobal("fetch", async () => new Response(new Blob(["png"], { type: "image/png" })));
51 const capture = await capturePreviewAnnotationScreenshot(annotation);
30 it("preserves the PNG bytes, MIME type, and filename without fetching", async () => {
31 const fetch = vi.fn().mockRejectedValue(new TypeError("Blocked by connect-src"));
32 vi.stubGlobal("fetch", fetch);
33 const capture = capturePreviewAnnotationScreenshot(annotation);
5234 expect(capture.status).toBe("captured");
35 if (capture.status !== "captured") throw new Error("Expected a screenshot file");
36 expect(capture.file.name).toBe("preview-annotation-annotation_1.png");
37 expect(capture.file.type).toBe("image/png");
38 expect(Buffer.from(await capture.file.arrayBuffer()).toString("hex")).toBe(
39 "89504e470d0a1a0a0000000d4948445200000001000000010804000000b51c0c020000000b4944415478da63fcff1f0003030200efa2a75b0000000049454e44ae426082",
40 );
41 expect(fetch).not.toHaveBeenCalled();
5342 });
5443
55 it("reports none when the annotation carries no crop", async () => {
56 const capture = await capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
44 it("reports none when the annotation carries no crop", () => {
45 const capture = capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
5746 expect(capture).toEqual({ status: "none" });
5847 });
5948
60 it("fails instead of hanging when the crop never arrives", async () => {
61 vi.useFakeTimers();
62 vi.stubGlobal("fetch", () => new Promise<Response>(() => {}));
63 const capturePromise = capturePreviewAnnotationScreenshot(annotation, 1_000);
64 await vi.advanceTimersByTimeAsync(1_000);
65 expect(await capturePromise).toEqual({ status: "failed" });
66 });
67
68 it("fails when the crop fetch throws", async () => {
69 vi.stubGlobal("fetch", async () => {
70 throw new Error("data url unreadable");
71 });
72 expect(await capturePreviewAnnotationScreenshot(annotation)).toEqual({ status: "failed" });
73 });
49 it.each(["data:image/jpeg;base64,AA==", "data:image/png;base64,", "data:image/png;base64,%%%"])(
50 "reports a malformed screenshot as failed: %s",
51 (dataUrl) => {
52 const picked = { ...annotation, screenshot: { ...annotation.screenshot!, dataUrl } };
53 expect(capturePreviewAnnotationScreenshot(picked)).toEqual({ status: "failed" });
54 },
55 );
7456});
apps/web/src/lib/previewAnnotation.ts+13-32
......@@ -1,49 +1,30 @@
11import type { PreviewAnnotationPayload } from "@t3tools/contracts";
22
3async function previewAnnotationScreenshotFile(
4 annotation: PreviewAnnotationPayload,
5): Promise<File | null> {
6 if (!annotation.screenshot) return null;
7 const response = await fetch(annotation.screenshot.dataUrl);
8 const blob = await response.blob();
9 return new File([blob], `preview-annotation-${annotation.id}.png`, {
10 type: blob.type || "image/png",
11 });
12}
13
14/** Upper bound on turning a picked element's crop into a composer attachment. */
15const PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS = 5_000;
3import { dataUrlToFile } from "./imageCompression";
164
175export type PreviewAnnotationCapture =
186 /** The crop is ready to attach. */
197 | { readonly status: "captured"; readonly file: File }
208 /** The pick carried no crop, which is normal for comment-only annotations. */
219 | { readonly status: "none" }
22 /** The crop stalled or threw. Send the annotation without it. */
10 /** The crop could not be decoded. Send the annotation without it. */
2311 | { readonly status: "failed" };
2412
25/**
26 * Bounded wrapper around `previewAnnotationScreenshotFile`. The picker holds the
27 * composer while this runs, so it must always settle: a stalled crop resolves as
28 * `failed` instead of leaving the caller waiting.
29 */
30export async function capturePreviewAnnotationScreenshot(
13const PNG_DATA_URL_PREFIX = "data:image/png;base64,";
14
15/** Decode Electron's PNG crop locally; fetching a data URL violates desktop connect-src. */
16export function capturePreviewAnnotationScreenshot(
3117 annotation: PreviewAnnotationPayload,
32 timeoutMs: number = PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS,
33): Promise<PreviewAnnotationCapture> {
18): PreviewAnnotationCapture {
3419 if (!annotation.screenshot) return { status: "none" };
35 let timer: ReturnType<typeof setTimeout> | undefined;
3620 try {
37 const file = await Promise.race([
38 previewAnnotationScreenshotFile(annotation),
39 new Promise<null>((resolve) => {
40 timer = setTimeout(() => resolve(null), timeoutMs);
41 }),
42 ]);
43 return file ? { status: "captured", file } : { status: "failed" };
21 const { dataUrl } = annotation.screenshot;
22 if (!dataUrl.startsWith(PNG_DATA_URL_PREFIX)) {
23 return { status: "failed" };
24 }
25 const file = dataUrlToFile(dataUrl, `preview-annotation-${annotation.id}.png`, "image/png");
26 return file.size > 0 ? { status: "captured", file } : { status: "failed" };
4427 } catch {
4528 return { status: "failed" };
46 } finally {
47 clearTimeout(timer);
4829 }
4930}