authorgravatar for yashafaid@gmail.comCestercian <yashafaid@gmail.com> 2026-09-20 03:39:10+05:30
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-19 15:09:10-07:00
log52d08a14b9e475371658fabbb12c056c3d3dad67
treefe4eb466d801801aba3f44c9d0b0b3941be913d6
parent63ff33756c47f127391962458d8388a647e8ed44
signature Signed by PGP key B5690EEEBB952194

fix(desktop): include SnapShot app text for Flatpak and GTK4 (#12635)

Co-authored-by: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

4 files changed, 224 insertions(+), 11 deletions(-)

apps/desktop/src/snapShot/DesktopSnapShot.test.ts+138
...@@ -29,6 +29,7 @@ beforeEach(() => {...@@ -29,6 +29,7 @@ beforeEach(() => {
29 transitionCapturePageMock.mockReset().mockResolvedValue(undefined);29 transitionCapturePageMock.mockReset().mockResolvedValue(undefined);
30 transitionSnapshotMock.mockReset().mockResolvedValue(undefined);30 transitionSnapshotMock.mockReset().mockResolvedValue(undefined);
31 prepareCaptureRevealMock.mockReset();31 prepareCaptureRevealMock.mockReset();
32 accessibilityListMock.mockReset().mockResolvedValue([]);
32});33});
3334
34const {35const {
...@@ -39,6 +40,7 @@ const {...@@ -39,6 +40,7 @@ const {
39 accessibilityProcessCoolMock,40 accessibilityProcessCoolMock,
40 accessibilityProcessReadMock,41 accessibilityProcessReadMock,
41 accessibilityByPidMock,42 accessibilityByPidMock,
43 accessibilityListMock,
42 accessibilityForegroundMock,44 accessibilityForegroundMock,
43 accessibilityTrustedMock,45 accessibilityTrustedMock,
44 allWindowsMock,46 allWindowsMock,
...@@ -87,6 +89,7 @@ const {...@@ -87,6 +89,7 @@ const {
87 }89 }
88 >(),90 >(),
89 accessibilityByPidMock: vi.fn(),91 accessibilityByPidMock: vi.fn(),
92 accessibilityListMock: vi.fn(),
90 accessibilityForegroundMock: vi.fn(),93 accessibilityForegroundMock: vi.fn(),
91 accessibilityTrustedMock: vi.fn((_prompt = false) => true),94 accessibilityTrustedMock: vi.fn((_prompt = false) => true),
92 allWindowsMock: vi.fn(95 allWindowsMock: vi.fn(
...@@ -430,6 +433,7 @@ import * as SnapShotAccessibility from "./SnapShotAccessibility.ts";...@@ -430,6 +433,7 @@ import * as SnapShotAccessibility from "./SnapShotAccessibility.ts";
430// Tests hand it this stand-in so the mocks above drive window lookups.433// Tests hand it this stand-in so the mocks above drive window lookups.
431const accessibilityApp = {434const accessibilityApp = {
432 byPid: accessibilityByPidMock,435 byPid: accessibilityByPidMock,
436 list: accessibilityListMock,
433 foreground: accessibilityForegroundMock,437 foreground: accessibilityForegroundMock,
434} as unknown as Parameters<typeof SnapShotAccessibility.readAccessibleWindowContextWithApp>[0];438} as unknown as Parameters<typeof SnapShotAccessibility.readAccessibleWindowContextWithApp>[0];
435const readAccessibleWindowContext = (439const readAccessibleWindowContext = (
...@@ -2654,6 +2658,140 @@ it.each([...@@ -2654,6 +2658,140 @@ it.each([
2654 }2658 }
2655});2659});
26562660
2661it("reads Flatpak app text from the AT-SPI proxy when the compositor PID misses", async () => {
2662 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2663 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2664 const tree = vi.fn(async () => ({
2665 name: "Issue — Zen Browser",
2666 children: [{ name: "New Tab", children: [] }],
2667 }));
2668 const proxyWindow = {
2669 role: "window",
2670 name: "Issue — Zen Browser",
2671 bounds,
2672 tree,
2673 children: async () => [],
2674 };
2675 accessibilityByPidMock
2676 .mockReset()
2677 .mockRejectedValue(
2678 new Error("XA11Y_SELECTOR_NOT_MATCHED: No element matched selector: application[pid=207651]"),
2679 );
2680 accessibilityListMock.mockReset().mockResolvedValue([
2681 {
2682 pid: 1,
2683 children: async () => [
2684 {
2685 role: "window",
2686 name: "Files",
2687 bounds,
2688 tree: async () => ({ name: "Files", children: [] }),
2689 },
2690 ],
2691 },
2692 { pid: 207646, children: async () => [proxyWindow] },
2693 ]);
2694 try {
2695 assert.strictEqual(
2696 await readAccessibleWindowText(
2697 {
2698 title: "Issue — Zen Browser",
2699 bounds: { x: 0, y: 0, width: 1_920, height: 1_048 },
2700 owner: { processId: 207651 },
2701 },
2702 "linux",
2703 "Issue — Zen Browser",
2704 ),
2705 "Issue — Zen Browser\nNew Tab",
2706 );
2707 assert.deepEqual(accessibilityByPidMock.mock.calls, [[207651, { timeout: 0 }]]);
2708 assert.lengthOf(accessibilityListMock.mock.calls, 1);
2709 assert.lengthOf(tree.mock.calls, 1);
2710 } finally {
2711 accessibilityListMock.mockReset().mockResolvedValue([]);
2712 vi.unstubAllEnvs();
2713 }
2714});
2715
2716it("reads GTK4 app text from one unnamed PID-scoped window of the captured size", async () => {
2717 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2718 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2719 const tree = vi.fn(async () => ({
2720 name: "System Monitor",
2721 children: [{ name: "CPU", children: [] }],
2722 }));
2723 accessibilityByPidMock.mockReset().mockResolvedValue({
2724 children: async () => [
2725 {
2726 role: "group",
2727 name: null,
2728 bounds,
2729 tree,
2730 children: async () => [
2731 { role: "page_tab_list", name: "Processes", bounds, children: async () => [] },
2732 ],
2733 },
2734 ],
2735 });
2736 accessibilityListMock.mockReset().mockResolvedValue([]);
2737 try {
2738 assert.strictEqual(
2739 await readAccessibleWindowText(
2740 {
2741 title: "System Monitor",
2742 bounds: { x: 12, y: 48, width: 1_920, height: 1_048 },
2743 owner: { processId: 210600 },
2744 },
2745 "linux",
2746 "System Monitor",
2747 ),
2748 "System Monitor\nCPU",
2749 );
2750 assert.deepEqual(accessibilityByPidMock.mock.calls, [[210600, { timeout: 0 }]]);
2751 assert.lengthOf(accessibilityListMock.mock.calls, 0);
2752 assert.lengthOf(tree.mock.calls, 1);
2753 } finally {
2754 accessibilityListMock.mockReset().mockResolvedValue([]);
2755 vi.unstubAllEnvs();
2756 }
2757});
2758
2759it("does not guess GTK4 app text when two unnamed PID windows share the captured size", async () => {
2760 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2761 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2762 const tree = vi.fn(async () => ({ value: "Wrong window", children: [] }));
2763 accessibilityByPidMock.mockReset().mockResolvedValue({
2764 children: async () => [
2765 { role: "group", name: null, bounds, tree },
2766 { role: "group", name: null, bounds: { ...bounds, x: 12 }, tree },
2767 ],
2768 });
2769 accessibilityListMock.mockReset().mockResolvedValue([
2770 {
2771 pid: 99,
2772 children: async () => [{ role: "window", name: "System Monitor", bounds, tree }],
2773 },
2774 ]);
2775 try {
2776 assert.isUndefined(
2777 await readAccessibleWindowText(
2778 {
2779 title: "System Monitor",
2780 bounds: { x: 12, y: 48, width: 1_920, height: 1_048 },
2781 owner: { processId: 210600 },
2782 },
2783 "linux",
2784 "System Monitor",
2785 ),
2786 );
2787 assert.lengthOf(accessibilityListMock.mock.calls, 0);
2788 assert.lengthOf(tree.mock.calls, 0);
2789 } finally {
2790 accessibilityListMock.mockReset().mockResolvedValue([]);
2791 vi.unstubAllEnvs();
2792 }
2793});
2794
2657it.each([20, 1_350, 2_999])(2795it.each([20, 1_350, 2_999])(
2658 "includes accessibility text as soon as a %d ms read completes",2796 "includes accessibility text as soon as a %d ms read completes",
2659 async (duration) => {2797 async (duration) => {
apps/desktop/src/snapShot/SnapShotAccessibility.ts+28-4
...@@ -97,6 +97,23 @@ function accessibilityReadSnapshot(...@@ -97,6 +97,23 @@ function accessibilityReadSnapshot(
97 ) as CapturedWindowAccessibilityContext;97 ) as CapturedWindowAccessibilityContext;
98}98}
9999
100type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];
101
102async function windowsForPid(
103 App: AccessibilityApp,
104 processId: number,
105): Promise<readonly AccessibilityElement[]> {
106 return await App.byPid(processId, { timeout: 0 })
107 .then((app) => app.children())
108 .catch(() => []);
109}
110
111/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
112async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
113 const apps = await App.list().catch(() => []);
114 return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
115}
116
100async function readCapturedWindowAccessibility(117async function readCapturedWindowAccessibility(
101 App: AccessibilityApp,118 App: AccessibilityApp,
102 request: SnapShotAccessibilityRequest,119 request: SnapShotAccessibilityRequest,
...@@ -105,17 +122,24 @@ async function readCapturedWindowAccessibility(...@@ -105,17 +122,24 @@ async function readCapturedWindowAccessibility(
105): Promise<CapturedWindowAccessibilityContext | undefined> {122): Promise<CapturedWindowAccessibilityContext | undefined> {
106 const { active, platform, sourceTitle, imageSize } = request;123 const { active, platform, sourceTitle, imageSize } = request;
107 const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;124 const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
108 const windows =125 const pidWindows =
109 foreground !== undefined126 foreground !== undefined
110 ? foreground.pid === active.owner.processId127 ? foreground.pid === active.owner.processId
111 ? [foreground.asElement()]128 ? [foreground.asElement()]
112 : []129 : []
113 : await (await App.byPid(active.owner.processId, { timeout: 0 })).children();130 : await windowsForPid(App, active.owner.processId);
114 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";131 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
132 const captured = {
133 title: active.title,
134 sourceTitle,
135 bounds: active.bounds,
136 clientBounds: active.clientBounds,
137 };
115 const window = findAccessibleWindow(138 const window = findAccessibleWindow(
116 windows,139 pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
117 { title: active.title, sourceTitle, bounds: active.bounds, clientBounds: active.clientBounds },140 captured,
118 matchMode,141 matchMode,
142 { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
119 );143 );
120 if (!window) {144 if (!window) {
121 onStarted();145 onStarted();
apps/desktop/src/snapShot/snapShot.test.ts+39
...@@ -464,6 +464,45 @@ describe("findAccessibleWindow", () => {...@@ -464,6 +464,45 @@ describe("findAccessibleWindow", () => {
464 expect(findAccessibleWindow(windows, { ...captured, title: "" }, "wayland")).toBeUndefined();464 expect(findAccessibleWindow(windows, { ...captured, title: "" }, "wayland")).toBeUndefined();
465 });465 });
466466
467 it("accepts one PID-scoped untitled window whose bounds match", () => {
468 const windows = [{ name: null, bounds: captured.bounds }];
469
470 expect(
471 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
472 ).toBe(windows[0]);
473 expect(findAccessibleWindow(windows, captured, "wayland")).toBeUndefined();
474 });
475
476 it("rejects ambiguous PID-scoped untitled windows even when bounds match", () => {
477 const windows = [
478 { name: null, bounds: captured.bounds },
479 { name: "", bounds: { ...captured.bounds, x: 0, y: 0 } },
480 ];
481
482 expect(
483 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
484 ).toBeUndefined();
485 });
486
487 it("does not fall back to a differently titled PID-scoped window with matching bounds", () => {
488 const windows = [{ name: "Preferences", bounds: captured.bounds }];
489
490 expect(
491 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
492 ).toBeUndefined();
493 });
494
495 it("does not use unique bounds when a titled match is already ambiguous", () => {
496 const windows = [
497 { name: "Editor", bounds: captured.bounds },
498 { name: "Editor", bounds: { ...captured.bounds, x: 0, y: 0 } },
499 ];
500
501 expect(
502 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
503 ).toBeUndefined();
504 });
505
467 it.each(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])(506 it.each(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])(
468 "ignores a leading Wayland title spinner frame %s",507 "ignores a leading Wayland title spinner frame %s",
469 (frame) => {508 (frame) => {
apps/desktop/src/snapShot/snapShot.ts+19-7
...@@ -480,6 +480,7 @@ export function findAccessibleWindow<...@@ -480,6 +480,7 @@ export function findAccessibleWindow<
480 readonly clientBounds?: WindowBounds | undefined;480 readonly clientBounds?: WindowBounds | undefined;
481 },481 },
482 matchMode: "screen-bounds" | "wayland" = "screen-bounds",482 matchMode: "screen-bounds" | "wayland" = "screen-bounds",
483 options: { readonly allowUntitledUniqueBounds?: boolean } = {},
483): T | undefined {484): T | undefined {
484 const normalizeTitle = (value: string) => {485 const normalizeTitle = (value: string) => {
485 const title = value.trim();486 const title = value.trim();
...@@ -489,7 +490,6 @@ export function findAccessibleWindow<...@@ -489,7 +490,6 @@ export function findAccessibleWindow<
489 const titles = new Set(490 const titles = new Set(
490 [captured.title, captured.sourceTitle ?? ""].map(normalizeTitle).filter(Boolean),491 [captured.title, captured.sourceTitle ?? ""].map(normalizeTitle).filter(Boolean),
491 );492 );
492 if (titles.size === 0) return undefined;
493 // Wayland accessibility providers can expose window size without a screen position.493 // Wayland accessibility providers can expose window size without a screen position.
494 const boundsKeys =494 const boundsKeys =
495 matchMode === "wayland"495 matchMode === "wayland"
...@@ -499,19 +499,31 @@ export function findAccessibleWindow<...@@ -499,19 +499,31 @@ export function findAccessibleWindow<
499 matchMode === "wayland" && captured.clientBounds499 matchMode === "wayland" && captured.clientBounds
500 ? [captured.bounds, captured.clientBounds]500 ? [captured.bounds, captured.clientBounds]
501 : [captured.bounds];501 : [captured.bounds];
502 const matches = windows.filter((window) => {502 const matchesBounds = (window: T) => {
503 const bounds = window.bounds;503 const bounds = window.bounds;
504 return (504 return (
505 titles.has(normalizeTitle(window.name ?? "")) &&
506 bounds !== null &&505 bounds !== null &&
507 candidateBounds.some((candidate) =>506 candidateBounds.some((candidate) =>
508 boundsKeys.every((key) => Math.abs(bounds[key] - candidate[key]) <= 2),507 boundsKeys.every((key) => Math.abs(bounds[key] - candidate[key]) <= 2),
509 )508 )
510 );509 );
511 });510 };
512 if (matches.length === 1) return matches[0];511 if (titles.size > 0) {
513 const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true);512 const matches = windows.filter(
514 return activeMatches.length === 1 ? activeMatches[0] : undefined;513 (window) => titles.has(normalizeTitle(window.name ?? "")) && matchesBounds(window),
514 );
515 if (matches.length === 1) return matches[0];
516 const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true);
517 if (activeMatches.length === 1) return activeMatches[0];
518 if (matches.length > 1) return undefined;
519 }
520 // GTK4/libadwaita often exposes the frame as an unnamed group. A PID-scoped
521 // lookup can accept the one window whose bounds match; size-only guesses cannot.
522 if (!options.allowUntitledUniqueBounds) return undefined;
523 const boundsMatches = windows.filter(
524 (window) => normalizeTitle(window.name ?? "") === "" && matchesBounds(window),
525 );
526 return boundsMatches.length === 1 ? boundsMatches[0] : undefined;
515}527}
516528
517const ELECTRON_KEY_NAMES: Readonly<Record<string, string>> = {529const ELECTRON_KEY_NAMES: Readonly<Record<string, string>> = {