authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-09-12 00:24:39-03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-12 00:24:39-03:00
loge816064945144957b6eb9b268912a98b0555644b
tree606dfbba67d1a5c4d0e583eef3c37c61e8b6f854
parent8eec78cf839a6ce507c085f0b4c42042831407c7
signature Signed by PGP key B5690EEEBB952194

fix(desktop): keep preview keystrokes out of the composer (#11354)

Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

4 files changed, 766 insertions(+), 137 deletions(-)

apps/desktop/src/preview/Manager.test.ts+174-74
......@@ -1,3 +1,4 @@
1import * as NodeVM from "node:vm";
12import { it as effectIt } from "@effect/vitest";
23import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
34import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts";
......@@ -3925,29 +3926,92 @@ describe("PreviewManager", () => {
39253926 withManager((manager) =>
39263927 Effect.gen(function* () {
39273928 let failKeyDown = false;
3929 let routeToIframe = false;
3930 let interruptFrameKeyDown = false;
3931 let holdKeyUp = false;
3932 let releaseKeyUp: (() => void) | undefined;
3933 let notifyKeyUpQueued: (() => void) | undefined;
3934 const keyUpQueued = new Promise<void>((resolve) => {
3935 notifyKeyUpQueued = resolve;
3936 });
3937 const listeners = new Map<string, (event: unknown) => void>();
3938 const eventCounts = new Map<string, number>();
3939 const animationFrames = new Map<number, () => void>();
3940 let animationFrameId = 0;
3941 const renderFrame = () => {
3942 const callbacks = [...animationFrames.values()];
3943 animationFrames.clear();
3944 callbacks.forEach((callback) => callback());
3945 };
3946 const frameContext = NodeVM.createContext({
3947 performance: { eventCounts },
3948 requestAnimationFrame: (callback: () => void) => {
3949 const id = ++animationFrameId;
3950 animationFrames.set(id, callback);
3951 return id;
3952 },
3953 cancelAnimationFrame: (id: number) => animationFrames.delete(id),
3954 window: {
3955 addEventListener: (type: string, listener: (event: unknown) => void) =>
3956 listeners.set(type, listener),
3957 removeEventListener: (type: string) => listeners.delete(type),
3958 },
3959 });
3960 const frame = {
3961 executeJavaScript: vi.fn(async (expression: string) =>
3962 NodeVM.runInContext(expression, frameContext),
3963 ),
3964 };
39283965 let humanInput: ((_event: unknown, signal: unknown) => void) | undefined;
3929 const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
3930 if (
3931 failKeyDown &&
3932 method === "Input.dispatchKeyEvent" &&
3933 (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown")
3934 ) {
3935 throw new Error("key dispatch failed");
3936 }
3937 if (
3938 method === "Input.dispatchKeyEvent" &&
3939 (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown")
3940 ) {
3941 humanInput?.(
3942 {},
3943 {
3944 kind: "key",
3945 key: params["key"],
3946 code: params["code"] ?? "Digit1",
3947 },
3948 );
3966 const sendCommand = vi.fn(
3967 async (method: string, params?: Record<string, unknown>, sessionId?: string) => {
3968 if (method === "Runtime.evaluate") {
3969 if (params?.["returnByValue"] === true) return { result: { value: { ok: true } } };
3970 return {
3971 result:
3972 routeToIframe && !sessionId
3973 ? { objectId: "focused-iframe-object" }
3974 : { subtype: "null" },
3975 };
3976 }
3977 if (method === "DOM.describeNode") return { node: { frameId: "focused-frame" } };
3978 if (method === "Target.getTargets")
3979 return {
3980 targetInfos: [
3981 { targetId: "unrelated-frame", type: "iframe" },
3982 { targetId: "focused-frame", type: "iframe" },
3983 ],
3984 };
3985 if (method === "Target.attachToTarget") return { sessionId: "child-session" };
3986 if (method === "Input.dispatchKeyEvent" && params?.["type"] !== "keyUp") {
3987 if (failKeyDown) throw new Error("key dispatch failed");
3988 if (interruptFrameKeyDown)
3989 humanInput?.({}, { kind: "pointer", x: 80, y: 40, button: 0 });
3990 }
3991 return undefined;
3992 },
3993 );
3994 const sendInputEvent = vi.fn((input: Electron.KeyboardInputEvent) => {
3995 const signal = {
3996 kind: "key",
3997 key: input.keyCode,
3998 code: input.keyCode === "!" ? "Digit1" : `Key${input.keyCode.toUpperCase()}`,
3999 };
4000 if (input.type === "keyUp") {
4001 const deliver = () => {
4002 eventCounts.set("keyup", (eventCounts.get("keyup") ?? 0) + 1);
4003 renderFrame();
4004 };
4005 if (holdKeyUp) {
4006 releaseKeyUp = deliver;
4007 notifyKeyUpQueued?.();
4008 } else {
4009 queueMicrotask(deliver);
4010 }
39494011 }
3950 return method === "Runtime.evaluate" ? { result: { value: { ok: true } } } : undefined;
4012 if (input.type !== "keyDown") return;
4013 if (failKeyDown) throw new Error("key dispatch failed");
4014 humanInput?.({}, signal);
39514015 });
39524016 const restoreFocus = vi.fn();
39534017 const focus = vi.fn();
......@@ -3958,6 +4022,7 @@ describe("PreviewManager", () => {
39584022 } as never);
39594023 fromId.mockReturnValue({
39604024 id: 42,
4025 mainFrame: { framesInSubtree: [frame] },
39614026 isDestroyed: () => false,
39624027 getType: () => "webview",
39634028 getURL: () => "https://example.com",
......@@ -3965,6 +4030,7 @@ describe("PreviewManager", () => {
39654030 isLoading: () => false,
39664031 isDevToolsOpened: () => false,
39674032 focus,
4033 sendInputEvent,
39684034 getZoomFactor: () => 1,
39694035 setZoomFactor: vi.fn(),
39704036 setAudioMuted: vi.fn(),
......@@ -4003,13 +4069,6 @@ describe("PreviewManager", () => {
40034069 ([method, params]) =>
40044070 method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === true,
40054071 );
4006 const keyDownIndex = calls.findIndex(
4007 ([method, params]) =>
4008 method === "Input.dispatchKeyEvent" && params?.["type"] === "keyDown",
4009 );
4010 const keyUpIndex = calls.findIndex(
4011 ([method, params]) => method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp",
4012 );
40134072 const focusOffIndex = calls.findIndex(
40144073 ([method, params]) =>
40154074 method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === false,
......@@ -4037,61 +4096,102 @@ describe("PreviewManager", () => {
40374096 expect(clearOnlyEvaluation).toBeDefined();
40384097 expect(methods).not.toContain("Input.insertText");
40394098 expect(enableIndex).toBeGreaterThanOrEqual(0);
4040 expect(focus).toHaveBeenCalledOnce();
4041 expect(restoreFocus).toHaveBeenCalledOnce();
4042 expect(methods).toContain("Page.bringToFront");
4099 expect(methods).not.toContain("Page.bringToFront");
4100 expect(methods).not.toContain("Input.dispatchKeyEvent");
40434101 expect(enableIndex).toBeLessThan(focusOnIndex);
4044 expect(focusOnIndex).toBeLessThan(keyDownIndex);
4045 expect(keyDownIndex).toBeLessThan(keyUpIndex);
4046 expect(keyUpIndex).toBeLessThan(focusOffIndex);
4047 expect(
4048 calls.filter(
4049 ([method, params]) =>
4050 method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp",
4051 ),
4052 ).toHaveLength(1);
4102 expect(sendCommand.mock.invocationCallOrder[focusOnIndex]).toBeLessThan(
4103 sendInputEvent.mock.invocationCallOrder[0]!,
4104 );
4105 expect(sendInputEvent.mock.invocationCallOrder[2]).toBeLessThan(
4106 sendCommand.mock.invocationCallOrder[focusOffIndex]!,
4107 );
4108 expect(sendInputEvent.mock.calls.map(([input]) => input.type)).toEqual([
4109 "keyDown",
4110 "char",
4111 "keyUp",
4112 ]);
40534113 expect(sendCommand).toHaveBeenCalledWith("Input.setIgnoreInputEvents", { ignore: false });
4114 expect(listeners.size).toBe(0);
4115 expect(animationFrames.size).toBe(0);
40544116
40554117 sendCommand.mockClear();
4056 failKeyDown = true;
4057 const failedPress = yield* Effect.exit(manager.automationPress("tab_input", { key: "y" }));
4058
4059 expect(Exit.isFailure(failedPress)).toBe(true);
4060 expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", {
4061 type: "keyUp",
4062 key: "y",
4063 code: "KeyY",
4064 modifiers: 0,
4065 windowsVirtualKeyCode: 89,
4066 location: 0,
4067 isKeypad: false,
4118 sendInputEvent.mockClear();
4119 getFocusedWebContents.mockReturnValue(null);
4120 holdKeyUp = true;
4121 const backgroundPress = yield* manager
4122 .automationPress("tab_input", { key: "x" })
4123 .pipe(Effect.forkChild({ startImmediately: true }));
4124 yield* Effect.promise(() => keyUpQueued);
4125 expect(sendCommand).not.toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", {
4126 enabled: false,
40684127 });
4128 renderFrame();
4129 expect(animationFrames.size).toBe(1);
4130 releaseKeyUp?.();
4131 yield* Fiber.join(backgroundPress);
4132 expect(listeners.size).toBe(0);
4133 expect(animationFrames.size).toBe(0);
40694134 expect(sendCommand).toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", {
40704135 enabled: false,
40714136 });
4072 expect(restoreFocus).toHaveBeenCalledTimes(2);
4073 expect(
4074 sendCommand.mock.calls.filter(
4075 ([method, params]) =>
4076 method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp",
4077 ),
4078 ).toHaveLength(1);
4137 holdKeyUp = false;
4138
4139 // Both native failures and expected-input matching must leave focus emulation off.
4140 for (const key of ["y", "!"]) {
4141 sendCommand.mockClear();
4142 sendInputEvent.mockClear();
4143 failKeyDown = key === "y";
4144 const exit = yield* Effect.exit(manager.automationPress("tab_input", { key }));
4145 expect(Exit.isFailure(exit)).toBe(failKeyDown);
4146 expect(sendInputEvent.mock.calls.map(([input]) => input.type)).toEqual(
4147 failKeyDown ? ["keyDown", "keyUp"] : ["keyDown", "char", "keyUp"],
4148 );
4149 expect(sendCommand).toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", {
4150 enabled: false,
4151 });
4152 }
40794153
4080 sendCommand.mockClear();
4081 failKeyDown = false;
4082 yield* manager.automationPress("tab_input", { key: "!" });
4083 expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", {
4084 type: "keyDown",
4085 key: "!",
4086 code: "Digit1",
4087 modifiers: 0,
4088 windowsVirtualKeyCode: 49,
4089 location: 0,
4090 isKeypad: false,
4091 text: "!",
4092 unmodifiedText: "!",
4093 });
4094 expect(restoreFocus).toHaveBeenCalledTimes(3);
4154 routeToIframe = true;
4155 sendInputEvent.mockClear();
4156 for (const outcome of ["success", "failure", "interrupted"]) {
4157 sendCommand.mockClear();
4158 failKeyDown = outcome === "failure";
4159 interruptFrameKeyDown = outcome === "interrupted";
4160 const exit = yield* Effect.exit(
4161 manager.automationPress("tab_input", {
4162 key: outcome === "success" ? "Enter" : "x",
4163 }),
4164 );
4165 expect(Exit.isSuccess(exit)).toBe(outcome === "success");
4166 if (outcome === "interrupted" && Exit.isFailure(exit)) {
4167 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
4168 _tag: "PreviewAutomationControlInterruptedError",
4169 });
4170 }
4171 expect(sendInputEvent).not.toHaveBeenCalled();
4172 expect(sendCommand).toHaveBeenCalledWith("Target.attachToTarget", {
4173 targetId: "focused-frame",
4174 flatten: true,
4175 });
4176 expect(
4177 sendCommand.mock.calls
4178 .filter(([method]) => method === "Input.dispatchKeyEvent")
4179 .map(([, params, sessionId]) => ({ type: params?.["type"], sessionId })),
4180 ).toEqual([
4181 { type: "keyDown", sessionId: "child-session" },
4182 { type: "keyUp", sessionId: "child-session" },
4183 ]);
4184 expect(sendCommand).toHaveBeenCalledWith(
4185 "Emulation.setFocusEmulationEnabled",
4186 { enabled: false },
4187 "child-session",
4188 );
4189 expect(sendCommand).toHaveBeenCalledWith("Target.detachFromTarget", {
4190 sessionId: "child-session",
4191 });
4192 }
4193 expect(focus).not.toHaveBeenCalled();
4194 expect(restoreFocus).not.toHaveBeenCalled();
40954195 }),
40964196 ),
40974197 );
apps/desktop/src/preview/Manager.ts+343-61
......@@ -5,6 +5,7 @@
55 * elements live in the renderer; we only attach listeners and forward state
66 * here). Single layer-scoped browser session partition.
77 */
8import * as NodeCrypto from "node:crypto";
89import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
910import type {
1011 DesktopPreviewAnnotationTheme,
......@@ -74,7 +75,11 @@ import {
7475} from "./GuestProtocol.ts";
7576import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts";
7677import { playwrightInjectedRuntimeInstallExpression } from "./PlaywrightInjectedRuntime.ts";
77import { makePreviewAutomationKeySequence } from "./PreviewKeyboard.ts";
78import {
79 makePreviewAutomationKeySequence,
80 makePreviewAutomationNativeKeySequence,
81 previewAutomationEditingCommandExpression,
82} from "./PreviewKeyboard.ts";
7883import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts";
7984
8085export type PreviewNavStatus =
......@@ -1391,6 +1396,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
13911396 type SendCommand = (
13921397 method: string,
13931398 commandParams?: Record<string, unknown>,
1399 sessionId?: string,
13941400 ) => Effect.Effect<unknown, PreviewManagerError>;
13951401
13961402 const prepareAutomationInput = Effect.fn("PreviewManager.prepareAutomationInput")(function* (
......@@ -1410,7 +1416,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
14101416 tabId: string,
14111417 wc: Electron.WebContents,
14121418 action: string,
1413 use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect<A, PreviewManagerError>,
1419 use: (
1420 send: SendCommand,
1421 sendCleanup: SendCommand,
1422 checkControl: Effect.Effect<void, PreviewManagerError>,
1423 ) => Effect.Effect<A, PreviewManagerError>,
14141424 ) {
14151425 const sequence = yield* nextCounter(actionSequenceRef);
14161426 const startedAt = yield* currentIso;
......@@ -1426,28 +1436,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
14261436 const control = yield* ensureControlSession(wc);
14271437 const execute = Effect.fn("PreviewManager.executeControlAction")(function* () {
14281438 yield* update(tabId, { controller: "agent" });
1439 const checkControl = Effect.gen(function* () {
1440 const currentEpoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0;
1441 if (currentEpoch !== epoch) {
1442 return yield* new PreviewAutomationControlInterruptedError({
1443 operation: action,
1444 tabId,
1445 webContentsId: wc.id,
1446 });
1447 }
1448 });
14291449 const send: SendCommand = Effect.fn("PreviewManager.sendCommand")(
1430 function* (method, commandParams) {
1431 const before = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0;
1432 if (before !== epoch) {
1433 return yield* new PreviewAutomationControlInterruptedError({
1434 operation: action,
1435 tabId,
1436 webContentsId: wc.id,
1437 });
1438 }
1450 function* (method, commandParams, sessionId) {
1451 yield* checkControl;
14391452 const result = yield* attemptPromise(
14401453 { operation: `${action}.${method}`, tabId, webContentsId: wc.id },
1441 () => control.debugger.sendCommand(method, commandParams),
1454 () =>
1455 sessionId === undefined
1456 ? control.debugger.sendCommand(method, commandParams)
1457 : control.debugger.sendCommand(method, commandParams, sessionId),
14421458 );
1443 const after = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0;
1444 if (after !== epoch) {
1445 return yield* new PreviewAutomationControlInterruptedError({
1446 operation: action,
1447 tabId,
1448 webContentsId: wc.id,
1449 });
1450 }
1459 yield* checkControl;
14511460 return result;
14521461 },
14531462 );
......@@ -1455,18 +1464,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
14551464 // control epoch. Otherwise a partially dispatched input can leave Chromium
14561465 // with a held key or focus emulation enabled for subsequent actions.
14571466 const sendCleanup: SendCommand = Effect.fn("PreviewManager.sendCleanupCommand")(
1458 function* (method, commandParams) {
1467 function* (method, commandParams, sessionId) {
14591468 return yield* attemptPromise(
14601469 {
14611470 operation: `${action}.cleanup.${method}`,
14621471 tabId,
14631472 webContentsId: wc.id,
14641473 },
1465 () => control.debugger.sendCommand(method, commandParams),
1474 () =>
1475 sessionId === undefined
1476 ? control.debugger.sendCommand(method, commandParams)
1477 : control.debugger.sendCommand(method, commandParams, sessionId),
14661478 );
14671479 },
14681480 );
1469 return yield* use(send, sendCleanup);
1481 return yield* use(send, sendCleanup, checkControl);
14701482 });
14711483 const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* (
14721484 exit: Exit.Exit<A, PreviewManagerError>,
......@@ -3912,56 +3924,326 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
39123924 );
39133925 });
39143926
3927 const withNativeKeyReceipt = Effect.fn("PreviewManager.withNativeKeyReceipt")(function* (
3928 tabId: string,
3929 wc: Electron.WebContents,
3930 dispatch: Effect.Effect<void, PreviewManagerError>,
3931 checkControl: Effect.Effect<void, PreviewManagerError>,
3932 ) {
3933 const context = { operation: "automationPress.awaitNativeKey", tabId, webContentsId: wc.id };
3934 const evaluate = (frame: Electron.WebFrameMain, expression: string) =>
3935 attemptPromise(context, () => frame.executeJavaScript(expression));
3936 const { frames, receiptKey } = yield* Effect.acquireRelease(
3937 attempt(context, () => ({
3938 frames: wc.mainFrame.framesInSubtree,
3939 receiptKey: JSON.stringify(`__t3NativeKey_${NodeCrypto.randomUUID()}`),
3940 })),
3941 ({ frames, receiptKey }) =>
3942 Effect.all(
3943 frames.map((frame) =>
3944 evaluate(frame, `globalThis[${receiptKey}]?.dispose()`).pipe(
3945 Effect.timeoutOption(1_000),
3946 Effect.ignore,
3947 ),
3948 ),
3949 { concurrency: "unbounded", discard: true },
3950 ),
3951 );
3952 yield* Effect.gen(function* () {
3953 for (const frame of frames) {
3954 yield* checkControl;
3955 yield* evaluate(
3956 frame,
3957 `(() => {
3958 const receiptKey = ${receiptKey};
3959 const counts = performance.eventCounts;
3960 if (!counts) throw new Error("Native key delivery counters are unavailable.");
3961 const keyUpsBefore = counts.get("keyup") ?? 0;
3962 const keyDownsBefore = counts.get("keydown") ?? 0;
3963 let settle;
3964 let animationFrame = 0;
3965 const promise = new Promise(resolve => { settle = resolve; });
3966 const finish = delivered => {
3967 cancelAnimationFrame(animationFrame);
3968 window.removeEventListener("pagehide", onPageHide, true);
3969 settle(delivered);
3970 };
3971 // Chromium counts trusted keys before dispatching page listeners,
3972 // so stopImmediatePropagation cannot hide completed input.
3973 const observe = () => {
3974 if ((counts.get("keyup") ?? 0) > keyUpsBefore) finish(true);
3975 else animationFrame = requestAnimationFrame(observe);
3976 };
3977 const onPageHide = () => finish(
3978 (counts.get("keyup") ?? 0) > keyUpsBefore ||
3979 (counts.get("keydown") ?? 0) > keyDownsBefore,
3980 );
3981 globalThis[receiptKey] = { promise, dispose: () => {
3982 finish(false);
3983 delete globalThis[receiptKey];
3984 }};
3985 window.addEventListener("pagehide", onPageHide, true);
3986 animationFrame = requestAnimationFrame(observe);
3987 })()`,
3988 );
3989 }
3990 yield* checkControl;
3991 yield* dispatch;
3992 yield* attemptPromise(context, () =>
3993 Promise.any(
3994 frames.map(async (frame) => {
3995 const delivered: unknown = await frame.executeJavaScript(
3996 `globalThis[${receiptKey}]?.promise`,
3997 );
3998 if (delivered !== true)
3999 throw new Error(
4000 "The preview document changed before native key delivery was confirmed.",
4001 );
4002 }),
4003 ),
4004 );
4005 yield* checkControl;
4006 }).pipe(
4007 Effect.timeout(5_000),
4008 Effect.catchTags({
4009 TimeoutError: () =>
4010 Effect.fail(new PreviewAutomationTimeoutError({ tabId, timeoutMs: 5_000 })),
4011 }),
4012 );
4013 }, Effect.scoped);
4014
4015 const resolveKeyboardTarget = Effect.fn("PreviewManager.resolveKeyboardTarget")(function* (
4016 tabId: string,
4017 send: SendCommand,
4018 sendCleanup: SendCommand,
4019 checkControl: Effect.Effect<void, PreviewManagerError>,
4020 ) {
4021 const context = { operation: "automationPress.resolveFocusedFrame", tabId };
4022 let sessionId: string | undefined;
4023 let contextId: number | undefined;
4024 while (true) {
4025 const evaluated = (yield* send(
4026 "Runtime.evaluate",
4027 {
4028 expression: `(() => {
4029 let element = document.activeElement;
4030 while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
4031 return element?.tagName === "IFRAME" ? element : null;
4032 })()`,
4033 ...(contextId === undefined ? {} : { contextId }),
4034 },
4035 sessionId,
4036 )) as { result?: { objectId?: string; subtype?: string } };
4037 if (evaluated.result?.subtype === "null") break;
4038 const objectId = evaluated.result?.objectId;
4039 if (!objectId)
4040 return yield* new PreviewOperationError({
4041 ...context,
4042 cause: new Error("The focused preview frame could not be resolved."),
4043 });
4044 const described = (yield* send("DOM.describeNode", { objectId }, sessionId).pipe(
4045 Effect.ensuring(
4046 sendCleanup("Runtime.releaseObject", { objectId }, sessionId).pipe(Effect.ignore),
4047 ),
4048 )) as { node?: { frameId?: string } };
4049 const frameId = described.node?.frameId;
4050 if (!frameId)
4051 return yield* new PreviewOperationError({
4052 ...context,
4053 cause: new Error("The focused preview iframe is unavailable."),
4054 });
4055 const targets = (yield* send("Target.getTargets")) as {
4056 targetInfos?: ReadonlyArray<{ targetId: string; type: string }>;
4057 };
4058 if (
4059 targets.targetInfos?.some(
4060 (target) => target.type === "iframe" && target.targetId === frameId,
4061 )
4062 ) {
4063 yield* checkControl;
4064 // Register cleanup before checking the epoch again: a successful
4065 // attach must be released even when human input interrupts its reply.
4066 sessionId = yield* Effect.acquireRelease(
4067 sendCleanup("Target.attachToTarget", { targetId: frameId, flatten: true }).pipe(
4068 Effect.flatMap((response) =>
4069 attempt(context, () => {
4070 const attached = response as { sessionId?: string };
4071 if (!attached.sessionId)
4072 throw new Error("The focused preview iframe could not be attached.");
4073 return attached.sessionId;
4074 }),
4075 ),
4076 ),
4077 (attachedSessionId) =>
4078 sendCleanup("Target.detachFromTarget", { sessionId: attachedSessionId }).pipe(
4079 Effect.ignore,
4080 ),
4081 );
4082 yield* checkControl;
4083 contextId = undefined;
4084 } else {
4085 const world = (yield* send(
4086 "Page.createIsolatedWorld",
4087 {
4088 frameId,
4089 worldName: "t3-preview-key-target",
4090 },
4091 sessionId,
4092 )) as { executionContextId?: number };
4093 if (typeof world.executionContextId !== "number")
4094 return yield* new PreviewOperationError({
4095 ...context,
4096 cause: new Error("The focused preview iframe context is unavailable."),
4097 });
4098 contextId = world.executionContextId;
4099 }
4100 }
4101 return { sessionId, contextId };
4102 });
4103
39154104 const performAutomationPress = Effect.fn("PreviewManager.performAutomationPress")(function* (
39164105 tabId: string,
39174106 wc: Electron.WebContents,
39184107 input: PreviewAutomationPressInput,
39194108 send: SendCommand,
39204109 sendCleanup: SendCommand,
4110 checkControl: Effect.Effect<void, PreviewManagerError>,
39214111 ) {
39224112 yield* prepareAutomationInput(send, false);
3923 const keySequence = makePreviewAutomationKeySequence(input, {
4113 const keySequence = makePreviewAutomationNativeKeySequence(input, {
39244114 isMac: hostPlatform === "darwin",
39254115 });
3926 const previouslyFocused = yield* attempt(
3927 { operation: "automationPress.getFocusedWebContents", tabId, webContentsId: wc.id },
3928 () => webContents.getFocusedWebContents(),
3929 );
3930 let keyDownAttempted = false;
3931 const releaseInput = Effect.gen(function* () {
3932 if (keyDownAttempted) {
3933 yield* sendCleanup("Input.dispatchKeyEvent", keySequence.keyUp).pipe(Effect.ignore);
3934 }
3935 yield* sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe(
3936 Effect.ignore,
4116 // CDP keyboard dispatch follows the embedder's focused renderer, and
4117 // WebContents.focus() is a no-op for webview guests. Native input targets
4118 // this guest's widget directly, so Enter cannot submit the host composer.
4119 yield* Effect.gen(function* () {
4120 const { sessionId, contextId } = yield* resolveKeyboardTarget(
4121 tabId,
4122 send,
4123 sendCleanup,
4124 checkControl,
39374125 );
3938 if (previouslyFocused && previouslyFocused.id !== wc.id && !previouslyFocused.isDestroyed()) {
3939 yield* attempt(
3940 {
3941 operation: "automationPress.restoreFocusedWebContents",
4126 // Only descendant renderer sessions bypass Chromium's desktop focus lookup.
4127 if (sessionId) {
4128 const keys = makePreviewAutomationKeySequence(input, { isMac: hostPlatform === "darwin" });
4129 yield* Effect.acquireRelease(Effect.void, () =>
4130 sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }, sessionId).pipe(
4131 Effect.ignore,
4132 ),
4133 );
4134 yield* send("Emulation.setFocusEmulationEnabled", { enabled: true }, sessionId);
4135 yield* Effect.acquireRelease(Effect.void, () =>
4136 sendCleanup("Input.dispatchKeyEvent", keys.keyUp, sessionId).pipe(Effect.ignore),
4137 );
4138 yield* send("Input.dispatchKeyEvent", keys.keyDown, sessionId);
4139 return;
4140 }
4141 if (keySequence.commands?.length) {
4142 const context = {
4143 operation: "automationPress.editFocusedFrame",
4144 tabId,
4145 webContentsId: wc.id,
4146 };
4147 const evaluate = (expression: string, cleanup = false) =>
4148 evaluateWithDebugger(
39424149 tabId,
3943 webContentsId: previouslyFocused.id,
3944 },
3945 () => previouslyFocused.focus(),
3946 ).pipe(Effect.ignore);
4150 (method, params) =>
4151 (cleanup ? sendCleanup : send)(method, {
4152 ...params,
4153 ...(contextId === undefined ? {} : { contextId }),
4154 }),
4155 expression,
4156 true,
4157 );
4158 const clipboardData = keySequence.commands.includes("paste")
4159 ? yield* attemptPromise(context, async () => {
4160 const formats: Array<{ type: string; data: string }> = [];
4161 for (const item of await clipboard.read()) {
4162 for (const type of item.types) {
4163 if (type.startsWith("electron ")) continue;
4164 const blob = await item.getType(type);
4165 if (!("arrayBuffer" in blob)) continue;
4166 formats.push({
4167 type,
4168 data: type.startsWith("text/")
4169 ? await blob.text()
4170 : Buffer.from(await blob.arrayBuffer()).toString("base64"),
4171 });
4172 }
4173 }
4174 return formats;
4175 })
4176 : [];
4177 yield* checkControl;
4178 const expression = previewAutomationEditingCommandExpression(
4179 input,
4180 keySequence,
4181 clipboardData,
4182 );
4183 const selectionKey = yield* encodeJson(
4184 context,
4185 `__t3EditingSelection_${NodeCrypto.randomUUID()}`,
4186 );
4187 // Editing requires an active document. Preserve the target
4188 // and selection across focus handlers without focusing the desktop.
4189 yield* Effect.acquireUseRelease(
4190 evaluate(`(() => {
4191 let element = document.activeElement;
4192 while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
4193 const selection = document.getSelection();
4194 const range = selection?.rangeCount ? selection.getRangeAt(0).cloneRange() : null;
4195 const backward = selection?.direction === "backward";
4196 const start = element?.selectionStart;
4197 const end = element?.selectionEnd;
4198 const direction = element?.selectionDirection;
4199 globalThis[${selectionKey}] = () => {
4200 element?.focus({ preventScroll: true });
4201 if (typeof start === "number") element.setSelectionRange(start, end, direction);
4202 else {
4203 selection?.removeAllRanges();
4204 if (range && backward) selection.setBaseAndExtent(
4205 range.endContainer, range.endOffset, range.startContainer, range.startOffset,
4206 );
4207 else if (range) selection.addRange(range);
4208 }
4209 };
4210 })()`),
4211 () =>
4212 Effect.gen(function* () {
4213 yield* send("Emulation.setFocusEmulationEnabled", { enabled: true });
4214 yield* evaluate(`globalThis[${selectionKey}]();${expression}`);
4215 }),
4216 () => evaluate(`delete globalThis[${selectionKey}]`, true).pipe(Effect.ignore),
4217 );
4218 yield* checkControl;
4219 return;
39474220 }
3948 });
3949
3950 // Focus the guest WebContents itself, not its containing BrowserWindow. This
3951 // activates native keyboard behavior for hidden/background previews without
3952 // changing which thread is mounted in the UI. Restore the previous renderer
3953 // after dispatch so automation never leaves the app's input focus behind.
3954 yield* Effect.gen(function* () {
3955 yield* attempt(
3956 { operation: "automationPress.focusWebContents", tabId, webContentsId: wc.id },
3957 () => wc.focus(),
3958 );
3959 yield* send("Page.bringToFront");
39604221 yield* send("Emulation.setFocusEmulationEnabled", { enabled: true });
3961 yield* expectAgentInput(tabId, keySequence.signal);
3962 keyDownAttempted = true;
3963 yield* send("Input.dispatchKeyEvent", keySequence.keyDown);
3964 }).pipe(Effect.ensuring(releaseInput));
4222 yield* withNativeKeyReceipt(
4223 tabId,
4224 wc,
4225 Effect.gen(function* () {
4226 yield* expectAgentInput(tabId, keySequence.signal);
4227 yield* attempt(
4228 { operation: "automationPress.sendInputEvent", tabId, webContentsId: wc.id },
4229 () => {
4230 try {
4231 wc.sendInputEvent(keySequence.keyDown);
4232 if (keySequence.char) wc.sendInputEvent(keySequence.char);
4233 } finally {
4234 wc.sendInputEvent(keySequence.keyUp);
4235 }
4236 },
4237 );
4238 }),
4239 checkControl,
4240 );
4241 }).pipe(
4242 Effect.scoped,
4243 Effect.ensuring(
4244 sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe(Effect.ignore),
4245 ),
4246 );
39654247 });
39664248
39674249 const automationPress = Effect.fn("PreviewManager.automationPress")(function* (
......@@ -3969,8 +4251,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
39694251 input: PreviewAutomationPressInput,
39704252 ) {
39714253 const wc = yield* requireWebContents(tabId);
3972 yield* withControlSession(tabId, wc, "press", (send, sendCleanup) =>
3973 performAutomationPress(tabId, wc, input, send, sendCleanup),
4254 yield* withControlSession(tabId, wc, "press", (send, sendCleanup, checkControl) =>
4255 performAutomationPress(tabId, wc, input, send, sendCleanup, checkControl),
39744256 );
39754257 });
39764258
apps/desktop/src/preview/PreviewKeyboard.test.ts+88-1
......@@ -1,6 +1,9 @@
11import { describe, expect, it } from "vite-plus/test";
22
3import { makePreviewAutomationKeySequence } from "./PreviewKeyboard.ts";
3import {
4 makePreviewAutomationKeySequence,
5 makePreviewAutomationNativeKeySequence,
6} from "./PreviewKeyboard.ts";
47
58describe("preview keyboard packets", () => {
69 it("includes the Chromium virtual key code and Enter text", () => {
......@@ -94,4 +97,88 @@ describe("preview keyboard packets", () => {
9497 });
9598 expect(sequence.signal).toEqual({ kind: "key", key: "!", code: "Digit1" });
9699 });
100
101 it.each([
102 ["Enter", "\r"],
103 ["z", "z"],
104 ])("converts %s into native down, char, and up packets", (key, text) => {
105 const sequence = makePreviewAutomationNativeKeySequence({ key });
106 const shared = { keyCode: key, modifiers: [], skipIfUnhandled: true };
107 expect(sequence.keyDown).toEqual({ type: "keyDown", ...shared });
108 expect(sequence.char).toEqual({ type: "char", ...shared, keyCode: text });
109 expect(sequence.keyUp).toEqual({ type: "keyUp", ...shared });
110 });
111
112 it("suppresses text for shortcuts and retains macOS editing commands", () => {
113 const sequence = makePreviewAutomationNativeKeySequence(
114 { key: "a", modifiers: ["Meta"] },
115 { isMac: true },
116 );
117 expect(sequence.keyDown).toEqual({
118 type: "keyDown",
119 keyCode: "a",
120 modifiers: ["meta"],
121 skipIfUnhandled: true,
122 });
123 expect(sequence.char).toBeUndefined();
124 expect(sequence.commands).toEqual(["selectAll"]);
125 });
126
127 it.each([
128 ["ArrowLeft", "Left"],
129 ["ArrowRight", "Right"],
130 ["ArrowUp", "Up"],
131 ["ArrowDown", "Down"],
132 ])("maps %s to Electron's %s accelerator", (key, keyCode) => {
133 const sequence = makePreviewAutomationNativeKeySequence({ key });
134 expect(sequence.keyDown.keyCode).toBe(keyCode);
135 expect(sequence.keyUp.keyCode).toBe(keyCode);
136 expect(sequence.signal.key).toBe(key);
137 expect(sequence.char).toBeUndefined();
138 });
139
140 it("matches native uppercase key signals without inventing shortcut modifiers", () => {
141 const plain = makePreviewAutomationNativeKeySequence({ key: "X" });
142 expect(plain.signal).toEqual({ kind: "key", key: "x", code: "KeyX" });
143 expect(plain.char?.keyCode).toBe("X");
144 const shortcut = makePreviewAutomationNativeKeySequence({ key: "A", modifiers: ["Control"] });
145 expect(shortcut.signal).toEqual({ kind: "key", key: "a", code: "KeyA" });
146 expect(shortcut.keyDown.modifiers).toEqual(["control"]);
147 expect(shortcut.char).toBeUndefined();
148 expect(
149 makePreviewAutomationNativeKeySequence({ key: "X", modifiers: ["Shift"] }).signal,
150 ).toEqual({
151 kind: "key",
152 key: "X",
153 code: "KeyX",
154 });
155 });
156
157 it("matches native signals for Unicode text and literal spaces", () => {
158 const unicode = makePreviewAutomationNativeKeySequence({ key: "é" });
159 expect(unicode.signal).toEqual({ kind: "key", key: "", code: "" });
160 expect(unicode.char?.keyCode).toBe("é");
161 expect(makePreviewAutomationNativeKeySequence({ key: " " }).signal).toEqual({
162 kind: "key",
163 key: " ",
164 code: "Space",
165 });
166 });
167
168 it("preserves text and editing commands for isolated child renderer targets", () => {
169 const text = makePreviewAutomationKeySequence({ key: "é" });
170 expect(text.keyDown).toMatchObject({ type: "keyDown", text: "é", key: "é" });
171 expect(text.keyUp).toMatchObject({ type: "keyUp", key: "é" });
172 const shortcut = makePreviewAutomationKeySequence(
173 { key: "a", modifiers: ["Meta"] },
174 { isMac: true },
175 );
176 expect(shortcut.keyDown).toMatchObject({
177 type: "rawKeyDown",
178 modifiers: 4,
179 commands: ["selectAll"],
180 });
181 expect(shortcut.keyDown).not.toHaveProperty("text");
182 expect(shortcut.keyDown).not.toHaveProperty("nativeVirtualKeyCode");
183 });
97184});
apps/desktop/src/preview/PreviewKeyboard.ts+161-1
......@@ -131,7 +131,7 @@ const modifierMask = (modifiers: PreviewAutomationPressInput["modifiers"]): numb
131131 }, 0);
132132
133133function resolveKeyDefinition(input: PreviewAutomationPressInput): KeyDefinition {
134 const named = NAMED_KEYS[input.key];
134 const named = NAMED_KEYS[input.key === " " ? "Space" : input.key];
135135 if (named) return named;
136136
137137 const functionKey = /^F([1-9]|1[0-2])$/.exec(input.key);
......@@ -201,3 +201,163 @@ export function makePreviewAutomationKeySequence(
201201 signal: { kind: "key", key: definition.key, code: definition.code },
202202 };
203203}
204
205/** Root CDP input can retarget the embedder; native packets address the guest widget. */
206export function makePreviewAutomationNativeKeySequence(
207 input: PreviewAutomationPressInput,
208 options?: { readonly isMac?: boolean },
209) {
210 const { keyDown, signal } = makePreviewAutomationKeySequence(input, options);
211 const modifiers = (
212 [
213 [1, "alt"],
214 [2, "control"],
215 [4, "meta"],
216 [8, "shift"],
217 ] as const
218 )
219 .filter(([mask]) => keyDown.modifiers & mask)
220 .map(([, modifier]) => modifier);
221 const shared = {
222 keyCode: keyDown.key.startsWith("Arrow") ? keyDown.key.slice(5) : keyDown.key,
223 modifiers,
224 skipIfUnhandled: true as const,
225 };
226 // Electron lowercases unshifted letters and reports no key for Unicode accelerators.
227 const key =
228 keyDown.windowsVirtualKeyCode === 0 && keyDown.key.length === 1
229 ? ""
230 : /^[A-Z]$/.test(keyDown.key) && !modifiers.includes("shift")
231 ? keyDown.key.toLowerCase()
232 : keyDown.key;
233 return {
234 keyDown: { type: "keyDown" as const, ...shared },
235 ...(keyDown.text ? { char: { type: "char" as const, ...shared, keyCode: keyDown.text } } : {}),
236 keyUp: { type: "keyUp" as const, ...shared },
237 ...(keyDown.commands ? { commands: keyDown.commands } : {}),
238 signal: { ...signal, key },
239 };
240}
241
242/** Keep macOS editing shortcuts inside the target page without native focus. */
243export function previewAutomationEditingCommandExpression(
244 input: PreviewAutomationPressInput,
245 sequence: ReturnType<typeof makePreviewAutomationNativeKeySequence>,
246 clipboardData: ReadonlyArray<{ readonly type: string; readonly data: string }> = [],
247): string {
248 const definition = resolveKeyDefinition(input);
249 const event = {
250 key: definition.key,
251 code: definition.code,
252 keyCode: definition.keyCode,
253 which: definition.keyCode,
254 location: definition.location ?? 0,
255 altKey: input.modifiers?.includes("Alt") ?? false,
256 ctrlKey: input.modifiers?.includes("Control") ?? false,
257 metaKey: input.modifiers?.includes("Meta") ?? false,
258 shiftKey: input.modifiers?.includes("Shift") ?? false,
259 bubbles: true,
260 cancelable: true,
261 composed: true,
262 };
263 return `(() => {
264 let element = document.activeElement;
265 while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
266 if (!element) return;
267 const event = ${JSON.stringify(event)};
268 try {
269 if (!element.dispatchEvent(new KeyboardEvent("keydown", event))) return;
270 for (const command of ${JSON.stringify(sequence.commands ?? [])}) {
271 // Main-process clipboard reads also work on insecure HTTP previews.
272 // Let the page's paste handler consume the clipboard MIME formats.
273 if (command === "paste") {
274 const transfer = new DataTransfer();
275 for (const { type, data } of ${JSON.stringify(clipboardData)}) {
276 if (type === "text/html") {
277 // Match native paste sanitization before page handlers or insertion.
278 const container = document.createElement("div");
279 container.setHTML(data);
280 transfer.setData(type, container.innerHTML);
281 } else if (type.startsWith("text/")) transfer.setData(type, data);
282 else {
283 const bytes = Uint8Array.from(atob(data), character => character.charCodeAt(0));
284 transfer.items.add(new File([bytes], "clipboard", { type }));
285 }
286 }
287 if (!element.dispatchEvent(new ClipboardEvent("paste", {
288 clipboardData: transfer, bubbles: true, cancelable: true, composed: true,
289 }))) continue;
290 const text = transfer.getData("text/plain");
291 if (!element.dispatchEvent(new InputEvent("beforeinput", {
292 inputType: "insertFromPaste", data: text, dataTransfer: transfer,
293 bubbles: true, cancelable: true, composed: true,
294 }))) continue;
295 const html = element.isContentEditable ? transfer.getData("text/html") : "";
296 document.execCommand(html ? "insertHTML" : "insertText", false, html || text);
297 continue;
298 }
299 const inputType = command === "deleteToBeginningOfLine" ? "deleteSoftLineBackward"
300 : command === "undo" ? "historyUndo"
301 : command === "redo" ? "historyRedo" : null;
302 // execCommand emits input without beforeinput. Let controlled editors
303 // perform the edit before applying the browser's default operation.
304 if (inputType && !element.dispatchEvent(new InputEvent("beforeinput", {
305 inputType, bubbles: true, cancelable: true, composed: true,
306 }))) continue;
307 const selection = document.getSelection();
308 if (command === "deleteToBeginningOfLine") {
309 const collapsed = typeof element.selectionStart === "number"
310 ? element.selectionStart === element.selectionEnd
311 : selection?.isCollapsed;
312 if (collapsed) selection?.modify("extend", "backward", "lineboundary");
313 document.execCommand("delete");
314 } else if (command.startsWith("moveTo")) {
315 const selectionElement = selection?.anchorNode?.nodeType === Node.ELEMENT_NODE
316 ? selection.anchorNode : selection?.anchorNode?.parentElement;
317 const editable = element.isContentEditable || selectionElement?.isContentEditable ||
318 (((element instanceof HTMLInputElement && element.selectionStart !== null) ||
319 element instanceof HTMLTextAreaElement) &&
320 !element.readOnly && !element.disabled);
321 if (!editable && (command === "moveToBeginningOfDocument" || command === "moveToEndOfDocument")) {
322 let scrollable = element === document.body ? selectionElement ?? element : element;
323 while (scrollable && !(scrollable.scrollHeight > scrollable.clientHeight &&
324 /^(auto|scroll|overlay)$/.test(getComputedStyle(scrollable).overflowY))) {
325 scrollable = scrollable.parentElement ?? scrollable.getRootNode().host;
326 }
327 scrollable ??= document.scrollingElement;
328 if (scrollable) scrollable.scrollTop = command === "moveToBeginningOfDocument"
329 ? 0 : scrollable.scrollHeight;
330 continue;
331 }
332 const direction = command.includes("Beginning") ? "backward"
333 : command.includes("Left") ? "left"
334 : command.includes("Right") ? "right" : "forward";
335 selection?.modify(
336 command.endsWith("AndModifySelection") ? "extend" : "move",
337 direction,
338 command.includes("Document") ? "documentboundary" : "lineboundary",
339 );
340 if (element instanceof HTMLInputElement && element.selectionStart !== null) {
341 if (command.includes("Left") || command.includes("Beginning")) element.scrollLeft = 0;
342 else element.scrollLeft = element.scrollWidth;
343 }
344 // Programmatic selection changes do not reveal the caret like native editing commands.
345 if (editable && command.includes("Document")) {
346 const beginning = command.includes("Beginning");
347 if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
348 element.scrollTop = beginning ? 0 : element.scrollHeight;
349 } else {
350 const caretElement = selection?.focusNode?.nodeType === Node.ELEMENT_NODE
351 ? selection.focusNode : selection?.focusNode?.parentElement;
352 caretElement?.scrollIntoView({ block: beginning ? "start" : "end", inline: "nearest" });
353 }
354 }
355 } else {
356 document.execCommand(command);
357 }
358 }
359 } finally {
360 element.dispatchEvent(new KeyboardEvent("keyup", event));
361 }
362 })()`;
363}