From ef7014d851f56bb037a9da963095ffd883c7fa08 Mon Sep 17 00:00:00 2001 From: Wout Stiens <71498452+StiensWout@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:52:36 +0200 Subject: [PATCH] fix(preview): restore recording and macOS rendering after Electron 43 (#9001) Co-authored-by: Guillermo Casanova --- apps/desktop/src/ipc/methods/preview.ts | 15 +- apps/desktop/src/preview/Manager.test.ts | 531 +++++++++++++----- apps/desktop/src/preview/Manager.ts | 277 +++++++-- apps/web/src/browser/HostedBrowserWebview.tsx | 6 +- apps/web/src/browser/browserRecording.test.ts | 255 +++++++-- apps/web/src/browser/browserRecording.ts | 266 +++++++-- .../browser/hostedBrowserWebviewStyle.test.ts | 22 +- .../src/browser/hostedBrowserWebviewStyle.ts | 27 +- .../components/preview/PreviewView.test.tsx | 1 + .../src/components/preview/PreviewView.tsx | 5 +- packages/contracts/src/ipc.ts | 18 +- 11 files changed, 1078 insertions(+), 345 deletions(-) diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 718ebd2a3cf3ee1656bc7c1ab122607166f2706d..5229d36c31f14b8c75288bdf34a97d5a38df2722 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -11,7 +11,6 @@ import { DesktopPreviewConfigInputSchema, DesktopPreviewNavigateInputSchema, DesktopPreviewRecordingArtifactSchema, - DesktopPreviewRecordingSourceSchema, DesktopPreviewRecordingSaveInputSchema, DesktopPreviewRegisterWebviewInputSchema, DesktopPreviewScreenshotArtifactSchema, @@ -174,15 +173,11 @@ export const cancelPickElement = tabMethod( "desktop.ipc.preview.cancelPickElement", (manager, tabId) => manager.cancelPickElement(tabId), ); -export const startRecording = DesktopIpc.makeIpcMethod({ - channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL, - payload: DesktopPreviewTabInputSchema, - result: DesktopPreviewRecordingSourceSchema, - handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) { - const manager = yield* PreviewManager.PreviewManager; - return yield* manager.startRecording(tabId); - }), -}); +export const startRecording = tabMethod( + IpcChannels.PREVIEW_RECORDING_START_CHANNEL, + "desktop.ipc.preview.startRecording", + (manager, tabId) => manager.startRecording(tabId), +); export const stopRecording = tabMethod( IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, "desktop.ipc.preview.stopRecording", diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 0cd85c710eaa1fe67235eb38dcc5781aef36bca8..75271d76386a1cb5a44b9ec99a0bf491eee1dd97 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1,4 +1,5 @@ import { it as effectIt } from "@effect/vitest"; +import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; @@ -123,7 +124,7 @@ const { } = vi.hoisted(() => ({ browserWindowConstructor: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), - fromId: vi.fn((_id?: number) => null), + fromId: vi.fn<(_id?: number) => Electron.WebContents | null>((_id?: number) => null), getFocusedWebContents: vi.fn(() => null), mkdir: vi.fn((_path: string) => undefined), showItemInFolder: vi.fn(), @@ -209,14 +210,52 @@ interface TestCapturedPreviewImage { readonly getSize: () => { readonly width: number; readonly height: number }; } +type TestDisplayMediaHandler = ( + request: { readonly frame: { readonly frameTreeNodeId: number } | null }, + callback: (streams: { video?: unknown }) => void, +) => void; + +interface TestHostWebContents { + readonly id: number; + readonly mainFrame: { readonly frameTreeNodeId: number }; + readonly executeJavaScript: ReturnType; + readonly isDestroyed: () => boolean; + readonly session: { + readonly setDisplayMediaRequestHandler: ReturnType; + }; + readonly displayMediaHandler: () => TestDisplayMediaHandler | undefined; +} + +type TestPreviewWebContents = Electron.WebContents & { + readonly setBackgroundThrottling: ReturnType void>>; +}; + +const makeTestHostWebContents = (): TestHostWebContents => { + let handler: TestDisplayMediaHandler | undefined; + return { + id: 7, + mainFrame: { frameTreeNodeId: 7 }, + executeJavaScript: vi.fn(async () => true), + isDestroyed: () => false, + session: { + setDisplayMediaRequestHandler: vi.fn((next: TestDisplayMediaHandler) => { + handler = next; + }), + }, + displayMediaHandler: () => handler, + }; +}; + const makeTestPreviewWebContents = ( capturePage: () => Promise, id = 42, -) => - ({ + hostWebContents: TestHostWebContents = makeTestHostWebContents(), +) => { + const setBackgroundThrottling = vi.fn<(enabled: boolean) => void>(); + return { id, - hostWebContents: { id: 7 }, - getMediaSourceId: vi.fn(() => `tab:${id}`), + mainFrame: { routingId: id }, + hostWebContents, executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })), isDestroyed: () => false, getType: () => "webview", @@ -226,6 +265,7 @@ const makeTestPreviewWebContents = ( getZoomFactor: () => 1, setZoomFactor: vi.fn(), setAudioMuted: vi.fn(), + setBackgroundThrottling, isCurrentlyAudible: () => false, on: vi.fn(), off: vi.fn(), @@ -241,7 +281,45 @@ const makeTestPreviewWebContents = ( off: vi.fn(), }, capturePage, - }) as never; + } as unknown as TestPreviewWebContents; +}; + +/** Two ready tabs (41, 42) sharing one window, so they contend for the single display-media slot. */ +const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("unused-recording-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const host = makeTestHostWebContents(); + const destroyedIds = new Set(); + const makeWebContents = (id: number) => + Object.assign(makeTestPreviewWebContents(capturePage, id, host), { + executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })), + isDestroyed: () => destroyedIds.has(id), + }); + const webContentsById = new Map([ + [41, makeWebContents(41)], + [42, makeWebContents(42)], + ]); + fromId.mockImplementation((id) => + id === undefined ? null : (webContentsById.get(id) ?? null), + ); + yield* manager.createTab("tab_race_a"); + yield* manager.createTab("tab_race_b"); + yield* manager.registerWebview("tab_race_a", 41); + yield* manager.registerWebview("tab_race_b", 42); + const grants: Array<{ video?: unknown }> = []; + return { + host, + grants, + destroy: (id: number) => destroyedIds.add(id), + takeGrant: (frame = host.mainFrame) => + host.displayMediaHandler()?.({ frame }, (value) => { + grants.push(value); + }), + }; + }); const TEST_FAVICON = "data:image/png;base64,cG5n"; @@ -1865,7 +1943,7 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("keeps window unthrottled until the final frame capture stops", () => + effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () => withManager((manager) => Effect.gen(function* () { const setBackgroundThrottling = vi.fn(); @@ -1873,9 +1951,12 @@ describe("PreviewManager", () => { toJPEG: () => Buffer.from("recording-frame"), getSize: () => ({ width: 1280, height: 720 }), })); + const host = makeTestHostWebContents(); + const firstWebContents = makeTestPreviewWebContents(capturePage, 41, host); + const secondWebContents = makeTestPreviewWebContents(capturePage, 42, host); const webContentsById = new Map([ - [41, makeTestPreviewWebContents(capturePage, 41)], - [42, makeTestPreviewWebContents(capturePage, 42)], + [41, firstWebContents], + [42, secondWebContents], ]); fromId.mockImplementation((id) => id === undefined ? null : (webContentsById.get(id) ?? null), @@ -1892,14 +1973,21 @@ describe("PreviewManager", () => { } as never); yield* manager.startRecording("tab_capture_throttling_1"); + // The first renderer takes its grant, freeing the arm slot for the second tab. + host.displayMediaHandler()?.({ frame: host.mainFrame }, () => {}); yield* manager.startRecording("tab_capture_throttling_2"); expect(setBackgroundThrottling.mock.calls).toEqual([[false]]); + expect(firstWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]); + expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]); yield* manager.stopRecording("tab_capture_throttling_1"); expect(setBackgroundThrottling.mock.calls).toEqual([[false]]); + expect(firstWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]); yield* manager.stopRecording("tab_capture_throttling_2"); expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); }), ), ); @@ -1953,6 +2041,48 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("rolls back window throttling when a recorded guest cannot be unthrottled", () => + withManager((manager) => + Effect.gen(function* () { + const setWindowBackgroundThrottling = vi.fn(); + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("recording-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const wc = makeTestPreviewWebContents(capturePage); + fromId.mockReturnValue(wc); + + yield* manager.createTab("tab_guest_throttling_failure"); + yield* manager.registerWebview("tab_guest_throttling_failure", 42); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { setBackgroundThrottling: setWindowBackgroundThrottling }, + } as never); + + wc.setBackgroundThrottling.mockImplementationOnce(() => { + throw new Error("guest throttling update failed"); + }); + const failedStart = yield* Effect.exit( + manager.startRecording("tab_guest_throttling_failure"), + ); + expect(Exit.isFailure(failedStart)).toBe(true); + expect(setWindowBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + + yield* manager.startRecording("tab_guest_throttling_failure"); + yield* manager.stopRecording("tab_guest_throttling_failure"); + expect(setWindowBackgroundThrottling.mock.calls).toEqual([ + [false], + [true], + [false], + [true], + ]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [false], [true]]); + }), + ), + ); + effectIt.effect("does not publish a replacement window when capture reconciliation fails", () => withManager((manager) => Effect.gen(function* () { @@ -2031,9 +2161,10 @@ describe("PreviewManager", () => { toJPEG: () => Buffer.from("recording-frame"), getSize: () => ({ width: 1280, height: 720 }), })); + const host = makeTestHostWebContents(); const webContentsById = new Map([ - [42, makeTestPreviewWebContents(capturePage, 42)], - [43, makeTestPreviewWebContents(capturePage, 43)], + [42, makeTestPreviewWebContents(capturePage, 42, host)], + [43, makeTestPreviewWebContents(capturePage, 43, host)], ]); fromId.mockImplementation((id) => id === undefined ? null : (webContentsById.get(id) ?? null), @@ -2065,6 +2196,10 @@ describe("PreviewManager", () => { yield* Effect.yieldNow; yield* Effect.yieldNow; + const grants: Array<{ video?: unknown }> = []; + host.displayMediaHandler()?.({ frame: host.mainFrame }, (value) => grants.push(value)); + expect(grants).toEqual([{}]); + yield* manager.setMainWindow({ isDestroyed: () => false, once: vi.fn(), @@ -2075,7 +2210,60 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("returns native media sources for concurrent preview recordings", () => + effectIt.effect("does not arm recording after the main window closes during warmup", () => + withManager((manager) => + Effect.gen(function* () { + let closeMainWindow: (() => void) | undefined; + let finishWarmup!: (image: TestCapturedPreviewImage) => void; + let markWarmupStarted!: () => void; + const warmupStarted = new Promise((resolve) => { + markWarmupStarted = resolve; + }); + const capturedImage = { + toJPEG: () => Buffer.from("recording-frame"), + getSize: () => ({ width: 1280, height: 720 }), + }; + const capturePage = vi.fn( + () => + new Promise((resolve) => { + markWarmupStarted(); + finishWarmup = resolve; + }), + ); + const host = makeTestHostWebContents(); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage, 42, host)); + + yield* manager.createTab("tab_window_close_warmup"); + yield* manager.registerWebview("tab_window_close_warmup", 42); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn((event: string, listener: () => void) => { + if (event === "closed") closeMainWindow = listener; + }), + webContents: { setBackgroundThrottling: vi.fn() }, + } as never); + + const start = yield* manager + .startRecording("tab_window_close_warmup") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => warmupStarted); + closeMainWindow?.(); + finishWarmup(capturedImage); + + const exit = yield* Fiber.await(start); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewMainWindowClosedError", + tabId: "tab_window_close_warmup", + }); + } + expect(host.session.setDisplayMediaRequestHandler).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("grants each concurrent preview recording its own tab frame", () => withManager((manager) => Effect.gen(function* () { const firstJpeg = Buffer.from("first-recording-frame"); @@ -2090,6 +2278,8 @@ describe("PreviewManager", () => { })); const firstSendCommand = vi.fn(async () => undefined); const secondSendCommand = vi.fn(async () => undefined); + // Both webviews live in the same window, so they share one display-media handler. + const host = makeTestHostWebContents(); const makeWebContents = ( id: number, capturePage: typeof firstCapturePage, @@ -2097,8 +2287,8 @@ describe("PreviewManager", () => { ) => ({ id, - hostWebContents: { id: 7 }, - getMediaSourceId: vi.fn(() => `tab:${id}`), + mainFrame: { routingId: id }, + hostWebContents: host, executeJavaScript: vi.fn(async () => id === 41 ? { width: 800, height: 600 } : { width: 390, height: 844 }, ), @@ -2110,6 +2300,7 @@ describe("PreviewManager", () => { getZoomFactor: () => 1, setZoomFactor: vi.fn(), setAudioMuted: vi.fn(), + setBackgroundThrottling: vi.fn(), isCurrentlyAudible: () => false, on: vi.fn(), off: vi.fn(), @@ -2137,15 +2328,19 @@ describe("PreviewManager", () => { yield* manager.createTab("tab_2"); yield* manager.registerWebview("tab_1", 41); yield* manager.registerWebview("tab_2", 42); - const sources = yield* Effect.all( - [manager.startRecording("tab_1"), manager.startRecording("tab_2")], - { concurrency: 2 }, - ); - expect(sources).toEqual([ - { sourceId: "tab:41", width: 800, height: 600 }, - { sourceId: "tab:42", width: 390, height: 844 }, - ]); + const grants: Array<{ video?: unknown }> = []; + const takeGrant = () => + host.displayMediaHandler()?.({ frame: host.mainFrame }, (value) => { + grants.push(value); + }); + + yield* manager.startRecording("tab_1"); + takeGrant(); + yield* manager.startRecording("tab_2"); + takeGrant(); + expect(grants).toEqual([{ video: { routingId: 41 } }, { video: { routingId: 42 } }]); + expect(firstCapturePage).toHaveBeenCalledOnce(); expect(secondCapturePage).toHaveBeenCalledOnce(); expect(firstSendCommand).not.toHaveBeenCalledWith( @@ -2165,147 +2360,185 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("requests display media with a fresh renderer gesture", () => + withManager((manager) => + Effect.gen(function* () { + const { host, takeGrant } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + + expect(host.executeJavaScript).toHaveBeenCalledWith( + expect.stringContaining(DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER), + true, + ); + expect(host.executeJavaScript).toHaveBeenCalledWith( + expect.stringContaining("tab_race_a"), + true, + ); + takeGrant(); + yield* manager.stopRecording("tab_race_a"); + }), + ), + ); + + // Runs on the real clock: an earlier queueing design only settled under TestClock and stalled the + // losing start forever in the desktop app. + effectIt.live("settles both starts when two tabs race for the capture stream", () => + withManager((manager) => + Effect.gen(function* () { + const { host, grants, takeGrant } = yield* setupRecordingRaceTabs(manager); + + const exits = yield* Effect.all( + [ + Effect.exit(manager.startRecording("tab_race_a")), + Effect.exit(manager.startRecording("tab_race_b")), + ], + { concurrency: 2 }, + ); + + const [exitA, exitB] = exits; + // Exactly one start owns the stream; the other fails fast instead of hanging. + expect(exits.filter(Exit.isSuccess)).toHaveLength(1); + const loserExit = Exit.isSuccess(exitA) ? exitB : exitA; + if (Exit.isSuccess(loserExit)) return; + expect(Option.getOrThrow(Cause.findErrorOption(loserExit.cause))).toMatchObject({ + _tag: "PreviewRecordingArmConflictError", + }); + + // The single grant goes to the tab that actually won the slot, never the other one. + takeGrant(); + expect(grants).toEqual([{ video: { routingId: Exit.isSuccess(exitA) ? 41 : 42 } }]); + expect(host.session.setDisplayMediaRequestHandler).toHaveBeenCalledOnce(); + }), + ), + ); + + effectIt.effect("releases an armed slot that the renderer never redeemed", () => + withManager((manager) => + Effect.gen(function* () { + const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + const blocked = yield* Effect.exit(manager.startRecording("tab_race_b")); + if (Exit.isSuccess(blocked)) throw new Error("expected the second tab to be refused"); + expect(Option.getOrThrow(Cause.findErrorOption(blocked.cause))).toMatchObject({ + _tag: "PreviewRecordingArmConflictError", + tabId: "tab_race_b", + armedTabId: "tab_race_a", + }); + + // Nothing ever captured the armed tab, so the slot goes stale and stops blocking starts. + yield* TestClock.adjust(10_000); + yield* manager.startRecording("tab_race_b"); + takeGrant(); + expect(grants).toEqual([{ video: { routingId: 42 } }]); + + yield* manager.stopRecording("tab_race_a"); + yield* manager.stopRecording("tab_race_b"); + }), + ), + ); + + effectIt.effect("denies a display-media request that arrives after the arm went stale", () => + withManager((manager) => + Effect.gen(function* () { + const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + yield* TestClock.adjust(10_000); + // The handler cannot read a clock, so the expiry fiber must have dropped the frame. + takeGrant(); + expect(grants).toEqual([{}]); + + yield* manager.stopRecording("tab_race_a"); + }), + ), + ); + + effectIt.effect("only lets the host frame that armed a recording claim its stream", () => + withManager((manager) => + Effect.gen(function* () { + const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + takeGrant({ frameTreeNodeId: 999 }); + takeGrant(); + + expect(grants).toEqual([{}, { video: { routingId: 41 } }]); + yield* manager.stopRecording("tab_race_a"); + }), + ), + ); + + effectIt.effect("reclaims the arm slot from a destroyed webContents", () => + withManager((manager) => + Effect.gen(function* () { + const { grants, takeGrant, destroy } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + destroy(41); + yield* manager.startRecording("tab_race_b"); + takeGrant(); + expect(grants).toEqual([{ video: { routingId: 42 } }]); + + yield* manager.stopRecording("tab_race_b"); + }), + ), + ); + effectIt.effect("continues native recording when the source warmup fails", () => withManager((manager) => Effect.gen(function* () { const capturePage = vi.fn(async () => { throw new Error("source is not ready"); }); - const getMediaSourceId = vi.fn(() => "tab:42"); - const webContents = Object.assign(makeTestPreviewWebContents(capturePage), { + const host = makeTestHostWebContents(); + const webContents = Object.assign(makeTestPreviewWebContents(capturePage, 42, host), { executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })), - getMediaSourceId, }); fromId.mockReturnValue(webContents); yield* manager.createTab("tab_recording_warmup_failure"); yield* manager.registerWebview("tab_recording_warmup_failure", 42); - expect(yield* manager.startRecording("tab_recording_warmup_failure")).toEqual({ - sourceId: "tab:42", - width: 1280, - height: 720, - }); + yield* manager.startRecording("tab_recording_warmup_failure"); expect(capturePage).toHaveBeenCalledTimes(2); - expect(getMediaSourceId).toHaveBeenCalledOnce(); + + // The armed tab answers exactly one display-media request, then further requests are denied. + const handler = host.displayMediaHandler(); + const streams: Array<{ video?: unknown }> = []; + handler?.({ frame: host.mainFrame }, (value) => streams.push(value)); + handler?.({ frame: host.mainFrame }, (value) => streams.push(value)); + expect(streams).toEqual([{ video: { routingId: 42 } }, {}]); yield* manager.stopRecording("tab_recording_warmup_failure"); }), ), ); - effectIt.effect("reports invalid native recording dimensions as a structured error", () => - withManager((manager) => - Effect.gen(function* () { - const capturePage = vi.fn(async () => ({ - toJPEG: () => Buffer.from("unused-recording-frame"), - getSize: () => ({ width: 1280, height: 720 }), - })); - const getMediaSourceId = vi.fn(() => "tab:42"); - const webContents = Object.assign(makeTestPreviewWebContents(capturePage), { - executeJavaScript: vi.fn(async () => ({ width: 0, height: 720 })), - getMediaSourceId, - }); - fromId.mockReturnValue(webContents); - - yield* manager.createTab("tab_invalid_recording_size"); - yield* manager.registerWebview("tab_invalid_recording_size", 42); - const exit = yield* Effect.exit(manager.startRecording("tab_invalid_recording_size")); - - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isSuccess(exit)) return; - expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ - _tag: "PreviewRecordingSourceSizeUnavailableError", - tabId: "tab_invalid_recording_size", - webContentsId: 42, - }); - expect(getMediaSourceId).not.toHaveBeenCalled(); - }), - ), - ); - - effectIt.effect("keeps a newer recording lease when an earlier start fails", () => - withManager((manager) => - Effect.gen(function* () { - const setBackgroundThrottling = vi.fn(); - const capturePage = vi.fn(async () => ({ - toJPEG: () => Buffer.from("unused-recording-frame"), - getSize: () => ({ width: 1280, height: 720 }), - })); - let markMeasurementStarted!: () => void; - const measurementStarted = new Promise((resolve) => { - markMeasurementStarted = resolve; - }); - let rejectFirstMeasurement!: (error: Error) => void; - const executeJavaScript = vi - .fn<(expression: string, userGesture?: boolean) => Promise>() - .mockImplementationOnce(() => { - markMeasurementStarted(); - return new Promise((_, reject) => { - rejectFirstMeasurement = reject; - }); - }) - .mockResolvedValue({ width: 1280, height: 720 }); - const webContents = Object.assign(makeTestPreviewWebContents(capturePage), { - executeJavaScript, - }); - fromId.mockReturnValue(webContents); - - yield* manager.createTab("tab_recording_start_race"); - yield* manager.registerWebview("tab_recording_start_race", 42); - yield* manager.setMainWindow({ - isDestroyed: () => false, - once: vi.fn(), - webContents: { setBackgroundThrottling }, - } as never); - - const firstStart = yield* manager - .startRecording("tab_recording_start_race") - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Effect.promise(() => measurementStarted); - const secondStart = yield* manager - .startRecording("tab_recording_start_race") - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Effect.yieldNow; - expect(executeJavaScript).toHaveBeenCalledOnce(); - - rejectFirstMeasurement(new Error("first measurement failed")); - const firstExit = yield* Fiber.await(firstStart); - expect(Exit.isFailure(firstExit)).toBe(true); - expect(yield* Fiber.join(secondStart)).toEqual({ - sourceId: "tab:42", - width: 1280, - height: 720, - }); - - yield* manager.stopRecording("tab_recording_start_race"); - expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true], [false], [true]]); - }), - ), - ); - effectIt.effect("serializes recording source acquisition with webview replacement", () => withManager((manager) => Effect.gen(function* () { - const capturePage = vi.fn(async () => ({ + const capturedImage = { toJPEG: () => Buffer.from("unused-recording-frame"), getSize: () => ({ width: 1280, height: 720 }), - })); - let markMeasurementStarted!: () => void; - const measurementStarted = new Promise((resolve) => { - markMeasurementStarted = resolve; - }); - let finishMeasurement!: (size: { readonly width: number; readonly height: number }) => void; - const executeJavaScript = vi.fn( - () => - new Promise<{ readonly width: number; readonly height: number }>((resolve) => { - markMeasurementStarted(); - finishMeasurement = resolve; - }), - ); - const initialWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 42), { - executeJavaScript, + }; + let markWarmupStarted!: () => void; + const warmupStarted = new Promise((resolve) => { + markWarmupStarted = resolve; }); + let finishWarmup!: (image: TestCapturedPreviewImage) => void; + const capturePage = vi + .fn<() => Promise>() + .mockImplementationOnce( + () => + new Promise((resolve) => { + markWarmupStarted(); + finishWarmup = resolve; + }), + ) + .mockResolvedValue(capturedImage); + const initialWebContents = makeTestPreviewWebContents(capturePage, 42); const replacementOn = vi.fn(); const replacementWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 43), { on: replacementOn, @@ -2321,22 +2554,25 @@ describe("PreviewManager", () => { const start = yield* manager .startRecording("tab_recording_replacement_race") .pipe(Effect.forkChild({ startImmediately: true })); - yield* Effect.promise(() => measurementStarted); + yield* Effect.promise(() => warmupStarted); const replacement = yield* manager .registerWebview("tab_recording_replacement_race", 43) .pipe(Effect.forkChild({ startImmediately: true })); yield* Effect.yieldNow; expect(replacementOn).not.toHaveBeenCalled(); - finishMeasurement({ width: 1280, height: 720 }); - expect(yield* Fiber.join(start)).toEqual({ - sourceId: "tab:42", - width: 1280, - height: 720, - }); + finishWarmup(capturedImage); + yield* Fiber.join(start); yield* Fiber.join(replacement); expect(replacementOn).toHaveBeenCalled(); + expect(initialWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]); + expect(replacementWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]); yield* manager.stopRecording("tab_recording_replacement_race"); + expect(initialWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + expect(replacementWebContents.setBackgroundThrottling.mock.calls).toEqual([ + [false], + [true], + ]); }), ), ); @@ -2345,7 +2581,9 @@ describe("PreviewManager", () => { withManager((manager) => Effect.gen(function* () { const setBackgroundThrottling = vi.fn(); - const mainWindowWebContents = { setBackgroundThrottling }; + const mainWindowWebContents = Object.assign(makeTestHostWebContents(), { + setBackgroundThrottling, + }); const jpeg = Buffer.from("shared-preview-frame"); const capturePage = vi.fn(async () => ({ toJPEG: () => jpeg, @@ -2353,8 +2591,8 @@ describe("PreviewManager", () => { })); fromId.mockReturnValue({ id: 42, + mainFrame: { routingId: 42 }, hostWebContents: mainWindowWebContents, - getMediaSourceId: vi.fn(() => "tab:42"), executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })), isDestroyed: () => false, getType: () => "webview", @@ -2364,6 +2602,7 @@ describe("PreviewManager", () => { getZoomFactor: () => 1, setZoomFactor: vi.fn(), setAudioMuted: vi.fn(), + setBackgroundThrottling: vi.fn(), isCurrentlyAudible: () => false, on: vi.fn(), off: vi.fn(), diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 7d11b2614c1e01db47f40ef17f4e45bc83091918..8ee312110d863927ca13811b15b0213aeaeceb5f 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -5,6 +5,7 @@ * elements live in the renderer; we only attach listeners and forward state * here). Single layer-scoped browser session partition. */ +import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; import type { DesktopPreviewAnnotationTheme, DesktopPreviewAutomationStatus, @@ -16,7 +17,6 @@ import type { PreviewAnnotationSubmissionResult, DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, - DesktopPreviewRecordingSource, DesktopPreviewScreenshotArtifact, DesktopPreviewTabDefaults, PreviewAutomationClickInput, @@ -109,8 +109,8 @@ const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_INTERACTIVE_ELEMENTS = 200; const MAX_SCREENSHOT_WIDTH = 1280; -const RECORDING_SOURCE_SIZE_EXPRESSION = - "({ width: Math.round(globalThis.innerWidth), height: Math.round(globalThis.innerHeight) })"; +/** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */ +const RECORDING_ARM_GRACE_MS = 10_000; const PICTURE_IN_PICTURE_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12); const PICTURE_IN_PICTURE_JPEG_QUALITY = 80; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; @@ -122,6 +122,8 @@ const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; const AGENT_CURSOR_CLICK_LEAD_MS = 40; +const requestRecordingCaptureExpression = (tabId: string): string => + `globalThis[${JSON.stringify(DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER)}]?.(${JSON.stringify(tabId)}) === true`; const encodeUnknownJson = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)); const DEFAULT_ANNOTATION_THEME: DesktopPreviewAnnotationTheme = { colorScheme: "light", @@ -390,6 +392,7 @@ type FrameCaptureConsumer = "picture-in-picture" | "recording"; interface FrameCaptureSession { readonly scope: Scope.Closeable | null; readonly consumers: ReadonlySet; + readonly unthrottledWebContentsIds: ReadonlySet; readonly lastPictureInPictureFrame: Buffer | null; } @@ -400,6 +403,14 @@ interface PictureInPictureSession { readonly initializationScope: Scope.Closeable; } +/** The tab whose frame the next `getDisplayMedia()` request is allowed to capture. */ +interface PendingRecording { + readonly tabId: string; + readonly webContents: Electron.WebContents; + readonly requestingFrameTreeNodeId: number; + readonly armedAtMillis: number; +} + interface PickSession { readonly cancel: Effect.Effect; } @@ -591,6 +602,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); const closingTabIdsRef = yield* Ref.make>(new Set()); + // Tab recording uses `setDisplayMediaRequestHandler` because Electron's legacy + // `getMediaSourceId` + `chromeMediaSource: "tab"` capture path was removed upstream + // (electron#44618) and now always rejects with NotAllowedError. + let pendingRecording: PendingRecording | null = null; + const displayMediaHandlerSessions = new WeakSet(); let frameCaptureWindowOpen = true; let currentMainWindow: BrowserWindow | undefined; let mainWindowCleanupFiber: Fiber.Fiber | undefined; @@ -665,6 +681,65 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (Option.isNone(mainWindow)) return; yield* setWindowBackgroundThrottling(mainWindow.value, enabled); }); + const setFrameCaptureWebContentsBackgroundThrottling = Effect.fnUntraced(function* ( + wc: Electron.WebContents, + enabled: boolean, + ) { + if (wc.isDestroyed()) return; + yield* attempt( + { + operation: "frameCapture.setBackgroundThrottling", + webContentsId: wc.id, + }, + () => wc.setBackgroundThrottling(enabled), + ); + }); + const restoreFrameCaptureWebContentsBackgroundThrottling = Effect.fnUntraced(function* ( + webContentsIds: ReadonlySet, + ) { + yield* Effect.forEach( + webContentsIds, + (webContentsId) => { + const wc = webContents.fromId(webContentsId); + if (!wc || wc.isDestroyed()) return Effect.void; + return setFrameCaptureWebContentsBackgroundThrottling(wc, true).pipe( + Effect.retry({ times: 2 }), + Effect.catch((error) => + Effect.logWarning("Failed to restore preview webview frame capture throttling.", { + webContentsId, + error, + }), + ), + ); + }, + { concurrency: "unbounded", discard: true }, + ); + }); + const keepFrameCaptureWebContentsUnthrottled = Effect.fnUntraced(function* ( + tabId: string, + wc: Electron.WebContents, + ) { + yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => { + const current = sessions.get(tabId); + if (!current || current.unthrottledWebContentsIds.has(wc.id)) { + return Effect.succeed([undefined, sessions] as const); + } + return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe( + Effect.map( + () => + [ + undefined, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + ...current, + unthrottledWebContentsIds: new Set([...current.unthrottledWebContentsIds, wc.id]), + }); + }), + ] as const, + ), + ); + }); + }); const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* ( tabId: string, consumer: FrameCaptureConsumer, @@ -694,6 +769,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const remainingSessions = replaceMap(sessions, (copy) => { copy.delete(tabId); }); + yield* restoreFrameCaptureWebContentsBackgroundThrottling( + current.unthrottledWebContentsIds, + ); if (remainingSessions.size === 0) { yield* setFrameCaptureBackgroundThrottling(true).pipe( Effect.retry({ times: 2 }), @@ -713,6 +791,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const stopAllRecordings = Effect.fn("PreviewManager.stopAllRecordings")(function* () { + pendingRecording = null; const sessions = yield* SynchronizedRef.get(frameCaptureSessionsRef); yield* Effect.forEach(sessions.keys(), (tabId) => stopFrameCapture(tabId, "recording"), { concurrency: "unbounded", @@ -1889,6 +1968,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; + clearPendingRecording(tabId); yield* Effect.all( [ cancelPickElement(tabId), @@ -1980,6 +2060,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attached = yield* Ref.get(attachedRef); const annotationTheme = yield* Ref.get(annotationThemeRef); const currentAttachment = attached.get(webContentsId); + yield* keepFrameCaptureWebContentsUnthrottled(tabId, wc); if (tab.webContentsId === webContentsId && currentAttachment?.webContents === wc) { // The guest we already own re-announced itself, so nothing about the tab // changed. Only push its zoom back down — Chromium may have just handed @@ -1996,6 +2077,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ? tab.webContentsId : null; if (replacedWebContentsId !== null) { + // The replaced guest can no longer redeem a display-media grant. + clearPendingRecording(tabId); yield* Effect.all( [ detachControlSession(replacedWebContentsId), @@ -2728,7 +2811,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ) { // Recording keeps only the activity lease. Picture-in-picture owns the // capturePage loop and tolerates transient compositor warmup failures. - yield* requireWebContents(tabId); const captureNextFrame = Effect.sleep(PICTURE_IN_PICTURE_FRAME_INTERVAL_MS).pipe( Effect.andThen(capturePreviewFrame(tabId)), Effect.catch((error) => @@ -2749,11 +2831,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { return yield* new PreviewTabNotFoundError({ tabId }); } + const wc = yield* requireWebContents(tabId); const current = sessions.get(tabId); if (current) { if (current.consumers.has(consumer)) { return [false, sessions] as const; } + if (!current.unthrottledWebContentsIds.has(wc.id)) { + yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false); + } let scope = current.scope; if (consumer === "picture-in-picture" && scope === null) { scope = yield* Scope.fork(parentScope, "sequential"); @@ -2766,6 +2852,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ...current, scope, consumers: new Set([...current.consumers, consumer]), + unthrottledWebContentsIds: new Set([...current.unthrottledWebContentsIds, wc.id]), }); }), ] as const; @@ -2773,6 +2860,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (sessions.size === 0) { yield* setFrameCaptureBackgroundThrottling(false); } + yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe( + Effect.onError(() => + sessions.size === 0 + ? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore) + : Effect.void, + ), + ); const scope = consumer === "picture-in-picture" ? yield* Scope.fork(parentScope, "sequential") : null; if (scope !== null) { @@ -2784,6 +2878,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, { scope, consumers: new Set([consumer]), + unthrottledWebContentsIds: new Set([wc.id]), lastPictureInPictureFrame: null, }); }), @@ -3099,6 +3194,83 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* Effect.failCause(initializationExit.cause); }); + /** Only drops the armed target when it still belongs to `tabId`, so tabs cannot clobber each other. */ + const clearPendingRecording = (tabId: string) => { + if (pendingRecording?.tabId === tabId) pendingRecording = null; + }; + + /** + * Claims the single arm slot for `tabId`. A display-media request carries no tab identity, so the + * slot is exclusive: a second tab arming before the first request lands would redirect the first + * renderer's stream. Rather than queue (which can only ever stall a start), a colliding start + * fails fast and the renderer can retry. An arm the renderer never redeemed goes stale after + * `RECORDING_ARM_GRACE_MS` so it cannot hold the slot forever. + */ + const armPendingRecording = Effect.fn("PreviewManager.armPendingRecording")(function* ( + tabId: string, + wc: Electron.WebContents, + requestingFrameTreeNodeId: number, + ) { + const now = yield* Clock.currentTimeMillis; + const previous = pendingRecording; + if ( + previous !== null && + previous.tabId !== tabId && + !previous.webContents.isDestroyed() && + now - previous.armedAtMillis < RECORDING_ARM_GRACE_MS + ) { + return yield* new PreviewRecordingArmConflictError({ + tabId, + webContentsId: wc.id, + armedTabId: previous.tabId, + }); + } + const armed: PendingRecording = { + tabId, + webContents: wc, + requestingFrameTreeNodeId, + armedAtMillis: now, + }; + pendingRecording = armed; + // The handler callback is sync and cannot read a clock, so expiry is driven from here. + // Identity compare: a re-arm replaces the object, and this fiber must not clobber it. + yield* Effect.forkIn( + Effect.sleep(RECORDING_ARM_GRACE_MS).pipe( + Effect.andThen( + Effect.sync(() => { + if (pendingRecording === armed) pendingRecording = null; + }), + ), + ), + parentScope, + ); + }); + + // Installed once per session: answers the renderer's `getDisplayMedia()` with the tab that + // `startRecording` armed, and denies anything else so pages cannot capture on their own. + const installDisplayMediaRequestHandler = (session: Session) => { + if (displayMediaHandlerSessions.has(session)) return; + displayMediaHandlerSessions.add(session); + session.setDisplayMediaRequestHandler((request, callback) => { + const armed = pendingRecording; + if (!armed) { + callback({}); + return; + } + if (armed.webContents.isDestroyed()) { + pendingRecording = null; + callback({}); + return; + } + if (request.frame?.frameTreeNodeId !== armed.requestingFrameTreeNodeId) { + callback({}); + return; + } + pendingRecording = null; + callback({ video: armed.webContents.mainFrame }); + }); + }; + const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) { return yield* new PreviewTabNotFoundError({ tabId }); @@ -3112,31 +3284,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (requestWebContents === null) { return yield* new PreviewMainWindowClosedError({ tabId }); } - const measuredSize = yield* attemptPromise( - { - operation: "recording.measureSource", - tabId, - webContentsId: wc.id, - }, - () => wc.executeJavaScript(RECORDING_SOURCE_SIZE_EXPRESSION, true), - ); - if ( - typeof measuredSize !== "object" || - measuredSize === null || - !("width" in measuredSize) || - !("height" in measuredSize) || - typeof measuredSize.width !== "number" || - typeof measuredSize.height !== "number" || - !Number.isInteger(measuredSize.width) || - !Number.isInteger(measuredSize.height) || - measuredSize.width <= 0 || - measuredSize.height <= 0 - ) { - return yield* new PreviewRecordingSourceSizeUnavailableError({ - tabId, - webContentsId: wc.id, - }); - } yield* attemptPromise( { operation: "recording.warmSource", @@ -3152,25 +3299,44 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function webContentsId: wc.id, }); } - const sourceId = yield* attempt( + if (!frameCaptureWindowOpen || requestWebContents.isDestroyed()) { + return yield* new PreviewMainWindowClosedError({ tabId }); + } + installDisplayMediaRequestHandler(requestWebContents.session); + yield* armPendingRecording(tabId, wc, requestWebContents.mainFrame.frameTreeNodeId); + const captureRequested = yield* attemptPromise( { - operation: "recording.getMediaSourceId", + operation: "recording.requestCapture", tabId, - webContentsId: wc.id, + webContentsId: requestWebContents.id, }, - () => wc.getMediaSourceId(requestWebContents), + () => + requestWebContents.executeJavaScript(requestRecordingCaptureExpression(tabId), true), ); - return { - sourceId, - width: measuredSize.width, - height: measuredSize.height, - } satisfies DesktopPreviewRecordingSource; - }).pipe(Effect.onError(() => stopFrameCapture(tabId, "recording").pipe(Effect.ignore))), + if (captureRequested !== true) { + return yield* new PreviewRecordingCaptureUnavailableError({ + tabId, + webContentsId: requestWebContents.id, + }); + } + }).pipe( + Effect.onError(() => { + clearPendingRecording(tabId); + return stopFrameCapture(tabId, "recording").pipe(Effect.ignore); + }), + ), ); }); const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { - yield* withTabLifecycleLock(tabId, stopFrameCapture(tabId, "recording")); + // Clearing runs under the tab lock so it cannot land before an in-flight start arms. + yield* withTabLifecycleLock( + tabId, + Effect.suspend(() => { + clearPendingRecording(tabId); + return stopFrameCapture(tabId, "recording"); + }), + ); }); const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* ( @@ -3980,12 +4146,28 @@ export class PreviewMainWindowClosedError extends Schema.TaggedErrorClass()( - "PreviewRecordingSourceSizeUnavailableError", - { tabId: Schema.String, webContentsId: Schema.Number }, +export class PreviewRecordingArmConflictError extends Schema.TaggedErrorClass()( + "PreviewRecordingArmConflictError", + { + tabId: Schema.String, + webContentsId: Schema.Number, + armedTabId: Schema.String, + }, ) { override get message(): string { - return `Preview media source dimensions are unavailable for tab ${this.tabId}`; + return `Preview tab ${this.armedTabId} is still claiming the capture stream, so recording could not start for tab ${this.tabId}`; + } +} + +export class PreviewRecordingCaptureUnavailableError extends Schema.TaggedErrorClass()( + "PreviewRecordingCaptureUnavailableError", + { + tabId: Schema.String, + webContentsId: Schema.Number, + }, +) { + override get message(): string { + return `Preview recording capture is unavailable for tab ${this.tabId} in WebContents ${this.webContentsId}`; } } @@ -4196,7 +4378,8 @@ export const PreviewManagerError = Schema.Union([ PreviewWebContentsNotFoundError, PreviewWebviewNotInitializedError, PreviewMainWindowClosedError, - PreviewRecordingSourceSizeUnavailableError, + PreviewRecordingArmConflictError, + PreviewRecordingCaptureUnavailableError, PreviewOperationError, PreviewArtifactPathOutsideDirectoryError, PreviewArtifactImageLoadError, @@ -4274,9 +4457,7 @@ export class PreviewManager extends Context.Service< readonly copyArtifactToClipboard: (path: string) => Effect.Effect; readonly openPictureInPicture: (tabId: string) => Effect.Effect; readonly closePictureInPicture: (tabId: string) => Effect.Effect; - readonly startRecording: ( - tabId: string, - ) => Effect.Effect; + readonly startRecording: (tabId: string) => Effect.Effect; readonly stopRecording: (tabId: string) => Effect.Effect; readonly saveRecording: ( tabId: string, diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 74db764005c09df20bb61e636d39531a60bb23f6..77c65264aa946c4abf1444ff3884d22b2c07f3cb 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -6,7 +6,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { previewBridge } from "~/components/preview/previewBridge"; import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; -import { cn } from "~/lib/utils"; +import { cn, isMacPlatform } from "~/lib/utils"; import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore"; import { useActiveBrowserRecordingTabIds } from "./browserRecording"; @@ -241,6 +241,10 @@ export function HostedBrowserWebview(props: { const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, renderingActive, + // Electron 43 can permanently blank a macOS webview after `visibility: hidden`. + // Inactive macOS guests intentionally remain paintable offscreen; other platforms still + // suspend them, and automation continues to see the macOS guests as inactive. + keepPaintableWhenInactive: isMacPlatform(navigator.platform), cornerRadius: presentation.cornerRadius, rect: lastRect, hiddenSize, diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index f45a69c601c9066dadc78c65ef75efde82b9f912..49145f314e98c1992f87c20a42b6ceb603018dd4 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -1,37 +1,57 @@ -import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { + DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER, + EnvironmentId, + ThreadId, +} from "@t3tools/contracts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -const { clientSettings, events, getUserMedia, registrySet, save, startScreencast, stopScreencast } = - vi.hoisted(() => { - const events: string[] = []; - return { - clientSettings: { browserRecordingFrameRate: 30 as 30 | 60 }, - events, - getUserMedia: vi.fn(), - registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { - events.push( - value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, - ); - }), - save: vi.fn(async (tabId: string) => ({ - id: "recording-test", - tabId, - path: "/tmp/recording-test.webm", - mimeType: "video/webm" as const, - sizeBytes: 0, - createdAt: "2026-06-26T00:00:00.000Z", - })), - startScreencast: vi.fn(async (tabId: string) => { - events.push("start-screencast"); - return { sourceId: `source:${tabId}`, width: 1000, height: 620 }; - }), - stopScreencast: vi.fn(async () => undefined), - }; - }); +const { + clientSettings, + events, + getDisplayMedia, + registrySet, + requestDisplayMediaCapture, + save, + startScreencast, + stopScreencast, +} = vi.hoisted(() => { + const events: string[] = []; + return { + clientSettings: { browserRecordingFrameRate: 30 as 30 | 60 }, + events, + getDisplayMedia: vi.fn(), + requestDisplayMediaCapture: vi.fn((_tabId: string) => undefined), + registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { + events.push( + value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, + ); + }), + save: vi.fn(async (tabId: string) => ({ + id: "recording-test", + tabId, + path: "/tmp/recording-test.webm", + mimeType: "video/webm" as const, + sizeBytes: 0, + createdAt: "2026-06-26T00:00:00.000Z", + })), + startScreencast: vi.fn(async (_tabId: string) => { + events.push("start-screencast"); + }), + stopScreencast: vi.fn(async () => undefined), + }; +}); vi.mock("~/components/preview/previewBridge", () => ({ previewBridge: { - recording: { onFrame: vi.fn(), save, startScreencast, stopScreencast }, + recording: { + onFrame: vi.fn(), + save, + startScreencast: async (tabId: string) => { + await startScreencast(tabId); + requestDisplayMediaCapture(tabId); + }, + stopScreencast, + }, }, })); @@ -50,6 +70,7 @@ import { BrowserRecordingCaptureTimeoutError, BrowserRecordingConflictError, BrowserRecordingFormatUnavailableError, + BrowserRecordingStartCancelledError, findActiveBrowserRecordingRuntimeTabId, readActiveBrowserRecordingTabIds, readActiveBrowserRecordingTargets, @@ -122,10 +143,16 @@ describe("browser recording", () => { }); vi.stubGlobal("cancelAnimationFrame", vi.fn()); vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder); - getUserMedia.mockResolvedValue({ + getDisplayMedia.mockResolvedValue({ getTracks: () => [{ stop: vi.fn() }], }); - vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + requestDisplayMediaCapture.mockImplementation((tabId: string) => { + const trigger = Reflect.get(globalThis, DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER); + if (typeof trigger !== "function" || trigger(tabId) !== true) { + throw new Error(`No pending display-media capture for ${tabId}.`); + } + }); + vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } }); useBrowserSurfaceStore.setState({ activityByTabId: {}, byTabId: {} }); }); @@ -142,13 +169,20 @@ describe("browser recording", () => { expect(startupEvents).toEqual(["publish:recording-tab", "start-screencast"]); }); + it("routes gesture-free starts through the desktop capture trigger", async () => { + await startBrowserRecording("automation-recording-tab"); + + expect(requestDisplayMediaCapture).toHaveBeenCalledWith("automation-recording-tab"); + expect(getDisplayMedia).toHaveBeenCalledOnce(); + await stopBrowserRecording("automation-recording-tab"); + }); + it("paints and holds a hidden browser surface for the recording lifetime", async () => { startScreencast.mockImplementationOnce(async (tabId: string) => { expect(animationFrameCount).toBe(2); expect(useBrowserSurfaceStore.getState().activityByTabId[tabId]).toBe(1); - return { sourceId: `source:${tabId}`, width: 1000, height: 620 }; }); - getUserMedia.mockImplementationOnce(async () => { + getDisplayMedia.mockImplementationOnce(async () => { expect(animationFrameCount).toBe(2); expect(useBrowserSurfaceStore.getState().activityByTabId["background-tab"]).toBe(1); return { getTracks: () => [{ stop: vi.fn() }] }; @@ -178,26 +212,16 @@ describe("browser recording", () => { await stopBrowserRecording("hidden-window-tab"); }); - it("records the native tab stream at the preview's current dimensions", async () => { + it("records the native tab stream armed by the main process", async () => { const stopTrack = vi.fn(); const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream; - getUserMedia.mockResolvedValueOnce(stream); + getDisplayMedia.mockResolvedValueOnce(stream); await startBrowserRecording("recording-tab"); - expect(getUserMedia).toHaveBeenCalledWith({ + expect(getDisplayMedia).toHaveBeenCalledWith({ audio: false, - video: { - mandatory: { - chromeMediaSource: "tab", - chromeMediaSourceId: "source:recording-tab", - minWidth: 1000, - maxWidth: 1000, - minHeight: 620, - maxHeight: 620, - maxFrameRate: 30, - }, - }, + video: { frameRate: { max: 30 } }, }); expect(FakeMediaRecorder.instances[0]?.stream).toBe(stream); @@ -210,18 +234,16 @@ describe("browser recording", () => { await startBrowserRecording("recording-tab"); - expect(getUserMedia).toHaveBeenCalledWith({ + expect(getDisplayMedia).toHaveBeenCalledWith({ audio: false, - video: { - mandatory: expect.objectContaining({ maxFrameRate: 60 }), - }, + video: { frameRate: { max: 60 } }, }); await stopBrowserRecording("recording-tab"); }); it("stops the native stream when MediaRecorder cleanup fails", async () => { const stopTrack = vi.fn(); - getUserMedia.mockResolvedValueOnce({ + getDisplayMedia.mockResolvedValueOnce({ getTracks: () => [{ stop: stopTrack }], }); @@ -285,7 +307,7 @@ describe("browser recording", () => { }); it("releases the native capture lease when stream acquisition fails", async () => { - getUserMedia.mockRejectedValueOnce(new Error("capture failed")); + getDisplayMedia.mockRejectedValueOnce(new Error("capture failed")); await expect(startBrowserRecording("recording-tab")).rejects.toMatchObject({ operation: "capture-media-stream", @@ -300,7 +322,7 @@ describe("browser recording", () => { vi.useFakeTimers(); let finishCapture!: (stream: MediaStream) => void; const stopTrack = vi.fn(); - getUserMedia.mockImplementationOnce( + getDisplayMedia.mockImplementationOnce( () => new Promise((resolve) => { finishCapture = resolve; @@ -308,7 +330,7 @@ describe("browser recording", () => { ); const startPromise = startBrowserRecording("recording-tab"); - await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce()); const rejection = expect(startPromise).rejects.toMatchObject({ _tag: "BrowserRecordingCaptureTimeoutError", tabId: "recording-tab", @@ -356,6 +378,124 @@ describe("browser recording", () => { expect(save).toHaveBeenCalledTimes(2); }); + it("serializes display media grants for concurrent recording starts", async () => { + let finishFirstCapture!: (stream: MediaStream) => void; + const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream; + getDisplayMedia + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishFirstCapture = resolve; + }), + ) + .mockResolvedValueOnce(stream); + + const firstStart = startBrowserRecording("recording-tab"); + await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce()); + const secondStart = startBrowserRecording("recording-tab-2"); + await vi.waitFor(() => expect(readActiveBrowserRecordingTabIds().size).toBe(2)); + + expect(startScreencast).toHaveBeenCalledTimes(1); + finishFirstCapture(stream); + await Promise.all([firstStart, secondStart]); + + expect(startScreencast.mock.calls).toEqual([["recording-tab"], ["recording-tab-2"]]); + expect(getDisplayMedia).toHaveBeenCalledTimes(2); + await Promise.all([ + stopBrowserRecording("recording-tab"), + stopBrowserRecording("recording-tab-2"), + ]); + }); + + it("cancels a queued recording when stopped before its media grant", async () => { + let finishFirstCapture!: (stream: MediaStream) => void; + const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream; + getDisplayMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + finishFirstCapture = resolve; + }), + ); + + const firstStart = startBrowserRecording("recording-tab"); + await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce()); + const secondStart = startBrowserRecording("recording-tab-2"); + await vi.waitFor(() => expect(readActiveBrowserRecordingTabIds().size).toBe(2)); + + const secondStop = stopBrowserRecording("recording-tab-2"); + await expect(secondStart).rejects.toBeInstanceOf(BrowserRecordingStartCancelledError); + await expect(secondStop).resolves.toBeNull(); + expect(startScreencast).toHaveBeenCalledTimes(1); + + finishFirstCapture(stream); + await firstStart; + await stopBrowserRecording("recording-tab"); + expect(getDisplayMedia).toHaveBeenCalledOnce(); + }); + + it("latches a stop that arrives before the start becomes queued", async () => { + let releaseDelayedPaint!: (timestamp: number) => void; + let frameId = 0; + vi.stubGlobal( + "requestAnimationFrame", + vi.fn((callback: FrameRequestCallback) => { + frameId += 1; + if (frameId === 1) releaseDelayedPaint = callback; + else callback(frameId); + return frameId; + }), + ); + let finishBlockingCapture!: (stream: MediaStream) => void; + const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream; + getDisplayMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + finishBlockingCapture = resolve; + }), + ); + + const delayedStart = startBrowserRecording("delayed-tab"); + await vi.waitFor(() => + expect(readActiveBrowserRecordingTabIds().has("delayed-tab")).toBe(true), + ); + const blockingStart = startBrowserRecording("blocking-tab"); + await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce()); + + const delayedStop = stopBrowserRecording("delayed-tab"); + releaseDelayedPaint(1); + + await expect(delayedStart).rejects.toBeInstanceOf(BrowserRecordingStartCancelledError); + await expect(delayedStop).resolves.toBeNull(); + expect(startScreencast).toHaveBeenCalledOnce(); + + finishBlockingCapture(stream); + await blockingStart; + await stopBrowserRecording("blocking-tab"); + }); + + it("finishes an uncontended pre-grant start before stopping", async () => { + const animationFrames: FrameRequestCallback[] = []; + vi.stubGlobal( + "requestAnimationFrame", + vi.fn((callback: FrameRequestCallback) => { + animationFrames.push(callback); + return animationFrames.length; + }), + ); + + const startPromise = startBrowserRecording("recording-tab"); + await vi.waitFor(() => + expect(readActiveBrowserRecordingTabIds().has("recording-tab")).toBe(true), + ); + const stopPromise = stopBrowserRecording("recording-tab"); + expect(startScreencast).not.toHaveBeenCalled(); + + animationFrames.shift()?.(1); + animationFrames.shift()?.(2); + await startPromise; + await expect(stopPromise).resolves.toMatchObject({ tabId: "recording-tab" }); + }); + it("keeps a recording reachable through its runtime id after a server epoch changes", async () => { const threadRef = { environmentId: EnvironmentId.make("environment-recording"), @@ -387,7 +527,6 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); - return { sourceId: "source:recording-tab", width: 1000, height: 620 }; }); const firstStart = startBrowserRecording("recording-tab"); @@ -452,7 +591,6 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); - return { sourceId: "source:recording-tab", width: 1000, height: 620 }; }); const startPromise = startBrowserRecording("recording-tab"); @@ -476,7 +614,6 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); - return { sourceId: "source:recording-tab", width: 1000, height: 620 }; }); const firstStart = startBrowserRecording("recording-tab"); @@ -503,7 +640,6 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); - return { sourceId: "source:recording-tab", width: 1000, height: 620 }; }); stopScreencast.mockRejectedValueOnce(new Error("initial stop failed")); @@ -537,7 +673,6 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); - return { sourceId: "source:recording-tab", width: 1000, height: 620 }; }); const startPromise = startBrowserRecording("recording-tab"); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index e242f25c7d2dc0c7dbeb3ffca80599e2fc3520f6..73bc2708ddf6eae211d627ea7152d72e7c8e4b80 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -1,8 +1,5 @@ -import type { - DesktopPreviewRecordingArtifact, - DesktopPreviewRecordingSource, - ScopedThreadRef, -} from "@t3tools/contracts"; +import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; +import type { DesktopPreviewRecordingArtifact, ScopedThreadRef } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import * as Schema from "effect/Schema"; import { Atom } from "effect/unstable/reactivity"; @@ -36,6 +33,17 @@ export class BrowserRecordingConflictError extends Schema.TaggedErrorClass()( + "BrowserRecordingStartCancelledError", + { + tabId: Schema.String, + }, +) { + override get message(): string { + return `Browser recording start was cancelled for tab ${this.tabId}.`; + } +} + export class BrowserRecordingFormatUnavailableError extends Schema.TaggedErrorClass()( "BrowserRecordingFormatUnavailableError", { tabId: Schema.String }, @@ -82,9 +90,21 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass; + readonly cancelBeforeGrant: () => void; + readonly setQueuedForGrant: (queued: boolean) => void; +} type BrowserRecordingLifecycle = - | { readonly phase: "starting" } + | StartingBrowserRecordingLifecycle | { readonly phase: "recording" } | { readonly phase: "stopping"; @@ -124,6 +144,53 @@ export function useActiveBrowserRecordingTabIds(): ReadonlySet { } const activeRecordings = new Map(); +let displayMediaGrantTail = Promise.resolve(); +let displayMediaGrantQueueDepth = 0; + +const makeStartingBrowserRecordingLifecycle = (): StartingBrowserRecordingLifecycle => { + let signalCancellation!: () => void; + const cancelledBeforeGrantSignal = new Promise((resolve) => { + signalCancellation = resolve; + }); + const lifecycle: StartingBrowserRecordingLifecycle = { + phase: "starting", + queuedForGrant: null, + grantStarted: false, + stopRequestedBeforeGrant: false, + cancelledBeforeGrant: false, + cancelledBeforeGrantSignal, + cancelBeforeGrant: () => { + // Queue position is unknown during paint/settings warmup. Keep the stop request so a start + // that later turns out to be contended can still be cancelled before native capture. + lifecycle.stopRequestedBeforeGrant = true; + if (lifecycle.queuedForGrant && !lifecycle.grantStarted && !lifecycle.cancelledBeforeGrant) { + lifecycle.cancelledBeforeGrant = true; + signalCancellation(); + } + }, + setQueuedForGrant: (queued) => { + lifecycle.queuedForGrant = queued; + if (queued && lifecycle.stopRequestedBeforeGrant) lifecycle.cancelBeforeGrant(); + }, + }; + return lifecycle; +}; + +const queueDisplayMediaGrant = ( + useGrant: () => Promise, +): { readonly queued: boolean; readonly result: Promise } => { + const queued = displayMediaGrantQueueDepth > 0; + displayMediaGrantQueueDepth += 1; + const result = displayMediaGrantTail.then(useGrant); + const settleGrant = () => { + displayMediaGrantQueueDepth -= 1; + }; + displayMediaGrantTail = result.then( + () => settleGrant(), + () => settleGrant(), + ); + return { queued, result }; +}; const publishActiveRecordingTabIds = (): void => { appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { @@ -184,23 +251,12 @@ const createMediaRecorder = (stream: MediaStream): MediaRecorder => { return mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream); }; -const captureTabMediaStream = ( - source: DesktopPreviewRecordingSource, - frameRate: number, -): Promise => - navigator.mediaDevices.getUserMedia({ +const captureTabMediaStream = (frameRate: number): Promise => + // The desktop main process routes this request to the tab that `startScreencast` armed, so the + // stream already arrives at that tab's native size and needs no source or dimension constraints. + navigator.mediaDevices.getDisplayMedia({ audio: false, - video: { - mandatory: { - chromeMediaSource: "tab", - chromeMediaSourceId: source.sourceId, - minWidth: source.width, - maxWidth: source.width, - minHeight: source.height, - maxHeight: source.height, - maxFrameRate: frameRate, - }, - } as unknown as MediaTrackConstraints, + video: { frameRate: { max: frameRate } }, }); const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise => { @@ -216,14 +272,75 @@ const stopMediaStream = (stream: MediaStream | null): void => { for (const track of stream?.getTracks() ?? []) track.stop(); }; +interface PendingTabMediaCapture { + readonly start: () => void; +} + +const pendingTabMediaCaptures = new Map(); + +const prepareTabMediaCapture = (tabId: string, frameRate: number) => { + let acceptStream = true; + let capturedStream: MediaStream | null = null; + let resolveCapture!: (stream: MediaStream | PromiseLike) => void; + let rejectCapture!: (cause: unknown) => void; + const capturePromise = new Promise((resolve, reject) => { + resolveCapture = resolve; + rejectCapture = reject; + }).then((stream) => { + capturedStream = stream; + if (!acceptStream) { + stopMediaStream(stream); + capturedStream = null; + } + return stream; + }); + const pending: PendingTabMediaCapture = { + start: () => { + try { + // Electron invokes this callback through executeJavaScript(..., true), so even automated + // and delayed queued starts satisfy getDisplayMedia's transient-activation requirement. + resolveCapture(captureTabMediaStream(frameRate)); + } catch (cause) { + rejectCapture(cause); + } + }, + }; + pendingTabMediaCaptures.set(tabId, pending); + return { + capturePromise, + cancel: () => { + acceptStream = false; + if (capturedStream) { + stopMediaStream(capturedStream); + capturedStream = null; + } + if (pendingTabMediaCaptures.get(tabId) === pending) pendingTabMediaCaptures.delete(tabId); + void capturePromise.catch(() => undefined); + }, + }; +}; + +const triggerTabMediaCapture = (tabId: unknown): boolean => { + if (typeof tabId !== "string") return false; + const pending = pendingTabMediaCaptures.get(tabId); + if (!pending) return false; + pendingTabMediaCaptures.delete(tabId); + pending.start(); + return true; +}; + +Object.defineProperty(globalThis, DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER, { + configurable: true, + value: triggerTabMediaCapture, +}); + const captureTabMediaStreamWithTimeout = async ( tabId: string, - source: DesktopPreviewRecordingSource, - frameRate: number, + capturePromise: Promise, ): Promise => { let acceptStream = true; let timeoutId: number | null = null; - const streamPromise = captureTabMediaStream(source, frameRate).then((stream) => { + const streamPromise = capturePromise.then((stream) => { if (!acceptStream) stopMediaStream(stream); return stream; }); @@ -382,6 +499,7 @@ export async function startBrowserRecording( const startupSettled = new Promise((resolve) => { settleStartup = resolve; }); + const startingLifecycle = makeStartingBrowserRecordingLifecycle(); const releaseSurfaceActivity = acquireBrowserSurfaceActivity(tabId); const recording: ActiveRecording = { tabId, @@ -393,7 +511,7 @@ export async function startBrowserRecording( releaseSurfaceActivity, stream: null, recorder: null, - lifecycle: { phase: "starting" }, + lifecycle: startingLifecycle, }; activeRecordings.set(tabId, recording); publishActiveRecordingTabIds(); @@ -402,23 +520,9 @@ export async function startBrowserRecording( () => getClientSettings().browserRecordingFrameRate, ); const [frameRate] = await Promise.all([frameRatePromise, waitForBrowserRecordingPaint()]); - let source: DesktopPreviewRecordingSource; - try { - source = await bridge.recording.startScreencast(tabId); - } catch (cause) { - if (!isRecordingStarting(recording)) { - throw recordingStartupCancelledError(recording, cause); - } - clearActiveRecording(recording); - throw new BrowserRecordingOperationError({ - operation: "start-screencast", - tabId, - cause, - }); - } const throwIfStartupCancelled = async (): Promise => { - // A stop requested during startup should let startup finish so the - // caller receives a real artifact. Only replacement/removal cancels it. + // Once a grant starts, a stop lets startup finish so the caller receives an artifact. + // Only a contended start can be cancelled before it reaches native capture. if (activeRecordings.get(tabId) === recording) return; try { await bridge.recording.stopScreencast(tabId); @@ -434,30 +538,67 @@ export async function startBrowserRecording( } throw recordingStartupCancelledError(recording); }; - await throwIfStartupCancelled(); - try { - recording.stream = await captureTabMediaStreamWithTimeout(tabId, source, frameRate); - } catch (cause) { - const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); - if (isBrowserRecordingCaptureTimeoutError(cause) && cleanupCause === undefined) throw cause; - throw new BrowserRecordingOperationError({ - operation: "capture-media-stream", - tabId, - cause: - cleanupCause === undefined - ? cause - : new AggregateError( - [cause, cleanupCause], - `Browser media capture and cleanup failed for tab ${tabId}.`, - { cause }, - ), - }); - } + // The desktop process exposes one display-media grant at a time. Keep only the + // arm-to-capture handoff exclusive; acquired streams can record concurrently. + const grant = queueDisplayMediaGrant(async () => { + if (startingLifecycle.cancelledBeforeGrant) { + throw new BrowserRecordingStartCancelledError({ tabId }); + } + startingLifecycle.grantStarted = true; + await throwIfStartupCancelled(); + const capture = prepareTabMediaCapture(tabId, frameRate); + try { + await bridge.recording.startScreencast(tabId); + } catch (cause) { + capture.cancel(); + if (!isRecordingStarting(recording)) { + throw recordingStartupCancelledError(recording, cause); + } + clearActiveRecording(recording); + throw new BrowserRecordingOperationError({ + operation: "start-screencast", + tabId, + cause, + }); + } + try { + await throwIfStartupCancelled(); + } catch (cause) { + capture.cancel(); + throw cause; + } + try { + recording.stream = await captureTabMediaStreamWithTimeout(tabId, capture.capturePromise); + return recording.stream; + } catch (cause) { + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + if (isBrowserRecordingCaptureTimeoutError(cause) && cleanupCause === undefined) throw cause; + throw new BrowserRecordingOperationError({ + operation: "capture-media-stream", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser media capture and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); + } + }); + startingLifecycle.setQueuedForGrant(grant.queued); + const stream = await Promise.race([ + grant.result, + startingLifecycle.cancelledBeforeGrantSignal.then(() => { + throw new BrowserRecordingStartCancelledError({ tabId }); + }), + ]); await throwIfStartupCancelled(); let recorder: MediaRecorder; try { - recorder = createMediaRecorder(recording.stream); + recorder = createMediaRecorder(stream); recording.recorder = recorder; recorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) chunks.push(event.data); @@ -639,6 +780,7 @@ export function stopBrowserRecording( const recording = activeRecordings.get(tabId); if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; + if (recording.lifecycle.phase === "starting") recording.lifecycle.cancelBeforeGrant(); const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index c2c78f4ac61c12db55dcb4cc3494a191fc31ea48..69216796af9fac0b9424581d42a311ea45b373be 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -61,7 +61,7 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { }); }); - it("keeps an active background task paintable offscreen", () => { + it("keeps an active background task paintable behind the app", () => { const style = resolveHostedBrowserWebviewWrapperStyle({ active: false, renderingActive: true, @@ -69,6 +69,26 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { hiddenSize: { width: 1280, height: 800 }, }); + expect(style).toEqual({ + left: 0, + top: 0, + width: 1280, + height: 800, + zIndex: -1, + pointerEvents: "none", + visibility: "visible", + }); + }); + + it("keeps an inactive webview paintable without marking it as rendering-active", () => { + const style = resolveHostedBrowserWebviewWrapperStyle({ + active: false, + renderingActive: false, + keepPaintableWhenInactive: true, + rect: null, + hiddenSize: { width: 1280, height: 800 }, + }); + expect(style).toEqual({ left: HIDDEN_BROWSER_WEBVIEW_OFFSET, top: HIDDEN_BROWSER_WEBVIEW_OFFSET, diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index 1e6ff0beabe5149f2a218dcd52252268bacb0834..a59a4a8b0083f55f56727475f271e4a6ee3bd441 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -21,11 +21,19 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; readonly renderingActive: boolean; + readonly keepPaintableWhenInactive?: boolean; readonly cornerRadius?: number; readonly rect: BrowserSurfaceRect | null; readonly hiddenSize: HostedBrowserWebviewSize; }): HostedBrowserWebviewWrapperStyle { - const { active, cornerRadius = 0, hiddenSize, rect, renderingActive } = input; + const { + active, + cornerRadius = 0, + hiddenSize, + keepPaintableWhenInactive = false, + rect, + renderingActive, + } = input; if (active && rect) { return { left: rect.x, @@ -38,6 +46,21 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { }; } + if (renderingActive) { + // Electron stops compositing a guest that is fully outside the window, even + // when background throttling is disabled. Keep capture-active guests inside + // the viewport but behind the app so recordings receive complete frames. + return { + left: 0, + top: 0, + width: hiddenSize.width, + height: hiddenSize.height, + zIndex: -1, + pointerEvents: "none", + visibility: "visible", + }; + } + return { left: HIDDEN_BROWSER_WEBVIEW_OFFSET, top: HIDDEN_BROWSER_WEBVIEW_OFFSET, @@ -45,6 +68,6 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { height: hiddenSize.height, zIndex: -1, pointerEvents: "none", - visibility: renderingActive ? "visible" : "hidden", + visibility: keepPaintableWhenInactive ? "visible" : "hidden", }; } diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 2ace29ae5c415c4ea9add863c65ca1ef8651ef44..fd6ac25ceceda83a25d950e6ef700003e646edf8 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -149,6 +149,7 @@ vi.mock("~/state/use-atom-command", () => ({ vi.mock("~/browser/browserRecording", () => ({ findActiveBrowserRecordingRuntimeTabId: vi.fn(() => null), + isBrowserRecordingStartCancelledError: vi.fn(() => false), startBrowserRecording: vi.fn(), stopBrowserRecording: vi.fn(), useActiveBrowserRecordingTabIds: () => new Set(), diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 9a812923717ad9fbc73b4ba0999104a14f768e2e..063314863fec92d0692f18f0d86f2a8136acb1c1 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -55,6 +55,7 @@ import { ZoomIndicator } from "./ZoomIndicator"; import { AgentBrowserCursor } from "./AgentBrowserCursor"; import { findActiveBrowserRecordingRuntimeTabId, + isBrowserRecordingStartCancelledError, startBrowserRecording, stopBrowserRecording, useActiveBrowserRecordingTabIds, @@ -398,10 +399,12 @@ export function PreviewView({ } if (record) { void startBrowserRecording(runtimeTabId, threadRef, tabId).catch((error) => { + const description = error instanceof Error ? error.message : "An error occurred."; + if (isBrowserRecordingStartCancelledError(error)) return; toastManager.add({ type: "error", title: "Unable to start recording", - description: error instanceof Error ? error.message : "An error occurred.", + description, }); }); return; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index b784792783300e145f6415bb8af4ab0cd95c0538..7de366524713aa60f30071d33931f38064c27cb2 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -734,19 +734,6 @@ export const DesktopPreviewRecordingFrameSchema: Schema.Codec = - Schema.Struct({ - sourceId: Schema.String, - width: Schema.Int.check(Schema.isGreaterThan(0)), - height: Schema.Int.check(Schema.isGreaterThan(0)), - }); - export interface DesktopPreviewRecordingArtifact { id: string; tabId: string; @@ -1173,6 +1160,9 @@ export interface DesktopBridge { preview?: DesktopPreviewBridge; } +/** Renderer callback invoked by Electron with a fresh user gesture before display-media capture. */ +export const DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER = "__t3DesktopPreviewRecordingCapture"; + export interface DesktopPreviewBridge { createTab: (tabId: string, defaults?: DesktopPreviewTabDefaults) => Promise; closeTab: (tabId: string) => Promise; @@ -1228,7 +1218,7 @@ export interface DesktopPreviewBridge { close: (tabId: string) => Promise; }; recording: { - startScreencast: (tabId: string) => Promise; + startScreencast: (tabId: string) => Promise; stopScreencast: (tabId: string) => Promise; save: ( tabId: string, -- 2.54.0