authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-03 22:39:44-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-03 22:39:44-07:00
logbf5c146be914c9b596cdc45d343228271115c099
tree712cf357514ffd891f5cf4f684ab9058a5b7f609
parentee7b49d638e412590b09b415cc48399d51644b60
signature Signed by PGP key B5690EEEBB952194

fix(mobile): keep dictation running across navigation behind an edge pill (#15502)

Co-authored-by: Bil0000 <62337003+Bil0000@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

16 files changed, 1306 insertions(+), 305 deletions(-)

apps/mobile/src/App.tsx+16-12
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1import * as Linking from "expo-linking";1import * as Linking from "expo-linking";
2import * as SplashScreen from "expo-splash-screen";2import * as SplashScreen from "expo-splash-screen";
3import { useEffect } from "react";3import { useEffect } from "react";
4import { StatusBar, View } from "react-native";4import { StatusBar } from "react-native";
5import { GestureHandlerRootView } from "react-native-gesture-handler";5import { GestureHandlerRootView } from "react-native-gesture-handler";
6import { KeyboardProvider } from "react-native-keyboard-controller";6import { KeyboardProvider } from "react-native-keyboard-controller";
7import { SafeAreaProvider } from "react-native-safe-area-context";7import { SafeAreaProvider } from "react-native-safe-area-context";
...@@ -23,6 +23,8 @@ import { OverlayPortalHost } from "./components/OverlayPortal";...@@ -23,6 +23,8 @@ import { OverlayPortalHost } from "./components/OverlayPortal";
23import { shouldHandleAppLink } from "./lib/appLinking";23import { shouldHandleAppLink } from "./lib/appLinking";
24import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme";24import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme";
25import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator";25import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator";
26import { VoiceInputProvider } from "./features/voice-input/VoiceInputProvider";
27import { GlobalVoiceInputControl } from "./features/voice-input/GlobalVoiceInputControl";
2628
27import "../global.css";29import "../global.css";
2830
...@@ -76,22 +78,24 @@ function AppContent() {...@@ -76,22 +78,24 @@ function AppContent() {
76 <GestureHandlerRootView className="flex-1">78 <GestureHandlerRootView className="flex-1">
77 <KeyboardProvider statusBarTranslucent>79 <KeyboardProvider statusBarTranslucent>
78 <SafeAreaProvider>80 <SafeAreaProvider>
79 <StatusBar barStyle={themeAppearance === "dark" ? "light-content" : "dark-content"} />81 <VoiceInputProvider>
80 {/* The navigation theme drives the NATIVE header appearance: native-stack82 <StatusBar barStyle={themeAppearance === "dark" ? "light-content" : "dark-content"} />
83 {/* The navigation theme drives the NATIVE header appearance: native-stack
81 forwards `dark` as the nav bar's overrideUserInterfaceStyle. Without84 forwards `dark` as the nav bar's overrideUserInterfaceStyle. Without
82 this, React Navigation defaults to its light theme and every native85 this, React Navigation defaults to its light theme and every native
83 header (glass buttons, title, materials) is forced light even when86 header (glass buttons, title, materials) is forced light even when
84 the system is in dark mode. */}87 the system is in dark mode. */}
85 <View style={{ flex: 1 }}>88 <GlobalVoiceInputControl>
86 <IncomingShareProvider>89 <IncomingShareProvider>
87 <Navigation linking={appLinking} theme={navigationTheme} />90 <Navigation linking={appLinking} theme={navigationTheme} />
88 </IncomingShareProvider>91 </IncomingShareProvider>
89 <ConfirmDialogHost />92 <ConfirmDialogHost />
90 <ThreadArrangementHost />93 <ThreadArrangementHost />
91 </View>94 </GlobalVoiceInputControl>
92 {/* Anchored-menu overlays render here — in-window, so the95 {/* Anchored-menu overlays render here — in-window, so the
93 keyboard stays up while a dropdown is open. */}96 keyboard stays up while a dropdown is open. */}
94 <OverlayPortalHost />97 <OverlayPortalHost />
98 </VoiceInputProvider>
95 </SafeAreaProvider>99 </SafeAreaProvider>
96 </KeyboardProvider>100 </KeyboardProvider>
97 </GestureHandlerRootView>101 </GestureHandlerRootView>
apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx+36-20
...@@ -15,7 +15,6 @@ import {...@@ -15,7 +15,6 @@ import {
15import {15import {
16 useCallback,16 useCallback,
17 useEffect,17 useEffect,
18 useId,
19 useLayoutEffect,18 useLayoutEffect,
20 useMemo,19 useMemo,
21 useRef,20 useRef,
...@@ -63,6 +62,8 @@ import {...@@ -63,6 +62,8 @@ import {
63} from "./scheduledTaskDraft";62} from "./scheduledTaskDraft";
64import { settingsTargetsForProject } from "./settings-environment-filter.logic";63import { settingsTargetsForProject } from "./settings-environment-filter.logic";
65import { useScheduledTaskEditor } from "./scheduled-task-editor";64import { useScheduledTaskEditor } from "./scheduled-task-editor";
65import { scheduledTaskEditorSessionAtom } from "./scheduled-task-editor-state";
66import { appAtomRegistry } from "../../state/atom-registry";
66import {67import {
67 formatNextScheduledTaskRun,68 formatNextScheduledTaskRun,
68 formatScheduledTaskInterval,69 formatScheduledTaskInterval,
...@@ -372,14 +373,21 @@ export function SettingsScheduledTaskEditRouteScreen() {...@@ -372,14 +373,21 @@ export function SettingsScheduledTaskEditRouteScreen() {
372}373}
373374
374function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }) {375function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }) {
375 const { editor, setEditor, hasChanges, draftForEnvironment } = useScheduledTaskEditor();376 const {
377 editor,
378 voiceOwnerKey,
379 readEditor,
380 setEditor,
381 startEditor,
382 hasChanges,
383 draftForEnvironment,
384 } = useScheduledTaskEditor();
376 const { availableTargets } = useSettingsEnvironmentFilter();385 const { availableTargets } = useSettingsEnvironmentFilter();
377 const navigation = useNavigation();386 const navigation = useNavigation();
378 const insets = useSafeAreaInsets();387 const insets = useSafeAreaInsets();
379 const [saving, setSaving] = useState(false);388 const [saving, setSaving] = useState(false);
380 const [saved, setSaved] = useState(false);389 const [saved, setSaved] = useState(false);
381 const voiceOwnerId = useId();390 const ownerKey = editor ? `${voiceOwnerKey}:${editor.environmentId}` : null;
382 const ownerKey = editor ? `${voiceOwnerId}:${editor.environmentId}` : null;
383 const prompt = editor?.draft.prompt ?? "";391 const prompt = editor?.draft.prompt ?? "";
384 const [selectionState, setSelectionState] = useState<{392 const [selectionState, setSelectionState] = useState<{
385 readonly ownerKey: string | null;393 readonly ownerKey: string | null;
...@@ -402,32 +410,40 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }...@@ -402,32 +410,40 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }
402 );410 );
403 const voiceInput = useVoiceInputController({411 const voiceInput = useVoiceInputController({
404 ownerKey,412 ownerKey,
405 draftMessage: prompt,413 label: editor?.draft.title.trim() || title,
414 subscribeToDraftChanges: (onChange) =>
415 appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange),
406 selection,416 selection,
417 readDraftMessage: () => {
418 const current = readEditor();
419 return current?.environmentId === editor?.environmentId
420 ? (current?.draft.prompt ?? null)
421 : null;
422 },
407 onChangeSelection: setSelection,423 onChangeSelection: setSelection,
408 onChangeDraftMessage: setPrompt,424 onChangeDraftMessage: (text) => {
425 if (readEditor()?.environmentId === editor?.environmentId) setPrompt(text);
426 },
409 disabled: saving,427 disabled: saving,
410 });428 });
411 const preventRemove = !saved && (hasChanges || saving || voiceInput.isBusy);429 const preventRemove = !saved && (hasChanges || saving);
412 usePreventRemove(preventRemove, ({ data }) => {430 usePreventRemove(preventRemove, ({ data }) => {
413 if (saving) {431 if (saving) {
414 Alert.alert("Saving task", "Wait for the task to finish saving before leaving.");432 Alert.alert("Saving task", "Wait for the task to finish saving before leaving.");
415 return;433 return;
416 }434 }
417 Alert.alert(435 Alert.alert("Discard changes?", "Your unsaved changes will be lost.", [
418 "Discard changes?",436 { text: "Keep editing", style: "cancel" },
419 voiceInput.isBusy437 {
420 ? "Your dictation and unsaved changes will be lost."438 text: "Discard changes",
421 : "Your unsaved changes will be lost.",439 style: "destructive",
422 [440 onPress: () => {
423 { text: "Keep editing", style: "cancel" },441 if (voiceInput.isBusy) voiceInput.cancel();
424 {442 startEditor(null);
425 text: "Discard changes",443 navigation.dispatch(data.action);
426 style: "destructive",
427 onPress: () => navigation.dispatch(data.action),
428 },444 },
429 ],445 },
430 );446 ]);
431 });447 });
432 useEffect(() => {448 useEffect(() => {
433 if (!saved) return;449 if (!saved) return;
apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts created+57
...@@ -0,0 +1,57 @@
1import { EnvironmentId } from "@t3tools/contracts";
2import { afterEach, describe, expect, it } from "vite-plus/test";
3
4import { appAtomRegistry } from "../../state/atom-registry";
5import { createDraft } from "./scheduledTaskDraft";
6import {
7 readScheduledTaskEditor,
8 scheduledTaskEditorSessionAtom,
9 startScheduledTaskEditor,
10 updateScheduledTaskEditor,
11} from "./scheduled-task-editor-state";
12
13afterEach(() => appAtomRegistry.set(scheduledTaskEditorSessionAtom, null));
14
15describe("scheduled task voice draft", () => {
16 const editor = {
17 environmentId: EnvironmentId.make("environment"),
18 environmentLabel: "Environment",
19 draft: createDraft(null, null),
20 };
21
22 it("keeps the prompt outside the screen and retains its original change baseline", () => {
23 startScheduledTaskEditor(editor);
24 const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!;
25 const ownerKey = `scheduled-task:${original.id}`;
26 updateScheduledTaskEditor(
27 (current) =>
28 current && {
29 ...current,
30 draft: { ...current.draft, prompt: "spoken prompt" },
31 },
32 null,
33 );
34
35 expect(readScheduledTaskEditor(ownerKey, null)?.draft.prompt).toBe("spoken prompt");
36 expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.initial).toBe(editor);
37 });
38
39 it("rejects an old voice target when another task opens in the same environment", () => {
40 startScheduledTaskEditor(editor);
41 const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!;
42 startScheduledTaskEditor({ ...editor, draft: { ...editor.draft, prompt: "other task" } });
43
44 expect(readScheduledTaskEditor(`scheduled-task:${original.id}`, editor)).toBeNull();
45 expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current?.draft.prompt).toBe(
46 "other task",
47 );
48 });
49
50 it("does not restore the default prompt after the editor is discarded", () => {
51 startScheduledTaskEditor(null);
52 const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!;
53 expect(readScheduledTaskEditor(`scheduled-task:${session.id}`, editor)).toBeNull();
54 updateScheduledTaskEditor((current) => current, editor);
55 expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current).toBeNull();
56 });
57});
apps/mobile/src/features/settings/scheduled-task-editor-state.ts created+55
...@@ -0,0 +1,55 @@
1import type { EnvironmentId } from "@t3tools/contracts";
2import { Atom } from "effect/unstable/reactivity";
3
4import { appAtomRegistry } from "../../state/atom-registry";
5import type { ScheduledTaskDraft } from "./scheduledTaskDraft";
6
7export type ScheduledTaskEditor = {
8 readonly environmentId: EnvironmentId;
9 readonly environmentLabel: string;
10 readonly draft: ScheduledTaskDraft;
11};
12
13type ScheduledTaskEditorSession = {
14 readonly id: number;
15 readonly initial: ScheduledTaskEditor | null;
16 readonly current: ScheduledTaskEditor | null;
17};
18
19export const scheduledTaskEditorSessionAtom = Atom.make<ScheduledTaskEditorSession | null>(
20 null,
21).pipe(Atom.keepAlive);
22let nextEditorId = 0;
23
24export function startScheduledTaskEditor(next: ScheduledTaskEditor | null): void {
25 appAtomRegistry.set(scheduledTaskEditorSessionAtom, {
26 id: ++nextEditorId,
27 initial: next,
28 current: next,
29 });
30}
31
32export function updateScheduledTaskEditor(
33 update:
34 | ScheduledTaskEditor
35 | null
36 | ((current: ScheduledTaskEditor | null) => ScheduledTaskEditor | null),
37 defaultEditor: ScheduledTaskEditor | null,
38): void {
39 const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom);
40 const previous = session ? session.current : defaultEditor;
41 appAtomRegistry.set(scheduledTaskEditorSessionAtom, {
42 id: session?.id ?? 0,
43 initial: session ? session.initial : previous,
44 current: typeof update === "function" ? update(previous) : update,
45 });
46}
47
48export function readScheduledTaskEditor(
49 voiceOwnerKey: string,
50 defaultEditor: ScheduledTaskEditor | null,
51): ScheduledTaskEditor | null {
52 const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom);
53 if (`scheduled-task:${session?.id ?? 0}` !== voiceOwnerKey) return null;
54 return session ? session.current : defaultEditor;
55}
apps/mobile/src/features/settings/scheduled-task-editor.tsx+40-24
...@@ -1,10 +1,10 @@...@@ -1,10 +1,10 @@
1import { useAtomValue } from "@effect/atom-react";
1import type { EnvironmentId } from "@t3tools/contracts";2import type { EnvironmentId } from "@t3tools/contracts";
2import {3import {
3 createContext,4 createContext,
4 use,5 use,
5 useMemo,6 useMemo,
6 useCallback,7 useCallback,
7 useState,
8 type Dispatch,8 type Dispatch,
9 type ReactNode,9 type ReactNode,
10 type SetStateAction,10 type SetStateAction,
...@@ -20,14 +20,18 @@ import {...@@ -20,14 +20,18 @@ import {
20import { useSettingsEnvironmentFilter } from "./settings-environment-filter";20import { useSettingsEnvironmentFilter } from "./settings-environment-filter";
21import { settingsTargetsForProject } from "./settings-environment-filter.logic";21import { settingsTargetsForProject } from "./settings-environment-filter.logic";
2222
23type ScheduledTaskEditor = {23import {
24 readonly environmentId: EnvironmentId;24 scheduledTaskEditorSessionAtom,
25 readonly environmentLabel: string;25 startScheduledTaskEditor,
26 readonly draft: ScheduledTaskDraft;26 updateScheduledTaskEditor,
27};27 readScheduledTaskEditor,
28 type ScheduledTaskEditor,
29} from "./scheduled-task-editor-state";
2830
29const ScheduledTaskEditorContext = createContext<{31const ScheduledTaskEditorContext = createContext<{
30 readonly editor: ScheduledTaskEditor | null;32 readonly editor: ScheduledTaskEditor | null;
33 readonly voiceOwnerKey: string;
34 readonly readEditor: () => ScheduledTaskEditor | null;
31 readonly setEditor: Dispatch<SetStateAction<ScheduledTaskEditor | null>>;35 readonly setEditor: Dispatch<SetStateAction<ScheduledTaskEditor | null>>;
32 readonly startEditor: (editor: ScheduledTaskEditor | null) => void;36 readonly startEditor: (editor: ScheduledTaskEditor | null) => void;
33 readonly hasChanges: boolean;37 readonly hasChanges: boolean;
...@@ -76,12 +80,10 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R...@@ -76,12 +80,10 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R
76 : null,80 : null,
77 [defaultTarget, draftForEnvironment],81 [defaultTarget, draftForEnvironment],
78 );82 );
79 const [session, setSession] = useState<{83 const session = useAtomValue(scheduledTaskEditorSessionAtom);
80 readonly initial: ScheduledTaskEditor | null;84 const voiceOwnerKey = `scheduled-task:${session?.id ?? 0}`;
81 readonly current: ScheduledTaskEditor | null;85 const editor = session ? session.current : defaultEditor;
82 } | null>(null);86 const initial = session ? session.initial : defaultEditor;
83 const editor = session?.current ?? defaultEditor;
84 const initial = session?.initial ?? defaultEditor;
85 const hasChanges = Boolean(87 const hasChanges = Boolean(
86 initial &&88 initial &&
87 editor &&89 editor &&
...@@ -90,23 +92,37 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R...@@ -90,23 +92,37 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R
90 );92 );
91 const setEditor = useCallback<Dispatch<SetStateAction<ScheduledTaskEditor | null>>>(93 const setEditor = useCallback<Dispatch<SetStateAction<ScheduledTaskEditor | null>>>(
92 (update) => {94 (update) => {
93 setSession((current) => {95 updateScheduledTaskEditor(update, defaultEditor);
94 const previous = current?.current ?? defaultEditor;
95 return {
96 initial: current?.initial ?? previous,
97 current: typeof update === "function" ? update(previous) : update,
98 };
99 });
100 },96 },
101 [defaultEditor],97 [defaultEditor],
102 );98 );
103 const startEditor = useCallback((next: ScheduledTaskEditor | null) => {99 const startEditor = startScheduledTaskEditor;
104 setSession({ initial: next, current: next });
105 }, []);
106 const resetEditor = useCallback(() => startEditor(defaultEditor), [defaultEditor, startEditor]);100 const resetEditor = useCallback(() => startEditor(defaultEditor), [defaultEditor, startEditor]);
101 const readEditor = useCallback(
102 () => readScheduledTaskEditor(voiceOwnerKey, defaultEditor),
103 [defaultEditor, voiceOwnerKey],
104 );
107 const value = useMemo(105 const value = useMemo(
108 () => ({ editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment }),106 () => ({
109 [editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment],107 editor,
108 voiceOwnerKey,
109 readEditor,
110 setEditor,
111 startEditor,
112 hasChanges,
113 resetEditor,
114 draftForEnvironment,
115 }),
116 [
117 editor,
118 voiceOwnerKey,
119 readEditor,
120 setEditor,
121 startEditor,
122 hasChanges,
123 resetEditor,
124 draftForEnvironment,
125 ],
110 );126 );
111 return <ScheduledTaskEditorContext value={value}>{children}</ScheduledTaskEditorContext>;127 return <ScheduledTaskEditorContext value={value}>{children}</ScheduledTaskEditorContext>;
112}128}
apps/mobile/src/features/threads/NewTaskDraftScreen.tsx+4-1
...@@ -96,6 +96,7 @@ import {...@@ -96,6 +96,7 @@ import {
96 captureComposerDraftInsertion,96 captureComposerDraftInsertion,
97 countComposerDraftAttachmentsAfterSelection,97 countComposerDraftAttachmentsAfterSelection,
98 getComposerDraftSnapshot,98 getComposerDraftSnapshot,
99 composerDraftsAtom,
99 mergeComposerDraftContent,100 mergeComposerDraftContent,
100 restoreComposerDraftSnapshot,101 restoreComposerDraftSnapshot,
101 updateComposerDraftSettings,102 updateComposerDraftSettings,
...@@ -497,7 +498,9 @@ export function NewTaskDraftScreen(props: {...@@ -497,7 +498,9 @@ export function NewTaskDraftScreen(props: {
497 });498 });
498 const voiceInput = useVoiceInputController({499 const voiceInput = useVoiceInputController({
499 ownerKey: flow.draftKey,500 ownerKey: flow.draftKey,
500 draftMessage: flow.prompt,501 label: selectedProject ? `New task in ${selectedProject.title}` : "New task",
502 readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null),
503 subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
501 selection: composerMenu.selection,504 selection: composerMenu.selection,
502 disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting,505 disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting,
503 onChangeDraftMessage: flow.setPrompt,506 onChangeDraftMessage: flow.setPrompt,
apps/mobile/src/features/threads/ThreadComposer.tsx+9-3
...@@ -54,9 +54,13 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme";...@@ -54,9 +54,13 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme";
54import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/remoteRegistration";54import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/remoteRegistration";
55import { scopedThreadKey } from "../../lib/scopedEntities";55import { scopedThreadKey } from "../../lib/scopedEntities";
56import {56import {
57 getComposerDraftSnapshot,
58 composerDraftsAtom,
59 setComposerDraftText,
57 composerContextImportsAtom,60 composerContextImportsAtom,
58 countComposerDraftAttachmentsAfterSelection,61 countComposerDraftAttachmentsAfterSelection,
59} from "../../state/use-composer-drafts";62} from "../../state/use-composer-drafts";
63import { appAtomRegistry } from "../../state/atom-registry";
60import type { ComposerDocumentAttachment } from "../../lib/composerContext";64import type { ComposerDocumentAttachment } from "../../lib/composerContext";
61import { useProject, useThreadShells } from "../../state/entities";65import { useProject, useThreadShells } from "../../state/entities";
62import { scopeProjectRef } from "@t3tools/client-runtime/environment";66import { scopeProjectRef } from "@t3tools/client-runtime/environment";
...@@ -504,10 +508,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -504,10 +508,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
504 usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined,508 usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined,
505 });509 });
506 const voiceInput = useVoiceInputController({510 const voiceInput = useVoiceInputController({
507 ownerKey: composerOwnerKey,511 ownerKey: composerDraftKey,
508 draftMessage: props.draftMessage,512 label: props.selectedThread.title || "Untitled thread",
513 readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text,
514 subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
509 selection: composerMenu.selection,515 selection: composerMenu.selection,
510 onChangeDraftMessage: props.onChangeDraftMessage,516 onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text),
511 onChangeSelection: composerMenu.onSelectionChange,517 onChangeSelection: composerMenu.onSelectionChange,
512 });518 });
513 const voicePresentation = resolveVoiceComposerPresentation(519 const voicePresentation = resolveVoiceComposerPresentation(
apps/mobile/src/features/threads/ThreadDetailScreen.tsx+6-1
...@@ -132,6 +132,7 @@ import { ComposerQueuedEditBanner } from "./ComposerQueuedEdit";...@@ -132,6 +132,7 @@ import { ComposerQueuedEditBanner } from "./ComposerQueuedEdit";
132import { useThreadQueuedCount } from "./ThreadQueueControl";132import { useThreadQueuedCount } from "./ThreadQueueControl";
133import type { ThreadContentPresentation } from "./threadContentPresentation";133import type { ThreadContentPresentation } from "./threadContentPresentation";
134import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow";134import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow";
135import { useGlobalVoiceInput } from "../voice-input/VoiceInputProvider";
135136
136export interface ThreadDetailScreenProps {137export interface ThreadDetailScreenProps {
137 readonly worktreeSetup?: WorktreeSetupCardProps | null;138 readonly worktreeSetup?: WorktreeSetupCardProps | null;
...@@ -304,6 +305,7 @@ const USER_INPUT_TOGGLE_TIMING = {...@@ -304,6 +305,7 @@ const USER_INPUT_TOGGLE_TIMING = {
304305
305export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) {306export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) {
306 const navigation = useNavigation();307 const navigation = useNavigation();
308 const { session: voiceInputSession } = useGlobalVoiceInput();
307 const reportedModelSelection = useThreadReportedModelSelection({309 const reportedModelSelection = useThreadReportedModelSelection({
308 environmentId: props.environmentId,310 environmentId: props.environmentId,
309 threadId: props.selectedThread.id,311 threadId: props.selectedThread.id,
...@@ -1169,7 +1171,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread...@@ -1169,7 +1171,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
1169 >1171 >
1170 <ComposerQueuedEditBanner1172 <ComposerQueuedEditBanner
1171 saving={props.isSavingQueuedEdit}1173 saving={props.isSavingQueuedEdit}
1172 onCancel={props.onCancelQueuedRunEdit}1174 onCancel={() => {
1175 voiceInputSession.cancel(props.composerDraftKey);
1176 props.onCancelQueuedRunEdit();
1177 }}
1173 />1178 />
1174 </Animated.View>1179 </Animated.View>
1175 ) : null}1180 ) : null}
apps/mobile/src/features/voice-input/ComposerDictationControl.tsx+30-3
...@@ -157,8 +157,10 @@ export function ComposerDictationToolbar(props: {...@@ -157,8 +157,10 @@ export function ComposerDictationToolbar(props: {
157const WaveformBar = memo(function WaveformBar(props: {157const WaveformBar = memo(function WaveformBar(props: {
158 readonly audioLevels: SharedValue<number[]>;158 readonly audioLevels: SharedValue<number[]>;
159 readonly sampleIndex: number;159 readonly sampleIndex: number;
160 readonly height?: number;
160}) {161}) {
161 const { audioLevels, sampleIndex } = props;162 const { audioLevels, sampleIndex } = props;
163 const height = props.height ?? WAVEFORM_BAR_HEIGHT;
162 const animatedStyle = useAnimatedStyle(() => {164 const animatedStyle = useAnimatedStyle(() => {
163 const level = audioLevels.value[sampleIndex] ?? 0;165 const level = audioLevels.value[sampleIndex] ?? 0;
164 return {166 return {
...@@ -166,8 +168,7 @@ const WaveformBar = memo(function WaveformBar(props: {...@@ -166,8 +168,7 @@ const WaveformBar = memo(function WaveformBar(props: {
166 transform: [168 transform: [
167 {169 {
168 scaleY: withTiming(170 scaleY: withTiming(
169 (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) /171 (WAVEFORM_MIN_BAR_HEIGHT + level * (height - WAVEFORM_MIN_BAR_HEIGHT)) / height,
170 WAVEFORM_BAR_HEIGHT,
171 WAVEFORM_TIMING,172 WAVEFORM_TIMING,
172 ),173 ),
173 },174 },
...@@ -178,11 +179,37 @@ const WaveformBar = memo(function WaveformBar(props: {...@@ -178,11 +179,37 @@ const WaveformBar = memo(function WaveformBar(props: {
178 return (179 return (
179 <Animated.View180 <Animated.View
180 className="w-0.5 rounded-full bg-foreground"181 className="w-0.5 rounded-full bg-foreground"
181 style={[{ height: WAVEFORM_BAR_HEIGHT }, animatedStyle]}182 style={[{ height }, animatedStyle]}
182 />183 />
183 );184 );
184});185});
185186
187const COMPACT_WAVEFORM_BAR_COUNT = 8;
188const COMPACT_WAVEFORM_BAR_HEIGHT = 14;
189
190/** A fixed-size waveform of the latest samples, for tight spaces like the global dictation pill. */
191export const CompactVoiceWaveform = memo(function CompactVoiceWaveform(props: {
192 readonly audioLevels: SharedValue<number[]>;
193}) {
194 return (
195 <View
196 accessibilityElementsHidden
197 importantForAccessibility="no-hide-descendants"
198 className="flex-row items-center gap-0.5"
199 style={{ height: COMPACT_WAVEFORM_BAR_HEIGHT }}
200 >
201 {Array.from({ length: COMPACT_WAVEFORM_BAR_COUNT }, (_, index) => (
202 <WaveformBar
203 key={index}
204 audioLevels={props.audioLevels}
205 height={COMPACT_WAVEFORM_BAR_HEIGHT}
206 sampleIndex={VOICE_WAVEFORM_SAMPLE_COUNT - COMPACT_WAVEFORM_BAR_COUNT + index}
207 />
208 ))}
209 </View>
210 );
211});
212
186const VoiceWaveform = memo(function VoiceWaveform(props: {213const VoiceWaveform = memo(function VoiceWaveform(props: {
187 readonly audioLevels: SharedValue<number[]>;214 readonly audioLevels: SharedValue<number[]>;
188}) {215}) {
apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx created+259
...@@ -0,0 +1,259 @@
1import { useEffect, useRef, useState, type ReactNode, type RefObject } from "react";
2import {
3 ActivityIndicator,
4 Linking,
5 Platform,
6 Pressable,
7 useWindowDimensions,
8 View,
9} from "react-native";
10import Animated, {
11 Easing,
12 Extrapolation,
13 FadeIn,
14 interpolate,
15 ReduceMotion,
16 useAnimatedStyle,
17 useSharedValue,
18 withTiming,
19} from "react-native-reanimated";
20import { useSafeAreaInsets } from "react-native-safe-area-context";
21import { FullWindowOverlay } from "react-native-screens";
22
23import { AppText as Text } from "../../components/AppText";
24import { SymbolView } from "../../components/AppSymbol";
25import { useAndroidControlSizing } from "../../components/useAndroidControlSizing";
26import { cn } from "../../lib/cn";
27import { CompactVoiceWaveform } from "./ComposerDictationControl";
28import { useGlobalVoiceInput } from "./VoiceInputProvider";
29import { resolveVoiceComposerPresentation } from "./voiceInputPresentation";
30
31const PILL_HEIGHT = 36;
32const COLLAPSED_WIDTH = 64;
33const EXPANDED_MAX_WIDTH = 280;
34// Clears Home's floating bottom toolbar (56) and a collapsed thread composer (60).
35const IOS_BOTTOM_CHROME_CLEARANCE = 64;
36const MORPH_TIMING = {
37 duration: 260,
38 easing: Easing.out(Easing.cubic),
39 reduceMotion: ReduceMotion.System,
40} as const;
41const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System);
42
43/**
44 * Keeps an off-screen dictation reachable as a pill on the screen's trailing
45 * edge. Wraps the app content so any touch that starts outside the pill,
46 * including a scroll, collapses it without claiming the touch.
47 */
48export function GlobalVoiceInputControl(props: { readonly children: ReactNode }) {
49 const voice = useGlobalVoiceInput();
50 const collapseRef = useRef<(() => void) | null>(null);
51 const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds);
52 const visible =
53 presentation.statusLabel !== null &&
54 !(voice.ownerKey && voice.focusedOwners.has(voice.ownerKey));
55 // Mounted only while visible, so each dictation starts collapsed.
56 const pill = visible ? <EdgeDictationPill collapseRef={collapseRef} /> : null;
57 return (
58 <>
59 <View
60 className="flex-1"
61 onStartShouldSetResponderCapture={() => {
62 collapseRef.current?.();
63 return false;
64 }}
65 >
66 {props.children}
67 </View>
68 {pill && Platform.OS === "ios" ? <FullWindowOverlay>{pill}</FullWindowOverlay> : pill}
69 </>
70 );
71}
72
73function EdgeDictationPill(props: { readonly collapseRef: RefObject<(() => void) | null> }) {
74 const voice = useGlobalVoiceInput();
75 const insets = useSafeAreaInsets();
76 const { width } = useWindowDimensions();
77 const { fabSize } = useAndroidControlSizing();
78 const [expanded, setExpanded] = useState(false);
79 const progress = useSharedValue(0);
80 useEffect(() => {
81 progress.value = withTiming(expanded ? 1 : 0, MORPH_TIMING);
82 }, [expanded, progress]);
83 const { collapseRef } = props;
84 useEffect(() => {
85 if (!expanded) return;
86 collapseRef.current = () => setExpanded(false);
87 return () => {
88 collapseRef.current = null;
89 };
90 }, [collapseRef, expanded]);
91
92 const expandedWidth = Math.min(width - insets.left - insets.right - 32, EXPANDED_MAX_WIDTH);
93 // Each face keeps its final width and stays pinned to the edge, so the
94 // morphing pill reveals the expanded row instead of reflowing it.
95 const pillStyle = useAnimatedStyle(() => ({
96 width: interpolate(progress.value, [0, 1], [COLLAPSED_WIDTH, expandedWidth]),
97 }));
98 const collapsedStyle = useAnimatedStyle(() => ({
99 opacity: interpolate(progress.value, [0, 0.4], [1, 0], Extrapolation.CLAMP),
100 }));
101 const expandedStyle = useAnimatedStyle(() => ({
102 opacity: interpolate(progress.value, [0.3, 1], [0, 1], Extrapolation.CLAMP),
103 }));
104
105 // Sit just above the bottom chrome: the toolbar or composer on iOS, the
106 // new-thread FAB on Android.
107 const bottom =
108 Platform.OS === "android"
109 ? Math.max(insets.bottom, 16) + 16 + fabSize + 12
110 : Math.max(insets.bottom, 12) + IOS_BOTTOM_CHROME_CLEARANCE;
111
112 const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds);
113 const phase = voice.state.phase;
114 const isError = phase === "error";
115 const openSettings = isError && voice.state.errorAction === "settings";
116 const label = voice.label ?? "Draft";
117 const elapsedLabel = `${Math.floor(voice.elapsedSeconds / 60)}:${String(voice.elapsedSeconds % 60).padStart(2, "0")}`;
118
119 return (
120 <View pointerEvents="box-none" className="absolute inset-0">
121 <Animated.View
122 entering={ENTERING}
123 className="absolute rounded-l-full border border-r-0 border-border bg-card shadow-md shadow-black/10"
124 style={[{ bottom, right: insets.right, height: PILL_HEIGHT }, pillStyle]}
125 >
126 <View className="flex-1 overflow-hidden rounded-l-full">
127 <Animated.View
128 accessibilityElementsHidden={expanded}
129 importantForAccessibility={expanded ? "no-hide-descendants" : "auto"}
130 pointerEvents={expanded ? "none" : "auto"}
131 className="absolute bottom-0 right-0 top-0"
132 style={[{ width: COLLAPSED_WIDTH }, collapsedStyle]}
133 >
134 <Pressable
135 accessibilityLabel={`${presentation.statusLabel}, dictating into ${label}`}
136 accessibilityHint="Shows dictation controls"
137 accessibilityRole="button"
138 className="flex-1 flex-row items-center justify-center gap-1.5 active:opacity-70"
139 hitSlop={{ top: 6, bottom: 6, left: 6 }}
140 onPress={() => setExpanded(true)}
141 >
142 {isError ? (
143 <SymbolView
144 name="exclamationmark.circle"
145 size={16}
146 tintColorClassName="accent-danger-foreground"
147 type="monochrome"
148 />
149 ) : phase === "recording" ? (
150 <>
151 <View className="size-2 rounded-full bg-danger-foreground" />
152 <Text
153 className="text-xs text-foreground"
154 style={{ fontVariant: ["tabular-nums"] }}
155 >
156 {elapsedLabel}
157 </Text>
158 </>
159 ) : (
160 <ActivityIndicator size="small" colorClassName="accent-icon-muted" />
161 )}
162 </Pressable>
163 </Animated.View>
164
165 <Animated.View
166 accessibilityElementsHidden={!expanded}
167 importantForAccessibility={expanded ? "auto" : "no-hide-descendants"}
168 pointerEvents={expanded ? "auto" : "none"}
169 className="absolute bottom-0 right-0 top-0 flex-row items-center gap-1.5 pl-1 pr-1.5"
170 style={[{ width: expandedWidth }, expandedStyle]}
171 >
172 <Pressable
173 accessibilityLabel={isError ? "Dismiss voice input error" : "Cancel dictation"}
174 accessibilityRole="button"
175 className="size-[30px] items-center justify-center active:opacity-70"
176 onPress={voice.cancel}
177 >
178 <SymbolView
179 name="xmark"
180 size={13}
181 tintColorClassName="accent-icon-muted"
182 type="monochrome"
183 />
184 </Pressable>
185 <Pressable
186 accessibilityLabel={isError ? (presentation.statusLabel ?? label) : label}
187 accessibilityHint="Hides dictation controls"
188 accessibilityRole="button"
189 className="min-w-0 flex-1 justify-center self-stretch active:opacity-70"
190 onPress={() => setExpanded(false)}
191 >
192 <Text
193 className={cn(
194 "text-xs",
195 isError ? "text-danger-foreground" : "font-t3-medium text-foreground",
196 )}
197 numberOfLines={1}
198 >
199 {isError ? presentation.statusLabel : label}
200 </Text>
201 </Pressable>
202 {phase === "recording" ? (
203 <>
204 <CompactVoiceWaveform audioLevels={voice.audioLevels} />
205 <Text
206 className="text-xs text-foreground-muted"
207 style={{ fontVariant: ["tabular-nums"] }}
208 >
209 {elapsedLabel}
210 </Text>
211 <Pressable
212 accessibilityLabel="Finish dictation"
213 accessibilityRole="button"
214 className="size-[30px] items-center justify-center active:opacity-70"
215 onPress={voice.stop}
216 >
217 <View className="size-[24px] items-center justify-center rounded-full bg-primary">
218 <SymbolView
219 name="checkmark"
220 size={12}
221 weight="semibold"
222 tintColorClassName="accent-primary-foreground"
223 type="monochrome"
224 />
225 </View>
226 </Pressable>
227 </>
228 ) : isError ? (
229 voice.isAvailable ? (
230 <Pressable
231 accessibilityLabel={openSettings ? "Open microphone settings" : "Retry dictation"}
232 accessibilityRole="button"
233 className="size-[30px] items-center justify-center active:opacity-70"
234 onPress={() => {
235 if (!openSettings) {
236 void voice.session.retry();
237 return;
238 }
239 voice.cancel();
240 void Linking.openSettings();
241 }}
242 >
243 <SymbolView name="mic" size={16} tintColorClassName="accent-icon" />
244 </Pressable>
245 ) : null
246 ) : (
247 <View
248 accessibilityLabel={presentation.statusLabel ?? undefined}
249 className="size-[30px] items-center justify-center"
250 >
251 <ActivityIndicator size="small" colorClassName="accent-icon-muted" />
252 </View>
253 )}
254 </Animated.View>
255 </View>
256 </Animated.View>
257 </View>
258 );
259}
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx created+222
...@@ -0,0 +1,222 @@
1import {
2 RecordingPresets,
3 requestRecordingPermissionsAsync,
4 setAudioModeAsync,
5 setIsAudioActiveAsync,
6 useAudioRecorder,
7 type RecordingStatus,
8} from "expo-audio";
9import { File } from "expo-file-system";
10import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
11import {
12 createContext,
13 use,
14 useCallback,
15 useEffect,
16 useId,
17 useRef,
18 useState,
19 type ReactNode,
20} from "react";
21import { AppState } from "react-native";
22import { useSharedValue } from "react-native-reanimated";
23
24import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
25import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
26import {
27 VOICE_RECORDING_LIMIT_SECONDS,
28 voiceInputBlocksSubmission,
29 type VoiceInputState,
30} from "@t3tools/client-runtime/voice-input";
31import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
32import { VoiceInputSession } from "./voiceInputSession";
33
34const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
35const VOICE_METERING_INTERVAL_MS = 80;
36const VOICE_RECORDING_OPTIONS = {
37 ...RecordingPresets.HIGH_QUALITY,
38 isMeteringEnabled: true,
39};
40
41async function releaseVoiceRecordingAudio(): Promise<void> {
42 try {
43 await setAudioModeAsync({ allowsRecording: false });
44 } finally {
45 // Expo does not deactivate AVAudioSession when recording stops or its
46 // category changes. Explicit deactivation resumes interrupted app audio.
47 await setIsAudioActiveAsync(false);
48 }
49}
50
51async function configureVoiceRecordingAudio(): Promise<void> {
52 try {
53 await setAudioModeAsync({
54 allowsRecording: true,
55 interruptionMode: "doNotMix",
56 playsInSilentMode: true,
57 shouldPlayInBackground: false,
58 });
59 await setIsAudioActiveAsync(true);
60 } catch (error) {
61 try {
62 await releaseVoiceRecordingAudio();
63 } catch {
64 // Keep the setup error. The controller has not started a recorder yet.
65 }
66 throw error;
67 }
68}
69
70const VoiceInputContext = createContext<ReturnType<typeof useVoiceInputRuntime> | null>(null);
71
72export function VoiceInputProvider({ children }: { readonly children: ReactNode }) {
73 const runtime = useVoiceInputRuntime();
74 return <VoiceInputContext value={runtime}>{children}</VoiceInputContext>;
75}
76
77export function useGlobalVoiceInput() {
78 const context = use(VoiceInputContext);
79 if (!context) throw new Error("Voice input provider is missing.");
80 return context;
81}
82
83function useVoiceInputRuntime() {
84 const [{ state, ownerKey, label }, setState] = useState({
85 state: INITIAL_STATE,
86 ownerKey: null as string | null,
87 label: null as string | null,
88 });
89 const [focusedOwners, setFocusedOwners] = useState<ReadonlySet<string>>(new Set());
90 const setOwnerFocused = useCallback((key: string, focused: boolean) => {
91 setFocusedOwners((current) => {
92 const next = new Set(current);
93 if (focused) next.add(key);
94 else next.delete(key);
95 return next;
96 });
97 }, []);
98 const [elapsedSeconds, setElapsedSeconds] = useState(0);
99 const keepAwakeId = useId();
100 const keepAwakeSessionRef = useRef(0);
101 const elapsedSecondsRef = useRef(0);
102 const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
103 const audioLevels = useSharedValue(audioLevelsRef.current);
104 const sessionRef = useRef<VoiceInputSession | null>(null);
105 const handleRecorderStatus = useCallback((status: RecordingStatus) => {
106 sessionRef.current?.controller.handleRecorderStatus({
107 isFinished: status.isFinished,
108 hasError: status.hasError || status.mediaServicesDidReset === true,
109 error: status.error,
110 url: status.url,
111 });
112 }, []);
113 const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus);
114
115 if (!sessionRef.current) {
116 sessionRef.current = new VoiceInputSession({
117 recorder,
118 getTranscriber: getLocalVoiceTranscriber,
119 requestPermission: async () => {
120 const permission = await requestRecordingPermissionsAsync();
121 return { granted: permission.granted, canAskAgain: permission.canAskAgain };
122 },
123 configureRecording: configureVoiceRecordingAudio,
124 releaseRecording: releaseVoiceRecordingAudio,
125 deleteRecording: (uri) => new File(uri).delete(),
126 onStateChange: (nextState) =>
127 setState({
128 state: nextState,
129 ownerKey: sessionRef.current?.ownerKey ?? null,
130 label: sessionRef.current?.label ?? null,
131 }),
132 });
133 }
134
135 const session = sessionRef.current;
136 const controller = session.controller;
137
138 useEffect(() => {
139 const subscription = AppState.addEventListener("change", (nextState) => {
140 // iOS reports `inactive` while its permission dialog is open. Only the
141 // real background state cancels preparation; recorder status handles
142 // calls and route interruptions during capture.
143 if (nextState === "background") controller.appMovedToBackground();
144 });
145 return () => subscription.remove();
146 }, [controller]);
147
148 useEffect(() => () => controller.dispose(), [controller]);
149
150 useEffect(() => {
151 if (state.phase !== "recording") return;
152
153 const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`;
154 const activation = activateKeepAwakeAsync(tag);
155 void activation.catch(() => {});
156 return () => {
157 // Release after activation settles, even if the recording ends immediately.
158 void activation.then(() => deactivateKeepAwake(tag)).catch(() => {});
159 };
160 }, [keepAwakeId, state.phase]);
161
162 useEffect(() => {
163 if (state.phase !== "preparing" && state.phase !== "recording") return;
164
165 if (audioLevelsRef.current.some((level) => level !== 0)) {
166 audioLevelsRef.current = Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0);
167 audioLevels.value = audioLevelsRef.current;
168 }
169 if (elapsedSecondsRef.current !== 0) {
170 elapsedSecondsRef.current = 0;
171 setElapsedSeconds(0);
172 }
173 if (state.phase !== "recording") return;
174
175 const sampleRecording = () => {
176 if (controller.currentState.phase !== "recording") return;
177 const status = recorder.getStatus();
178 if (!status.isRecording) return;
179
180 const level = normalizeVoiceInputDecibels(status.metering);
181 const history = audioLevelsRef.current;
182 if (level !== 0 || history.some((sample) => sample !== 0)) {
183 const nextLevels = [...history.slice(1), level];
184 audioLevelsRef.current = nextLevels;
185 audioLevels.value = nextLevels;
186 }
187
188 const nextElapsedSeconds = Math.min(
189 VOICE_RECORDING_LIMIT_SECONDS,
190 Math.max(0, Math.floor(status.durationMillis / 1_000)),
191 );
192 if (nextElapsedSeconds !== elapsedSecondsRef.current) {
193 elapsedSecondsRef.current = nextElapsedSeconds;
194 setElapsedSeconds(nextElapsedSeconds);
195 }
196 };
197
198 sampleRecording();
199 const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS);
200 return () => clearInterval(intervalId);
201 }, [audioLevels, controller, recorder, state.phase]);
202
203 const stop = useCallback(() => controller.stop(), [controller]);
204 const cancel = useCallback(() => controller.cancel(), [controller]);
205
206 return {
207 // Store screenshots show the dictation button even on simulators, whose
208 // on-device transcription is unavailable.
209 isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null,
210 state,
211 audioLevels,
212 elapsedSeconds,
213 isBusy: voiceInputBlocksSubmission(state),
214 ownerKey,
215 label,
216 focusedOwners,
217 setOwnerFocused,
218 session,
219 stop,
220 cancel,
221 };
222}
apps/mobile/src/features/voice-input/useVoiceInputController.ts+53-209
...@@ -1,235 +1,79 @@...@@ -1,235 +1,79 @@
1import {
2 RecordingPresets,
3 requestRecordingPermissionsAsync,
4 setAudioModeAsync,
5 setIsAudioActiveAsync,
6 useAudioRecorder,
7 type RecordingStatus,
8} from "expo-audio";
9import { File } from "expo-file-system";
10import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
11import { useFocusEffect } from "@react-navigation/native";1import { useFocusEffect } from "@react-navigation/native";
12import { useCallback, useEffect, useId, useRef, useState } from "react";2import { useCallback, useEffect, useRef } from "react";
13import { AppState } from "react-native";
14import { useSharedValue } from "react-native-reanimated";
15
16import type { ComposerEditorSelection } from "../../components/ComposerEditor";
17import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
18import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
19import {3import {
20 VoiceInputController,
21 VOICE_RECORDING_LIMIT_SECONDS,
22 voiceInputBlocksSubmission,4 voiceInputBlocksSubmission,
23 voiceInputFreezesEditor,
24 type VoiceDraftSnapshot,
25 type VoiceInputState,5 type VoiceInputState,
26} from "@t3tools/client-runtime/voice-input";6} from "@t3tools/client-runtime/voice-input";
27import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
287
29const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };8import type { ComposerEditorSelection } from "../../components/ComposerEditor";
30const VOICE_METERING_INTERVAL_MS = 80;9import { useGlobalVoiceInput } from "./VoiceInputProvider";
31const VOICE_RECORDING_OPTIONS = {10import { createVoiceInputTarget } from "./voiceInputSession";
32 ...RecordingPresets.HIGH_QUALITY,
33 isMeteringEnabled: true,
34};
35
36async function releaseVoiceRecordingAudio(): Promise<void> {
37 try {
38 await setAudioModeAsync({ allowsRecording: false });
39 } finally {
40 // Expo does not deactivate AVAudioSession when recording stops or its
41 // category changes. Explicit deactivation resumes interrupted app audio.
42 await setIsAudioActiveAsync(false);
43 }
44}
4511
46async function configureVoiceRecordingAudio(): Promise<void> {12const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
47 try {
48 await setAudioModeAsync({
49 allowsRecording: true,
50 interruptionMode: "doNotMix",
51 playsInSilentMode: true,
52 shouldPlayInBackground: false,
53 });
54 await setIsAudioActiveAsync(true);
55 } catch (error) {
56 try {
57 await releaseVoiceRecordingAudio();
58 } catch {
59 // Keep the setup error. The controller has not started a recorder yet.
60 }
61 throw error;
62 }
63}
6413
65export function useVoiceInputController(input: {14export function useVoiceInputController(input: {
66 readonly ownerKey: string | null;15 readonly ownerKey: string | null;
67 readonly draftMessage: string;16 /** Shown by the global dictation pill when this composer is off screen. */
17 readonly label: string;
18 readonly readDraftMessage: () => string | null;
19 readonly subscribeToDraftChanges: (onChange: () => void) => () => void;
68 readonly selection: ComposerEditorSelection;20 readonly selection: ComposerEditorSelection;
69 readonly disabled?: boolean;21 readonly disabled?: boolean;
70 readonly onChangeDraftMessage: (value: string) => void;22 readonly onChangeDraftMessage: (value: string) => void;
71 readonly onChangeSelection: (selection: ComposerEditorSelection) => void;23 readonly onChangeSelection: (selection: ComposerEditorSelection) => void;
72}) {24}) {
73 const [state, setState] = useState<VoiceInputState>(INITIAL_STATE);25 const global = useGlobalVoiceInput();
74 const [elapsedSeconds, setElapsedSeconds] = useState(0);26 const { setOwnerFocused, session } = global;
75 const keepAwakeId = useId();27 const latestInput = useRef(input);
76 const keepAwakeSessionRef = useRef(0);28 latestInput.current = input;
77 const elapsedSecondsRef = useRef(0);29 const mounted = useRef(true);
78 const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
79 const audioLevels = useSharedValue(audioLevelsRef.current);
80 const controllerRef = useRef<VoiceInputController | null>(null);
81 const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage });
82 const revisionRef = useRef(0);
83 if (
84 previousDraftRef.current.ownerKey !== input.ownerKey ||
85 previousDraftRef.current.text !== input.draftMessage
86 ) {
87 previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage };
88 revisionRef.current += 1;
89 }
90 const latestInputRef = useRef(input);
91 latestInputRef.current = input;
92
93 const handleRecorderStatus = useCallback((status: RecordingStatus) => {
94 controllerRef.current?.handleRecorderStatus({
95 isFinished: status.isFinished,
96 hasError: status.hasError || status.mediaServicesDidReset === true,
97 error: status.error,
98 url: status.url,
99 });
100 }, []);
101 const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus);
102
103 if (!controllerRef.current) {
104 controllerRef.current = new VoiceInputController({
105 recorder,
106 getTranscriber: getLocalVoiceTranscriber,
107 requestPermission: async () => {
108 const permission = await requestRecordingPermissionsAsync();
109 return { granted: permission.granted, canAskAgain: permission.canAskAgain };
110 },
111 configureRecording: configureVoiceRecordingAudio,
112 releaseRecording: releaseVoiceRecordingAudio,
113 deleteRecording: (uri) => new File(uri).delete(),
114 readDraft: (): VoiceDraftSnapshot | null => {
115 const current = latestInputRef.current;
116 if (!current.ownerKey) return null;
117 return {
118 ownerKey: current.ownerKey,
119 text: current.draftMessage,
120 selection: current.selection,
121 revision: revisionRef.current,
122 };
123 },
124 commitDraft: (text, selection) => {
125 const current = latestInputRef.current;
126 current.onChangeSelection(selection);
127 current.onChangeDraftMessage(text);
128 },
129 onStateChange: setState,
130 });
131 }
132
133 const controller = controllerRef.current;
134 const previousOwnerRef = useRef(input.ownerKey);
135 useEffect(() => {
136 if (previousOwnerRef.current === input.ownerKey) return;
137 previousOwnerRef.current = input.ownerKey;
138 controller.ownerChanged();
139 }, [controller, input.ownerKey]);
140
141 useFocusEffect(
142 useCallback(
143 () => () => {
144 controller.dispose();
145 },
146 [controller],
147 ),
148 );
149
150 useEffect(() => {
151 const subscription = AppState.addEventListener("change", (nextState) => {
152 // iOS reports `inactive` while its permission dialog is open. Only the
153 // real background state cancels preparation; recorder status handles
154 // calls and route interruptions during capture.
155 if (nextState === "background") controller.appMovedToBackground();
156 });
157 return () => subscription.remove();
158 }, [controller]);
159
160 useEffect(() => () => controller.dispose(), [controller]);
161
162 useEffect(() => {30 useEffect(() => {
163 if (state.phase !== "recording") return;31 mounted.current = true;
164
165 const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`;
166 const activation = activateKeepAwakeAsync(tag);
167 void activation.catch(() => {});
168 return () => {32 return () => {
169 // Release after activation settles, even if the recording ends immediately.33 mounted.current = false;
170 void activation.then(() => deactivateKeepAwake(tag)).catch(() => {});
171 };
172 }, [keepAwakeId, state.phase]);
173
174 useEffect(() => {
175 if (state.phase !== "preparing" && state.phase !== "recording") return;
176
177 if (audioLevelsRef.current.some((level) => level !== 0)) {
178 audioLevelsRef.current = Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0);
179 audioLevels.value = audioLevelsRef.current;
180 }
181 if (elapsedSecondsRef.current !== 0) {
182 elapsedSecondsRef.current = 0;
183 setElapsedSeconds(0);
184 }
185 if (state.phase !== "recording") return;
186
187 const sampleRecording = () => {
188 if (controller.currentState.phase !== "recording") return;
189 const status = recorder.getStatus();
190 if (!status.isRecording) return;
191
192 const level = normalizeVoiceInputDecibels(status.metering);
193 const history = audioLevelsRef.current;
194 if (level !== 0 || history.some((sample) => sample !== 0)) {
195 const nextLevels = [...history.slice(1), level];
196 audioLevelsRef.current = nextLevels;
197 audioLevels.value = nextLevels;
198 }
199
200 const nextElapsedSeconds = Math.min(
201 VOICE_RECORDING_LIMIT_SECONDS,
202 Math.max(0, Math.floor(status.durationMillis / 1_000)),
203 );
204 if (nextElapsedSeconds !== elapsedSecondsRef.current) {
205 elapsedSecondsRef.current = nextElapsedSeconds;
206 setElapsedSeconds(nextElapsedSeconds);
207 }
208 };34 };
35 }, []);
20936
210 sampleRecording();37 useFocusEffect(
211 const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS);38 useCallback(() => {
212 return () => clearInterval(intervalId);39 const ownerKey = input.ownerKey;
213 }, [audioLevels, controller, recorder, state.phase]);40 if (!ownerKey) return;
41 setOwnerFocused(ownerKey, true);
42 return () => setOwnerFocused(ownerKey, false);
43 }, [input.ownerKey, setOwnerFocused]),
44 );
21445
215 const start = useCallback(() => {46 const start = useCallback(() => {
216 if (!latestInputRef.current.disabled) void controller.start();47 const captured = latestInput.current;
217 }, [controller]);48 if (!captured.ownerKey || captured.disabled) return;
218 const stop = useCallback(() => controller.stop(), [controller]);49 void session.start({
219 const cancel = useCallback(() => controller.cancel(), [controller]);50 ...createVoiceInputTarget(
22051 captured.ownerKey,
52 captured.readDraftMessage,
53 (text, selection) => {
54 captured.onChangeDraftMessage(text);
55 if (mounted.current && latestInput.current.ownerKey === captured.ownerKey) {
56 latestInput.current.onChangeSelection(selection);
57 }
58 },
59 captured.selection,
60 captured.subscribeToDraftChanges,
61 ),
62 label: captured.label,
63 });
64 }, [session]);
65 const state = global.ownerKey === input.ownerKey ? global.state : IDLE_STATE;
66 const isBusy = voiceInputBlocksSubmission(state);
221 return {67 return {
222 // Store screenshots show the dictation button even on simulators, whose68 isAvailable: global.isAvailable && (!global.isBusy || global.ownerKey === input.ownerKey),
223 // on-device transcription is unavailable.
224 isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null,
225 state,69 state,
226 audioLevels,70 audioLevels: global.audioLevels,
227 elapsedSeconds,71 elapsedSeconds: global.elapsedSeconds,
228 isBusy: voiceInputBlocksSubmission(state),72 isBusy,
229 freezesEditor: voiceInputFreezesEditor(state),73 freezesEditor: isBusy,
230 blocksSubmission: voiceInputBlocksSubmission(state),74 blocksSubmission: isBusy,
231 start,75 start,
232 stop,76 stop: global.stop,
233 cancel,77 cancel: global.cancel,
234 };78 };
235}79}
apps/mobile/src/features/voice-input/voiceInputSession.test.ts created+272
...@@ -0,0 +1,272 @@
1import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
2import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input";
3import { Atom, AtomRegistry } from "effect/unstable/reactivity";
4import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller";
5
6import {
7 createVoiceInputTarget,
8 VoiceInputSession,
9 type VoiceInputTarget,
10} from "./voiceInputSession";
11
12function createTarget(
13 ownerKey: string,
14 readText: () => string | null,
15 commit: VoiceInputTarget["commitDraft"],
16 selection: { start: number; end: number },
17) {
18 return createVoiceInputTarget(ownerKey, readText, commit, selection, () => () => {});
19}
20
21function createSession() {
22 const recorder = {
23 uri: "file:///voice.m4a",
24 prepareToRecordAsync: vi.fn(async () => {}),
25 record: vi.fn(),
26 stop: vi.fn(async () => {}),
27 };
28 const prepare = vi.fn(async (): Promise<PreparedVoiceTranscription> => ({
29 locale: "en-US",
30 transcribe: async () => "spoken text",
31 }));
32 const session = new VoiceInputSession({
33 recorder,
34 getTranscriber: () => ({ prepare }),
35 requestPermission: async () => ({ granted: true, canAskAgain: true }),
36 configureRecording: async () => {},
37 releaseRecording: vi.fn(async () => {}),
38 deleteRecording: vi.fn(),
39 onStateChange: vi.fn(),
40 });
41 return { session, recorder, prepare };
42}
43
44describe("global voice input", () => {
45 beforeEach(() => resetVoiceInputGlobalsForTests());
46
47 it.each([
48 { selection: { start: 6, end: 11 }, expected: "hello spoken text", cursor: 17 },
49 { selection: { start: 6, end: 6 }, expected: "hello spoken text world", cursor: 18 },
50 ])(
51 "keeps the starting selection $selection after navigation",
52 async ({ selection, expected, cursor }) => {
53 const { session } = createSession();
54 const commit = vi.fn();
55 await session.start(createTarget("first", () => "hello world", commit, selection));
56 await session.start(
57 createTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }),
58 );
59 await session.controller.stop();
60 expect(commit).toHaveBeenCalledWith(expected, {
61 start: cursor,
62 end: cursor,
63 });
64 },
65 );
66
67 it("appends to the starting draft after its screen leaves and another draft opens", async () => {
68 const { session, recorder } = createSession();
69 const drafts = new Map([
70 ["first", "original prompt"],
71 ["second", "other prompt"],
72 ]);
73 let visibleDraft = "first";
74 const targetKey = visibleDraft;
75 await session.start(
76 createTarget(
77 targetKey,
78 () => drafts.get(targetKey) ?? null,
79 (text) => drafts.set(targetKey, text),
80 { start: 15, end: 15 },
81 ),
82 );
83 visibleDraft = "second";
84 expect(session.controller.currentState.phase).toBe("recording");
85 expect(recorder.stop).not.toHaveBeenCalled();
86 await session.controller.stop();
87
88 expect(drafts.get("first")).toBe("original prompt spoken text");
89 expect(drafts.get(visibleDraft)).toBe("other prompt");
90 expect(session.controller.currentState.phase).toBe("idle");
91 });
92
93 it.each(["preparing", "recording", "transcribing"] as const)(
94 "keeps one recorder and its original target during %s",
95 async (phase) => {
96 const preparation = Promise.withResolvers<PreparedVoiceTranscription>();
97 const preparationEntered = Promise.withResolvers<void>();
98 const transcription = Promise.withResolvers<string>();
99 const transcriptionEntered = Promise.withResolvers<void>();
100 const { session, recorder, prepare } = createSession();
101 prepare.mockImplementationOnce(() => {
102 preparationEntered.resolve();
103 return preparation.promise;
104 });
105 const firstCommit = vi.fn();
106 const secondCommit = vi.fn();
107 const starting = session.start(
108 createTarget("first", () => "first", firstCommit, { start: 5, end: 5 }),
109 );
110 await preparationEntered.promise;
111 let stopping: Promise<void> | null = null;
112 if (phase !== "preparing") {
113 preparation.resolve({
114 locale: "en-US",
115 transcribe: () => {
116 transcriptionEntered.resolve();
117 return transcription.promise;
118 },
119 });
120 await starting;
121 }
122 if (phase === "transcribing") {
123 stopping = session.controller.stop();
124 await transcriptionEntered.promise;
125 }
126 await session.start(
127 createTarget("second", () => "second", secondCommit, { start: 6, end: 6 }),
128 );
129 expect(session.ownerKey).toBe("first");
130 expect(session.controller.currentState.phase).toBe(phase);
131 expect(prepare).toHaveBeenCalledTimes(1);
132 preparation.resolve({ locale: "en-US", transcribe: async () => "spoken text" });
133 await starting;
134 transcription.resolve("spoken text");
135 await (stopping ?? session.controller.stop());
136 expect(recorder.record).toHaveBeenCalledTimes(1);
137 expect(firstCommit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 });
138 expect(secondCommit).not.toHaveBeenCalled();
139 },
140 );
141
142 it.each(["changed", "removed"] as const)(
143 "does not overwrite a %s starting draft",
144 async (change) => {
145 const { session } = createSession();
146 let text: string | null = "first";
147 const commit = vi.fn();
148 await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 }));
149 text = change === "removed" ? null : "edited prompt";
150 await session.controller.stop();
151 expect(commit).not.toHaveBeenCalled();
152 expect(session.controller.currentState.error).toContain("draft changed");
153 },
154 );
155
156 it("finishes the original draft at the recording limit while it is off screen", async () => {
157 const { session, recorder } = createSession();
158 const commit = vi.fn();
159 await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 }));
160 await session.controller.handleRecorderStatus({
161 isFinished: true,
162 hasError: false,
163 error: null,
164 url: recorder.uri,
165 });
166 expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 });
167 expect(session.controller.currentState.phase).toBe("idle");
168 });
169
170 it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => {
171 const registry = AtomRegistry.make();
172 const draft = Atom.make("hello world");
173 const unsubscribe = vi.fn();
174 const { session } = createSession();
175 const commit = vi.fn();
176 const target = createVoiceInputTarget(
177 "first",
178 () => registry.get(draft),
179 commit,
180 { start: 6, end: 6 },
181 (onChange) => {
182 const stop = registry.subscribe(draft, onChange);
183 return () => {
184 stop();
185 unsubscribe();
186 };
187 },
188 );
189 await session.start(target);
190 registry.set(draft, "changed");
191 registry.set(draft, "hello world");
192 await session.controller.stop();
193 expect(commit).not.toHaveBeenCalled();
194 expect(session.controller.currentState.error).toContain("draft changed");
195 expect(unsubscribe).toHaveBeenCalledTimes(1);
196 registry.dispose();
197 });
198
199 it("stops recording when its queued edit is discarded", async () => {
200 const { session, recorder } = createSession();
201 const commit = vi.fn();
202 let draft: string | null = "queued prompt";
203 const ownerKey = "thread~queued-edit~run";
204 await session.start(createTarget(ownerKey, () => draft, commit, { start: 13, end: 13 }));
205 session.cancel(ownerKey);
206 draft = null;
207 await session.controller.stop();
208 expect(recorder.stop).toHaveBeenCalledTimes(1);
209 expect(session.controller.currentState).toEqual({
210 phase: "idle",
211 error: null,
212 errorAction: null,
213 });
214 expect(commit).not.toHaveBeenCalled();
215 });
216
217 it.each(["complete", "cancel"] as const)(
218 "releases draft observation after %s",
219 async (finish) => {
220 const { session } = createSession();
221 const unsubscribe = vi.fn();
222 const subscribe = vi.fn(() => unsubscribe);
223 await session.start(
224 createVoiceInputTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }, subscribe),
225 );
226 expect(subscribe).toHaveBeenCalledTimes(1);
227 if (finish === "complete") await session.controller.stop();
228 else session.cancel("first");
229 expect(unsubscribe).toHaveBeenCalledTimes(1);
230 },
231 );
232
233 it("keeps another prompt's recording when a queued edit is discarded", async () => {
234 const { session, recorder } = createSession();
235 const commit = vi.fn();
236 await session.start(createTarget("other prompt", () => "hello", commit, { start: 5, end: 5 }));
237 session.cancel("thread~queued-edit~run");
238 expect(recorder.stop).not.toHaveBeenCalled();
239 expect(session.controller.currentState.phase).toBe("recording");
240 await session.controller.stop();
241 expect(commit).toHaveBeenCalledWith("hello spoken text", { start: 17, end: 17 });
242 });
243
244 it("waits for canceled native work before starting a recording for another draft", async () => {
245 const preparation = Promise.withResolvers<PreparedVoiceTranscription>();
246 const preparationEntered = Promise.withResolvers<void>();
247 const { session, recorder, prepare } = createSession();
248 prepare.mockImplementationOnce(() => {
249 preparationEntered.resolve();
250 return preparation.promise;
251 });
252 const oldCommit = vi.fn();
253 const nextCommit = vi.fn();
254 const firstStart = session.start(
255 createTarget("first", () => "first", oldCommit, { start: 5, end: 5 }),
256 );
257 await preparationEntered.promise;
258 session.cancel("first");
259 const nextStart = session.start(
260 createTarget("second", () => "second", nextCommit, { start: 6, end: 6 }),
261 );
262 expect(prepare).toHaveBeenCalledTimes(1);
263 expect(recorder.record).not.toHaveBeenCalled();
264 preparation.resolve({ locale: "en-US", transcribe: async () => "old transcript" });
265 await firstStart;
266 await nextStart;
267 await session.controller.stop();
268 expect(recorder.record).toHaveBeenCalledTimes(1);
269 expect(oldCommit).not.toHaveBeenCalled();
270 expect(nextCommit).toHaveBeenCalledWith("second spoken text", { start: 18, end: 18 });
271 });
272});
apps/mobile/src/features/voice-input/voiceInputSession.ts created+88
...@@ -0,0 +1,88 @@
1import {
2 VoiceInputController,
3 voiceInputBlocksSubmission,
4 type VoiceDraftSnapshot,
5 type VoiceInputControllerDependencies,
6} from "@t3tools/client-runtime/voice-input";
7
8export type VoiceInputTarget = {
9 readonly ownerKey: string;
10 /** Names the draft being dictated into while its composer is off screen. */
11 readonly label?: string;
12 readonly readDraft: () => VoiceDraftSnapshot | null;
13 readonly commitDraft: VoiceInputControllerDependencies["commitDraft"];
14 readonly subscribe: () => () => void;
15};
16
17export function createVoiceInputTarget(
18 ownerKey: string,
19 readText: () => string | null,
20 commitDraft: VoiceInputTarget["commitDraft"],
21 selection: VoiceDraftSnapshot["selection"],
22 subscribeToChanges: (onChange: () => void) => () => void,
23): VoiceInputTarget {
24 let revision = 0;
25 return {
26 ownerKey,
27 readDraft: () => {
28 const text = readText();
29 if (text === null) return null;
30 return { ownerKey, text, selection, revision };
31 },
32 commitDraft,
33 subscribe: () => {
34 let previousText = readText();
35 return subscribeToChanges(() => {
36 const text = readText();
37 if (text !== previousText) {
38 previousText = text;
39 revision += 1;
40 }
41 });
42 },
43 };
44}
45
46export class VoiceInputSession {
47 readonly controller: VoiceInputController;
48 private target: VoiceInputTarget | null = null;
49 private unsubscribe: (() => void) | null = null;
50
51 constructor(dependencies: Omit<VoiceInputControllerDependencies, "readDraft" | "commitDraft">) {
52 this.controller = new VoiceInputController({
53 ...dependencies,
54 onStateChange: (state) => {
55 if (!voiceInputBlocksSubmission(state)) {
56 this.unsubscribe?.();
57 this.unsubscribe = null;
58 }
59 dependencies.onStateChange(state);
60 },
61 readDraft: () => this.target?.readDraft() ?? null,
62 commitDraft: (text, selection) => this.target?.commitDraft(text, selection),
63 });
64 }
65
66 get ownerKey(): string | null {
67 return this.target?.ownerKey ?? null;
68 }
69
70 get label(): string | null {
71 return this.target?.label ?? null;
72 }
73
74 cancel(ownerKey: string | null): void {
75 if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel();
76 }
77
78 retry(): Promise<void> {
79 return this.target ? this.start(this.target) : Promise.resolve();
80 }
81
82 start(target: VoiceInputTarget): Promise<void> {
83 if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve();
84 this.target = target;
85 this.unsubscribe = target.subscribe();
86 return this.controller.start();
87 }
88}
packages/client-runtime/src/voice-input/controller.test.ts+115-10
...@@ -298,11 +298,21 @@ describe("VoiceInputController", () => {...@@ -298,11 +298,21 @@ describe("VoiceInputController", () => {
298 });298 });
299299
300 it.each(["cancel", "dispose", "ownerChanged"] as const)(300 it.each(["cancel", "dispose", "ownerChanged"] as const)(
301 "holds the session after %s until non-abortable transcription settles",301 "queues the next recording after %s until transcription and audio cleanup settle",
302 async (action) => {302 async (action) => {
303 const transcription = deferred<string>();303 const transcription = deferred<string>();
304 const transcriptionEntered = deferred<AbortSignal>();304 const transcriptionEntered = deferred<AbortSignal>();
305 const audioRelease = deferred<void>();
306 const audioReleaseEntered = deferred<void>();
307 const releaseRecording = vi
308 .fn<() => Promise<void>>()
309 .mockRejectedValueOnce(new Error("busy"))
310 .mockImplementationOnce(() => {
311 audioReleaseEntered.resolve(undefined);
312 return audioRelease.promise;
313 });
305 const harness = createHarness({314 const harness = createHarness({
315 releaseRecording,
306 getTranscriber: () => ({316 getTranscriber: () => ({
307 prepare: async () =>317 prepare: async () =>
308 preparedTranscription((_uri, { signal }) => {318 preparedTranscription((_uri, { signal }) => {
...@@ -321,20 +331,31 @@ describe("VoiceInputController", () => {...@@ -321,20 +331,31 @@ describe("VoiceInputController", () => {
321 harness.controller[action]();331 harness.controller[action]();
322 expect(signal.aborted).toBe(true);332 expect(signal.aborted).toBe(true);
323333
324 const next = createHarness();334 const prepare = vi.fn(async () => preparedTranscription());
325 await next.controller.start();335 const next = createHarness({ getTranscriber: () => ({ prepare }) });
326 expect(next.controller.currentState.error).toContain("already active");336 const nextStart = next.controller.start();
337 expect(next.controller.currentState).toEqual({
338 phase: "preparing",
339 error: null,
340 errorAction: null,
341 });
342 expect(prepare).not.toHaveBeenCalled();
327 expect(next.recorder.record).not.toHaveBeenCalled();343 expect(next.recorder.record).not.toHaveBeenCalled();
328344
329 transcription.resolve("late text");345 transcription.resolve("late text");
346 await audioReleaseEntered.promise;
347 expect(prepare).not.toHaveBeenCalled();
348 expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled();
349 audioRelease.resolve(undefined);
330 await stopping;350 await stopping;
351 await nextStart;
331352
332 expect(harness.commits).toEqual([]);353 expect(harness.commits).toEqual([]);
333 expect(harness.deleted).toEqual(["file:///voice.m4a"]);354 expect(harness.deleted).toEqual(["file:///voice.m4a"]);
334 expect(harness.controller.currentState.phase).toBe("idle");355 expect(harness.controller.currentState.phase).toBe("idle");
335356
336 await next.controller.start();
337 expect(next.controller.currentState.phase).toBe("recording");357 expect(next.controller.currentState.phase).toBe("recording");
358 expect(prepare).toHaveBeenCalledTimes(1);
338 await next.controller.interruptRecording();359 await next.controller.interruptRecording();
339 },360 },
340 );361 );
...@@ -431,7 +452,7 @@ describe("VoiceInputController", () => {...@@ -431,7 +452,7 @@ describe("VoiceInputController", () => {
431 expect(harness.controller.currentState.error).toContain("draft changed");452 expect(harness.controller.currentState.error).toContain("draft changed");
432 });453 });
433454
434 it("keeps the app-wide session locked until canceled preparation settles", async () => {455 it("queues the next recording until canceled preparation settles", async () => {
435 const preparation = deferred<PreparedVoiceTranscription>();456 const preparation = deferred<PreparedVoiceTranscription>();
436 const preparationEntered = deferred<AbortSignal>();457 const preparationEntered = deferred<AbortSignal>();
437 const first = createHarness({458 const first = createHarness({
...@@ -447,21 +468,105 @@ describe("VoiceInputController", () => {...@@ -447,21 +468,105 @@ describe("VoiceInputController", () => {
447 first.controller.cancel();468 first.controller.cancel();
448 expect(signal.aborted).toBe(true);469 expect(signal.aborted).toBe(true);
449470
450 const blocked = createHarness();471 const next = createHarness();
451 await blocked.controller.start();472 const nextStart = next.controller.start();
452 expect(blocked.controller.currentState.error).toContain("already active");473 expect(next.controller.currentState.phase).toBe("preparing");
474 expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled();
453475
454 preparation.resolve(preparedTranscription());476 preparation.resolve(preparedTranscription());
455 await firstStart;477 await firstStart;
456 expect(first.recorder.record).not.toHaveBeenCalled();478 expect(first.recorder.record).not.toHaveBeenCalled();
457 blocked.controller.cancel();479 await nextStart;
480 expect(next.controller.currentState.phase).toBe("recording");
481 await next.controller.interruptRecording();
482 });
483
484 it("still rejects a second start while the first recording is active", async () => {
485 const first = createHarness();
486 await first.controller.start();
487 const next = createHarness();
488 await next.controller.start();
489 expect(next.controller.currentState.error).toContain("already active");
490 expect(next.recorder.record).not.toHaveBeenCalled();
491 await first.controller.interruptRecording();
492 });
493
494 it("does not start a canceled waiter or let it release another recording", async () => {
495 const preparation = deferred<PreparedVoiceTranscription>();
496 const preparationEntered = deferred<void>();
497 const first = createHarness({
498 getTranscriber: () => ({
499 prepare: () => {
500 preparationEntered.resolve(undefined);
501 return preparation.promise;
502 },
503 }),
504 });
505 const firstStart = first.controller.start();
506 await preparationEntered.promise;
507 first.controller.dispose();
508
509 const next = createHarness();
510 const canceledStart = next.controller.start();
511 next.controller.cancel();
512 const nextStart = next.controller.start();
513 preparation.resolve(preparedTranscription());
514 await Promise.all([firstStart, canceledStart, nextStart]);
515
516 expect(next.recorder.record).toHaveBeenCalledTimes(1);
517 expect(next.deleted).toEqual([]);
518 const blocked = createHarness();
519 await blocked.controller.start();
520 expect(blocked.controller.currentState.error).toContain("already active");
521 await next.controller.interruptRecording();
522 });
458523
524 it("releases an abandoned recording even if its native recorder was already disposed", async () => {
525 const harness = createHarness();
526 await harness.controller.start();
527 harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released"));
528 Object.defineProperty(harness.recorder, "uri", {
529 get() {
530 throw new Error("recorder released");
531 },
532 });
533
534 await expect(harness.controller.interruptRecording()).resolves.toBeUndefined();
535 expect(harness.deleted).toEqual(["file:///voice.m4a"]);
459 const next = createHarness();536 const next = createHarness();
460 await next.controller.start();537 await next.controller.start();
461 expect(next.controller.currentState.phase).toBe("recording");538 expect(next.controller.currentState.phase).toBe("recording");
462 await next.controller.interruptRecording();539 await next.controller.interruptRecording();
463 });540 });
464541
542 it("keeps the new abort controller when restarting the same composer after cancellation", async () => {
543 const transcription = deferred<string>();
544 const transcriptionEntered = deferred<void>();
545 const prepare = vi
546 .fn<VoiceTranscriber["prepare"]>()
547 .mockResolvedValueOnce(
548 preparedTranscription(() => {
549 transcriptionEntered.resolve(undefined);
550 return transcription.promise;
551 }),
552 )
553 .mockResolvedValue(preparedTranscription());
554 const harness = createHarness({ getTranscriber: () => ({ prepare }) });
555 await harness.controller.start();
556 const stopping = harness.controller.stop();
557 await transcriptionEntered.promise;
558 harness.controller.cancel();
559 const restarting = harness.controller.start();
560 transcription.resolve("late text");
561 await Promise.all([stopping, restarting]);
562 await harness.controller.stop();
563
564 expect(harness.commits).toEqual([
565 { text: "hello new text", selection: { start: 14, end: 14 } },
566 ]);
567 expect(harness.controller.currentState.phase).toBe("idle");
568 });
569
465 it("does not start the microphone for an owner that changed during preparation", async () => {570 it("does not start the microphone for an owner that changed during preparation", async () => {
466 const preparation = deferred<PreparedVoiceTranscription>();571 const preparation = deferred<PreparedVoiceTranscription>();
467 const preparationEntered = deferred<void>();572 const preparationEntered = deferred<void>();
packages/client-runtime/src/voice-input/controller.ts+44-22
...@@ -121,18 +121,28 @@ export function resolveTranscriptCommit(...@@ -121,18 +121,28 @@ export function resolveTranscriptCommit(
121 };121 };
122}122}
123123
124let activeSession: symbol | null = null;124type VoiceInputSession = {
125 abandoned: boolean;
126 readonly released: Promise<void>;
127 readonly release: () => void;
128};
129
130let activeSession: VoiceInputSession | null = null;
125let activeTranscriptionOperation: Promise<unknown> | null = null;131let activeTranscriptionOperation: Promise<unknown> | null = null;
126132
127function acquireSession(): symbol | null {133function acquireSession(): VoiceInputSession | null {
128 if (activeSession) return null;134 if (activeSession) return null;
129 const token = Symbol("voice-input-session");135 const { promise, resolve } = Promise.withResolvers<void>();
136 const token = { abandoned: false, released: promise, release: resolve };
130 activeSession = token;137 activeSession = token;
131 return token;138 return token;
132}139}
133140
134function releaseSession(token: symbol | null): void {141function releaseSession(token: VoiceInputSession | null): void {
135 if (token && activeSession === token) activeSession = null;142 if (token && activeSession === token) {
143 activeSession = null;
144 token.release();
145 }
136}146}
137147
138async function runTranscriptionOperation<T>(operation: () => Promise<T>): Promise<T> {148async function runTranscriptionOperation<T>(operation: () => Promise<T>): Promise<T> {
...@@ -177,7 +187,7 @@ export class VoiceInputController {...@@ -177,7 +187,7 @@ export class VoiceInputController {
177 private readonly dependencies: VoiceInputControllerDependencies;187 private readonly dependencies: VoiceInputControllerDependencies;
178 private state: VoiceInputState = IDLE_STATE;188 private state: VoiceInputState = IDLE_STATE;
179 private operationToken = 0;189 private operationToken = 0;
180 private sessionToken: symbol | null = null;190 private sessionToken: VoiceInputSession | null = null;
181 private transcription: PreparedVoiceTranscription | null = null;191 private transcription: PreparedVoiceTranscription | null = null;
182 private transcriptionAbortController: AbortController | null = null;192 private transcriptionAbortController: AbortController | null = null;
183 private capturedDraft: VoiceDraftSnapshot | null = null;193 private capturedDraft: VoiceDraftSnapshot | null = null;
...@@ -201,19 +211,28 @@ export class VoiceInputController {...@@ -201,19 +211,28 @@ export class VoiceInputController {
201 this.setError("This draft is no longer available.", "retry");211 this.setError("This draft is no longer available.", "retry");
202 return;212 return;
203 }213 }
204 const sessionToken = acquireSession();214 let sessionToken: VoiceInputSession | null = null;
205 if (!sessionToken) {
206 this.setError("Another voice recording is already active.", "retry");
207 return;
208 }
209
210 this.sessionToken = sessionToken;
211 const operationToken = ++this.operationToken;215 const operationToken = ++this.operationToken;
212 const abortController = new AbortController();
213 this.transcriptionAbortController = abortController;
214 this.setState({ phase: "preparing", error: null, errorAction: null });216 this.setState({ phase: "preparing", error: null, errorAction: null });
215217
216 try {218 try {
219 for (
220 let previousSession = activeSession;
221 previousSession?.abandoned;
222 previousSession = activeSession
223 ) {
224 await previousSession.released;
225 if (!this.isCurrent(operationToken)) return;
226 }
227 sessionToken = acquireSession();
228 if (!sessionToken) {
229 this.setError("Another voice recording is already active.", "retry");
230 return;
231 }
232 this.sessionToken = sessionToken;
233 const abortController = new AbortController();
234 this.transcriptionAbortController = abortController;
235
217 const transcriber = this.dependencies.getTranscriber();236 const transcriber = this.dependencies.getTranscriber();
218 if (!transcriber) {237 if (!transcriber) {
219 this.setError("Voice transcription is not available.", null);238 this.setError("Voice transcription is not available.", null);
...@@ -260,10 +279,12 @@ export class VoiceInputController {...@@ -260,10 +279,12 @@ export class VoiceInputController {
260 if (this.isCurrent(operationToken))279 if (this.isCurrent(operationToken))
261 this.setError("Could not start voice recording.", "retry");280 this.setError("Could not start voice recording.", "retry");
262 } finally {281 } finally {
263 if (this.isCurrent(operationToken) && this.state.phase === "error") {282 if (sessionToken && this.sessionToken === sessionToken) {
264 await this.releaseResources();283 if (this.isCurrent(operationToken) && this.state.phase === "error") {
265 } else if (!this.isCurrent(operationToken) && !this.finishing) {284 await this.releaseResources();
266 await this.releaseResources();285 } else if (!this.isCurrent(operationToken) && !this.finishing) {
286 await this.releaseResources();
287 }
267 }288 }
268 }289 }
269 }290 }
...@@ -426,9 +447,7 @@ export class VoiceInputController {...@@ -426,9 +447,7 @@ export class VoiceInputController {
426 );447 );
427 try {448 try {
428 await this.dependencies.recorder.stop();449 await this.dependencies.recorder.stop();
429 this.rememberRecordingUri(this.dependencies.recorder.uri);
430 } catch {450 } catch {
431 this.rememberRecordingUri(this.dependencies.recorder.uri);
432 } finally {451 } finally {
433 await this.releaseResources();452 await this.releaseResources();
434 }453 }
...@@ -436,7 +455,9 @@ export class VoiceInputController {...@@ -436,7 +455,9 @@ export class VoiceInputController {
436455
437 private async releaseResources(): Promise<void> {456 private async releaseResources(): Promise<void> {
438 this.rememberRecordingUri(this.recordingUri);457 this.rememberRecordingUri(this.recordingUri);
439 this.rememberRecordingUri(this.dependencies.recorder.uri);458 try {
459 this.rememberRecordingUri(this.dependencies.recorder.uri);
460 } catch {}
440 this.recordingUri = null;461 this.recordingUri = null;
441 for (const uri of this.ownedRecordingUris) {462 for (const uri of this.ownedRecordingUris) {
442 try {463 try {
...@@ -470,6 +491,7 @@ export class VoiceInputController {...@@ -470,6 +491,7 @@ export class VoiceInputController {
470491
471 private invalidateOperation(): void {492 private invalidateOperation(): void {
472 this.operationToken += 1;493 this.operationToken += 1;
494 if (this.sessionToken) this.sessionToken.abandoned = true;
473 this.transcriptionAbortController?.abort();495 this.transcriptionAbortController?.abort();
474 }496 }
475497