authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 09:52:04-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 09:52:04-07:00
logc71ca063dfcd8c2e3dcb2bd92adbb2180cbbece3
tree402bc6d998175fe7a43fac2ba3142f33e7ab813c
parent1c33f3b2b2976b7f4e26698ed69c549de687a97e
signature Signed by PGP key B5690EEEBB952194

perf(mobile): defer audio recorder creation until dictation (#15248)

3 files changed, 307 insertions(+), 15 deletions(-)

apps/mobile/src/features/voice-input/VoiceInputProvider.tsx+38-15
...@@ -1,9 +1,10 @@...@@ -1,9 +1,10 @@
1import {1import {
2 AudioModule,
2 RecordingPresets,3 RecordingPresets,
3 requestRecordingPermissionsAsync,4 requestRecordingPermissionsAsync,
4 setAudioModeAsync,5 setAudioModeAsync,
5 setIsAudioActiveAsync,6 setIsAudioActiveAsync,
6 useAudioRecorder,7 type RecorderState,
7 type RecordingStatus,8 type RecordingStatus,
8} from "expo-audio";9} from "expo-audio";
9import { File } from "expo-file-system";10import { File } from "expo-file-system";
...@@ -18,7 +19,7 @@ import {...@@ -18,7 +19,7 @@ import {
18 useState,19 useState,
19 type ReactNode,20 type ReactNode,
20} from "react";21} from "react";
21import { AppState } from "react-native";22import { AppState, Platform } from "react-native";
22import { useSharedValue } from "react-native-reanimated";23import { useSharedValue } from "react-native-reanimated";
2324
24import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";25import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
...@@ -28,14 +29,23 @@ import {...@@ -28,14 +29,23 @@ import {
28 voiceInputBlocksSubmission,29 voiceInputBlocksSubmission,
29 type VoiceInputState,30 type VoiceInputState,
30} from "@t3tools/client-runtime/voice-input";31} from "@t3tools/client-runtime/voice-input";
32import { createLazyVoiceRecorder, type LazyVoiceRecorder } from "./lazyVoiceRecorder";
31import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";33import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
32import { VoiceInputSession } from "./voiceInputSession";34import { VoiceInputSession } from "./voiceInputSession";
3335
34const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };36const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
35const VOICE_METERING_INTERVAL_MS = 80;37const VOICE_METERING_INTERVAL_MS = 80;
38// The native constructor takes platform-flattened options, as `useAudioRecorder`
39// builds them with expo-audio's internal `createRecordingOptions`.
40const { ios: IOS_RECORDING_OPTIONS, android: ANDROID_RECORDING_OPTIONS } =
41 RecordingPresets.HIGH_QUALITY;
36const VOICE_RECORDING_OPTIONS = {42const VOICE_RECORDING_OPTIONS = {
37 ...RecordingPresets.HIGH_QUALITY,43 extension: RecordingPresets.HIGH_QUALITY.extension,
44 sampleRate: RecordingPresets.HIGH_QUALITY.sampleRate,
45 numberOfChannels: RecordingPresets.HIGH_QUALITY.numberOfChannels,
46 bitRate: RecordingPresets.HIGH_QUALITY.bitRate,
38 isMeteringEnabled: true,47 isMeteringEnabled: true,
48 ...(Platform.OS === "ios" ? IOS_RECORDING_OPTIONS : ANDROID_RECORDING_OPTIONS),
39};49};
4050
41async function releaseVoiceRecordingAudio(): Promise<void> {51async function releaseVoiceRecordingAudio(): Promise<void> {
...@@ -102,17 +112,22 @@ function useVoiceInputRuntime() {...@@ -102,17 +112,22 @@ function useVoiceInputRuntime() {
102 const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));112 const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
103 const audioLevels = useSharedValue(audioLevelsRef.current);113 const audioLevels = useSharedValue(audioLevelsRef.current);
104 const sessionRef = useRef<VoiceInputSession | null>(null);114 const sessionRef = useRef<VoiceInputSession | null>(null);
105 const handleRecorderStatus = useCallback((status: RecordingStatus) => {115 const recorderRef = useRef<LazyVoiceRecorder<RecorderState> | null>(null);
106 sessionRef.current?.controller.handleRecorderStatus({116
107 isFinished: status.isFinished,117 if (!sessionRef.current || !recorderRef.current) {
108 hasError: status.hasError || status.mediaServicesDidReset === true,118 // The native recorder is created when dictation starts, not on app launch.
109 error: status.error,119 const recorder = createLazyVoiceRecorder({
110 url: status.url,120 create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
121 onStatus: (status: RecordingStatus) => {
122 sessionRef.current?.controller.handleRecorderStatus({
123 isFinished: status.isFinished,
124 hasError: status.hasError || status.mediaServicesDidReset === true,
125 error: status.error,
126 url: status.url,
127 });
128 },
111 });129 });
112 }, []);130 recorderRef.current = recorder;
113 const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus);
114
115 if (!sessionRef.current) {
116 sessionRef.current = new VoiceInputSession({131 sessionRef.current = new VoiceInputSession({
117 recorder,132 recorder,
118 getTranscriber: getLocalVoiceTranscriber,133 getTranscriber: getLocalVoiceTranscriber,
...@@ -134,6 +149,7 @@ function useVoiceInputRuntime() {...@@ -134,6 +149,7 @@ function useVoiceInputRuntime() {
134149
135 const session = sessionRef.current;150 const session = sessionRef.current;
136 const controller = session.controller;151 const controller = session.controller;
152 const recorder = recorderRef.current;
137153
138 useEffect(() => {154 useEffect(() => {
139 const subscription = AppState.addEventListener("change", (nextState) => {155 const subscription = AppState.addEventListener("change", (nextState) => {
...@@ -145,7 +161,14 @@ function useVoiceInputRuntime() {...@@ -145,7 +161,14 @@ function useVoiceInputRuntime() {
145 return () => subscription.remove();161 return () => subscription.remove();
146 }, [controller]);162 }, [controller]);
147163
148 useEffect(() => () => controller.dispose(), [controller]);164 useEffect(
165 () => () => {
166 // Dispose first so an active recording is stopped before the recorder is released.
167 controller.dispose();
168 recorder.release();
169 },
170 [controller, recorder],
171 );
149172
150 useEffect(() => {173 useEffect(() => {
151 if (state.phase !== "recording") return;174 if (state.phase !== "recording") return;
...@@ -175,7 +198,7 @@ function useVoiceInputRuntime() {...@@ -175,7 +198,7 @@ function useVoiceInputRuntime() {
175 const sampleRecording = () => {198 const sampleRecording = () => {
176 if (controller.currentState.phase !== "recording") return;199 if (controller.currentState.phase !== "recording") return;
177 const status = recorder.getStatus();200 const status = recorder.getStatus();
178 if (!status.isRecording) return;201 if (!status?.isRecording) return;
179202
180 const level = normalizeVoiceInputDecibels(status.metering);203 const level = normalizeVoiceInputDecibels(status.metering);
181 const history = audioLevelsRef.current;204 const history = audioLevelsRef.current;
apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts created+181
...@@ -0,0 +1,181 @@
1import { describe, expect, it, vi } from "vite-plus/test";
2
3import { VoiceInputController } from "@t3tools/client-runtime/voice-input";
4
5import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
6
7type Status = { readonly isFinished: boolean };
8type State = { readonly isRecording: boolean };
9
10function deferred() {
11 let resolve!: () => void;
12 const promise = new Promise<void>((resolvePromise) => {
13 resolve = resolvePromise;
14 });
15 return { promise, resolve };
16}
17
18function createNativeRecorder(uri: string) {
19 const listeners = new Set<(status: Status) => void>();
20 return {
21 uri: null as string | null,
22 prepared: uri,
23 prepareToRecordAsync: vi.fn(async function (this: { uri: string | null; prepared: string }) {
24 this.uri = this.prepared;
25 }),
26 record: vi.fn(),
27 stop: vi.fn(async (): Promise<void> => undefined),
28 getStatus: vi.fn((): State => ({ isRecording: true })),
29 addListener: vi.fn((_event: "recordingStatusUpdate", listener: (status: Status) => void) => {
30 listeners.add(listener);
31 return { remove: () => listeners.delete(listener) };
32 }),
33 release: vi.fn(),
34 emit: (status: Status) => listeners.forEach((listener) => listener(status)),
35 };
36}
37
38function createHarness() {
39 const created: Array<ReturnType<typeof createNativeRecorder>> = [];
40 const statuses: Status[] = [];
41 const recorder = createLazyVoiceRecorder({
42 create: () => {
43 const native = createNativeRecorder(`file:///voice-${created.length}.m4a`);
44 created.push(native);
45 return native;
46 },
47 onStatus: (status) => statuses.push(status),
48 });
49 return { recorder, created, statuses };
50}
51
52describe("createLazyVoiceRecorder", () => {
53 it("does not create a native recorder until a recording prepares", async () => {
54 const { recorder, created } = createHarness();
55
56 expect(recorder.uri).toBeNull();
57 expect(recorder.getStatus()).toBeNull();
58 await recorder.stop();
59 recorder.release();
60 expect(created).toHaveLength(0);
61
62 await recorder.prepareToRecordAsync();
63 recorder.record({ forDuration: 300 });
64
65 expect(created).toHaveLength(1);
66 expect(created[0]!.record).toHaveBeenCalledWith({ forDuration: 300 });
67 expect(recorder.uri).toBe("file:///voice-0.m4a");
68 expect(recorder.getStatus()).toEqual({ isRecording: true });
69 });
70
71 it("reuses one native recorder across recordings", async () => {
72 const { recorder, created } = createHarness();
73
74 await recorder.prepareToRecordAsync();
75 await recorder.stop();
76 await recorder.prepareToRecordAsync();
77
78 expect(created).toHaveLength(1);
79 expect(created[0]!.prepareToRecordAsync).toHaveBeenCalledTimes(2);
80 });
81
82 it("forwards status events until the recorder is released", async () => {
83 const { recorder, created, statuses } = createHarness();
84 await recorder.prepareToRecordAsync();
85 const native = created[0]!;
86
87 native.emit({ isFinished: true });
88 recorder.release();
89 native.emit({ isFinished: false });
90
91 expect(statuses).toEqual([{ isFinished: true }]);
92 });
93
94 it("releases only after a pending stop settles and keeps the final uri", async () => {
95 const { recorder, created } = createHarness();
96 await recorder.prepareToRecordAsync();
97 const native = created[0]!;
98 const stopped = deferred();
99 native.stop.mockReturnValueOnce(stopped.promise);
100
101 const stopping = recorder.stop();
102 recorder.release();
103 await Promise.resolve();
104 expect(native.release).not.toHaveBeenCalled();
105 expect(recorder.uri).toBe("file:///voice-0.m4a");
106
107 stopped.resolve();
108 await stopping;
109 await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1));
110 expect(recorder.uri).toBe("file:///voice-0.m4a");
111 expect(recorder.getStatus()).toBeNull();
112 });
113
114 it("releases after a failed stop and creates a fresh recorder for the next start", async () => {
115 const { recorder, created } = createHarness();
116 await recorder.prepareToRecordAsync();
117 const first = created[0]!;
118 first.stop.mockRejectedValueOnce(new Error("not recording"));
119
120 await expect(recorder.stop()).rejects.toThrow("not recording");
121 recorder.release();
122 await vi.waitFor(() => expect(first.release).toHaveBeenCalledTimes(1));
123 expect(() => recorder.record({ forDuration: 1 })).toThrow("not prepared");
124
125 await recorder.prepareToRecordAsync();
126 expect(created).toHaveLength(2);
127 expect(recorder.uri).toBe("file:///voice-1.m4a");
128 });
129
130 describe("with the voice input controller", () => {
131 function createController(granted: boolean) {
132 const harness = createHarness();
133 const deleted: string[] = [];
134 const controller = new VoiceInputController({
135 recorder: harness.recorder,
136 getTranscriber: () => ({
137 prepare: async () => ({ locale: "en-US", transcribe: async () => "text" }),
138 }),
139 requestPermission: async () => ({ granted, canAskAgain: true }),
140 configureRecording: async () => undefined,
141 releaseRecording: async () => undefined,
142 deleteRecording: (uri) => deleted.push(uri),
143 readDraft: () => ({
144 ownerKey: "thread",
145 text: "",
146 selection: { start: 0, end: 0 },
147 revision: 0,
148 }),
149 commitDraft: () => undefined,
150 onStateChange: () => undefined,
151 });
152 return { ...harness, controller, deleted };
153 }
154
155 it("does not create a native recorder when microphone permission is denied", async () => {
156 const { controller, created } = createController(false);
157
158 await controller.start();
159
160 expect(controller.currentState.phase).toBe("error");
161 expect(created).toHaveLength(0);
162 });
163
164 it("stops, deletes, and releases the recorder created by a cancelled recording", async () => {
165 const { controller, recorder, created, deleted } = createController(true);
166
167 await controller.start();
168 expect(controller.currentState.phase).toBe("recording");
169 expect(created).toHaveLength(1);
170 const native = created[0]!;
171
172 controller.cancel();
173 recorder.release();
174 await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1));
175
176 expect(native.stop).toHaveBeenCalledTimes(1);
177 expect(deleted).toEqual(["file:///voice-0.m4a"]);
178 expect(controller.currentState.phase).toBe("idle");
179 });
180 });
181});
apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts created+88
...@@ -0,0 +1,88 @@
1import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input";
2
3type Subscription = { remove(): void };
4
5// The subset of expo-audio's native `AudioRecorder` the voice input hook uses.
6export interface NativeVoiceRecorder<TStatus, TRecorderState> {
7 readonly uri: string | null;
8 prepareToRecordAsync(): Promise<void>;
9 record(options: { readonly forDuration: number }): void;
10 stop(): Promise<void>;
11 getStatus(): TRecorderState;
12 addListener(
13 eventName: "recordingStatusUpdate",
14 listener: (status: TStatus) => void,
15 ): Subscription;
16 release(): void;
17}
18
19export interface LazyVoiceRecorder<TRecorderState> extends VoiceRecorder {
20 /** Status of the native recorder, or null before a recording has prepared one. */
21 getStatus(): TRecorderState | null;
22 /**
23 * Releases the native recorder after its pending prepare and stop calls settle.
24 * A later recording creates a new one, so replayed effects keep working.
25 */
26 release(): void;
27}
28
29/**
30 * Creates expo-audio's native `AudioRecorder` when a recording first prepares,
31 * instead of on composer render. Construction blocks the JS thread for tens of
32 * milliseconds on a cold composer, and most composer mounts never record.
33 */
34export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
35 readonly create: () => NativeVoiceRecorder<TStatus, TRecorderState>;
36 readonly onStatus: (status: TStatus) => void;
37}): LazyVoiceRecorder<TRecorderState> {
38 let recorder: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
39 let subscription: Subscription | null = null;
40 // A released recorder still answers `uri` so the controller can delete its file.
41 let retired: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
42 let retiredUri: string | null = null;
43 let pending: Promise<unknown> = Promise.resolve();
44
45 const track = <T>(promise: Promise<T>): Promise<T> => {
46 pending = Promise.allSettled([pending, promise]).then(() => undefined);
47 return promise;
48 };
49
50 return {
51 get uri() {
52 if (recorder) return recorder.uri;
53 return retired ? retired.uri : retiredUri;
54 },
55 prepareToRecordAsync() {
56 if (!recorder) {
57 recorder = input.create();
58 subscription = recorder.addListener("recordingStatusUpdate", input.onStatus);
59 }
60 return track(recorder.prepareToRecordAsync());
61 },
62 record(options) {
63 if (!recorder) throw new Error("Voice recorder is not prepared.");
64 recorder.record(options);
65 },
66 stop() {
67 return recorder ? track(recorder.stop()) : Promise.resolve();
68 },
69 getStatus() {
70 return recorder ? recorder.getStatus() : null;
71 },
72 release() {
73 const released = recorder;
74 if (!released) return;
75 recorder = null;
76 subscription?.remove();
77 subscription = null;
78 retired = released;
79 void pending.then(() => {
80 if (retired === released) {
81 retiredUri = released.uri;
82 retired = null;
83 }
84 released.release();
85 });
86 },
87 };
88}